docs: validate attachment lifecycle contract

.env.example

@@ -76,24 +76,12 @@ SEARXNG_INSTANCE=http://localhost:8080
 # Change this if another local service already uses 7000 (macOS AirPlay often does).
 # APP_PORT=7000
 
-# Optional HTTP address advertised in companion/mobile pairing codes. Set this
-# when Docker would otherwise advertise a container address or loopback. Use a
-# LAN or Tailscale IPv4 address, a single-label hostname, or an mDNS *.local
-# name that the phone can reach. HTTPS and public hostnames are not supported
-# by the current companion client. Do not include credentials, a path, query,
-# or fragment.
-# COMPANION_BASE_URL=http://192.168.1.50:7000
-
 # Development-only auth bypass for loopback requests.
 # Keep false for Docker, LAN, reverse proxy, and any shared deployment.
 # LOCALHOST_BYPASS=false
 
-# Mark session cookies Secure. Left unset, this follows the request scheme:
+# Mark session cookies Secure. Set true when Odysseus is served through HTTPS
-# an HTTPS login gets a Secure cookie, a plain-HTTP one does not. Set true to
+# by a trusted reverse proxy or private access gateway.
-# force it on, or false to force it off while you still serve plain HTTP.
-# Upgrading: this used to default to false. Drop a leftover SECURE_COOKIES=false
-# from your .env unless you still need that escape hatch — it keeps HTTPS logins
-# on a non-Secure cookie.
 # SECURE_COOKIES=true
 
 # Optional: pre-seed the first admin password during setup.
@@ -163,21 +151,6 @@ SEARXNG_INSTANCE=http://localhost:8080
 # Local HTTP setups may use the callback URL inferred by the application.
 # GOOGLE_OAUTH_REDIRECT_URI=https://your-domain.com/api/email/oauth/google/callback
 
-# Origin the MCP OAuth callback is sent back to, for remote (Streamable HTTP)
-# MCP servers that register it dynamically. Defaults to http://localhost:$APP_PORT,
-# which is right only when you reach Odysseus directly on that port. Set it for
-# HTTPS, reverse-proxy, hosted, and Docker installs — inside the container the
-# app always listens on 7000 and cannot see the host port map, so the default is
-# wrong there whenever APP_PORT is not 7000.
-#
-# Not for Google MCP servers. Those use Desktop App credentials, and Google only
-# accepts loopback redirect URIs for that client type, so a public origin here is
-# rejected with redirect_uri_mismatch. Leave it unset for a Google-only install:
-# the loopback default is what Google wants, and remote users finish through the
-# paste-back page, which never has to load the redirect.
-# https://developers.google.com/identity/protocols/oauth2/native-app
-# OAUTH_REDIRECT_BASE_URL=https://your-domain.com
-
 # ============================================================
 # Misc
 # ============================================================
@@ -216,7 +189,6 @@ SEARXNG_INSTANCE=http://localhost:8080
 # ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=26214400       # email compose attachment (25 MB)
 # ODYSSEUS_STT_MAX_AUDIO_BYTES=26214400                  # speech-to-text audio (25 MB)
 # ODYSSEUS_ICS_MAX_BYTES=10485760                        # calendar .ics import (10 MB)
-# ODYSSEUS_TTS_CACHE_MAX_BYTES=524288000                 # TTS cache (500 MB)
 
 # ============================================================
 # Host Docker access (explicit opt-in)

.gitattributes

@@ -15,13 +15,6 @@ docker/entrypoint.sh text eol=lf
 *.cmd text eol=crlf
 *.bat text eol=crlf
 
-# Vendored third-party bundles in static/lib/ are published minified artifacts
-# and must stay byte-identical to what npm ships — stripping trailing whitespace
-# to satisfy `git diff --check` would desync them from the upstream release. Turn
-# the whitespace check off for that tree instead, and keep the bundles out of
-# GitHub's language statistics.
-static/lib/** -whitespace linguist-vendored
-
 # Binary assets — never normalize.
 *.png  binary
 *.jpg  binary

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -26,18 +26,6 @@ body:
         - label: I am running the latest code from the `dev` branch (the default branch you get on clone, where fixes land first) and the bug still reproduces there. Please `git pull` the latest `dev` before filing.
           required: true
 
-  - type: input
-    id: revision
-    attributes:
-      label: Odysseus Revision
-      description: |
-        From the repository root (on the host when using Docker), run
-        `git show -s --abbrev=12 --format='%h (%cs)' HEAD`
-        and paste the output exactly.
-      placeholder: "1fef4929cf1d (2026-08-11)"
-    validations:
-      required: true
-
   - type: dropdown
     id: install-method
     attributes:

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -8,8 +8,8 @@ body:
       value: |
         **Before submitting:** search [open issues](https://github.com/odysseus-dev/odysseus/issues)
         and [discussions](https://github.com/odysseus-dev/odysseus/discussions) first.
-        Feature requests that duplicate [ROADMAP.md](https://github.com/odysseus-dev/odysseus/blob/main/ROADMAP.md)
+        The [roadmap](https://github.com/odysseus-dev/odysseus/blob/main/ROADMAP.md) is directional rather than a complete backlog.
-        or an existing open issue will be closed as duplicates.
+        Feature requests that duplicate an existing issue or accepted proposal may be closed as duplicates.
 
         If your idea needs community input before it becomes a concrete proposal,
         start a [discussion](https://github.com/odysseus-dev/odysseus/discussions/categories/ideas) instead.

.github/pull_request_template.md

@@ -28,7 +28,6 @@ Fixes #
 - [ ] This PR targets `dev`
 - [ ] My changes are limited to the scope described above — no unrelated refactors or whitespace changes mixed in.
 - [ ] I actually ran the app (`docker compose up` or `uvicorn app:app`) and verified the change works end-to-end. Type-checks and unit tests are not enough.
-- [ ] I did not run the app/runtime validation and stated that gap in **How to Test**. Leave this unchecked when the app-run box above is checked.
 
 ## How to Test
 

.github/scripts/check-issue-description.js

@@ -41,14 +41,6 @@ module.exports = async ({ github, context, core }) => {
       break;
 
     case 'bug': {
-      const revisionText = section('Odysseus Revision');
-      if (!/^[0-9a-f]{12} \(\d{4}-\d{2}-\d{2}\)$/i.test(revisionText)) {
-        failures.push(
-          '**Odysseus Revision** — paste the 12-character commit SHA and date, ' +
-          'for example `1fef4929cf1d (2026-08-11)`',
-        );
-      }
-
       if (!section('Install Method')) {
         failures.push('**Install Method** — select how you installed Odysseus');
       }
@@ -161,16 +153,6 @@ module.exports = async ({ github, context, core }) => {
     }
   }
 
-  const LABEL_BAD  = 'needs more info';
-  const LABEL_GOOD = 'ready for review';
-
-  // Closed issues are no longer awaiting review.
-  // This also prevents later edits to closed issues from restoring the label.
-  if (issue.state === 'closed') {
-    await dropLabel(LABEL_GOOD);
-    return;
-  }
-
   // ── Find existing bot comment to update in-place ──────────────────────────
   const MARKER = '<!-- issue-description-check -->';
   const { data: comments } = await github.rest.issues.listComments({
@@ -178,6 +160,9 @@ module.exports = async ({ github, context, core }) => {
   });
   const existing = comments.find(c => c.user.type === 'Bot' && c.body.includes(MARKER));
 
+  const LABEL_BAD  = 'needs more info';
+  const LABEL_GOOD = 'ready for review';
+
   if (failures.length === 0) {
     if (existing) {
       await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id });

.github/scripts/check-pr-description.js

@@ -21,11 +21,11 @@ module.exports = async ({ github, context, core }) => {
     return strip(m?.[0].replace(new RegExp(`#+\\s+${heading}`, 'i'), '') ?? '');
   }
 
-  const descriptionProblems = [];
+  const problems = [];
 
   // 1. Summary must be filled in.
   if (section('Summary').length < 20) {
-    descriptionProblems.push('**Summary** is empty or too short — describe what changed and why.');
+    problems.push('**Summary** is empty or too short — describe what changed and why.');
   }
 
   // 2. Linked Issue must reference a real issue. Accept a bare #NNN, a closing
@@ -34,18 +34,18 @@ module.exports = async ({ github, context, core }) => {
   const linkedSection = section('Linked Issue');
   const hasIssueRef = /#\d+\b/.test(linkedSection) || /\/issues\/\d+/.test(linkedSection);
   if (!linkedSection || !hasIssueRef) {
-    descriptionProblems.push('**Linked Issue** — add a reference like `Fixes #NNN`, a bare `#NNN`, or a link to the issue.');
+    problems.push('**Linked Issue** — add a reference like `Fixes #NNN`, a bare `#NNN`, or a link to the issue.');
   }
 
   // 3. At least one Type of Change box must be checked.
   const typeBlock = body.match(/##\s+Type of Change[\s\S]*?(?=\n##\s|$)/i)?.[0] ?? '';
   if (!/- \[x\]/i.test(typeBlock)) {
-    descriptionProblems.push('**Type of Change** — check at least one box.');
+    problems.push('**Type of Change** — check at least one box.');
   }
 
   // 4. Duplicate-search checklist item must be checked.
   if (!/- \[x\] I searched/i.test(body)) {
-    descriptionProblems.push('**Checklist** — check the duplicate-search box to confirm you searched existing issues and PRs.');
+    problems.push('**Checklist** — check the duplicate-search box to confirm you searched existing issues and PRs.');
   }
 
   // 5. How to Test must contain enough real detail for a reviewer to act on.
@@ -53,83 +53,7 @@ module.exports = async ({ github, context, core }) => {
   //    code block — so we only require non-trivial content, not a specific shape.
   const howTo = section('How to Test');
   if (howTo.length < 30) {
-    descriptionProblems.push('**How to Test** — explain how a reviewer can verify this change. Numbered steps, the commands you ran, or a short code block all work — give a sentence or two of real detail (not just "tested locally").');
+    problems.push('**How to Test** — explain how a reviewer can verify this change. Numbered steps, the commands you ran, or a short code block all work — give a sentence or two of real detail (not just "tested locally").');
-  }
-
-  // Classify paths from GitHub's API. This workflow runs in the privileged base
-  // context, so it must never check out or execute code from the PR branch.
-  const changedFiles = await github.paginate(github.rest.pulls.listFiles, {
-    owner, repo, pull_number: prNum, per_page: 100,
-  });
-  const changedPaths = changedFiles.map(file => file.filename);
-
-  function isUiSensitivePath(filename) {
-    const path = filename.toLowerCase();
-    return path.startsWith('static/')
-      || path.startsWith('templates/')
-      || /\.(?:html?|css|svg)$/.test(path);
-  }
-
-  function isDocsOnlyPath(filename) {
-    const path = filename.toLowerCase();
-    return /\.(?:md|mdx|rst|adoc|txt)$/.test(path)
-      || (path.startsWith('docs/') && !isUiSensitivePath(path));
-  }
-
-  function isRuntimeSensitivePath(filename) {
-    const path = filename.toLowerCase();
-    if (isUiSensitivePath(path)) return false;
-    if (path.startsWith('tests/') || path.startsWith('.github/')) return false;
-    return /^(?:app\.py|routes\/|services\/|src\/|core\/|mcp_servers\/|scripts\/|docker\/)/.test(path)
-      || /^(?:dockerfile|docker-compose.*\.ya?ml|requirements(?:-optional)?\.txt|pyproject\.toml|setup\.py)$/.test(path)
-      || /\.(?:py|sh|ps1|bat)$/.test(path);
-  }
-
-  let classification = 'tooling';
-  if (changedPaths.some(isUiSensitivePath)) {
-    classification = 'UI-sensitive';
-  } else if (changedPaths.some(isRuntimeSensitivePath)) {
-    classification = 'backend/runtime';
-  } else if (changedPaths.length > 0 && changedPaths.every(isDocsOnlyPath)) {
-    classification = 'docs-only';
-  }
-
-  const appRan = /- \[x\]\s+I actually ran the app\b/i.test(body);
-  const appNotRun = /- \[x\]\s+I did not run the app\/runtime validation\b/i.test(body);
-  // Anchor on the wording, not the template's emphasis: a ticked box the author
-  // retyped without the surrounding ** renders identically on the PR page, so
-  // treating it as unchecked is invisible from their side. Matches the two
-  // attestations above, which already ignore formatting.
-  const screenshotChecked = /- \[x\]\s+[*_]{0,2}Screenshot or short clip[*_]{0,2}/i.test(body);
-  const screenshotSection = section('Screenshots / clips');
-  const hasVisualEvidence = /!\[[^\]]*\]\([^)]+\)|<(?:img|video|source)\b[^>]*(?:src|href)=|https?:\/\/[^\s)]+/i.test(screenshotSection);
-  const evidenceGaps = [];
-  let needsRuntimeValidation = false;
-  let needsVisualEvidence = false;
-
-  if (classification === 'backend/runtime' || classification === 'UI-sensitive') {
-    if (appRan && appNotRun) {
-      needsRuntimeValidation = true;
-      evidenceGaps.push('The app-run and explicit not-run boxes are both checked. Select the one state that is true.');
-    } else if (!appRan) {
-      needsRuntimeValidation = true;
-      if (appNotRun) {
-        evidenceGaps.push('The author explicitly reports that app/runtime validation was not performed.');
-      } else {
-        evidenceGaps.push('App/runtime validation is not author-attested. Check the run box only after running it, or check the explicit not-run box and describe the gap.');
-      }
-    }
-  }
-
-  if (classification === 'UI-sensitive') {
-    if (!screenshotChecked) {
-      needsVisualEvidence = true;
-      evidenceGaps.push('The screenshot/clip checkbox is not checked for this UI-sensitive change.');
-    }
-    if (!hasVisualEvidence) {
-      needsVisualEvidence = true;
-      evidenceGaps.push('The Screenshots / clips section does not contain an actual attachment or link.');
-    }
   }
 
   // ── Comment ──────────────────────────────────────────────────────────────
@@ -138,43 +62,22 @@ module.exports = async ({ github, context, core }) => {
   });
   const existing = comments.find(c => (c.body ?? '').includes(MARKER));
 
-  if (descriptionProblems.length === 0 && evidenceGaps.length === 0) {
+  if (problems.length === 0) {
     if (existing) {
       await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id });
     }
   } else {
-    const commentLines = [MARKER];
+    const commentBody = [
-    if (descriptionProblems.length > 0) {
+      MARKER,
-      commentLines.push(
+      '⚠️ **PR description — action needed**',
-        '⚠️ **PR description — action needed**',
+      '',
-        '',
+      'The following required sections are missing or incomplete. Please update the PR description to address them:',
-        'The following required sections are missing or incomplete. Please update the PR description to address them:',
+      '',
-        '',
+      problems.map(p => `- ${p}`).join('\n'),
-        descriptionProblems.map(problem => `- ${problem}`).join('\n'),
-      );
-    } else {
-      commentLines.push(
-        '⚠️ **PR description is complete; validation evidence is still outstanding**',
-        '',
-        `Changed-file classification: **${classification}**.`,
-      );
-    }
-    if (evidenceGaps.length > 0) {
-      commentLines.push(
-        '',
-        '**Author-reported runtime / visual state**',
-        '',
-        evidenceGaps.map(gap => `- ${gap}`).join('\n'),
-        '',
-        'Checkboxes are author attestations. GitHub Actions results remain the execution evidence for CI; this check does not prove that a local command ran.',
-      );
-    }
-    commentLines.push(
       '',
       '---',
-      '_This comment updates automatically when the description or changed files change._',
+      '_This comment is deleted automatically once all sections are complete._',
-    );
+    ].join('\n');
-    const commentBody = commentLines.join('\n');
 
     if (existing) {
       await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body: commentBody });
@@ -194,47 +97,34 @@ module.exports = async ({ github, context, core }) => {
       return true;
     } catch (e) {
       if (e.status === 404) return false;
-      if (e.status === 403) {
-        core.warning(`Could not inspect label "${name}" — token lacks label read access; skipping.`);
-        return false;
-      }
       throw e;
     }
   }
 
-  async function setLabel(name, wanted) {
+  async function swapLabel(num, add, remove) {
-    if (wanted && await labelExists(name)) {
+    if (await labelExists(add)) {
       try {
-        await github.rest.issues.addLabels({ owner, repo, issue_number: prNum, labels: [name] });
+        await github.rest.issues.addLabels({ owner, repo, issue_number: num, labels: [add] });
       } catch (e) {
         // Fail soft on a token that can't write labels so a label permission
         // problem never masks the actual description verdict.
-        if (e.status !== 403 && e.status !== 404) throw e;
+        if (e.status !== 403) throw e;
-        core.warning(`Could not add "${name}" — label is unavailable or the token lacks label write access; skipping.`);
+        core.warning(`Could not add "${add}" — token lacks label write here; skipping.`);
       }
-    } else if (wanted) {
-      core.warning(`Label "${name}" does not exist in the repo — skipping. Create it once to enable labelling.`);
     } else {
-      try {
+      core.warning(`Label "${add}" does not exist in the repo — skipping. Create it once to enable labelling.`);
-        await github.rest.issues.removeLabel({ owner, repo, issue_number: prNum, name });
+    }
-      } catch (e) {
+    try {
-        if (e.status !== 404 && e.status !== 410 && e.status !== 403) throw e;
+      await github.rest.issues.removeLabel({ owner, repo, issue_number: num, name: remove });
-      }
+    } catch (e) {
+      if (e.status !== 404 && e.status !== 410 && e.status !== 403) throw e;
     }
   }
 
-  const descriptionComplete = descriptionProblems.length === 0;
+  if (problems.length === 0) {
-  const evidenceComplete = evidenceGaps.length === 0;
+    await swapLabel(prNum, 'ready for review', 'needs work');
-  const isDraft = Boolean(context.payload.pull_request.draft);
+  } else {
-  await setLabel(
+    await swapLabel(prNum, 'needs work', 'ready for review');
-    'ready for review',
+    core.setFailed(`PR description has ${problems.length} issue(s) — see bot comment for details.`);
-    descriptionComplete && evidenceComplete && !isDraft,
-  );
-  await setLabel('needs work', !descriptionComplete);
-  await setLabel('needs runtime validation', needsRuntimeValidation);
-  await setLabel('needs visual evidence', needsVisualEvidence);
-
-  if (!descriptionComplete) {
-    core.setFailed(`PR description has ${descriptionProblems.length} issue(s) — see bot comment for details.`);
   }
 };

.github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   push:
-    branches: [main, dev]
+    branches: [main]
   pull_request:
 
 # Least privilege: none of the jobs write to the repo.
@@ -103,7 +103,10 @@ jobs:
   python-tests:
     name: Python tests (pytest)
     runs-on: ubuntu-latest
-    # Make Python test validation authoritative for the configured scope.
+    # Informational for now: the suite has known flaky / environment-dependent
+    # failures (test isolation + embedding-model assertions). Tracked under the
+    # ROADMAP "fresh install smoke tests" item; make this required once green.
+    continue-on-error: true
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:

.github/workflows/issue-description-check.yml

@@ -2,7 +2,7 @@ name: ci / issue description check
 
 on:
   issues:
-    types: [opened, edited, reopened, closed]
+    types: [opened, edited, reopened]
 
 permissions:
   issues: write

.github/workflows/pr-description-check.yml

@@ -5,11 +5,7 @@ on:
   # works on fork PRs. Safe here: the checkout pins to the base branch (no fork
   # code runs) and the scripts only read context.payload and call the GitHub API.
   pull_request_target:  # zizmor: ignore[dangerous-triggers]
-    types: [opened, edited, synchronize, reopened, ready_for_review, converted_to_draft]
+    types: [opened, edited, synchronize, reopened, ready_for_review]
-
-concurrency:
-  group: pr-description-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
 
 # Default-deny at the workflow level; each job opts into only the scopes it needs.
 # Note: modifying a PR's labels/comments needs pull-requests:write even though the
@@ -63,14 +59,12 @@ jobs:
 
   check-mergeable:
     name: Flag unmergeable PRs
-    needs: check-description
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
       issues: write
-    # Run after description validation failures, but never from an obsolete
+    # Skip bots: they open PRs programmatically and have their own process.
-    # workflow run canceled by a newer PR event.
+    if: github.event.pull_request.user.type != 'Bot'
-    if: ${{ !cancelled() && github.event.pull_request.user.type != 'Bot' }}
     steps:
       - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3  # v9.0.0
         with:

ACKNOWLEDGMENTS.md

@@ -65,16 +65,6 @@ Vendored in `static/lib/` and served directly:
 | [jsPDF](https://github.com/parallax/jsPDF) (bundled in html2pdf) | PDF generation | MIT |
 | [html2canvas](https://github.com/niklasvh/html2canvas) (bundled in html2pdf) | DOM → canvas rasterization | MIT |
 | [node-qrcode](https://github.com/soldair/node-qrcode) (`qrcode.min.js`) | QR-code rendering (2FA setup) | MIT |
-| [KaTeX](https://github.com/KaTeX/KaTeX) v0.16.22 (`katex/katex.min.{js,css}` + `katex/fonts/*.woff2`) | Math typesetting | MIT ([`licenses/KaTeX-MIT-LICENSE.txt`](licenses/KaTeX-MIT-LICENSE.txt)) |
-| [Mermaid](https://github.com/mermaid-js/mermaid) v11.16.1 (`mermaid.min.js`) | Diagrams from text | MIT ([`licenses/Mermaid-MIT-LICENSE.txt`](licenses/Mermaid-MIT-LICENSE.txt)) |
-
-KaTeX and Mermaid are loaded on first use by `static/js/markdown.js` rather than
-from `index.html`, so a session that renders no math and no diagram never fetches
-either. Only the `.woff2` KaTeX fonts are shipped, matching `static/fonts/`; the
-`.woff` and `.ttf` variants its stylesheet also lists are never requested by a
-browser that supports `woff2`. The bundles are the published npm artifacts,
-unmodified — `.gitattributes` turns the whitespace check off for `static/lib/`
-so they can stay byte-identical to upstream.
 
 ## Front-end libraries loaded at runtime (CDN)
 
@@ -82,6 +72,8 @@ Referenced from `cdn.jsdelivr.net` / `cdnjs.cloudflare.com` at runtime — not v
 
 | Library | Purpose | License |
 |---|---|---|
+| [KaTeX](https://github.com/KaTeX/KaTeX) 0.16.22 | Math typesetting | MIT |
+| [Mermaid](https://github.com/mermaid-js/mermaid) 11 | Diagrams from text | MIT |
 | [Pyodide](https://github.com/pyodide/pyodide) 0.27.5 | In-browser Python runtime | MPL-2.0 |
 | [PDFObject](https://github.com/pipwerks/PDFObject) 2.1.1 | Inline PDF embedding | MIT |
 

README.md

@@ -1,3 +1,5 @@
+# Odysseus
+
 <p align="center">
   <img src="docs/odysseus-wordmark.png" alt="Odysseus" width="238">
 </p>
@@ -9,6 +11,8 @@
 <p align="center">
   <a href="#quick-start">Quick Start</a> ·
   <a href="docs/setup.md">Setup Guide</a> ·
+  <a href="docs/ARCHITECTURE.md">Architecture</a> ·
+  <a href="SECURITY.md">Security</a> ·
   <a href="CONTRIBUTING.md">Contributing</a> ·
   <a href="ROADMAP.md">Roadmap</a>
 </p>
@@ -51,31 +55,26 @@ Native installs, GPU notes, Windows/macOS instructions, HTTPS, and configuration
 
 ## Demo
 
-A full hover-to-play tour lives on the landing page: [`docs/index.html`](docs/index.html).
+Explore the interface through the [interactive product tour](docs/index.html).
 
 ## Contributing
 
-Help is welcome. The best entry points are fresh-install testing, provider setup bugs, mobile/editor polish, docs, and small focused refactors. See [CONTRIBUTING.md](CONTRIBUTING.md) and [ROADMAP.md](ROADMAP.md).
+Help is welcome. The best entry points are fresh-install testing, provider setup bugs, mobile/editor polish, documentation, and small focused refactors. Read the [contributing guide](CONTRIBUTING.md), review the [public roadmap](ROADMAP.md), and browse the open [GitHub issues](https://github.com/odysseus-dev/odysseus/issues).
 
 ## Security
 
-Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled, keep private data out of Git, and do not expose raw model/service ports publicly.
+Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled, keep private data out of Git, and do not expose raw model or service ports publicly. Read the [security policy](SECURITY.md) and the [deployment security guidance](docs/setup.md#security-notes).
-
-- Keep `AUTH_ENABLED=true` for any network-accessible deployment.
-- Keep `LOCALHOST_BYPASS=false` outside local development.
-
-Deployment details are in the [setup guide](docs/setup.md#security-notes).
 
 ## Star History
 
-<a href="https://star-history.dera.page/#odysseus-dev/odysseus&type=date&legend=top-left">
+<a href="https://www.star-history.com/?repos=odysseus-dev%2Fodysseus&type=date&legend=top-left">
  <picture>
-   <source media="(prefers-color-scheme: dark)" srcset="https://star-history.dera.page/svg?repos=odysseus-dev/odysseus&type=date&theme=dark&legend=top-left" />
+   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=odysseus-dev/odysseus&type=date&theme=dark&legend=top-left" />
-   <source media="(prefers-color-scheme: light)" srcset="https://star-history.dera.page/svg?repos=odysseus-dev/odysseus&type=date&legend=top-left" />
+   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=odysseus-dev/odysseus&type=date&legend=top-left" />
-   <img alt="Star History Chart" src="https://star-history.dera.page/svg?repos=odysseus-dev/odysseus&type=date&legend=top-left" />
+   <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=odysseus-dev/odysseus&type=date&legend=top-left" />
  </picture>
 </a>
 
 ## License
 
-AGPL-3.0-or-later -- see [LICENSE](LICENSE) and [ACKNOWLEDGMENTS.md](ACKNOWLEDGMENTS.md).
+Licensed under AGPL-3.0-or-later. See the [license](LICENSE) and [acknowledgments](ACKNOWLEDGMENTS.md).

ROADMAP.md

@@ -1,87 +1,55 @@
-# Roadmap / Help Wanted
+# Roadmap
 
-Odysseus is on a voyage, but not home yet. It works great for me (lol), but this ship is moving fast and feedback/help would be appreciated! (I don't know what I'm doing, help).
+This document provides a high-level view of the areas Odysseus is currently improving.
 
-If you see weird CSS, strange layout behavior, or a suspiciously murky corner of
+It is directional rather than exhaustive. Priorities may change as the project evolves, defects are discovered, and maintainers learn more from implementation work and user feedback.
-the codebase, you are probably right to stay away.
 
-## High Priority
+For current implementation work, see the open [GitHub issues](https://github.com/odysseus-dev/odysseus/issues). Accepted behaviour should be documented in the repository alongside the code.
 
-- SQUASH BUGS
+## Current priorities
-- Fresh install smoke tests on Linux, macOS, and Windows. Docker, native Python,
-  and WSL all need coverage.
 
-- Integration audit: do integrations even work? Confirm what works, what needs setup docs, and what should be removed or hidden. 
+### Reliability and setup
-- Cookbook reliability on other computers. This is probably the area most likely to need work across different machines, GPUs, drivers, shells, and Python environments.
-- Cookbook SGLang support across platforms. Make sure SGLang setup/serve works
-  predictably on Linux, Windows/WSL, macOS where possible, Docker, and common
-  NVIDIA/AMD hardware paths.
-- Deep Research model presets by hardware. Recommend approved model/parameter
-  profiles for small, medium, and large local setups so people with different
-  hardware can use Deep Research without guessing. Surface this either in Deep
-  Research settings or as a Cookbook scan/dropdown suggestion.
-- Cookbook model scan/download ranking. Prioritize newer architectures and
-  better hardware-fit models instead of scoring everything almost the same.
-  Ranking should account for architecture age, quant format, VRAM/RAM fit,
-  backend support, vision/mmproj requirements, and likely serve reliability.
-- Cookbook error feedback and logging. Failed downloads, dependency installs,
-  preflights, and serve jobs should show the actual command/output/error in the
-  UI, with copyable logs and clear next steps instead of just "crashed".
-- Agent prompt/context bloat. Agent mode is too heavy for smaller local models:
-  tool schemas, skills, memory, documents, and instructions can eat the context
-  before the user request really starts. We need slimmer prompts, better tool
-  selection, smaller default tool sets, and clearer guidance for models with
-  4k/8k/16k context windows.
-- Local model speculative decoding support. For Odysseus-tuned local models,
-  plan to ship or recommend a small same-tokenizer draft model when the serving
-  backend supports it. Early vLLM testing showed a generic `Qwen3-0.6B` draft
-  beside `Qwen3-8B` can materially reduce wall time, while an unsupported
-  DSpark conversion performed poorly. Treat this as a supported draft-model lane
-  first; keep MTP-specific packaging as future work only when the architecture
-  and runtime support are real. Judge this by time-to-success, tool correctness,
-  grammar, and unchanged target output, not tokens/sec alone.
-- Skill/tool prompt-injection audit. User-editable skills, notes, documents,
-  fetched pages, and memories should be treated as untrusted data. Keep testing
-  whether models follow malicious instructions from those surfaces.
-- Better degraded-state reporting for ChromaDB, SearXNG, email, ntfy, and provider probes.
-- Email performance audit. Fetching, searching, opening, deleting, and sending
-  email can feel slow, especially over IMAP/SMTP providers with high latency.
-  Need someone who knows mail performance to profile the current flow, identify
-  whether the bottleneck is IMAP folder select/fetch, cache invalidation,
-  attachment/body loading, SMTP handshakes, or frontend refresh behavior, then
-  propose safer caching/prefetch/batching without breaking multi-account state.
-- Provider setup/probing audit for Anthropic, Gemini, Groq, xAI, OpenRouter, OpenAI, and DeepSeek.
 
-## Refactor Targets
+- Improve fresh-install and smoke-test coverage across supported environments.
-- CSS cleanup. `static/style.css` basically Calypso's island atm.
+- Make provider setup, probing, and failure states more predictable.
-- Tour core helper. The onboarding tours have too much copy-pasted scaffolding; promote a shared `tour-core.js` helper before adding more tours.
+- Improve Cookbook reliability across hardware, operating systems, drivers, shells, and serving backends.
-- Modal/window positioning cleanup. Some window controls have improved, but the
+- Improve degraded-state reporting and recovery guidance when optional services are unavailable.
-  underlying popup/dropdown/fixed-position behavior is still too fragile.
-- Mobile media override discoverability. A lot of "CSS did not move" bugs are mobile `@media` overrides of the same selector; comments or linting around desktop/mobile paired rules would help.
-- Dead code pass for old routes, stale feature flags, and unused UI states.
 
-## Frontend
+### Local model workflows
 
-- Expand the Editor for quicker, more robust everyday use. Better file/document
+- Improve hardware-aware model recommendations and compatibility guidance.
-  handling, smoother window behavior, clearer save/export flows, stronger image
+- Evaluate serving optimizations, including speculative decoding, through reproducible benchmarks.
-  editing affordances, and fewer brittle edge cases.
+- Improve installation, preflight checks, logging, and error reporting for local model serving.
-- Better AI integration for Notes and Todos. Notes should be easier for the
+- Reduce prompt and context overhead for smaller local models.
-  agent to read, update, summarize, and turn into actions. Todos should be
-  assignable to an agent from the UI, possibly through a button, task action,
-  or dedicated skill/tool flow.
-- Mobile gallery/editor polish. Easier to launch/download inpaint model or any missing pieces.
-- Accessibility pass: keyboard navigation, focus states, contrast, reduced motion.
-- Improve empty states and error messages on fresh installs.
-- Tighten first-run setup, hints, and tours so they do not repeat or fight each other.
-- Vendor CDN assets eventually for a more fully self-hosted/offline mode.
 
-## Backend
+### Safety and resilience
 
-- More tests around endpoint probing and provider setup.
+- Continue hardening tool execution, filesystem access, credentials, networking, and destructive operations.
-- Better task scheduler defaults and visibility.
+- Treat content from documents, notes, memories, skills, and fetched pages as potentially untrusted.
-- Backup/restore guide and helper flow for `data/`.
+- Improve security-focused regression coverage and operational guidance.
-- Security hardening around admin-only tools and clear docs for their risk.
+- Review integrations that expand access to sensitive data or privileged operations.
 
-## Not The Focus Right Now
+### Product usability
 
-I prob shouldnt add more themes.
+- Improve first-run setup, onboarding, hints, and tours.
+- Improve accessibility, keyboard navigation, focus behaviour, contrast, and reduced-motion support.
+- Improve empty states, error messages, and recovery paths.
+- Strengthen Notes, Todos, Editor, mobile, and everyday workspace flows.
+
+### Architecture and maintainability
+
+- Reduce duplication and technical debt through focused, reviewable refactors.
+- Improve subsystem documentation as behaviour and architecture become stable.
+- Remove stale code, obsolete feature flags, and unsupported integrations.
+- Keep implementation decisions grounded in current code and verified behaviour.
+
+## Tracking work
+
+Concrete implementation tasks, defects, proposals, and technical investigations are tracked in:
+
+- [GitHub Issues](https://github.com/odysseus-dev/odysseus/issues)
+- [Contributing Guide](CONTRIBUTING.md)
+
+Maintainers may use additional private coordination tools for ownership, planning, and unresolved decisions.
+
+This roadmap is not a complete backlog or a guarantee that a particular item will be delivered.

SECURITY.md

@@ -10,7 +10,7 @@ Security fixes are handled on the default branch until formal releases are cut.
 
 - Keep `AUTH_ENABLED=true` for any network-accessible deployment.
 - Keep `LOCALHOST_BYPASS=false` outside local development.
-- Leave `SECURE_COOKIES` unset unless you need to override it: session cookies are marked `Secure` whenever the request arrives over HTTPS. Set `SECURE_COOKIES=true` to force it on (for a proxy Odysseus cannot see the scheme of), or `SECURE_COOKIES=false` to force it off while you still serve plain HTTP alongside HTTPS.
+- Set `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway.
 - Use HTTPS when exposing the app beyond localhost.
 - Put the authenticated Odysseus web/API entrypoint behind a trusted reverse proxy or private access layer such as Cloudflare Access, Tailscale, or a VPN.
 - Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only.
@@ -37,4 +37,6 @@ Only `.env.example`, docs, source, tests, and static assets should be committed.
 
 ## Reporting
 
-Please report vulnerabilities privately via GitHub security advisories if available, or by opening a minimal issue that does not disclose exploit details.
+Report security vulnerabilities privately through [GitHub Security Advisories](https://github.com/odysseus-dev/odysseus/security/advisories/new).
+
+Do not open a public issue or discussion, and do not disclose exploit details publicly.

THREAT_MODEL.md

@@ -37,7 +37,7 @@ Non-admin defaults are in `core/auth.py:DEFAULT_PRIVILEGES`. Tool enforcement is
 
 - **Sessions:** bcrypt passwords, 7-day session tokens stored atomically in `data/sessions.json` via `core/atomic_io.py`.
 - **2FA:** TOTP with 8 single-use backup codes. Verified after password check, before session issuance.
-- **Reserved usernames:** request sentinels and the Default/Local storage owner cannot be registered or renamed into. Defined in `core/auth.py:RESERVED_USERNAMES`.
+- **Reserved usernames:** `internal-tool`, `api`, `demo`, `system` cannot be registered or renamed into. Defined in `core/auth.py:RESERVED_USERNAMES`.
   - `internal-tool` is security-critical: `core/middleware.py:require_admin` treats any request where `request.state.current_user == "internal-tool"` as the in-process tool loopback and grants admin unconditionally. A real account with that name would silently pass every `require_admin` check.
 - **Orphan sessions:** `validate_token` re-checks that the user record still exists on every call. A deleted user's cookie is dropped on next request rather than continuing to authenticate.
 
@@ -68,14 +68,14 @@ External content that reaches the LLM is treated as untrusted via `src/prompt_se
 - `X-Content-Type-Options: nosniff` and `Referrer-Policy: no-referrer` everywhere.
 - **CSP:** nonce-based `script-src 'self' 'nonce-{nonce}' https://cdn.jsdelivr.net`. `style-src 'unsafe-inline'` is intentionally kept — `static/index.html` ships inline `<style>` blocks and JS modules set `style=""` attributes at runtime. Inline styles do not execute script so the risk is visual-only. Removing this requires templating the HTML files and auditing all JS-set style attributes.
 
+## Token-Supplied Model Endpoints
+
+Direct `/api/v1/chat` requests with a token-supplied `base_url` must use a public HTTP(S) endpoint. This restriction applies only to untrusted direct values; administrator-configured endpoints may intentionally use local or LAN URLs for private model providers.
+
 ## Known Gaps
 
 These are open, acknowledged, and contributor help is welcome:
 
 1. **No shell/filesystem sandbox.** The agent `bash` and `read_file`/`write_file` tools run as the app process user with no network egress filtering or filesystem confinement. A successful prompt-injection reaching a shell-enabled admin session can make outbound requests to internal services. See #1058 for the sandbox proposal.
 
-2. **SSRF via `/api/v1/chat` `base_url` parameter.** A chat-scoped API token can supply an arbitrary `base_url`; the server forwards the LLM request to that host without validating the scheme or address. PR #1039 fixes this.
+2. **Token scopes are coarse.** There is no way to grant a session a subset of the owning user's privileges. Companion/mobile tokens carry either `chat` or `admin` scope with no per-capability granularity.
-
-3. **`src/search/` partial consolidation.** `src.search.core` and `src.search.providers` correctly alias `services.search` via `sys.modules` replacement. `analytics`, `cache`, `content`, `query`, and `ranking` are still independent copies that can drift. The SSRF regression tests in `tests/test_webhook_ssrf_resilience.py` test `src.webhook_manager` directly (separate from search), so the safety net there is intact. See #1058.
-
-4. **Token scopes are coarse.** There is no way to grant a session a subset of the owning user's privileges. Companion/mobile tokens carry either `chat` or `admin` scope with no per-capability granularity.

app.py

@@ -67,13 +67,7 @@ from core.constants import (
     REQUEST_TIMEOUT, OPENAI_API_KEY, AUTH_FILE,
 )
 from core.database import SessionLocal, ApiToken
-from core.middleware import (
+from core.middleware import SecurityHeadersMiddleware, is_cors_preflight
-    SecurityHeadersMiddleware,
-    get_application_route_path,
-    is_cors_preflight,
-    path_is_route_or_child,
-    with_asgi_root_path,
-)
 from core.auth import AuthManager, normalize_known_username
 from core.exceptions import (
     SessionNotFoundError, InvalidFileUploadError,
@@ -84,7 +78,6 @@ import bcrypt as _bcrypt
 
 from src.app_helpers import abs_join, serve_html_with_nonce
 from src.generated_images import GENERATED_IMAGE_HEADERS, resolve_generated_image_path
-from src.owner_identity import auth_disabled
 from starlette.responses import RedirectResponse
 
 # ========= LOGGING =========
@@ -255,7 +248,7 @@ from routes.auth_routes import setup_auth_routes, SESSION_COOKIE
 
 auth_manager = AuthManager()
 app.state.auth_manager = auth_manager
-AUTH_ENABLED = not auth_disabled()
+AUTH_ENABLED = os.getenv("AUTH_ENABLED", "true").lower() != "false"
 LOCALHOST_BYPASS = os.getenv("LOCALHOST_BYPASS", "false").lower() == "true"
 if LOCALHOST_BYPASS:
     logger.warning("LOCALHOST_BYPASS is enabled, loopback requests bypass authentication. Do not expose this instance to a network.")
@@ -291,7 +284,7 @@ if AUTH_ENABLED:
     def _is_auth_exempt(path: str) -> bool:
         if path in AUTH_EXEMPT_EXACT:
             return True
-        if any(path_is_route_or_child(path, p) for p in AUTH_EXEMPT_PREFIXES):
+        if any(path.startswith(p) for p in AUTH_EXEMPT_PREFIXES):
             return True
         return any(p.match(path) for p in AUTH_EXEMPT_PATTERNS)
 
@@ -362,7 +355,7 @@ if AUTH_ENABLED:
 
     class AuthMiddleware(BaseHTTPMiddleware):
         async def dispatch(self, request: Request, call_next):
-            path = get_application_route_path(request.scope)
+            path = request.url.path
             # A genuine CORS preflight (OPTIONS + Access-Control-Request-Method)
             # carries no credentials by design and must reach CORSMiddleware to be
             # answered. AuthMiddleware is the outermost middleware, so gating the
@@ -406,10 +399,7 @@ if AUTH_ENABLED:
             if not auth_manager.is_configured:
                 # No users yet — redirect to login for first-time setup
                 if not path.startswith("/api/"):
-                    return RedirectResponse(
+                    return RedirectResponse(url="/login", status_code=302)
-                        url=with_asgi_root_path(request.scope, "/login"),
-                        status_code=302,
-                    )
                 return JSONResponse(status_code=401, content={"error": "Setup required"})
 
             # --- Bearer token auth (API tokens for external integrations) ---
@@ -471,10 +461,7 @@ if AUTH_ENABLED:
             if not auth_manager.validate_token(token):
                 if path.startswith("/api/"):
                     return JSONResponse(status_code=401, content={"error": "Not authenticated"})
-                return RedirectResponse(
+                return RedirectResponse(url="/login", status_code=302)
-                    url=with_asgi_root_path(request.scope, "/login"),
-                    status_code=302,
-                )
 
             # Attach current username to request state for downstream routes
             request.state.current_user = auth_manager.get_username_for_token(token)
@@ -643,24 +630,13 @@ app.include_router(auth_router)
 
 @app.post("/api/activity/heartbeat")
 async def activity_heartbeat():
-    from src.interactive_gate import (
+    from src.interactive_gate import mark_browser_activity
-        mark_browser_activity,
-        maybe_stop_background_tasks_for_heartbeat,
-    )
-
     await mark_browser_activity()
-
     async def _stop_background():
         try:
-            await maybe_stop_background_tasks_for_heartbeat(
+            await task_scheduler.stop_background_tasks_for_foreground(reason="browser heartbeat")
-                task_scheduler.stop_background_tasks_for_foreground
-            )
         except Exception:
-            logging.getLogger("app.foreground_gate").debug(
+            logging.getLogger("app.foreground_gate").debug("heartbeat task stop failed", exc_info=True)
-                "heartbeat task stop failed",
-                exc_info=True,
-            )
-
     asyncio.create_task(_stop_background())
     return {"ok": True}
 
@@ -716,7 +692,7 @@ from routes.history.history_routes import setup_history_routes
 app.include_router(setup_history_routes(session_manager, upload_handler=upload_handler))
 
 # Search
-from routes.search.search_routes import setup_search_routes
+from routes.search_routes import setup_search_routes
 app.include_router(setup_search_routes(config))
 
 # Presets
@@ -763,7 +739,7 @@ app.include_router(setup_stt_routes(stt_service))
 logger.info("STT service initialized (provider managed via settings)")
 
 # Documents (artifacts/canvas)
-from routes.document.document_routes import setup_document_routes
+from routes.document_routes import setup_document_routes
 document_router = setup_document_routes(session_manager, upload_handler)
 app.include_router(document_router)
 
@@ -784,7 +760,7 @@ from src.task_scheduler import TaskScheduler
 task_scheduler = TaskScheduler(session_manager)
 from src.event_bus import set_task_scheduler
 set_task_scheduler(task_scheduler)
-from routes.task.task_routes import setup_task_routes
+from routes.task_routes import setup_task_routes
 app.include_router(setup_task_routes(task_scheduler))
 
 from routes.assistant_routes import setup_assistant_routes
@@ -829,7 +805,7 @@ app.include_router(setup_font_routes())
 # MCP (Model Context Protocol)
 from src.mcp_manager import McpManager
 from src.agent_tools import set_mcp_manager
-from routes.mcp.mcp_routes import setup_mcp_routes
+from routes.mcp_routes import setup_mcp_routes
 
 mcp_manager = McpManager()
 set_mcp_manager(mcp_manager)
@@ -844,7 +820,7 @@ set_ai_rag_manager(rag_manager, personal_docs_mgr)
 logger.info("AI interaction tools initialized (session, memory, RAG, UI control)")
 
 # Webhooks
-from routes.webhook.webhook_routes import setup_webhook_routes
+from routes.webhook_routes import setup_webhook_routes
 app.include_router(setup_webhook_routes(webhook_manager, auth_manager, session_manager, api_key_manager))
 
 # API Tokens
@@ -876,7 +852,7 @@ app.include_router(setup_codex_routes(
 ))
 app.include_router(setup_claude_routes())
 
-from routes.vault.vault_routes import setup_vault_routes
+from routes.vault_routes import setup_vault_routes
 app.include_router(setup_vault_routes())
 
 # Contacts (CardDAV)

build-macos-app.sh

@@ -73,10 +73,6 @@ cat > "$APP/Contents/MacOS/$APP_NAME.tmpl" <<'LAUNCHER'
 INSTALL_DIR="__INSTALL_DIR__"
 PORT="__PORT__"
 URL="http://127.0.0.1:${PORT}"
-# uvicorn is started with --port below, but APP_PORT is what the app itself
-# reads when it needs to build a URL for this instance (internal_api_base(),
-# companion pairing, the MCP OAuth callback), so export it as well.
-export APP_PORT="$PORT"
 export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$PATH"
 
 UVICORN="$INSTALL_DIR/venv/bin/uvicorn"

companion/pairing.py

@@ -6,14 +6,11 @@ units so the route layer stays thin and the logic is directly testable.
 
 from __future__ import annotations
 
-import ipaddress
 import json
 import os
-import re
 import secrets
 import socket
 import uuid
-from urllib.parse import urlsplit
 
 import bcrypt
 
@@ -23,102 +20,6 @@ PAIRING_VERSION = 1
 COMPANION_SCOPE = "chat"
 
 
-_COMPANION_IPV4_NETWORKS = tuple(
-    ipaddress.ip_network(cidr)
-    for cidr in (
-        "10.0.0.0/8",
-        "100.64.0.0/10",
-        "127.0.0.0/8",
-        "169.254.0.0/16",
-        "172.16.0.0/12",
-        "192.168.0.0/16",
-    )
-)
-_DNS_LABEL_RE = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\Z")
-
-
-def _valid_companion_client_host(host: str) -> bool:
-    """Match the host forms supported by the current v1 Expo client."""
-    if not host or len(host) > 253 or not host.isascii() or "%" in host:
-        return False
-
-    try:
-        address = ipaddress.ip_address(host)
-    except ValueError:
-        labels = host.split(".")
-        if any(not _DNS_LABEL_RE.fullmatch(label) for label in labels):
-            return False
-        if any(label.startswith("xn--") for label in labels):
-            return False
-        # WHATWG URL parsers treat a decimal or ``0x`` single-label hostname
-        # as an IPv4 number even though Python's strict ``ipaddress`` parser
-        # rejects that spelling.  The v1 client interpolates this host back
-        # into a URL, so accepting e.g. ``134744072`` would make the phone send
-        # its bearer token to public 8.8.8.8.  Keep DNS labels unambiguous.
-        if len(labels) == 1 and (
-            labels[0].isdigit()
-            or re.fullmatch(r"0x[0-9a-f]*", labels[0]) is not None
-        ):
-            return False
-        return len(labels) == 1 or (len(labels) >= 2 and labels[-1] == "local")
-
-    return isinstance(address, ipaddress.IPv4Address) and any(
-        address in network for network in _COMPANION_IPV4_NETWORKS
-    )
-
-
-def parse_companion_base_url(value: str) -> tuple[str, int]:
-    """Validate a v1 companion address and return its legacy (host, port).
-
-    The deployed client understands only HTTP plus a LAN-style host and port.
-    Reject anything outside that exact contract instead of advertising a URL
-    the client would reject, downgrade, or interpret differently.
-    """
-    if not isinstance(value, str) or not value:
-        raise ValueError("COMPANION_BASE_URL must be a canonical HTTP LAN origin")
-    if not value.isascii():
-        raise ValueError("COMPANION_BASE_URL must contain only ASCII characters")
-    if any(
-        ord(char) <= 32 or ord(char) == 127 or char in {"\\", "%"}
-        for char in value
-    ):
-        raise ValueError(
-            "COMPANION_BASE_URL contains a forbidden character"
-        )
-
-    try:
-        parsed = urlsplit(value)
-        port = parsed.port
-    except ValueError as exc:
-        raise ValueError("COMPANION_BASE_URL must be a valid HTTP LAN origin") from exc
-
-    host = parsed.hostname
-    if parsed.scheme.lower() != "http" or not parsed.netloc or not host:
-        raise ValueError("COMPANION_BASE_URL must be a canonical HTTP LAN origin")
-    if parsed.username is not None or parsed.password is not None:
-        raise ValueError("COMPANION_BASE_URL must not contain credentials")
-    if parsed.path or parsed.query or parsed.fragment:
-        raise ValueError("COMPANION_BASE_URL must not contain a path, query, or fragment")
-    if port is not None and not 1 <= port <= 65535:
-        raise ValueError("COMPANION_BASE_URL port must be between 1 and 65535")
-    if not _valid_companion_client_host(host):
-        raise ValueError("COMPANION_BASE_URL host is not supported by companion v1")
-
-    netloc = f"{host}:{port}" if port is not None else host
-    origin = f"http://{netloc}"
-    if value != origin:
-        raise ValueError("COMPANION_BASE_URL must be a canonical HTTP LAN origin")
-    return host, port or 80
-
-
-def configured_companion_origin() -> tuple[str, int] | None:
-    """Return the validated operator-configured v1 address, if any."""
-    value = os.environ.get("COMPANION_BASE_URL")
-    if value is None or value == "":
-        return None
-    return parse_companion_base_url(value)
-
-
 def default_port() -> int:
     """Best guess at the port the server is reachable on. Callers that know the
     real request port should pass it explicitly."""

companion/routes.py

@@ -23,7 +23,7 @@ from fastapi import APIRouter, HTTPException, Request
 from fastapi.responses import HTMLResponse
 
 from core.middleware import require_admin
-from src.auth_helpers import _auth_disabled, get_current_user
+from src.auth_helpers import get_current_user
 
 from companion import pairing as _pairing
 
@@ -113,9 +113,8 @@ def setup_companion_routes() -> APIRouter:
         The stock /api/models route scopes to get_current_user, which for a
         bearer token is the sandboxed pseudo-user "api" (owns nothing). Here we
         scope to the token's real owner instead, plus legacy null-owner shared
-        rows -- the same rule as owner_filter. Explicit auth-disabled mode keeps
+        rows -- the same rule as owner_filter. Read-only; never returns api_key
-        the stock route's single-user all-endpoints view. Read-only; never
+        material.
-        returns api_key material.
         """
         require_models_scope(request)
         import json as _json
@@ -124,11 +123,6 @@ def setup_companion_routes() -> APIRouter:
         from src.endpoint_resolver import build_chat_url
 
         owner = token_owner(request)
-        single_user_mode = (
-            owner is None
-            and not getattr(request.state, "api_token", False)
-            and _auth_disabled()
-        )
         out = []
         db = SessionLocal()
         try:
@@ -139,7 +133,7 @@ def setup_companion_routes() -> APIRouter:
             if owner:
                 q = q.filter((ModelEndpoint.owner == owner) | (ModelEndpoint.owner == None))  # noqa: E711
             for ep in q.all():
-                if not single_user_mode and not owner_can_see(ep.owner, owner):
+                if not owner_can_see(ep.owner, owner):
                     continue
                 try:
                     model_ids = _json.loads(ep.cached_models) if ep.cached_models else []
@@ -200,27 +194,19 @@ def setup_companion_routes() -> APIRouter:
         the code works immediately, no restart. `?format=json` returns the
         payload for an in-app pairing screen."""
         require_admin(request)
-        try:
-            configured_origin = _pairing.configured_companion_origin()
-        except ValueError as exc:
-            raise HTTPException(500, str(exc)) from None
         owner = get_current_user(request)
         invalidate = getattr(request.app.state, "invalidate_token_cache", None)
         token_id, raw_token = mint_pairing_token(owner, invalidate)
 
-        if configured_origin:
+        hosts = _pairing.lan_ip_candidates()
-            host, port = configured_origin
+        host = hosts[0] if hosts else "127.0.0.1"
-            hosts = [host]
+        port = request.url.port or _pairing.default_port()
-        else:
-            hosts = _pairing.lan_ip_candidates()
-            host = hosts[0] if hosts else "127.0.0.1"
-            port = request.url.port or _pairing.default_port()
         payload = _pairing.pairing_payload(host, port, raw_token)
         qr = _pairing.pairing_qr_png_data_uri(payload)
         qr_ok = bool(qr and qr.startswith("data:image/png;base64,"))
 
         if (request.query_params.get("format") or "").lower() == "json":
-            response = {
+            return {
                 "host": host,
                 "port": port,
                 "token": raw_token,
@@ -229,7 +215,6 @@ def setup_companion_routes() -> APIRouter:
                 "payload": payload,
                 "qr": qr if qr_ok else None,
             }
-            return response
 
         import json as _json
         payload_json = _json.dumps(payload, separators=(",", ":"))

core/atomic_io.py

@@ -15,53 +15,31 @@ from __future__ import annotations
 
 import json
 import os
-import uuid
 from typing import Any, Optional
 
 
 def atomic_write_json(path: str, data: Any, *, indent: Optional[int] = None) -> None:
     """Atomically persist `data` as JSON at `path`.
 
-    The temp file uses a random suffix so two concurrent writers saving the
+    The temp file uses the live PID as a suffix so two processes saving the
-    same file don't collide on the rename target. A PID suffix does not do
+    same file (e.g. unit tests) don't collide on the rename target.
-    this: the PID is constant for the life of a process, so two writers on
-    the same path within one process (or one single-process container, where
-    the PID never changes at all) still race for the same temp file.
     """
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
-    tmp = f"{path}.tmp.{uuid.uuid4().hex}"
+    tmp = f"{path}.tmp.{os.getpid()}"
-
+    with open(tmp, "w", encoding="utf-8") as f:
-    try:
+        json.dump(data, f, indent=indent)
-        with open(tmp, "w", encoding="utf-8") as f:
+        f.flush()
-            json.dump(data, f, indent=indent)
+        os.fsync(f.fileno())
-            f.flush()
+    os.replace(tmp, path)
-            os.fsync(f.fileno())
-        os.replace(tmp, path)
-    finally:
-        # Directly unlink to avoid a check-then-act race condition.
-        # Swallows FileNotFoundError (on success path) and other cleanup OSErrors.
-        try:
-            os.unlink(tmp)
-        except OSError:
-            pass
 
 
 def atomic_write_text(path: str, text: str) -> None:
     if not isinstance(text, str):
         raise TypeError("atomic_write_text expects a string")
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
-    tmp = f"{path}.tmp.{uuid.uuid4().hex}"
+    tmp = f"{path}.tmp.{os.getpid()}"
-
+    with open(tmp, "w", encoding="utf-8") as f:
-    try:
+        f.write(text)
-        with open(tmp, "w", encoding="utf-8") as f:
+        f.flush()
-            f.write(text)
+        os.fsync(f.fileno())
-            f.flush()
+    os.replace(tmp, path)
-            os.fsync(f.fileno())
-        os.replace(tmp, path)
-    finally:
-        # Directly unlink to avoid a check-then-act race condition.
-        # Swallows FileNotFoundError (on success path) and other cleanup OSErrors.
-        try:
-            os.unlink(tmp)
-        except OSError:
-            pass

core/auth.py

@@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
 
 
 from core.atomic_io import atomic_write_json as _atomic_write_json  # noqa: E402
+from core.middleware import INTERNAL_TOOL_USER  # noqa: E402
 
 DEFAULT_PRIVILEGES = {
     "can_use_agent": True,
@@ -48,18 +49,24 @@ ADMIN_PRIVILEGES["allowed_models_restricted"] = False
 ADMIN_PRIVILEGES["block_all_models"] = False
 
 from src.constants import AUTH_FILE, PASSWORD_MIN_LENGTH
-from src.owner_identity import RESERVED_AUTH_USERNAMES
 DEFAULT_AUTH_PATH = AUTH_FILE
 TOKEN_TTL = 60 * 60 * 24 * 7  # 7 days
 
-# Usernames the auth + middleware layer reserves for request sentinels and
+# Usernames the auth + middleware layer reserve as internal "synthetic owner"
-# internal storage owners; they must never belong to a real login account.
+# sentinels; they must never belong to a real account. The most dangerous is
-# "internal-tool" is the most dangerous because `core.middleware.require_admin`
+# "internal-tool": `core.middleware.require_admin` treats any request whose
-# treats it as the in-process tool loopback. "api" collides with bearer-token
+# `current_user == "internal-tool"` as the in-process tool loopback and grants
-# attribution. "demo"/"system" are synthetic owners already special-cased by
+# admin, and because the cookie auth path sets `current_user` to the raw
-# scheduler/assistant/research paths. The Default/Local owner is a storage
+# username, an account literally named "internal-tool" would be silently
-# bucket for explicit auth-disabled no-login mode, not a login username.
+# treated as an admin by every `require_admin`-gated route. "api" collides with
-RESERVED_USERNAMES = frozenset(RESERVED_AUTH_USERNAMES)
+# the bearer-token owner-attribution sentinel. "demo"/"system" round out the
+# synthetic-owner set the rest of the codebase already special-cases (see
+# `_SYNTHETIC_OWNERS` in routes/assistant_routes.py and the matching guards in
+# src/task_scheduler.py / routes/research_routes.py) — a real account with one
+# of those names would be denied an assistant and inconsistently owner-scoped.
+# Refuse to create or rename into any of them so the sentinels can't be
+# impersonated. (Keep this in sync with that synthetic-owner set.)
+RESERVED_USERNAMES = frozenset({INTERNAL_TOOL_USER, "api", "demo", "system"})
 
 
 def normalize_known_username(users: Dict[str, Any], username: str | None) -> Optional[str]:

core/database.py

@@ -5,7 +5,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import Optional
 from urllib.parse import unquote, urlparse
-from sqlalchemy import DDL, event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, inspect, text
+from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text
 from sqlalchemy.engine import Engine, make_url
 from sqlalchemy.types import TypeDecorator
 from sqlalchemy.ext.declarative import declarative_base, declared_attr
@@ -430,93 +430,6 @@ class EmailAccount(TimestampMixin, Base):
     )
 
 
-class EmailAccountOwnerLock(Base):
-    """Durable per-owner mutex for email-account default mutations.
-
-    Row-locking databases serialize mutations by locking this row before they
-    inspect or stage EmailAccount changes.  SQLite uses ``BEGIN IMMEDIATE``
-    instead, because it ignores ``SELECT ... FOR UPDATE``; keeping the table in
-    the shared metadata still makes the non-SQLite path available without a
-    separate migration.  The empty key represents the normalized legacy /
-    unconfigured scope shared by ``owner IS NULL`` and ``owner = ''`` rows.
-    """
-    __tablename__ = "email_account_owner_locks"
-
-    owner_key = Column(String, primary_key=True)
-
-
-_EMAIL_ACCOUNT_DEFAULT_INDEX = "ux_email_accounts_one_default_per_owner"
-_EMAIL_ACCOUNT_DEFAULT_INDEX_DDL = {
-    "sqlite": (
-        f"CREATE UNIQUE INDEX IF NOT EXISTS {_EMAIL_ACCOUNT_DEFAULT_INDEX} "
-        "ON email_accounts (COALESCE(owner, '')) WHERE is_default = 1"
-    ),
-    "postgresql": (
-        f"CREATE UNIQUE INDEX IF NOT EXISTS {_EMAIL_ACCOUNT_DEFAULT_INDEX} "
-        "ON email_accounts ((COALESCE(owner, ''))) WHERE is_default IS TRUE"
-    ),
-}
-
-
-# SQLAlchemy cannot express one portable partial, functional index across the
-# two supported database families.  Register dialect-specific DDL so fresh
-# databases get the invariant as part of create_all(); the startup migration
-# below installs the same index on existing databases after normalizing legacy
-# duplicate rows.
-for _dialect_name, _index_ddl in _EMAIL_ACCOUNT_DEFAULT_INDEX_DDL.items():
-    event.listen(
-        EmailAccount.__table__,
-        "after_create",
-        DDL(_index_ddl).execute_if(dialect=_dialect_name),
-    )
-
-
-def lock_email_account_owner_mutations(db, *owners: str) -> None:
-    """Lock normalized email-account owner scopes in canonical order.
-
-    ``NULL`` and the empty string are one legacy/single-user owner partition,
-    matching the unique default-account index.  SQLite has only a database
-    writer reservation, while row-locking databases use durable mutex rows.
-    Sorting all requested owner keys keeps multi-owner operations such as user
-    rename from deadlocking with another mutation that requests the same keys
-    in the opposite order.
-    """
-    from sqlalchemy.exc import IntegrityError
-
-    owner_keys = sorted({owner or "" for owner in owners} or {""})
-    if db.get_bind().dialect.name == "sqlite":
-        db.execute(text("BEGIN IMMEDIATE"))
-        return
-
-    for owner_key in owner_keys:
-        lock_row = db.get(
-            EmailAccountOwnerLock,
-            owner_key,
-            with_for_update=True,
-        )
-        if lock_row is not None:
-            continue
-
-        inserted = False
-        try:
-            with db.begin_nested():
-                db.add(EmailAccountOwnerLock(owner_key=owner_key))
-                db.flush()
-                inserted = True
-        except IntegrityError:
-            # A competing transaction created the mutex row first.  Once its
-            # insert commits, lock that durable row before touching accounts.
-            pass
-
-        if not inserted:
-            (
-                db.query(EmailAccountOwnerLock)
-                .filter(EmailAccountOwnerLock.owner_key == owner_key)
-                .with_for_update()
-                .one()
-            )
-
-
 class ModelEndpoint(TimestampMixin, Base):
     """Admin-configured model endpoints. Models are auto-discovered via /v1/models."""
     __tablename__ = "model_endpoints"
@@ -1491,25 +1404,8 @@ def _migrate_assign_legacy_owner():
             with open(prefs_path, "r", encoding="utf-8") as f:
                 prefs = _json.load(f)
             if "_users" not in prefs and prefs:
-                # Flat format → nest ordinary preferences under the admin
+                # Flat format → nest under admin user
-                # user. Foreground fallback is an explicit per-owner opt-in,
+                new_prefs = {"_users": {admin_user: prefs}}
-                # so auth-disabled consent must remain inert at the flat root
-                # rather than becoming consent for the first named owner.
-                foreground_keys = {
-                    "foreground_fallback_enabled",
-                    "foreground_model_fallbacks",
-                }
-                named_prefs = {
-                    key: value
-                    for key, value in prefs.items()
-                    if key not in foreground_keys
-                }
-                new_prefs = {
-                    key: prefs[key]
-                    for key in foreground_keys
-                    if key in prefs
-                }
-                new_prefs["_users"] = {admin_user: named_prefs}
                 with open(prefs_path, "w", encoding="utf-8") as f:
                     _json.dump(new_prefs, f, indent=2)
                 logger.info(f"Migrated user_prefs.json to per-user format under '{admin_user}'")
@@ -1916,142 +1812,72 @@ class Integration(TimestampMixin, Base):
 
 
 
-def _migrate_email_account_default_invariant():
-    """Normalize legacy duplicates and install durable at-most-one enforcement.
-
-    Older databases only had a non-unique ``(owner, is_default)`` lookup index.
-    Keep the oldest default deterministically in each normalized owner scope,
-    then add the same partial functional unique index used for fresh schemas.
-    """
-    dialect_name = engine.dialect.name
-    index_ddl = _EMAIL_ACCOUNT_DEFAULT_INDEX_DDL.get(dialect_name)
-    if index_ddl is None:
-        logger.warning(
-            "Email-account default uniqueness is not available for database "
-            "dialect %s; mutations remain serialized but are not protected by "
-            "a database constraint",
-            dialect_name,
-        )
-        return
-
-    try:
-        with engine.begin() as conn:
-            if not inspect(conn).has_table(EmailAccount.__tablename__):
-                return
-            default_rows = conn.execute(text("""
-                SELECT id, owner
-                FROM email_accounts
-                WHERE is_default IS TRUE
-                ORDER BY
-                    COALESCE(owner, ''),
-                    CASE WHEN created_at IS NULL THEN 1 ELSE 0 END,
-                    created_at,
-                    id
-            """)).mappings()
-            seen_owner_keys = set()
-            duplicate_ids = []
-            for row in default_rows:
-                owner_key = row["owner"] or ""
-                if owner_key in seen_owner_keys:
-                    duplicate_ids.append(row["id"])
-                else:
-                    seen_owner_keys.add(owner_key)
-
-            for account_id in duplicate_ids:
-                conn.execute(
-                    text("UPDATE email_accounts SET is_default = :value WHERE id = :id"),
-                    {"value": False, "id": account_id},
-                )
-            conn.execute(text(index_ddl))
-
-        if duplicate_ids:
-            logger.warning(
-                "Normalized %d duplicate default email account(s) before "
-                "installing %s",
-                len(duplicate_ids),
-                _EMAIL_ACCOUNT_DEFAULT_INDEX,
-            )
-    except Exception:
-        # Starting without the constraint would silently retain the race this
-        # migration is intended to close.  Fail startup so an operator sees and
-        # can repair an incompatible schema instead of accepting unsafe writes.
-        logger.exception("Failed to enforce the email-account default invariant")
-        raise
-
-
 def _migrate_seed_email_account():
-    """Atomically seed one legacy default account when no account exists.
+    """If email_accounts is empty and settings.json has legacy flat imap_host/smtp_host
-
+    keys, create a single default account from them so nothing breaks for users who
-    Reading settings is intentionally done before taking the owner mutex.  The
+    upgraded. Safe to run repeatedly — it short-circuits once any row exists."""
-    decisive emptiness check and insert share one locked transaction, so two
-    application workers starting together cannot both seed a default row.
-    """
-    import json as _json
-    import uuid as _uuid
-
-    settings_file = Path(SETTINGS_FILE)
-    if not settings_file.exists():
-        return
     try:
-        s = _json.loads(settings_file.read_text(encoding="utf-8"))
+        with engine.connect() as conn:
-    except Exception:
+            tables = [r[0] for r in conn.execute(text(
-        return
+                "SELECT name FROM sqlite_master WHERE type='table' AND name='email_accounts'"
+            ))]
+            if "email_accounts" not in tables:
+                return
+            existing = conn.execute(text("SELECT COUNT(*) FROM email_accounts")).scalar() or 0
+            if existing > 0:
+                return
 
-    imap_host = (s.get("imap_host") or "").strip()
+        import json as _json
-    smtp_host = (s.get("smtp_host") or "").strip()
+        import uuid as _uuid
-    if not imap_host and not smtp_host:
+        from pathlib import Path
-        return
+        settings_file = Path(SETTINGS_FILE)
+        if not settings_file.exists():
+            return
+        try:
+            s = _json.loads(settings_file.read_text(encoding="utf-8"))
+        except Exception:
+            return
 
-    db = None
+        imap_host = (s.get("imap_host") or "").strip()
-    try:
+        smtp_host = (s.get("smtp_host") or "").strip()
-        if not inspect(engine).has_table(EmailAccount.__tablename__):
+        if not imap_host and not smtp_host:
-            return
+            return  # nothing to migrate
-        db = SessionLocal()
-        lock_email_account_owner_mutations(db, "")
-        existing = db.execute(text("SELECT COUNT(*) FROM email_accounts")).scalar() or 0
-        if existing > 0:
-            return
 
         now = utcnow_naive()
-        db.execute(text("""
+        with engine.begin() as conn:
-            INSERT INTO email_accounts
+            conn.execute(text("""
-              (id, owner, name, is_default, enabled,
+                INSERT INTO email_accounts
-               imap_host, imap_port, imap_user, imap_password, imap_starttls,
+                  (id, owner, name, is_default, enabled,
-               smtp_host, smtp_port, smtp_user, smtp_password,
+                   imap_host, imap_port, imap_user, imap_password, imap_starttls,
-               from_address, created_at, updated_at)
+                   smtp_host, smtp_port, smtp_user, smtp_password,
-            VALUES
+                   from_address, created_at, updated_at)
-              (:id, :owner, :name, :is_default, :enabled,
+                VALUES
-               :imap_host, :imap_port, :imap_user, :imap_password, :imap_starttls,
+                  (:id, :owner, :name, :is_default, :enabled,
-               :smtp_host, :smtp_port, :smtp_user, :smtp_password,
+                   :imap_host, :imap_port, :imap_user, :imap_password, :imap_starttls,
-               :from_address, :created_at, :updated_at)
+                   :smtp_host, :smtp_port, :smtp_user, :smtp_password,
-        """), {
+                   :from_address, :created_at, :updated_at)
-            "id": _uuid.uuid4().hex,
+            """), {
-            "owner": None,
+                "id": _uuid.uuid4().hex,
-            "name": "Default",
+                "owner": None,
-            "is_default": True,
+                "name": "Default",
-            "enabled": True,
+                "is_default": True,
-            "imap_host": imap_host,
+                "enabled": True,
-            "imap_port": int(s.get("imap_port") or 993),
+                "imap_host": imap_host,
-            "imap_user": s.get("imap_user") or "",
+                "imap_port": int(s.get("imap_port") or 993),
-            "imap_password": s.get("imap_password") or "",
+                "imap_user": s.get("imap_user") or "",
-            "imap_starttls": bool(s.get("imap_starttls", True)),
+                "imap_password": s.get("imap_password") or "",
-            "smtp_host": smtp_host,
+                "imap_starttls": bool(s.get("imap_starttls", True)),
-            "smtp_port": int(s.get("smtp_port") or 465),
+                "smtp_host": smtp_host,
-            "smtp_user": s.get("smtp_user") or "",
+                "smtp_port": int(s.get("smtp_port") or 465),
-            "smtp_password": s.get("smtp_password") or "",
+                "smtp_user": s.get("smtp_user") or "",
-            "from_address": s.get("email_from") or "",
+                "smtp_password": s.get("smtp_password") or "",
-            "created_at": now,
+                "from_address": s.get("email_from") or "",
-            "updated_at": now,
+                "created_at": now,
-        })
+                "updated_at": now,
-        db.commit()
+            })
-        logger.info("Seeded email_accounts 'Default' from settings.json")
+            logging.getLogger(__name__).info("Seeded email_accounts 'Default' from settings.json")
     except Exception as e:
-        if db is not None:
+        logging.getLogger(__name__).warning(f"seed email account migration: {e}")
-            db.rollback()
-        logger.warning("seed email account migration: %s", e)
-    finally:
-        if db is not None:
-            db.close()
 
 
 # WARNING: Foreign-key enforcement is enabled globally for all SQLite connections.
@@ -2134,7 +1960,6 @@ def init_db():
     _migrate_add_crew_member_id()
     _migrate_add_assistant_columns()
     _migrate_add_email_smtp_security()
-    _migrate_email_account_default_invariant()
     _migrate_seed_email_account()
     _migrate_add_calendar_metadata()
     _migrate_add_calendar_is_utc()

core/middleware.py

@@ -3,14 +3,10 @@
 
 import os
 import secrets
-from collections.abc import Mapping
 
 from fastapi import HTTPException, Request
 from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.responses import Response
-from starlette.routing import get_route_path
-
-from src.owner_identity import INTERNAL_TOOL_USER, auth_disabled
 
 
 # Per-process token that lets the in-app tool layer hit admin-gated
@@ -19,30 +15,8 @@ from src.owner_identity import INTERNAL_TOOL_USER, auth_disabled
 # same value from this module. Never persisted or exposed externally.
 INTERNAL_TOOL_TOKEN = os.environ.get("ODYSSEUS_INTERNAL_TOKEN") or secrets.token_hex(32)
 INTERNAL_TOOL_HEADER = "X-Odysseus-Internal-Token"
-
+# Pseudo-username on in-process tool-loopback requests; require_admin trusts it and it is reserved.
-
+INTERNAL_TOOL_USER = "internal-tool"
-def get_application_route_path(scope: Mapping[str, object]) -> str:
-    """Return the application-relative path used by Starlette routing.
-
-    Uvicorn prefixes ``scope["path"]`` with a configured ASGI ``root_path``;
-    Starlette removes that prefix before matching routes. Middleware policy
-    must use the same path form or a deployment prefix can change which policy
-    applies to an otherwise unchanged application route.
-    """
-    return get_route_path(scope)
-
-
-def with_asgi_root_path(scope: Mapping[str, object], path: str) -> str:
-    """Prefix an application path for a client-facing redirect target."""
-    root_path = scope.get("root_path", "")
-    if not isinstance(root_path, str) or not root_path:
-        return path
-    return f"{root_path.rstrip('/')}{path}"
-
-
-def path_is_route_or_child(path: str, prefix: str) -> bool:
-    """Return whether ``path`` is exactly ``prefix`` or below that route."""
-    return path == prefix or path.startswith(prefix + "/")
 
 
 def is_cors_preflight(method: str, headers) -> bool:
@@ -73,7 +47,7 @@ def require_admin(request: Request):
         pass
 
     auth_mgr = getattr(request.app.state, "auth_manager", None)
-    if auth_disabled():
+    if os.getenv("AUTH_ENABLED", "true").lower() == "false":
         return
     if not auth_mgr or not auth_mgr.is_configured:
         raise HTTPException(403, "Admin only")

core/models.py

@@ -8,11 +8,6 @@ These are simple datacontainers. All persistence is handled by SessionManager.
 from dataclasses import dataclass
 from typing import Dict, List, Any, Optional, TYPE_CHECKING
 
-from src.tool_approval_scopes import (
-    CHAT_SESSION_APPROVAL_CONTEXT_MARKER,
-    CHAT_SESSION_APPROVAL_DECISION,
-)
-
 if TYPE_CHECKING:
     from .session_manager import SessionManager
 
@@ -36,35 +31,6 @@ set_session_manager = set_session_manager_instance
 get_session_manager = get_session_manager_instance
 
 
-def _history_grants_chat_session_approval(
-    history: List["ChatMessage"],
-    session_id: str,
-) -> bool:
-    """Return whether this exact chat has a resolved session-scope grant."""
-
-    expected_session = str(session_id or "")
-    if not expected_session:
-        return False
-    for message in reversed(history or []):
-        metadata = getattr(message, "metadata", None)
-        if not isinstance(metadata, dict):
-            continue
-        tool_events = metadata.get("tool_events")
-        if not isinstance(tool_events, list):
-            continue
-        for event in reversed(tool_events):
-            ask_user = event.get("ask_user") if isinstance(event, dict) else None
-            if not isinstance(ask_user, dict):
-                continue
-            if (
-                ask_user.get("kind") == "tool_approval"
-                and ask_user.get("resolved") == CHAT_SESSION_APPROVAL_DECISION
-                and str(ask_user.get("session_id") or "") == expected_session
-            ):
-                return True
-    return False
-
-
 @dataclass
 class ChatMessage:
     """A single chat message."""
@@ -150,27 +116,11 @@ class Session:
         the model. Display/history-load paths use the raw ``history`` and are
         unaffected.
         """
-        messages = [
+        return [
             msg.to_dict()
             for msg in self.history
             if (msg.metadata or {}).get("source") != "slash"
         ]
-        if not _history_grants_chat_session_approval(self.history, self.id):
-            return messages
-
-        # Keep the grant close to the latest user request so route-neutral
-        # compaction/trimming preserves it. Copy the metadata instead of
-        # mutating the durable transcript object.
-        for index in range(len(messages) - 1, -1, -1):
-            if messages[index].get("role") != "user":
-                continue
-            message = dict(messages[index])
-            metadata = dict(message.get("metadata") or {})
-            metadata[CHAT_SESSION_APPROVAL_CONTEXT_MARKER] = True
-            message["metadata"] = metadata
-            messages[index] = message
-            break
-        return messages
 
     def get(self, key: str, default=None):
         """Dict-like access for compatibility."""

core/session_manager.py

@@ -14,8 +14,6 @@ import logging
 from datetime import datetime, timezone, timedelta
 from typing import Dict, Optional
 
-from sqlalchemy import func
-
 from .database import Session as DbSession, ChatMessage as DbChatMessage, Document as DbDocument, SessionLocal, utcnow_naive
 from .models import Session, ChatMessage
 from src.attachment_refs import persistable_message_content
@@ -94,28 +92,14 @@ class SessionManager:
         try:
             db_sessions = db.query(DbSession).filter(
                 DbSession.archived == False,
-                DbSession.messages.any(),
+                DbSession.message_count > 0,
             ).order_by(DbSession.last_accessed.desc()).limit(100).all()
 
-            # message_count is derived metadata and can drift after interrupted
-            # or legacy writes. Count only the bounded discovery set so startup
-            # remains metadata-only while lazy hydration sees an authoritative
-            # positive count for every discovered non-empty session.
-            message_counts = {}
-            if db_sessions:
-                message_counts = dict(
-                    db.query(DbChatMessage.session_id, func.count(DbChatMessage.id))
-                    .filter(DbChatMessage.session_id.in_([row.id for row in db_sessions]))
-                    .group_by(DbChatMessage.session_id)
-                    .all()
-                )
-
             loaded_count = 0
             for db_session in db_sessions:
                 try:
                     session = self._db_to_session_meta(db_session)
                     if session is not None:
-                        session.message_count = message_counts[db_session.id]
                         self.sessions[db_session.id] = session
                         loaded_count += 1
                 except Exception as e:
@@ -210,12 +194,7 @@ class SessionManager:
             is_important=getattr(db_session, 'is_important', False) or False,
         )
 
-        # The rows just loaded are the whole transcript, so they — not the
+        session.message_count = getattr(db_session, 'message_count', len(history))
-        # denormalized sessions.message_count column — are the truth for this
-        # cached object. get_session's hydration gate compares against this
-        # number; seeding it from a drifted column would ask for a reload that
-        # can never close the gap.
-        session.message_count = len(history)
         return session
 
     # ------------------------------------------------------------------
@@ -419,50 +398,30 @@ class SessionManager:
     # ------------------------------------------------------------------
 
     def get_session(self, session_id: str) -> Session:
-        """Get a session by ID, loading complete DB history when needed.
+        """Get a session by ID, loading from DB if needed.
 
-        Sessions seeded by ``load_sessions`` start with empty history, and a
+        Sessions seeded by `load_sessions` start with empty history. The
-        cached session can also become partially stale. Refresh metadata first,
+        first read here hydrates them with the message rows.
-        then hydrate whenever the cached transcript is short of the stored rows.
-        Model-send routes enter through this method before building context,
-        while paginated display history reads SQLite directly.
-
-        The gate compares against ``sync_session_metadata``'s reconciled count
-        (the real ``chat_messages`` total), never the denormalized column, so a
-        hydrate always closes the gap and the next read is a cache hit.
         """
         if session_id not in self.sessions:
             self._load_session_from_db(session_id)
+        else:
+            cached = self.sessions[session_id]
+            # Lazy hydrate: metadata-only entries get their messages on first read.
+            if not cached.history and getattr(cached, "message_count", 0) > 0:
+                self._load_session_from_db(session_id)
 
         # Keep model/endpoint metadata fresh. Endpoint deletion can clear the
-        # DB row while a session object is still cached in RAM. Refreshing first
+        # DB row while a session object is still cached in RAM.
-        # also exposes the authoritative message count before completeness is
-        # checked.
         self.sync_session_metadata(session_id)
 
-        cached = self.sessions[session_id]
-        cached_count = len(cached.history or [])
-        stored_count = int(getattr(cached, "message_count", 0) or 0)
-        if cached_count < stored_count:
-            self._load_session_from_db(session_id)
-
         # Update last_accessed
         self._touch_session(session_id)
 
         return self.sessions[session_id]
 
     def sync_session_metadata(self, session_id: str) -> bool:
-        """Refresh non-message session fields from the DB into the cached object.
+        """Refresh non-message session fields from the DB into the cached object."""
-
-        ``message_count`` is reconciled against the real ``chat_messages`` rows
-        rather than copied from the denormalized ``sessions.message_count``
-        column. That column drifts in normal operation — ``_persist_message``
-        swallows a failed insert but ``add_message`` has already appended in
-        memory, so the next successful persist writes rows+1, and a persist for
-        an uncached session writes 0. Hydration keys off this number: a
-        drifted-high column would reload the whole transcript on every warm
-        read, and a drifted-low one would leave the model a truncated one.
-        """
         session = self.sessions.get(session_id)
         if session is None:
             return False
@@ -485,11 +444,7 @@ class SessionManager:
             session.archived = db_session.archived
             session.owner = getattr(db_session, "owner", None)
             session.is_important = getattr(db_session, "is_important", False) or False
-            session.message_count = (
+            session.message_count = getattr(db_session, "message_count", session.message_count) or 0
-                db.query(DbChatMessage)
-                .filter(DbChatMessage.session_id == session_id)
-                .count()
-            )
             return True
         except Exception as e:
             logger.error(f"Error syncing session metadata {session_id}: {e}")

discovery/README.md

@@ -0,0 +1,34 @@
+# Odysseus discovery maps
+
+Compact, code-grounded discovery maps of cross-cutting systems in the checked-in Odysseus codebase. They preserve investigation context and open factual questions; they are not canonical subsystem specifications, a feature certification, or a substitute for normal testing.
+
+> [!IMPORTANT]
+> Checked-in code is the source of truth for current behaviour. Mature subsystem specifications, where they exist, are the canonical documentation of accepted subsystem behaviour. Check code, tests, and configuration before reconciling a discovery finding. Discovery remains non-canonical.
+
+## Explore the maps
+
+| Document | Purpose |
+|---|---|
+| [Current system map](system-map.md) | Records evidence locations, confirmed local observations, and factual open questions about subsystem boundaries. |
+| [Safety boundaries](safety-boundaries.md) | Records evidence about broad authority, safeguards, confirmed risks or gaps, and unverified behaviour. |
+
+## Working rules
+
+- **Trace the code first.** Confirm the current path in source before recording a claim.
+- **Promote selectively.** When an owning mature specification exists, add a fact only when it is verified, useful, and not already represented there.
+- **Record missing ownership.** When no owning specification exists, retain the verified finding in discovery and record missing documentation ownership as a follow-up.
+- **Retain uncertainty here.** Keep unresolved questions and useful investigation context in discovery rather than treating them as canonical truth.
+- **Keep specifications current-state only.** Do not record intentions, design direction, refactor plans, decision history, priority, ownership, or sequencing here.
+- **Investigate with cause.** Do not exhaustively revalidate existing functionality without a report, visible failure, relevant change, or high-authority review need.
+- **Review authority carefully.** Give execution, data access, external tools, credentials, destructive operations, and unattended work focused review.
+- **Use stable locations.** Cite modules, routes, classes, and functions instead of fragile line ranges or generated evidence tables.
+
+## Reconciliation flow
+
+1. Start with the relevant map and trace the cited code.
+2. Classify the finding against current source evidence and an owning mature specification where one exists.
+3. Promote only verified, useful facts that are missing from an existing owning specification.
+4. When no owning specification exists, retain the verified finding here and record missing documentation ownership as a follow-up; otherwise retain unresolved context here and correct stale wording.
+
+> [!NOTE]
+> This package intentionally contains no generator, validator, maturity scale, feature database, or parallel work tracker. The [architecture runtime inventory](./architecture-runtime-inventory.md) preserves dated structural metrics, investigation context, and historical planning as an explicitly non-canonical snapshot.

discovery/architecture-runtime-inventory.md

@@ -0,0 +1,331 @@
+# Architecture runtime inventory
+
+> [!WARNING]
+> This document is a dated structural snapshot, not a canonical runtime specification.
+> Counts, paths, and implementation details may drift as the repository changes.
+> Verify implementation-sensitive claims against the current code and tests.
+
+- **Branch:** `discovery`
+- **Commit:** `c762efe1c97a`
+- **Generated:** `2026-07-28T05:38:14+01:00`
+- **Historical context:** readability and refactor planning in [#4071](https://github.com/odysseus-dev/odysseus/issues/4071) and [#4082](https://github.com/odysseus-dev/odysseus/issues/4082)
+
+## Disposition
+
+> [!NOTE]
+> Reviewed for documentation classification. Stable runtime structure and
+> subsystem ownership have been transferred to the proposed canonical
+> destination, [`docs/ARCHITECTURE.md`](../docs/ARCHITECTURE.md), for
+> maintainer review.
+>
+> This document retains dated metrics, rankings, investigation context,
+> refactor-sensitive observations, and historical planning. Those contents are
+> non-canonical and belong under `discovery/`.
+
+| Content | Authority and destination |
+|---|---|
+| Stable runtime structure | Proposed canonical destination: `docs/ARCHITECTURE.md` |
+| Stable subsystem boundaries | Proposed canonical destination: `docs/ARCHITECTURE.md` |
+| Frontend module organization | `static/js/MODULE_SUMMARY.md` |
+| Counts, line totals, and rankings | This non-canonical inventory |
+| Investigation context and open questions | `discovery/` |
+| Refactor options and prioritization | Issues, Plane, or non-canonical discovery material |
+
+The transfer preserves the stable facts without promoting generated metrics or
+historical prioritization into canonical documentation.
+
+## Purpose
+
+This inventory provides a reviewable map of the current repository structure,
+large runtime modules, major subsystem boundaries, and refactor-sensitive areas.
+
+It does not:
+
+- define accepted subsystem behaviour;
+- certify runtime correctness;
+- prescribe a committed refactor sequence;
+- replace focused specifications, tests, or source review.
+
+For cross-cutting implementation evidence, see the [discovery maps](./README.md).
+
+## Top-level runtime structure
+
+| Area | Role |
+|---|---|
+| `app.py` | FastAPI application composition and entry point |
+| `launcher.py` | Application launch support |
+| `setup.py` | Native setup workflow |
+| `core/` | Authentication, middleware, persistence, sessions, and platform primitives |
+| `routes/` | HTTP and API route handlers |
+| `src/` | Application services, orchestration, tools, providers, and runtime helpers |
+| `services/` | Domain-oriented service packages |
+| `mcp_servers/` | Built-in MCP server implementations |
+| `scripts/` | CLI tools, diagnostics, maintenance, and migration helpers |
+| `static/` | No-build browser frontend and bundled assets |
+| `tests/` | Automated test suite and supporting test infrastructure |
+
+## Directory snapshot
+
+| Directory | Tracked files | Tracked Python files | Direct subdirectories |
+|---|---:|---:|---|
+| `src/` | 143 | 143 | `agent_tools/`, `model_capability_readers/`, `search/`, `tools/` |
+| `routes/` | 73 | 73 | `admin_wipe/`, `cleanup/`, `compare/`, `contacts/`, `gallery/`, `history/`, `memory/`, `note/`, `research/` |
+| `core/` | 11 | 11 | None |
+| `services/` | 42 | 40 | `docs/`, `faces/`, `hwfit/`, `memory/`, `research/`, `search/`, `shell/`, `stt/`, `tts/`, `youtube/` |
+| `mcp_servers/` | 5 | 5 | None |
+| `scripts/` | 44 | 17 | `_completion/`, `_lib/`, `demo_email/` |
+| `static/js/` | 154 | 0 | `calendar/`, `color/`, `compare/`, `editor/`, `emailLibrary/`, `markdown/`, `model/`, `research/`, `util/` |
+| `tests/` | 768 | 758 | `cli/`, `helpers/`, `streaming/`, `tools/` |
+
+> [!NOTE]
+> Counts in this table use `git ls-files`, so generated caches, virtual
+> environments, and other untracked local files are excluded.
+
+## Largest backend modules
+
+Large files are review signals, not proof that a module should be split.
+Coupling, ownership, import compatibility, tests, and runtime authority matter more
+than line count alone.
+
+| Rank | File | Lines | Classes | Top-level functions | Review signal |
+|---:|---|---:|---:|---:|---|
+| 1 | `routes/email_routes.py` | 6032 | 1 | 58 | High |
+| 2 | `src/agent_loop.py` | 5248 | 0 | 63 | High |
+| 3 | `routes/cookbook_routes.py` | 4545 | 0 | 16 | High |
+| 4 | `mcp_servers/email_server.py` | 2920 | 0 | 77 | High |
+| 5 | `src/llm_core.py` | 2895 | 3 | 85 | High |
+| 6 | `src/builtin_actions.py` | 2845 | 2 | 27 | High |
+| 7 | `routes/model_routes.py` | 2743 | 0 | 65 | High |
+| 8 | `src/task_scheduler.py` | 2627 | 1 | 8 | Medium |
+| 9 | `core/database.py` | 2562 | 28 | 67 | High |
+| 10 | `routes/gallery/gallery_routes.py` | 2325 | 0 | 16 | Medium |
+| 11 | `routes/chat_routes.py` | 2063 | 0 | 18 | Medium |
+| 12 | `routes/shell_routes.py` | 1971 | 1 | 21 | Medium |
+| 13 | `src/visual_report.py` | 1933 | 0 | 11 | Medium |
+| 14 | `routes/email_helpers.py` | 1888 | 3 | 48 | Medium |
+| 15 | `routes/document_routes.py` | 1810 | 0 | 5 | Medium |
+| 16 | `src/tools/cookbook.py` | 1705 | 0 | 34 | Medium |
+| 17 | `routes/calendar_routes.py` | 1667 | 2 | 19 | Medium |
+| 18 | `routes/skills_routes.py` | 1662 | 3 | 19 | Medium |
+| 19 | `src/tool_schemas.py` | 1595 | 0 | 3 | Medium |
+| 20 | `routes/email_pollers.py` | 1551 | 0 | 23 | Medium |
+
+The largest current backend concentrations include:
+
+- email routing and helper logic;
+- agent-loop orchestration;
+- Cookbook lifecycle and serving logic;
+- provider and model routing;
+- task scheduling;
+- shared database models and persistence helpers.
+
+These areas require focused ownership and compatibility analysis before structural
+changes are attempted.
+
+## Largest frontend modules
+
+| Rank | File | Lines |
+|---:|---|---:|
+| 1 | `static/style.css` | 41132 |
+| 2 | `static/js/document.js` | 11200 |
+| 3 | `static/js/emailLibrary.js` | 8505 |
+| 4 | `static/js/slashCommands.js` | 6520 |
+| 5 | `static/js/chat.js` | 6001 |
+| 6 | `static/js/settings.js` | 5819 |
+| 7 | `static/js/notes.js` | 5365 |
+| 8 | `static/app.js` | 4681 |
+| 9 | `static/js/cookbookRunning.js` | 4433 |
+| 10 | `static/js/galleryEditor.js` | 4386 |
+| 11 | `static/js/cookbookServe.js` | 4305 |
+| 12 | `static/js/calendar.js` | 3722 |
+| 13 | `static/js/cookbook.js` | 3677 |
+| 14 | `static/js/sessions.js` | 3665 |
+| 15 | `static/js/documentLibrary.js` | 3422 |
+| 16 | `static/js/tasks.js` | 3187 |
+| 17 | `static/js/admin.js` | 3144 |
+| 18 | `static/js/gallery.js` | 2958 |
+| 19 | `static/js/cookbook-hwfit.js` | 2826 |
+| 20 | `static/js/chatRenderer.js` | 2808 |
+
+The browser frontend remains a no-build ES-module application. Its current source
+tree is authoritative; the maintained structural summary is available in
+[`static/js/MODULE_SUMMARY.md`](../static/js/MODULE_SUMMARY.md).
+
+CSS modularization remains tracked separately in
+[#2617](https://github.com/odysseus-dev/odysseus/issues/2617).
+
+## Major subsystem boundaries
+
+| Subsystem | Primary implementation locations |
+|---|---|
+| Application startup | `app.py`, `src/app_initializer.py`, `core/` |
+| Authentication and sessions | `core/auth.py`, `core/middleware.py`, `core/session_manager.py`, `routes/auth_routes.py` |
+| Chat and streaming | `routes/chat_routes.py`, `routes/chat_helpers.py`, `src/chat_handler.py`, `src/chat_processor.py`, `src/llm_core.py` |
+| Agents and tools | `src/agent_loop.py`, `src/tool_execution.py`, `src/agent_tools/`, `src/tools/`, `src/tool_policy.py`, `src/tool_security.py` |
+| Models and providers | `routes/model_routes.py`, `src/model_discovery.py`, `src/model_capabilities.py`, `src/endpoint_resolver.py`, `src/llm_core.py` |
+| Cookbook and hardware fit | `routes/cookbook_routes.py`, `routes/cookbook_helpers.py`, `src/cookbook_serve_lifecycle.py`, `services/hwfit/` |
+| Search and research | `routes/search_routes.py`, `services/search/`, `routes/research/`, `services/research/`, `src/deep_research.py` |
+| Documents and retrieval | `routes/document_routes.py`, `src/document_processor.py`, `src/personal_docs.py`, `src/rag_manager.py`, `src/pdf_runtime.py` |
+| Memory and skills | `routes/memory/`, `services/memory/`, `routes/skills_routes.py` |
+| Email | `routes/email_routes.py`, `routes/email_helpers.py`, `routes/email_pollers.py`, `mcp_servers/email_server.py` |
+| Calendar, contacts, notes, and tasks | `routes/calendar_routes.py`, `routes/contacts/`, `routes/note/`, `routes/task_routes.py`, `src/task_scheduler.py` |
+| Media and speech | `routes/gallery/`, `routes/stt_routes.py`, `routes/tts_routes.py`, `services/stt/`, `services/tts/` |
+| Persistence and operations | `core/database.py`, `src/runtime_paths.py`, `src/bg_jobs.py`, `routes/backup_routes.py`, `routes/cleanup/` |
+
+For a broader evidence map, see
+[`system-map.md`](./system-map.md).
+
+## Refactor-sensitive areas
+
+### Shared persistence
+
+`core/database.py` is a central dependency containing models and shared persistence
+helpers. Changes can affect routes, services, background work, tests, migrations,
+and import compatibility.
+
+A split should not begin from file size alone. It requires:
+
+- an importer inventory;
+- model and helper ownership decisions;
+- migration compatibility checks;
+- stable re-export or migration strategy;
+- focused and full-suite validation.
+
+### Agent orchestration
+
+`src/agent_loop.py` coordinates model interaction, tool selection, policy decisions,
+multi-round execution, and background behaviour. Extraction work must preserve tool
+event semantics, policy enforcement, cancellation, and test patch points.
+
+Historical agent-loop modularization discussion is tracked in
+[#3266](https://github.com/odysseus-dev/odysseus/issues/3266).
+
+### Tool implementation boundaries
+
+Tool implementation is no longer represented by one proposed future package alone.
+Current responsibilities are distributed across:
+
+- `src/tool_execution.py`;
+- `src/tool_schemas.py`;
+- `src/tool_index.py`;
+- `src/tool_policy.py`;
+- `src/tool_security.py`;
+- `src/agent_tools/`;
+- `src/tools/`;
+- remaining compatibility surfaces such as `src/tool_implementations.py`.
+
+Historical tool modularization work is tracked in
+[#3629](https://github.com/odysseus-dev/odysseus/issues/3629).
+
+### Route ownership
+
+`routes/` now contains both flat modules and domain packages. Existing package
+boundaries should be extended only through focused changes. Broad mechanical route
+movement would affect registration, imports, tests, monkeypatch targets, and
+compatibility paths.
+
+### Frontend concentration
+
+The no-build frontend contains several large JavaScript modules and one central CSS
+file. Refactors should preserve module load order, global compatibility exports,
+DOM contracts, deep-link handling, and browser behaviour.
+
+## Non-implemented architecture options
+
+> [!NOTE]
+> The paths below are historical or possible design directions. They do not describe
+> the current repository and are not approved implementation plans.
+
+Earlier planning discussed:
+
+- renaming `app.py` to `main.py`;
+- moving agent orchestration into a new `src/agent/` package;
+- introducing broad `src/domain/`, `src/infra/`, `src/api/`, or `src/pkg/` layers;
+- moving all routes into domain subpackages;
+- splitting database models into a new infrastructure hierarchy.
+
+These options should be reconsidered against the current tree rather than copied
+forward as assumed targets.
+
+## Refactor guardrails
+
+- Keep structural changes behaviour-preserving.
+- Change one ownership boundary at a time.
+- Do not mix file movement with unrelated feature work.
+- Preserve existing import and monkeypatch paths where compatibility is required.
+- Identify focused tests before modifying high-authority modules.
+- Validate startup, imports, and affected runtime paths.
+- Avoid repository-wide package reorganizations without maintainer agreement.
+- Treat generated metrics as snapshots, not architectural decisions.
+
+## Reproduce the snapshot
+
+Run these commands from the repository root.
+
+```bash
+# Tracked directory totals
+for dir in src routes core services mcp_servers scripts static/js tests; do
+  files="$(git ls-files "$dir" | wc -l)"
+  python_files="$(git ls-files "$dir" '*.py' | wc -l)"
+
+  printf '%-14s tracked=%-5s python=%-5s\n' \
+    "$dir" \
+    "$files" \
+    "$python_files"
+done
+
+# Largest tracked backend files
+git ls-files \
+  'app.py' \
+  'launcher.py' \
+  'setup.py' \
+  'core/*.py' \
+  'core/**/*.py' \
+  'routes/*.py' \
+  'routes/**/*.py' \
+  'services/*.py' \
+  'services/**/*.py' \
+  'src/*.py' \
+  'src/**/*.py' \
+  'mcp_servers/*.py' \
+  'scripts/*.py' \
+  'scripts/**/*.py' |
+xargs wc -l |
+sort -nr |
+head -31
+
+# Largest tracked frontend source files
+git ls-files \
+  'static/*.js' \
+  'static/*.css' \
+  'static/*.html' \
+  'static/**/*.js' \
+  'static/**/*.css' \
+  'static/**/*.html' |
+grep -vE '\.min\.js$' |
+xargs wc -l |
+sort -nr |
+head -31
+```
+
+## Validation for architecture changes
+
+Use the smallest relevant checks first, then expand according to risk:
+
+```bash
+python3 -m compileall -q app.py core routes services src
+venv/bin/python -m pytest tests/<focused-test-file>.py -q
+venv/bin/python -m pytest -q
+```
+
+Startup, browser, Docker, and integration checks may also be required depending on
+the affected boundary.
+
+## Related documentation
+
+- [Documentation style](../docs/STYLE.md)
+- [Discovery maps](../discovery/README.md)
+- [Current system map](../discovery/system-map.md)
+- [Safety boundaries](../discovery/safety-boundaries.md)
+- [Frontend module summary](../static/js/MODULE_SUMMARY.md)
+- [Testing standard](../tests/TESTING_STANDARD.md)

discovery/safety-boundaries.md

@@ -0,0 +1,142 @@
+# Safety boundaries
+
+> [!IMPORTANT]
+> This non-canonical discovery map records code-grounded safeguards, confirmed risks or gaps, and unverified behaviour. Broad authority does not by itself establish a vulnerability. Verify the cited source before relying on a finding. No destructive test, external connection, or real credential was used for this map.
+
+## Navigate the boundaries
+
+- [Shell and subprocess execution](#shell-and-subprocess-execution)
+- [Filesystem access and workspace confinement](#filesystem-access-and-workspace-confinement)
+- [Agent-controlled tool dispatch](#agent-controlled-tool-dispatch)
+- [MCP and external tool servers](#mcp-and-external-tool-servers)
+- [Outbound network requests and URL validation](#outbound-network-requests-and-url-validation)
+- [Secrets, credentials, and vault sessions](#secrets-credentials-and-vault-sessions)
+- [Authentication and privileged administration](#authentication-and-privileged-administration)
+- [Deletion, wipe, backup, and restore](#deletion-wipe-backup-and-restore)
+- [Background jobs and unattended task execution](#background-jobs-and-unattended-task-execution)
+
+## Shell and subprocess execution
+
+- **Boundary:** Shell routes, agent `bash` and `python` tools, local model serving, and detached background jobs.
+
+- **Available authority:** Commands run as the application process user and can create child processes.
+
+- **User-controlled inputs:** Direct shell requests, model-produced tool arguments, scheduled-task prompts, and model-serving configuration.
+
+- **Current safeguards:** Agent dispatch applies owner/admin checks and tool policy; process helpers use timeouts or bounded background-job lifecycle where implemented.
+
+- **Confirmed risks or gaps:** Intentional authority with a confirmed gap: the agent shell starts in its workspace but is not sandboxed to it, and has no egress sandbox. This is documented in source and the threat model; it is not a newly demonstrated bypass.
+
+- **Unverified behaviour:** Role-gate and disabled-tool outcomes, direct shell-route behaviour, and timeout, cancellation, and output handling for foreground and detached processes remain unverified.
+
+## Filesystem access and workspace confinement
+
+- **Boundary:** Agent read, write, patch, listing, glob, and grep tools.
+
+- **Available authority:** Read and modify files within active workspace confinement or fallback allowlisted roots.
+
+- **User-controlled inputs:** Tool paths, patches, file contents, search patterns, and workspace selection passed into the tool dispatcher.
+
+- **Current safeguards:** [`src/tool_execution.py`](../src/tool_execution.py) resolves paths, blocks sensitive subpaths, applies allowlist containment, and tightens paths to the active workspace when one is bound. File tools use those resolvers.
+
+- **Confirmed risks or gaps:** Intentional authority with safeguards. The file-tool policy does not sandbox the shell; treating a workspace as a whole-process containment boundary would be incorrect.
+
+- **Unverified behaviour:** Traversal, symlink, sensitive-name, absolute-path, and workspace-switch behaviour remains unverified.
+
+## Agent-controlled tool dispatch
+
+- **Boundary:** Model output becomes native or parsed tool calls and is dispatched by the agent loop.
+
+- **Available authority:** The authority of every enabled tool, including privileged built-ins and external tools.
+
+- **User-controlled inputs:** Chat content, attached/retrieved content that may influence the model, tool arguments, per-request tool selection, and policy toggles.
+
+- **Current safeguards:** [`src/tool_security.py`](../src/tool_security.py) blocks protected tools for non-admin users and fails closed for malformed tool names; [`src/tool_policy.py`](../src/tool_policy.py) supports disabled and guide-only policy; prompt-security helpers label untrusted context.
+
+- **Confirmed risks or gaps:** Credible risk requiring verification: aliases, legacy text tools, native function calls, and MCP-qualified names must all reach the same policy outcome. The code has specific alias handling for email/MCP names, which makes this a sensitive compatibility seam.
+
+- **Unverified behaviour:** The current policy outcomes for owner role, request mode, disabled state, native versus parsed invocation, qualified aliases, and external-content entry points remain unverified.
+
+## MCP and external tool servers
+
+- **Boundary:** Configured MCP servers and their tools are exposed to the agent through the MCP manager and routes.
+
+- **Available authority:** Depends on the server: external network access, local process access, messaging, or data mutation may be delegated outside the application.
+
+- **User-controlled inputs:** Server configuration, remote OAuth completion, tool arguments, and model-selected MCP calls.
+
+- **Current safeguards:** MCP routes are registered through [`routes/mcp_routes.py`](../routes/mcp_routes.py); MCP-qualified tools are denied to non-admin users by [`src/tool_security.py`](../src/tool_security.py). OAuth state and token persistence are handled in [`src/mcp_oauth.py`](../src/mcp_oauth.py).
+
+- **Confirmed risks or gaps:** Credible risk requiring verification: an MCP server authority is broader than the application can infer from its tool name. This map does not establish a trust or approval model for server installation and individual tool invocation.
+
+- **Unverified behaviour:** Server onboarding, credential storage, server-origin trust, OAuth callback deployment, tool disablement, and invocation audit behaviour remain unverified.
+
+## Outbound network requests and URL validation
+
+- **Boundary:** Search/content fetch, research, webhooks, skill import, provider endpoints, and other HTTP clients.
+
+- **Available authority:** The application can make outbound requests from its network position.
+
+- **User-controlled inputs:** Search/fetch URLs, imported skill URLs, webhook configuration, and some endpoint settings.
+
+- **Current safeguards:** [`src/url_security.py`](../src/url_security.py) validates untrusted public HTTP URLs and fails closed on unsuitable schemes or private addresses. [`services/search/content.py`](../services/search/content.py) resolves and rejects non-public hosts, pins resolved addresses for fetches, caps bodies, and limits redirects.
+
+- **Confirmed risks or gaps:** Intentional split: administrator-created model endpoints may target private providers, while untrusted URLs use public-address checks. That distinction is required for self-hosted deployments but needs explicit call-site review.
+
+- **Unverified behaviour:** The URL-source classification for outbound clients and the current handling of redirects and DNS changes remain unverified.
+
+## Secrets, credentials, and vault sessions
+
+- **Boundary:** Application-managed encrypted secrets, API keys, provider credentials, and Bitwarden/Vaultwarden CLI sessions.
+
+- **Available authority:** Credentials unlock remote providers and connected personal services.
+
+- **User-controlled inputs:** Administrative configuration, login/unlock requests, imported settings, and agent vault tool arguments.
+
+- **Current safeguards:** [`src/secret_storage.py`](../src/secret_storage.py) uses a locally stored Fernet key with restrictive permissions for supported database secrets. Vault routes require an administrator, avoid passing master passwords in command arguments, and set restrictive permissions on the vault-session file.
+
+- **Confirmed risks or gaps:** Confirmed current boundary: vault session data is persisted through the vault path, not through [`src/secret_storage.py`](../src/secret_storage.py). This is an unresolved question about current security semantics, not a confirmed exposure.
+
+- **Unverified behaviour:** Current encryption-at-rest, owner scope, rotation, lock/logout, backup/restore, and log/tool-result exposure behaviour remains unverified.
+
+## Authentication and privileged administration
+
+- **Boundary:** Session authentication, API tokens, privileged routes, and internal tool loopback.
+
+- **Available authority:** Administrative identity can access execution, settings, integrations, data deletion, and secrets.
+
+- **User-controlled inputs:** Login/signup data, session cookies, API tokens, authentication configuration, and requests to privileged routes.
+
+- **Current safeguards:** [`core/auth.py`](../core/auth.py), [`core/middleware.py`](../core/middleware.py), and route-level checks establish identity and administrator gates. [`app.py`](../app.py) warns when localhost bypass is configured; [`SECURITY.md`](../SECURITY.md) documents deployment requirements.
+
+- **Confirmed risks or gaps:** Intentional authority with safeguards. Security depends on deployments keeping authentication enabled and internal services private; this map does not audit reverse-proxy or environment configuration.
+
+- **Unverified behaviour:** Setup, anonymous, non-admin, admin, token, and internal-loopback behaviour, including privileged-route gate consistency, remains unverified.
+
+## Deletion, wipe, backup, and restore
+
+- **Boundary:** Administrative wipe, cleanup, backup import/export, and the backup restore command.
+
+- **Available authority:** Delete or replace user data and credentials.
+
+- **User-controlled inputs:** Administrative HTTP requests, cleanup choices, backup payloads, archive paths, and restore command options.
+
+- **Current safeguards:** Administrative wipe routes use the administrative boundary. Cleanup exposes a preview route before mutation. The documented backup tool requires explicit restore confirmation, stages the old data directory, and validates archive members before extraction.
+
+- **Confirmed risks or gaps:** Intentional destructive authority. Backup archives contain secrets by design, as documented in [`docs/backup-restore.md`](../docs/backup-restore.md); this is an operator confidentiality responsibility, not a code defect established here.
+
+- **Unverified behaviour:** Role-gate, confirmation, archive-rejection, staged-recovery, and owner-isolation behaviour remains unverified. No destructive runtime test was performed.
+
+## Background jobs and unattended task execution
+
+- **Boundary:** Scheduled tasks, background-job monitor, startup tasks, and notification/delivery work that continue without an active browser request.
+
+- **Available authority:** Scheduled agent work can obtain model access and, for eligible owners, shell and file tools; task output can interact with connected services.
+
+- **User-controlled inputs:** Stored task prompt, schedule, model/crew selection, enabled-tool configuration, output target, and prior persisted state.
+
+- **Current safeguards:** [`src/task_scheduler.py`](../src/task_scheduler.py) serializes execution, records task runs, associates work with an owner, and applies the agent owner-based tool gate. [`src/bg_jobs.py`](../src/bg_jobs.py) keeps bounded state and can terminate overlong subprocess jobs.
+
+- **Confirmed risks or gaps:** Credible risk requiring verification: authority is inherited and exercised later, so changes to roles, task configuration, and disabled tools must be checked at execution time rather than assumed from task creation.
+
+- **Unverified behaviour:** Creation, editing, role-change, scheduling, cancellation, restart-recovery, and execution behaviour remains unverified, including whether current policy is re-evaluated before privileged action.

discovery/system-map.md

@@ -0,0 +1,139 @@
+# Current system map
+
+> [!NOTE]
+> This non-canonical discovery map is an evidence guide, not an exhaustive feature catalog or runtime certification. Verify the cited source before relying on a finding. Each section records local implementation observations, evidence locations, confirmed current problems, and unresolved factual questions.
+
+## Navigate the system
+
+- [Startup and application composition](#startup-and-application-composition)
+- [Frontend shell and browser interaction](#frontend-shell-and-browser-interaction)
+- [Chat, sessions, and streaming](#chat-sessions-and-streaming)
+- [Agents, tools, and execution](#agents-tools-and-execution)
+- [Models, providers, and local serving](#models-providers-and-local-serving)
+- [Search and research](#search-and-research)
+- [Documents, retrieval, and personal knowledge](#documents-retrieval-and-personal-knowledge)
+- [Memory and skills](#memory-and-skills)
+- [Email, calendar, contacts, notes, and tasks](#email-calendar-contacts-notes-and-tasks)
+- [Media, speech, and image work](#media-speech-and-image-work)
+- [Authentication, secrets, and privileged administration](#authentication-secrets-and-privileged-administration)
+- [Persistence, background work, and operations](#persistence-background-work-and-operations)
+
+## Startup and application composition
+
+- **How it works:** [`app.py`](../app.py) creates the application, mounts static assets, constructs shared services, registers route factories, and owns lifespan startup and shutdown. [`src/app_initializer.py`](../src/app_initializer.py) prepares application state; [`core/`](../core/) provides persistence, authentication, middleware, sessions, and platform helpers.
+
+- **Evidence locations:** [`app.py`](../app.py); [`src/app_initializer.py`](../src/app_initializer.py); [`core/database.py`](../core/database.py); [`core/auth.py`](../core/auth.py); [`core/middleware.py`](../core/middleware.py); [`routes/`](../routes/).
+
+- **Known problems:** None recorded by this mapping.
+
+- **Open question:** Which component currently owns startup and shutdown for each long-lived service?
+
+## Frontend shell and browser interaction
+
+- **How it works:** [`static/index.html`](../static/index.html) is served by the root and SPA deep-link routes in [`app.py`](../app.py); [`static/app.js`](../static/app.js), [`static/style.css`](../static/style.css), and [`static/js/`](../static/js/) implement the client surface.
+
+- **Evidence locations:** [`static/index.html`](../static/index.html); [`static/app.js`](../static/app.js); [`static/js/`](../static/js/); [`static/style.css`](../static/style.css); [`app.py`](../app.py) deep-link handlers.
+
+- **Known problems:** The `/backgrounds` route in [`app.py`](../app.py) calls `serve_html_with_nonce` for `static/backgrounds.html`, but that file is absent from [`static/`](../static/). This is a confirmed broken prototype route, not evidence about the rest of the frontend.
+
+- **Open question:** Is `/backgrounds` currently an intentionally supported route or an obsolete prototype?
+
+## Chat, sessions, and streaming
+
+- **How it works:** [`routes/chat_routes.py`](../routes/chat_routes.py) and [`routes/chat_helpers.py`](../routes/chat_helpers.py) coordinate requests, session state, and SSE delivery. [`src/chat_handler.py`](../src/chat_handler.py), [`src/chat_processor.py`](../src/chat_processor.py), [`src/llm_core.py`](../src/llm_core.py), and [`src/session_actions.py`](../src/session_actions.py) provide message preparation, provider interaction, and session operations.
+
+- **Evidence locations:** [`routes/chat_routes.py`](../routes/chat_routes.py); [`routes/chat_helpers.py`](../routes/chat_helpers.py); [`routes/session_routes.py`](../routes/session_routes.py); [`src/chat_handler.py`](../src/chat_handler.py); [`src/chat_processor.py`](../src/chat_processor.py); [`src/llm_core.py`](../src/llm_core.py); [`core/session_manager.py`](../core/session_manager.py).
+
+- **Known problems:** [`src/agent_loop.py`](../src/agent_loop.py) annotates `_resolved_tool_event_name` with `Any` but imports no `Any` and does not enable postponed annotation evaluation. Python evaluates that annotation while importing the module, so this is an import-time defect at the checked baseline.
+
+- **Open question:** No end-to-end provider or browser streaming run was performed for this map.
+
+## Agents, tools, and execution
+
+- **How it works:** [`src/agent_loop.py`](../src/agent_loop.py) drives multi-round tool use. [`src/tool_execution.py`](../src/tool_execution.py) dispatches calls and binds workspace context. [`src/agent_tools/`](../src/agent_tools/) contains individual implementations; [`src/tool_security.py`](../src/tool_security.py) and [`src/tool_policy.py`](../src/tool_policy.py) apply role and request policies. Long-running command work is represented by [`src/bg_jobs.py`](../src/bg_jobs.py).
+
+- **Evidence locations:** [`src/agent_loop.py`](../src/agent_loop.py); [`src/tool_execution.py`](../src/tool_execution.py); [`src/agent_tools/`](../src/agent_tools/); [`src/tool_security.py`](../src/tool_security.py); [`src/tool_policy.py`](../src/tool_policy.py); [`src/tool_schemas.py`](../src/tool_schemas.py); [`src/bg_jobs.py`](../src/bg_jobs.py).
+
+- **Known problems:** The import-time annotation defect above blocks the main agent/tool path. The shell is intentionally not a filesystem or network sandbox; that is an authority boundary, not by itself a vulnerability claim.
+
+- **Open question:** Which native, legacy, and MCP-qualified invocation paths reach each policy gate?
+
+## Models, providers, and local serving
+
+- **How it works:** Model routes delegate to discovery, capabilities, endpoint resolution, and LLM core modules. Cookbook routes and hardware-fit services handle model lifecycle and local-serving support.
+
+- **Evidence locations:** [`routes/model_routes.py`](../routes/model_routes.py); [`src/model_discovery.py`](../src/model_discovery.py); [`src/model_capabilities.py`](../src/model_capabilities.py); [`src/endpoint_resolver.py`](../src/endpoint_resolver.py); [`src/llm_core.py`](../src/llm_core.py); [`routes/cookbook_routes.py`](../routes/cookbook_routes.py); [`src/cookbook_serve_lifecycle.py`](../src/cookbook_serve_lifecycle.py); [`services/hwfit/`](../services/hwfit/).
+
+- **Known problems:** None recorded by this mapping.
+
+- **Open question:** Which endpoint inputs are administrator-created and permitted to use private provider addresses?
+
+## Search and research
+
+- **How it works:** HTTP search routes use [`services/search/`](../services/search/); research is exposed through [`routes/research/`](../routes/research/) and implemented in [`services/research/`](../services/research/), [`src/deep_research.py`](../src/deep_research.py), and related helpers. [`src/search/`](../src/search/) remains an import-compatibility layer for callers not yet moved to `services.search`.
+
+- **Evidence locations:** [`routes/search_routes.py`](../routes/search_routes.py); [`services/search/`](../services/search/); [`routes/research/research_routes.py`](../routes/research/research_routes.py); [`services/research/`](../services/research/); [`src/deep_research.py`](../src/deep_research.py); [`src/search/`](../src/search/).
+
+- **Known problems:** None recorded by this mapping.
+
+- **Open question:** No live provider request was made; provider configuration and network access remain unverified.
+
+## Documents, retrieval, and personal knowledge
+
+- **How it works:** Document routes coordinate upload handling, document processing, and editor actions. Personal-document and RAG modules use Chroma and embedding clients. PDF viewing uses the optional-dependency loader in [`src/pdf_runtime.py`](../src/pdf_runtime.py); form extraction and filling live separately in [`src/pdf_forms.py`](../src/pdf_forms.py) and [`src/pdf_form_doc.py`](../src/pdf_form_doc.py).
+
+- **Evidence locations:** [`routes/document_routes.py`](../routes/document_routes.py); [`src/upload_handler.py`](../src/upload_handler.py); [`src/document_processor.py`](../src/document_processor.py); [`src/document_actions.py`](../src/document_actions.py); [`src/personal_docs.py`](../src/personal_docs.py); [`src/rag_manager.py`](../src/rag_manager.py); [`src/embeddings.py`](../src/embeddings.py); [`src/pdf_runtime.py`](../src/pdf_runtime.py); [`src/pdf_forms.py`](../src/pdf_forms.py); [`src/pdf_form_doc.py`](../src/pdf_form_doc.py).
+
+- **Known problems:** PDF viewing/runtime loading and PDF form processing are separate implementations. That separation is confirmed and intentional in the source; it is not a defect without a reported behavioural failure.
+
+- **Open question:** Optional PDF dependencies and representative uploaded documents were not exercised.
+
+## Memory and skills
+
+- **How it works:** Memory routes use [`services/memory/`](../services/memory/) and vector helpers. Skills are exposed through [`routes/skills_routes.py`](../routes/skills_routes.py), stored and managed in [`services/memory/skills.py`](../services/memory/skills.py), and may be imported through [`services/memory/skill_importer.py`](../services/memory/skill_importer.py).
+
+- **Evidence locations:** [`routes/memory/memory_routes.py`](../routes/memory/memory_routes.py); [`services/memory/`](../services/memory/); [`src/memory.py`](../src/memory.py); [`src/memory_vector.py`](../src/memory_vector.py); [`routes/skills_routes.py`](../routes/skills_routes.py); [`services/memory/skills.py`](../services/memory/skills.py); [`services/memory/skill_importer.py`](../services/memory/skill_importer.py).
+
+- **Known problems:** None recorded by this mapping.
+
+- **Open question:** Which imported skill content can reach execution-capable paths, and which validation occurs before that point?
+
+## Email, calendar, contacts, notes, and tasks
+
+- **How it works:** Dedicated route modules own email, CalDAV calendar, CardDAV contacts, notes, and tasks. Supporting modules include email helpers and pollers, CalDAV sync and writeback, and the task scheduler.
+
+- **Evidence locations:** [`routes/email_routes.py`](../routes/email_routes.py); [`routes/calendar_routes.py`](../routes/calendar_routes.py); [`routes/contacts/contacts_routes.py`](../routes/contacts/contacts_routes.py); [`routes/note/note_routes.py`](../routes/note/note_routes.py); [`routes/task_routes.py`](../routes/task_routes.py); [`routes/assistant_routes.py`](../routes/assistant_routes.py); [`src/caldav_sync.py`](../src/caldav_sync.py); [`src/caldav_writeback.py`](../src/caldav_writeback.py); [`src/task_scheduler.py`](../src/task_scheduler.py).
+
+- **Known problems:** None recorded by this mapping.
+
+- **Open question:** External account behaviour, writeback, and delivery require controlled credentials and are not runtime-validated here.
+
+## Media, speech, and image work
+
+- **How it works:** Gallery and image routes coordinate media features. Service modules own speech and media integrations; [`src/generated_images.py`](../src/generated_images.py) and [`src/visual_report.py`](../src/visual_report.py) support artifact handling and presentation.
+
+- **Evidence locations:** [`routes/gallery/gallery_routes.py`](../routes/gallery/gallery_routes.py); [`routes/stt_routes.py`](../routes/stt_routes.py); [`routes/tts_routes.py`](../routes/tts_routes.py); [`src/generated_images.py`](../src/generated_images.py); [`services/stt/`](../services/stt/); [`services/tts/`](../services/tts/); [`services/faces/`](../services/faces/); [`src/visual_report.py`](../src/visual_report.py).
+
+- **Known problems:** None recorded by this mapping.
+
+- **Open question:** Hardware- and provider-dependent media workflows were not exercised.
+
+## Authentication, secrets, and privileged administration
+
+- **How it works:** [`core/auth.py`](../core/auth.py) and [`core/middleware.py`](../core/middleware.py) provide identity and request gates. [`src/secret_storage.py`](../src/secret_storage.py) encrypts application-managed database secrets with a local Fernet key. Vault handling is separate: [`routes/vault_routes.py`](../routes/vault_routes.py) and [`src/tools/vault.py`](../src/tools/vault.py) invoke the Bitwarden CLI and persist its session data in the application data area.
+
+- **Evidence locations:** [`core/auth.py`](../core/auth.py); [`core/middleware.py`](../core/middleware.py); [`routes/auth_routes.py`](../routes/auth_routes.py); [`routes/api_token_routes.py`](../routes/api_token_routes.py); [`src/secret_storage.py`](../src/secret_storage.py); [`routes/vault_routes.py`](../routes/vault_routes.py); [`src/tools/vault.py`](../src/tools/vault.py); [`routes/admin_wipe/admin_wipe_routes.py`](../routes/admin_wipe/admin_wipe_routes.py).
+
+- **Known problems:** Vault-command handling and local application secret storage are distinct paths with different storage mechanisms. This is a source-confirmed boundary, not evidence that either path is compromised.
+
+- **Open question:** What are the current confidentiality, ownership, rotation, and backup semantics for vault session data?
+
+## Persistence, background work, and operations
+
+- **How it works:** SQLite models and persistence are centred in [`core/database.py`](../core/database.py); managers use application data paths. The scheduler and background-job monitor can continue work outside a live browser request. Operational routes cover cleanup, backup, and administrative wipe; the repository also provides a backup script and user documentation.
+
+- **Evidence locations:** [`core/database.py`](../core/database.py); [`src/runtime_paths.py`](../src/runtime_paths.py); [`src/task_scheduler.py`](../src/task_scheduler.py); [`src/bg_jobs.py`](../src/bg_jobs.py); [`src/bg_monitor.py`](../src/bg_monitor.py); [`routes/backup_routes.py`](../routes/backup_routes.py); [`routes/cleanup/cleanup_routes.py`](../routes/cleanup/cleanup_routes.py); [`routes/admin_wipe/admin_wipe_routes.py`](../routes/admin_wipe/admin_wipe_routes.py); [`scripts/odysseus-backup`](../scripts/odysseus-backup); [`docs/backup-restore.md`](../docs/backup-restore.md).
+
+- **Known problems:** None recorded by this mapping.
+
+- **Open question:** What current behaviour applies to background execution, cancellation, retries, and authority inheritance?

docker-compose.gpu-amd.yml

@@ -46,11 +46,10 @@ services:
       - DATABASE_URL=${DATABASE_URL:-sqlite:///./data/app.db}
       - AUTH_ENABLED=${AUTH_ENABLED:-true}
       - LOCALHOST_BYPASS=${LOCALHOST_BYPASS:-false}
-      - COMPANION_BASE_URL=${COMPANION_BASE_URL:-}
       - ODYSSEUS_ADMIN_USER=${ODYSSEUS_ADMIN_USER:-admin}
       - ODYSSEUS_ADMIN_PASSWORD=${ODYSSEUS_ADMIN_PASSWORD:-}
       - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost,http://127.0.0.1}
-      - SECURE_COOKIES=${SECURE_COOKIES:-}
+      - SECURE_COOKIES=${SECURE_COOKIES:-false}
       - EMBEDDING_URL=${EMBEDDING_URL:-}
       - EMBEDDING_MODEL=${EMBEDDING_MODEL:-}
       - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
@@ -68,18 +67,12 @@ services:
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
-      - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}
       - GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID:-}
       - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-}
       - GOOGLE_OAUTH_REDIRECT_URI=${GOOGLE_OAUTH_REDIRECT_URI:-}
-      # Externally reachable origin for MCP OAuth callbacks. The container
-      # always listens on 7000 and cannot see the host port map above, so
-      # remote MCP OAuth needs this set whenever the browser reaches
-      # Odysseus on anything other than http://localhost:7000.
-      - OAUTH_REDIRECT_BASE_URL=${OAUTH_REDIRECT_BASE_URL:-}
       - TAVILY_API_KEY=${TAVILY_API_KEY:-}
       - SERPER_API_KEY=${SERPER_API_KEY:-}
       # PUID / PGID — the user/group the container drops to before
@@ -135,17 +128,12 @@ services:
           fi
           sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml
         fi
-        # Advisory: a settings file the migration cannot parse or rewrite must
-        # not be what stops searxng from booting. It explains itself on stderr
-        # and we carry on, letting searxng report anything genuinely wrong.
-        /usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py /etc/searxng/settings.yml || true
         exec /usr/local/searxng/entrypoint.sh
     ports:
       - "127.0.0.1:8080:8080"
     volumes:
       - searxng-data:/etc/searxng
       - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z
-      - ./scripts/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro,z
     environment:
       - SEARXNG_BASE_URL=http://localhost:8080/
       - SEARXNG_SECRET=${SEARXNG_SECRET:-}

docker-compose.gpu-nvidia.yml

@@ -45,11 +45,10 @@ services:
       - DATABASE_URL=${DATABASE_URL:-sqlite:///./data/app.db}
       - AUTH_ENABLED=${AUTH_ENABLED:-true}
       - LOCALHOST_BYPASS=${LOCALHOST_BYPASS:-false}
-      - COMPANION_BASE_URL=${COMPANION_BASE_URL:-}
       - ODYSSEUS_ADMIN_USER=${ODYSSEUS_ADMIN_USER:-admin}
       - ODYSSEUS_ADMIN_PASSWORD=${ODYSSEUS_ADMIN_PASSWORD:-}
       - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost,http://127.0.0.1}
-      - SECURE_COOKIES=${SECURE_COOKIES:-}
+      - SECURE_COOKIES=${SECURE_COOKIES:-false}
       - EMBEDDING_URL=${EMBEDDING_URL:-}
       - EMBEDDING_MODEL=${EMBEDDING_MODEL:-}
       - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
@@ -67,18 +66,12 @@ services:
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
-      - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}
       - GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID:-}
       - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-}
       - GOOGLE_OAUTH_REDIRECT_URI=${GOOGLE_OAUTH_REDIRECT_URI:-}
-      # Externally reachable origin for MCP OAuth callbacks. The container
-      # always listens on 7000 and cannot see the host port map above, so
-      # remote MCP OAuth needs this set whenever the browser reaches
-      # Odysseus on anything other than http://localhost:7000.
-      - OAUTH_REDIRECT_BASE_URL=${OAUTH_REDIRECT_BASE_URL:-}
       - TAVILY_API_KEY=${TAVILY_API_KEY:-}
       - SERPER_API_KEY=${SERPER_API_KEY:-}
       # PUID / PGID — the user/group the container drops to before
@@ -138,17 +131,12 @@ services:
           fi
           sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml
         fi
-        # Advisory: a settings file the migration cannot parse or rewrite must
-        # not be what stops searxng from booting. It explains itself on stderr
-        # and we carry on, letting searxng report anything genuinely wrong.
-        /usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py /etc/searxng/settings.yml || true
         exec /usr/local/searxng/entrypoint.sh
     ports:
       - "127.0.0.1:8080:8080"
     volumes:
       - searxng-data:/etc/searxng
       - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z
-      - ./scripts/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro,z
     environment:
       - SEARXNG_BASE_URL=http://localhost:8080/
       - SEARXNG_SECRET=${SEARXNG_SECRET:-}

docker-compose.yml

@@ -34,11 +34,10 @@ services:
       - DATABASE_URL=${DATABASE_URL:-sqlite:///./data/app.db}
       - AUTH_ENABLED=${AUTH_ENABLED:-true}
       - LOCALHOST_BYPASS=${LOCALHOST_BYPASS:-false}
-      - COMPANION_BASE_URL=${COMPANION_BASE_URL:-}
       - ODYSSEUS_ADMIN_USER=${ODYSSEUS_ADMIN_USER:-admin}
       - ODYSSEUS_ADMIN_PASSWORD=${ODYSSEUS_ADMIN_PASSWORD:-}
       - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost,http://127.0.0.1}
-      - SECURE_COOKIES=${SECURE_COOKIES:-}
+      - SECURE_COOKIES=${SECURE_COOKIES:-false}
       - EMBEDDING_URL=${EMBEDDING_URL:-}
       - EMBEDDING_MODEL=${EMBEDDING_MODEL:-}
       - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
@@ -56,18 +55,12 @@ services:
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
-      - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}
       - GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID:-}
       - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-}
       - GOOGLE_OAUTH_REDIRECT_URI=${GOOGLE_OAUTH_REDIRECT_URI:-}
-      # Externally reachable origin for MCP OAuth callbacks. The container
-      # always listens on 7000 and cannot see the host port map above, so
-      # remote MCP OAuth needs this set whenever the browser reaches
-      # Odysseus on anything other than http://localhost:7000.
-      - OAUTH_REDIRECT_BASE_URL=${OAUTH_REDIRECT_BASE_URL:-}
       - TAVILY_API_KEY=${TAVILY_API_KEY:-}
       - SERPER_API_KEY=${SERPER_API_KEY:-}
       # PUID / PGID — the user/group the container drops to before
@@ -116,17 +109,12 @@ services:
           fi
           sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml
         fi
-        # Advisory: a settings file the migration cannot parse or rewrite must
-        # not be what stops searxng from booting. It explains itself on stderr
-        # and we carry on, letting searxng report anything genuinely wrong.
-        /usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py /etc/searxng/settings.yml || true
         exec /usr/local/searxng/entrypoint.sh
     ports:
       - "127.0.0.1:8080:8080"
     volumes:
       - searxng-data:/etc/searxng
       - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z
-      - ./scripts/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro,z
     environment:
       - SEARXNG_BASE_URL=http://localhost:8080/
       - SEARXNG_SECRET=${SEARXNG_SECRET:-}

docs/ARCHITECTURE.md

@@ -0,0 +1,87 @@
+# Architecture
+
+> [!NOTE]
+> This document is the proposed canonical destination for stable high-level
+> architecture facts. It remains subject to maintainer review. Source code,
+> tests, and configuration remain authoritative for implementation-sensitive
+> behaviour.
+
+## Purpose
+
+This document identifies the stable runtime boundaries and primary ownership
+locations used to navigate and extend Odysseus.
+
+It intentionally excludes generated metrics, file-size rankings, refactor
+priorities, unresolved investigation findings, and proposed package layouts.
+
+## Runtime structure
+
+| Area | Responsibility |
+|---|---|
+| `app.py` | FastAPI application composition and primary application entry point |
+| `launcher.py` | Application launch support |
+| `setup.py` | Native setup workflow |
+| `core/` | Authentication, middleware, persistence, sessions, and platform primitives |
+| `routes/` | HTTP and API route handlers |
+| `src/` | Application orchestration, tools, providers, and runtime helpers |
+| `services/` | Domain-oriented service implementations |
+| `mcp_servers/` | Built-in MCP server implementations |
+| `scripts/` | CLI tools, diagnostics, maintenance, and migration helpers |
+| `static/` | No-build browser frontend and bundled assets |
+| `tests/` | Automated tests and supporting test infrastructure |
+
+## Subsystem boundaries
+
+| Subsystem | Primary implementation locations |
+|---|---|
+| Application startup | `app.py`, `src/app_initializer.py`, `core/` |
+| Authentication and sessions | `core/auth.py`, `core/middleware.py`, `core/session_manager.py`, `routes/auth_routes.py` |
+| Chat and streaming | `routes/chat_routes.py`, `routes/chat_helpers.py`, `src/chat_handler.py`, `src/chat_processor.py`, `src/llm_core.py` |
+| Agents and tools | `src/agent_loop.py`, `src/tool_execution.py`, `src/agent_tools/`, `src/tools/`, `src/tool_policy.py`, `src/tool_security.py` |
+| Models and providers | `routes/model_routes.py`, `src/model_discovery.py`, `src/model_capabilities.py`, `src/endpoint_resolver.py`, `src/llm_core.py` |
+| Cookbook and hardware fit | `routes/cookbook_routes.py`, `routes/cookbook_helpers.py`, `src/cookbook_serve_lifecycle.py`, `services/hwfit/` |
+| Search and research | `routes/search_routes.py`, `services/search/`, `routes/research/`, `services/research/`, `src/deep_research.py` |
+| Documents and retrieval | `routes/document_routes.py`, `src/document_processor.py`, `src/personal_docs.py`, `src/rag_manager.py`, `src/pdf_runtime.py` |
+| Memory and skills | `routes/memory/`, `services/memory/`, `routes/skills_routes.py` |
+| Email | `routes/email_routes.py`, `routes/email_helpers.py`, `routes/email_pollers.py`, `mcp_servers/email_server.py` |
+| Calendar, contacts, notes, and tasks | `routes/calendar_routes.py`, `routes/contacts/`, `routes/note/`, `routes/task_routes.py`, `src/task_scheduler.py` |
+| Media and speech | `routes/gallery/`, `routes/stt_routes.py`, `routes/tts_routes.py`, `services/stt/`, `services/tts/` |
+| Persistence and operations | `core/database.py`, `src/runtime_paths.py`, `src/bg_jobs.py`, `routes/backup_routes.py`, `routes/cleanup/` |
+
+## Architectural constraints
+
+- Preserve established import and compatibility paths unless a focused change
+  explicitly migrates them.
+- Keep HTTP concerns in route modules and reusable domain behaviour in runtime
+  or service modules.
+- Treat shared persistence, agent orchestration, tool execution, and application
+  startup as high-authority boundaries.
+- Change one ownership boundary at a time.
+- Do not mix structural movement with unrelated feature behaviour.
+- Validate affected imports, startup paths, compatibility surfaces, and tests.
+
+## Frontend
+
+The browser frontend is a no-build ES-module application under `static/`.
+
+Its maintained module-level structure is documented in
+[`static/js/MODULE_SUMMARY.md`](../static/js/MODULE_SUMMARY.md).
+
+## Investigation and snapshots
+
+Non-canonical investigation material is maintained under [`discovery/`](../discovery/).
+
+The following documents may contain dated observations, metrics, unresolved
+questions, or historical planning and must not be treated as specifications:
+
+- [`discovery/system-map.md`](../discovery/system-map.md)
+- [`discovery/architecture-runtime-inventory.md`](../discovery/architecture-runtime-inventory.md)
+
+## Documentation authority
+
+- Code, tests, and configuration define implemented behaviour.
+- Mature specifications define accepted subsystem behaviour where they exist.
+- Following maintainer acceptance, this document will define the high-level
+  architecture map.
+- Discovery documents preserve evidence and uncertainty but remain
+  non-canonical.

docs/STYLE.md

@@ -0,0 +1,72 @@
+# Documentation style
+
+This guide defines the shared structure and writing conventions for Odysseus documentation.
+
+## Principles
+
+- Write for a clear audience and purpose.
+- State whether a document is canonical, informational, a snapshot, or planning material.
+- Prefer current behaviour over historical explanation.
+- Link to source files, tests, issues, or other documentation when useful.
+- Separate verified behaviour from assumptions, open questions, and future work.
+- Keep headings descriptive and consistent.
+- Use Markdown callouts where status or risk must be visible.
+- Do not use emojis.
+
+## Document status
+
+Use a status callout near the top when the document is not normal canonical guidance.
+
+### Canonical documentation
+
+> [!IMPORTANT]
+> This document describes accepted current behaviour. Verify implementation-sensitive details against the current code and tests.
+
+### Discovery material
+
+> [!NOTE]
+> This is non-canonical discovery material. It records code-grounded observations and open questions.
+
+### Snapshot or inventory
+
+> [!WARNING]
+> This document is a dated snapshot. Counts, paths, and implementation details may drift as the codebase changes.
+
+### Planning material
+
+> [!NOTE]
+> This document records planning context. It does not define current runtime behaviour or guarantee future implementation.
+
+## Recommended structure
+
+Use the following sections where relevant:
+
+1. Title
+2. Purpose or status callout
+3. Scope
+4. Current behaviour or guidance
+5. Safety, limitations, or known gaps
+6. Validation or evidence
+7. Related documentation
+
+Not every document needs every section.
+
+## Writing style
+
+- Use concise sentences.
+- Prefer direct language.
+- Avoid jokes, filler, and informal warnings.
+- Avoid repeating the same guidance across several files.
+- Link to the owning document instead of duplicating large sections.
+- Use lists for procedures, requirements, and comparisons.
+- Use tables only when they improve scanning.
+- Use fenced code blocks with an appropriate language identifier.
+- Use relative repository links for internal files.
+
+## Authority
+
+The current code, tests, and configuration are the source of truth for implemented behaviour.
+
+Canonical documentation describes accepted behaviour and supported workflows.
+
+Discovery, inventory, and planning documents must identify themselves explicitly and must not silently become behavioural specifications.

docs/attachments.md

@@ -1,7 +1,13 @@
 # Attachment References and Upload Storage
 
-Odysseus stores uploaded bytes once under the configured upload directory and
+> [!NOTE]
-passes stable references through chat history, tools, and future artifact work.
+> This document records the current attachment-reference and upload-lifecycle
+> contract proposed for maintainer acceptance. Source code, tests, and
+> configuration remain authoritative for implementation-sensitive behaviour.
+
+Odysseus stores chat and document attachment bytes under the configured upload
+directory and passes stable references through chat history, document flows, and
+tool context.
 The goal is to avoid duplicating large inline media payloads in
 `chat_messages.content` or the SQLite FTS index.
 
@@ -54,6 +60,32 @@ External MCP/custom tools should treat the URI and attachment ID as the stable
 contract and request bytes through an owner-checked server path, not by assuming
 host filesystem layout.
 
+## Implementation evidence
+
+The current contract is implemented primarily through:
+
+- `src/upload_handler.py` for upload metadata, owner-aware resolution,
+  reservations, cleanup, and deletion;
+- `src/attachment_refs.py` for compact persisted references and search-index
+  sanitization;
+- `src/document_processor.py` for resolving attachments into chat/model context;
+- `src/tool_execution.py` for attachment manifests exposed to tools;
+- `routes/upload_routes.py` and `routes/document_helpers.py` for upload and
+  retrieval paths.
+
+Focused regression coverage includes:
+
+- `tests/test_attachment_refs.py`;
+- `tests/test_upload_handler_cleanup.py`;
+- `tests/test_replace_messages_upload_reservations.py`;
+- `tests/test_resolve_upload_path_nondict.py`;
+- `tests/test_chat_preprocess_tool_policy.py`;
+- the upload, attachment, and PDF-marker cases in
+  `tests/test_security_regressions.py`.
+
+These tests cover compact persistence, owner isolation, path containment,
+cleanup safety, reservation-before-write behaviour, and traversal resistance.
+
 ## Retention and Deletion
 
 Current retention behavior is conservative:

docs/pr-blocker-audit.md

@@ -181,8 +181,8 @@ Dirty, blocked, conflicting, and unknown merge states are shown as risk/caution
 ## Validation
 
 ```bash
-python3 -m py_compile scripts/pr_blocker_audit.py tests/test_pr_blocker_audit.py
+venv/bin/python -m py_compile scripts/pr_blocker_audit.py tests/test_pr_blocker_audit.py
-python3 -m pytest tests/test_pr_blocker_audit.py -q
+venv/bin/python -m pytest tests/test_pr_blocker_audit.py -q
 python3 scripts/pr_blocker_audit.py --help
 git diff --check
 ```

docs/setup.md

@@ -1,6 +1,28 @@
 # Odysseus Setup Guide
 
-This page keeps the detailed install, deployment, troubleshooting, and configuration notes out of the front README.
+This guide covers installation, deployment, troubleshooting, and configuration.
+For a minimal Docker installation, start with the
+[repository README](../README.md#quick-start).
+
+## On this page
+
+- [Quick Start](#quick-start)
+  - [Docker](#docker-recommended)
+  - [Native Linux and macOS](#native-linux--macos)
+  - [Apple Silicon](#apple-silicon)
+  - [Native Windows](#native-windows)
+- [Troubleshooting and advanced setup](#troubleshooting--advanced-setup)
+- [Security notes](#security-notes)
+- [Configuration](#configuration)
+- [Architecture](#architecture)
+- [Data and backups](#data)
+
+Related guidance:
+
+- [Security policy](../SECURITY.md)
+- [Architecture overview](ARCHITECTURE.md)
+- [Backup and restore guide](backup-restore.md)
+- [Contributing guide](../CONTRIBUTING.md)
 
 ## Quick Start
 
@@ -15,8 +37,8 @@ On first setup, Odysseus creates an admin account (`admin` unless
 For Docker installs, the same line is in `docker compose logs odysseus`.
 Use that for the first login, then change it in **Settings**.
 
-Contributing? See [CONTRIBUTING.md](../CONTRIBUTING.md) for setup, testing, and
+Contributing? See the [contributing guide](../CONTRIBUTING.md) for development
-pull request guidelines.
+setup, testing, and pull request guidelines.
 
 ### Docker (recommended)
 ```bash
@@ -309,32 +331,6 @@ container. Cookbook **Serve** is a separate workflow for serving downloaded
 models through Odysseus/llama.cpp, so Windows users with an existing Ollama
 install usually only need to add the endpoint in Settings.
 
-**Tool calls not firing on a manually-added Ollama `/v1` endpoint.** By
-design, a local Ollama `/v1` endpoint defaults to the conservative
-text-based (fenced-block) tool-calling path rather than native structured
-tool calls, since some locally-served models mishandle native schemas (see
-#1567). This is correct for most local setups, but if you know your specific
-model reliably supports native tool calling (check `ollama show <model>` for
-`tools` under Capabilities), you can opt that endpoint in explicitly. There
-is currently no UI control for this on manually-added endpoints (see #5192);
-the flag can still be set directly against the existing API, from a browser
-console on an authenticated admin session:
-
-```js
-fetch('/api/model-endpoints/<endpoint-id>', {
-  method: 'PATCH',
-  credentials: 'same-origin',
-  headers: {'Content-Type': 'application/json'},
-  body: JSON.stringify({supports_tools: true})
-}).then(r => r.json()).then(console.log)
-```
-
-Find `<endpoint-id>` by inspecting the `/api/model-endpoints` response (or
-your browser's network tab while Settings loads the endpoint list). Send
-`supports_tools: false` to disable native structured tool calls and force the
-conservative fenced/text path, or `supports_tools: null` to return the endpoint
-to the Auto heuristic.
-
 **Useful checks.**
 
 ```bash
@@ -441,19 +437,10 @@ A grab-bag of small gotchas that otherwise turn into long debugging sessions.
 | Package | Feature unlocked |
 |---------|-----------------|
 | `faster-whisper` | Local speech-to-text (microphone -> text) via the "local" STT provider. |
-| `kokoro`, `soundfile` | Local Kokoro-82M text-to-speech on a CUDA GPU. The pinned Kokoro release supports Odysseus installs on Python 3.11-3.12; these packages are intentionally skipped on Python 3.13+ (including the Python 3.14 container image). |
 | `ddgs` | DuckDuckGo as a search provider option. |
 | `PyMuPDF` | PDF page rendering in the side viewer panel and form-filling. (Note: AGPL-3.0) |
 | `markitdown` | Office/EPUB document text extraction (converts .docx/.xlsx/.pptx/.xls/.epub to Markdown). |
 
-Install the optional set only when you need these features:
-
-```bash
-pip install -r requirements-optional.txt
-```
-
-The default Docker image currently uses Python 3.14, while Kokoro 0.9.4 declares Python `>=3.10,<3.13`. Odysseus itself continues to support Python 3.11+, but this pinned optional local-TTS feature requires a native Python 3.11 or 3.12 environment. Kokoro declares `torch`, but the local provider only activates when that torch build has CUDA and a GPU is visible; install the CUDA build appropriate for your host. Browser and configured endpoint TTS remain available on Python 3.13+ and in the container image.
-
 ### Faster, reproducible installs with uv (optional)
 [uv](https://docs.astral.sh/uv/) works as a drop-in replacement for the
 venv + pip steps in the native install guides, no project changes are needed but this change results in faster installs along with a lockfile for reproducible environments. After [installing `uv`](https://docs.astral.sh/uv/getting-started/installation/), use:
@@ -476,15 +463,15 @@ uv pip sync requirements.lock                          # reproduce it exactly la
 ### Outlook / Office 365 email
 Odysseus email accounts currently use IMAP/SMTP username-password auth. Outlook
 and Microsoft 365 generally require OAuth instead, so normal Microsoft mailbox
-passwords will fail. See [docs/email-outlook.md](docs/email-outlook.md) for the
+passwords will fail. See the [Outlook and Microsoft 365 email guide](email-outlook.md)
-current limitation and the planned integration direction.
+for the current limitation and planned integration direction.
 
 ## Security Notes
 Odysseus is a self-hosted workspace with powerful local tools: shell access, file uploads, model downloads, web research, email/calendar integrations, and API tokens. Treat it like an admin console.
 
 - Keep `AUTH_ENABLED=true` for any network-accessible deployment.
 - Keep `LOCALHOST_BYPASS=false` outside local development.
-- Leave `SECURE_COOKIES` unset unless you need to override it: session cookies are marked `Secure` whenever the request arrives over HTTPS. Use `SECURE_COOKIES=true` to force it on for a proxy whose scheme Odysseus cannot see, or `SECURE_COOKIES=false` to force it off while you still serve plain HTTP alongside HTTPS.
+- Use `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway.
 - Do not expose it directly to the public internet without HTTPS and a trusted reverse proxy or private access layer.
 - Keep `.env`, `data/`, `logs/`, databases, uploads, generated media, backups, auth/session files, API keys, and model/provider tokens out of Git and private shares. They are ignored by default.
 - Review `data/auth.json` after first boot: disable open signup unless you intentionally want it, make only your own account admin, and keep demo/test accounts non-admin.
@@ -495,14 +482,6 @@ Odysseus is a self-hosted workspace with powerful local tools: shell access, fil
 - Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only. Expose only the authenticated Odysseus web/API entrypoint through your trusted proxy or private access layer.
 - Before publishing a fork, run `git status --short` and confirm no private files from `.env`, `data/`, `logs/`, uploads, backups, or local databases are staged.
 
-> **Upgrading an existing install:** `SECURE_COOKIES` used to default to
-> `false`, so an install set up before scheme derivation may still carry
-> `SECURE_COOKIES=false` in its own `.env`. That explicit value stays
-> authoritative, so HTTPS logins keep getting a non-`Secure` session cookie.
-> Pulling this change updates the tracked Compose files, but nothing rewrites
-> your `.env` — drop the line from it unless you deliberately serve plain HTTP
-> alongside HTTPS and want the escape hatch.
-
 ### Private or proxied deployments
 Odysseus serves plain HTTP on its app port. Docker Compose binds Odysseus and the bundled services to `127.0.0.1` by default, so a typical production/private setup is:
 
@@ -511,162 +490,9 @@ Odysseus serves plain HTTP on its app port. Docker Compose binds Odysseus and th
 3. Put the authenticated Odysseus web/API entrypoint behind that layer.
 4. Keep raw service and model ports internal-only.
 
-Cloudflare Access, Tailscale, Caddy, nginx, and Traefik can all fit this pattern; none are required by Odysseus. If your access layer reaches Odysseus on the same host, proxy to `http://127.0.0.1:7000` and keep `AUTH_ENABLED=true` and `LOCALHOST_BYPASS=false`. Any proxy that forwards `X-Forwarded-Proto: https` gets `Secure` session cookies without configuration, so `SECURE_COOKIES` only needs setting when you want to override that — force it on for a proxy that forwards no scheme at all, or off while you still serve plain HTTP.
+Cloudflare Access, Tailscale, Caddy, nginx, and Traefik can all fit this pattern; none are required by Odysseus. If your access layer reaches Odysseus on the same host, proxy to `http://127.0.0.1:7000` and keep `AUTH_ENABLED=true`, `LOCALHOST_BYPASS=false`, and `SECURE_COOKIES=true`.
 `ALLOWED_ORIGINS` lists exact permitted origins for cross-origin browser/API clients; ordinary same-origin reverse-proxy access usually does not need a special CORS entry.
 
-#### Faster over the network: HTTP/2
-
-The frontend is raw ES modules with no bundler, so a page load is a few hundred
-small same-origin requests. Over HTTP/1.1 browsers typically allow only a small
-number of concurrent connections per host (commonly around six), so many of
-those requests are serialized across multiple round trips. On localhost that
-costs almost nothing. Over a LAN, VPN, or remote link it can become a major
-part of load time, especially as latency increases.
-
-HTTP/2 multiplexes them onto one connection and the serialisation disappears.
-Odysseus needs no changes for this — uvicorn keeps speaking HTTP/1.1 on
-loopback and the proxy speaks HTTP/2 to the browser. Mainstream browsers
-negotiate HTTP/2 for normal web pages over TLS; they do not use the cleartext
-h2c mode here, so browser-facing HTTP/2 requires a certificate. The
-`--ssl-certfile` route in *HTTPS + LAN/Tailscale exposure* above gives you
-HTTPS but not HTTP/2 — uvicorn does not speak it.
-
-**1. Install Caddy.** See the [install docs](https://caddyserver.com/docs/install)
-for your platform; on macOS, `brew install caddy`.
-
-**2. Write a `Caddyfile`.** Pick the block that matches how you reach the
-machine. Replace `7000` if Odysseus listens elsewhere — the macOS start script
-uses `7860`.
-
-Public domain, Caddy obtains and renews the certificate itself:
-
-```
-odysseus.example.com {
-	reverse_proxy 127.0.0.1:7000
-}
-```
-
-Tailscale, no public DNS needed — `tailscale cert` issues a browser-trusted
-certificate for a tailnet name and writes `<domain>.crt` and `<domain>.key`:
-
-```bash
-tailscale cert myhost.tailnet-name.ts.net
-```
-
-```
-myhost.tailnet-name.ts.net {
-	tls /path/to/myhost.tailnet-name.ts.net.crt /path/to/myhost.tailnet-name.ts.net.key
-	reverse_proxy 127.0.0.1:7000
-}
-```
-
-LAN with your own certificate — same shape, your own files:
-
-```
-odysseus.lan {
-	tls /path/to/cert.pem /path/to/key.pem
-	reverse_proxy 127.0.0.1:7000
-}
-```
-
-Give `tls` absolute paths: a service starts in a working directory you did not
-choose. If port 443 is already taken, append a port to the site address
-(`odysseus.example.com:8443`) and use it in the URL. That alone does not free
-port 80 — Caddy still binds it for the HTTP-to-HTTPS redirect, and fails to
-start with `listen tcp :80: bind: address already in use` if something else
-holds it. Turn the redirect off with a global block at the top of the file:
-
-```
-{
-	auto_https disable_redirects
-}
-```
-
-**3. Run it in the foreground first:**
-
-```bash
-caddy run --config ./Caddyfile
-```
-
-Once that works, run it as a service:
-
-```bash
-brew services start caddy          # macOS — reads $(brew --prefix)/etc/Caddyfile, not ./Caddyfile
-sudo systemctl enable --now caddy  # Linux, if your package installed the unit
-```
-
-Odysseus's own service is unchanged; the proxy runs alongside it. Under Docker,
-run the proxy as another container, or on the host pointing at the published
-port.
-
-**4. Point Odysseus at the new origin** in `.env`, then restart it.
-
-A proxy that exposes the HTTPS request scheme to Odysseus needs no `SECURE_COOKIES` setting. Only force it on when the proxy cannot expose that scheme:
-
-```bash
-# only if the proxy cannot expose the external HTTPS scheme to Odysseus:
-SECURE_COOKIES=true
-# only if you use remote MCP servers with OAuth:
-OAUTH_REDIRECT_BASE_URL=https://odysseus.example.com
-```
-
-Gmail OAuth needs nothing here when the proxy runs on the same host: the
-redirect URI is built from the incoming request, and uvicorn rewrites the
-scheme from `X-Forwarded-Proto` for proxies it trusts — by default only
-`127.0.0.1`. A proxy in a separate container or on another machine is not
-trusted, so pin the URI there:
-
-```bash
-GOOGLE_OAUTH_REDIRECT_URI=https://odysseus.example.com/api/email/oauth/google/callback
-```
-
-(uvicorn's own `FORWARDED_ALLOW_IPS` widens that trust, but it has to be in the
-environment uvicorn starts with — `.env` is read by the app afterwards, too
-late for it to take effect.)
-
-**5. Confirm HTTP/2 is really on:**
-
-```bash
-curl -s -o /dev/null -w '%{http_version}\n' https://odysseus.example.com/
-# 2
-```
-
-The status code is not the thing to check here — a logged-out request redirects
-to the login page, so `curl -I` shows `HTTP/2 302`, and the `HTTP/2` prefix is
-the part that matters. The browser reports the same in the Network panel's
-Protocol column (`h2`); in Chrome and Firefox that column is hidden until you
-enable it by right-clicking the column headers.
-
-Three things bite when moving an existing install behind TLS:
-
-- Leave `SECURE_COOKIES` unset when Odysseus can see the external HTTPS scheme;
-  the cookie then follows the request automatically. If your proxy cannot expose
-  that scheme, set `SECURE_COOKIES=true` **at the same time** you stop serving
-  plain HTTP, not before. An explicit `true` applies to every login, so while an
-  HTTP entrypoint is still reachable the browser will reject the `Secure` cookie
-  there and login will appear to loop.
-- `OAUTH_REDIRECT_BASE_URL` defaults to `http://localhost:7000`. Unlike the
-  Gmail redirect URI it cannot be derived from a request — it is registered
-  with each MCP authorization server up front — so set it to the external
-  origin if you use remote MCP servers over OAuth.
-- Odysseus sends `Strict-Transport-Security` once it sees `X-Forwarded-Proto:
-  https`. HSTS applies to the whole hostname and ignores the port, so any other
-  plain-HTTP service on that same hostname becomes unreachable in browsers that
-  have visited Odysseus. Give Odysseus its own hostname, or strip the header at
-  the proxy (`header_down -Strict-Transport-Security` in Caddy).
-
-Server-sent events are not buffered by this configuration, so chat streaming
-arrives token by token; add `flush_interval -1` inside the `reverse_proxy`
-block if you want that pinned explicitly. nginx needs `proxy_buffering off;`
-for the same reason.
-
-Changing the external origin also affects state scoped to it. Service workers
-and their caches are origin-scoped, so moving to a different origin starts with
-a cold load. Cookies follow their own domain/path/security rules rather than
-being port-scoped: changing the hostname normally requires a new login, while
-changing only the scheme or port does not by itself guarantee that existing
-cookies disappear.
-
 Common internal-only ports from the default docs/compose setup:
 
 | Port | Service |
@@ -697,7 +523,7 @@ Key settings:
 | `AUTH_ENABLED` | `true` | Enable/disable login |
 | `LOCALHOST_BYPASS` | `false` | Development-only auth bypass for loopback requests. Keep false for shared/network deployments. |
 | `ALLOWED_ORIGINS` | `http://localhost,http://127.0.0.1` | Comma-separated exact permitted origins for cross-origin browser/API clients. |
-| `SECURE_COOKIES` | derived from the request scheme | Marks session cookies `Secure` on HTTPS requests. Set true to force it on, false to force it off. |
+| `SECURE_COOKIES` | `false` | Set true when serving Odysseus through HTTPS at a trusted proxy or private access gateway. |
 | `DATABASE_URL` | `sqlite:///./data/app.db` | Database connection string |
 | `CHROMADB_HOST` | `localhost` | ChromaDB host for vector memory. Docker overrides this to `chromadb`. |
 | `CHROMADB_PORT` | `8100` | ChromaDB port for manual host runs. Docker overrides this to `8000`. |
@@ -726,19 +552,16 @@ npx -y @playwright/mcp@latest --version
 That installs `@playwright/mcp` plus Playwright (~300MB total). Restart Odysseus and the server will register at startup.
 
 ## Architecture
-```
+
-app.py                   # FastAPI entry point
+For stable high-level runtime structure, subsystem boundaries, and documentation
-core/      auth, database, middleware, constants
+authority, see the [architecture overview](ARCHITECTURE.md).
-src/       llm_core, agent_loop, agent_tools, chat_processor, search/
+
-routes/    chat, session, document, memory, model … endpoints
+Source code, tests, and configuration remain authoritative for
-services/  docs, memory, search, hwfit (Cookbook) …
+implementation-sensitive behaviour.
-static/    index.html + app.js + style.css + js/ (modular front-end)
-docs/      landing page (index.html) + preview clips
-```
 
 ## Data
 All user data lives in `data/` (gitignored): `app.db` (sessions, messages, documents),
 `memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`.
 
-To back up or restore everything in `data/`, see the
+To protect or recover this data, follow the
-[Backup & Restore guide](backup-restore.md).
+[backup and restore guide](backup-restore.md).

launch-windows.ps1

@@ -163,10 +163,6 @@ if (Test-Path $cudaBase) {
 }
 
 # 7. Start the server (use `python -m uvicorn` - bare `uvicorn` may not be on PATH)
-# -Port only reaches uvicorn as a flag. Everything that builds a URL for this
-# instance - internal_api_base(), companion pairing, the MCP OAuth callback -
-# reads APP_PORT, so set it too or they all assume 7000.
-$env:APP_PORT = $Port
 Write-Step ("Starting Odysseus at http://{0}:{1}" -f $BindHost, $Port)
 Write-Host "Press Ctrl+C to stop."
 Write-Host ""

licenses/KaTeX-MIT-LICENSE.txt

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2013-2020 Khan Academy and other contributors
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

licenses/Mermaid-MIT-LICENSE.txt

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2014 - 2022 Knut Sveidqvist
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

mcp_servers/email_server.py

@@ -1802,6 +1802,7 @@ async def _ai_draft_reply_to_email(uid, folder="INBOX", reply_all=False, account
         from src.endpoint_resolver import (
             resolve_endpoint,
             resolve_utility_fallback_candidates,
+            resolve_chat_fallback_candidates,
         )
         from src.llm_core import llm_call_async_with_fallback
     except Exception as exc:
@@ -1842,6 +1843,13 @@ async def _ai_draft_reply_to_email(uid, folder="INBOX", reply_all=False, account
         utility_fallbacks = resolve_utility_fallback_candidates() or []
     for cand in utility_fallbacks:
         _add(*cand)
+    try:
+        chat_fallbacks = resolve_chat_fallback_candidates(owner=None) or []
+    except TypeError:
+        chat_fallbacks = resolve_chat_fallback_candidates() or []
+    for cand in chat_fallbacks:
+        _add(*cand)
+
     if not candidates:
         return {"error": "No LLM endpoint configured for AI reply"}
 

mcp_servers/memory_server.py

@@ -17,8 +17,6 @@ from mcp.types import Tool, TextContent
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
-from src.memory import MemoryStoreUnreadable
-
 server = Server("memory")
 
 # Late-initialized managers (set during first tool call)
@@ -31,10 +29,6 @@ _OWNER_SCOPE_ERROR = (
     "Error: Memory MCP owner is not configured for an owner-scoped memory store. "
     "Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool."
 )
-_UNREADABLE_STORE_ERROR = (
-    "Error: Memory store is temporarily unreadable — nothing was saved. "
-    "Repair or restore memory.json, then retry."
-)
 
 
 def _configured_owner() -> str | None:
@@ -57,21 +51,9 @@ def _owner_scoped_store(entries: list[dict]) -> bool:
     return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict))
 
 
-def _scope_entries(for_update: bool = False) -> tuple[str | None, list[dict], list[dict], str | None]:
+def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]:
-    """Return configured owner, all entries, visible entries, and optional error.
+    """Return configured owner, all entries, visible entries, and optional error."""
-
+    entries = _memory_manager.load_all()
-    ``for_update=True`` is for read-modify-write callers. They save the ``all
-    entries`` list back, so an unreadable store must be reported as an error
-    instead of degrading to ``[]`` — otherwise the save writes their one new
-    entry over the whole store (issue #5673).
-    """
-    if for_update:
-        try:
-            entries = _memory_manager.load_all_for_update()
-        except MemoryStoreUnreadable as e:
-            return None, [], [], f"{_UNREADABLE_STORE_ERROR} ({e})"
-    else:
-        entries = _memory_manager.load_all()
     owner = _configured_owner()
     if owner is None and _owner_scoped_store(entries):
         return None, entries, [], _OWNER_SCOPE_ERROR
@@ -179,7 +161,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
         category = arguments.get("category", "fact")
         if not text:
             return _text_result("Error: Memory text cannot be empty")
-        owner, memories, _visible, scope_error = _scope_entries(for_update=True)
+        owner, memories, _visible, scope_error = _scope_entries()
         if scope_error:
             return _text_result(scope_error)
         entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)

requirements-optional.txt

@@ -12,16 +12,6 @@
 # GPU-accelerated transcription — it's auto-detected, CPU is used otherwise.
 faster-whisper
 
-# Local text-to-speech via Kokoro-82M for the "local" TTS provider.
-# Kokoro 0.9.4 declares Python >=3.10,<3.13; Odysseus itself requires 3.11+,
-# so pip installs these extras on 3.11-3.12 and deliberately skips them on
-# Python 3.13+ (including the Python 3.14 container image). Kokoro declares
-# torch; the local provider still
-# requires a CUDA-enabled torch build and GPU at runtime. SoundFile is separate
-# in Kokoro's official install instructions and is not a transitive dependency.
-kokoro==0.9.4; python_version >= "3.11" and python_version < "3.13"
-soundfile; python_version >= "3.11" and python_version < "3.13"
-
 # DuckDuckGo as a search provider option.
 # Install if you want DDG in the search-provider dropdown.
 # Alternatives: SearXNG, Brave, Tavily, Serper, Google PSE.

requirements.txt

@@ -38,10 +38,7 @@ python-dateutil
 caldav
 cryptography
 bcrypt
-# Built-in servers use the v1 low-level Server decorator API. MCP SDK v2 is a
+mcp
-# breaking rewrite, so keep fresh installs on the maintained v1 line until the
-# servers are migrated together.
-mcp<2
 pyotp
 qrcode[pil]
 croniter

routes/assistant_routes.py

@@ -16,7 +16,7 @@ from pydantic import BaseModel
 
 from core.database import SessionLocal, CrewMember, ScheduledTask
 from src.auth_helpers import get_current_user
-from src.owner_identity import REQUEST_SENTINEL_OWNERS
+from core.auth import RESERVED_USERNAMES
 from src.task_scheduler import compute_next_run
 
 
@@ -90,12 +90,11 @@ def setup_assistant_routes(task_scheduler) -> APIRouter:
     # check-in tasks seeded. Hitting any /assistant route under one of these
     # used to seed a full CrewMember + Morning/Midday/Evening tasks under that
     # owner, which then double-fired alongside the real user's check-ins.
-    # REQUEST_SENTINEL_OWNERS covers request-only identities; Default/Local is a
+    # RESERVED_USERNAMES covers the same set; the `not owner` guard handles "".
-    # reserved login name but remains a valid storage owner.
 
     async def _get_or_create(owner: str) -> CrewMember:
         """Return the per-owner assistant CrewMember, creating it on demand."""
-        if not owner or owner in REQUEST_SENTINEL_OWNERS:
+        if not owner or owner in RESERVED_USERNAMES:
             raise HTTPException(status_code=400, detail=f"Cannot seed assistant for {owner!r}")
         db = SessionLocal()
         try:

routes/auth_routes.py

@@ -22,8 +22,6 @@ from src.settings import (
     load_features as _load_features,
     save_features as _save_features,
     DEFAULT_SETTINGS,
-    RETIRED_SETTING_KEYS,
-    without_retired_settings,
 )
 from src.integrations import (
     load_integrations,
@@ -86,33 +84,6 @@ class SetOpenRegistrationRequest(BaseModel):
 SESSION_COOKIE = "odysseus_session"
 
 
-def _secure_cookie(request: Request) -> bool:
-    """Decide the ``Secure`` attribute of the session cookie.
-
-    ``SECURE_COOKIES`` stays authoritative when it holds an explicit value:
-    ``true`` always marks the cookie Secure (the documented knob for a TLS
-    proxy), ``false`` never does, which is the escape hatch for an install
-    that still answers on plain HTTP alongside HTTPS. Anything else —
-    unset, or the present-but-empty value docker-compose injects for a
-    variable the host has not defined — derives it from the request, so an
-    HTTPS login gets a Secure cookie without any configuration.
-
-    Either the connection scheme or ``X-Forwarded-Proto`` saying https is
-    enough, which is the same test ``core/middleware.py`` applies before it
-    sends HSTS. Uvicorn's proxy-headers middleware already folds that header
-    into the scheme for the proxies it trusts, so reading it here only adds
-    the case of a terminator that is not on a trusted address; the cost is
-    that a client talking to the app directly can set the header and lock
-    its own session out over plain HTTP.
-    """
-    configured = os.getenv("SECURE_COOKIES", "").strip().lower()
-    if configured in ("true", "false"):
-        return configured == "true"
-    # A chained proxy sends a list — the client-facing hop comes first.
-    forwarded_proto = request.headers.get("x-forwarded-proto", "").split(",")[0]
-    return request.url.scheme == "https" or forwarded_proto.strip().lower() == "https"
-
-
 def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
     router = APIRouter(prefix="/api/auth", tags=["auth"])
 
@@ -186,7 +157,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             value=token,
             httponly=True,
             samesite="lax",
-            secure=_secure_cookie(request),
+            secure=os.getenv("SECURE_COOKIES", "false").lower() == "true",
             path="/",
         )
         if body.remember:
@@ -374,61 +345,9 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         # docs, email accounts, tasks, etc.
         try:
             from sqlalchemy import func
-            from core.database import (
+            from core.database import Base, SessionLocal
-                Base,
-                EmailAccount,
-                SessionLocal,
-                lock_email_account_owner_mutations,
-            )
             db = SessionLocal()
             try:
-                # Email-account defaults are protected by per-owner mutex rows.
-                # A rename crosses two owner partitions, so lock both in the
-                # shared helper's canonical order before inspecting either.
-                lock_email_account_owner_mutations(
-                    db, old_username, new_username
-                )
-
-                source_default_ids = [
-                    row[0]
-                    for row in (
-                        db.query(EmailAccount.id)
-                        .filter(
-                            func.lower(EmailAccount.owner) == old_username,
-                            EmailAccount.is_default == True,  # noqa: E712
-                        )
-                        .order_by(EmailAccount.created_at.asc(), EmailAccount.id.asc())
-                        .all()
-                    )
-                ]
-                destination_default_ids = [
-                    row[0]
-                    for row in (
-                        db.query(EmailAccount.id)
-                        .filter(
-                            func.lower(EmailAccount.owner) == new_username,
-                            EmailAccount.is_default == True,  # noqa: E712
-                        )
-                        .order_by(EmailAccount.created_at.asc(), EmailAccount.id.asc())
-                        .all()
-                    )
-                ]
-                if destination_default_ids:
-                    clear_default_ids = (
-                        destination_default_ids[1:] + source_default_ids
-                    )
-                else:
-                    clear_default_ids = source_default_ids[1:]
-                if clear_default_ids:
-                    (
-                        db.query(EmailAccount)
-                        .filter(EmailAccount.id.in_(clear_default_ids))
-                        .update(
-                            {EmailAccount.is_default: False},
-                            synchronize_session=False,
-                        )
-                    )
-
                 for mapper in Base.registry.mappers:
                     model = mapper.class_
                     if not hasattr(model, "owner"):
@@ -718,7 +637,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         a scrubbed copy with secret keys blanked. The frontend uses this
         for keybinds + TTS prefs, so it stays callable without admin."""
         user = _get_current_user(request)
-        settings = without_retired_settings(_load_settings())
+        settings = _load_settings()
         if user and auth_manager.is_admin(user):
             return settings
         return scrub_settings(settings)
@@ -738,8 +657,6 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             "agent_max_tool_calls": (0, 1000),  # 0 = unlimited
         }
         for key in DEFAULT_SETTINGS:
-            if key in RETIRED_SETTING_KEYS:
-                continue
             if key not in body:
                 continue
             val = body[key]
@@ -752,7 +669,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
                 val = max(lo, min(val, hi))
             current[key] = val
         _save_settings(current)
-        return without_retired_settings(current)
+        return current
 
     # ---- Integrations CRUD ----
 

routes/backup_routes.py

@@ -6,7 +6,6 @@ from datetime import datetime
 
 from fastapi import APIRouter, HTTPException, Request, Response
 from core.middleware import require_admin
-from services.memory import MemoryStoreUnreadable
 from src.auth_helpers import get_current_user
 from src.settings import load_settings, save_settings, load_features, save_features
 
@@ -77,15 +76,7 @@ def setup_backup_routes(memory_manager, preset_manager, skills_manager) -> APIRo
 
         # ── Memories ──
         if "memories" in body and isinstance(body["memories"], list):
-            # Strict load: importing on top of an unreadable store would write
+            existing = memory_manager.load_all()
-            # only the incoming rows back and drop everything already saved.
-            try:
-                existing = memory_manager.load_all_for_update()
-            except MemoryStoreUnreadable as e:
-                logger.error("Refusing to import memories: %s", e)
-                raise HTTPException(
-                    503, "Memory store is temporarily unreadable — nothing was imported."
-                )
             # Dedup against THIS user's own memories only. Using every tenant's
             # rows (load_all) meant a memory whose text matched any other
             # user's was silently skipped, so the importing user lost their own

routes/calendar_routes.py

@@ -10,7 +10,6 @@ from typing import Optional, List
 from fastapi import APIRouter, HTTPException, Request, UploadFile, File
 from pydantic import BaseModel
 from sqlalchemy import or_, and_
-from sqlalchemy.exc import IntegrityError
 from dateutil.rrule import rrulestr
 
 from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent
@@ -222,125 +221,22 @@ class EventUpdate(BaseModel):
 
 # ── Helpers ──
 
-_DEFAULT_CALENDAR_NAMESPACE = uuid.UUID("4840613a-9847-4a3b-bd75-19e6bc5fc3ce")
-
-
-def _default_calendar_id(owner: str, collision_index: int = 0) -> str:
-    """Return one stable primary-key candidate for an owner's lazy default.
-
-    Slot zero preserves the original owner-derived identifier.  Later slots
-    let a username be reused after its prior calendar was migrated to another
-    owner during a rename, without making concurrent first use choose random
-    and therefore divergent identifiers.
-    """
-    if collision_index == 0:
-        candidate_name = owner
-    else:
-        candidate_name = json.dumps(
-            [owner, collision_index],
-            ensure_ascii=False,
-            separators=(",", ":"),
-        )
-    return str(uuid.uuid5(_DEFAULT_CALENDAR_NAMESPACE, candidate_name))
-
-
-def _begin_sqlite_default_write(db) -> None:
-    """Serialize an absent-default check with other SQLite writers.
-
-    SQLite's default deferred transactions allow two workers to both read an
-    empty calendar set before either writes.  ``BEGIN IMMEDIATE`` acquires the
-    writer reservation before the second, authoritative lookup.  We issue it
-    only when the driver has not already opened a write transaction; a caller
-    with a pending write already owns the required reservation.
-    """
-    connection = db.connection()
-    dbapi_connection = connection.connection
-    driver_connection = getattr(
-        dbapi_connection,
-        "driver_connection",
-        dbapi_connection,
-    )
-    if not getattr(driver_connection, "in_transaction", False):
-        connection.exec_driver_sql("BEGIN IMMEDIATE")
-
-
 def _ensure_default_calendar(db, owner: str = None) -> CalendarCal:
-    """Return the owner's calendar, staging a default in the caller's transaction.
+    """Create default calendar if none exist for this owner."""
-
-    A stable owner-derived primary key makes concurrent first-use inserts
-    converge on one row on every SQL backend.  SQLite additionally serializes
-    the absent-row check because its deferred transactions otherwise permit
-    both workers to read the gap before either writes.  Other backends recover
-    a lost insert race inside a savepoint so the caller's event transaction
-    remains usable and atomic.
-    """
     owner = owner or FALLBACK_OWNER
     cal = db.query(CalendarCal).filter(CalendarCal.owner == owner).first()
-    if cal:
+    if not cal:
-        return cal
-
-    dialect = db.get_bind().dialect.name
-    if dialect == "sqlite":
-        _begin_sqlite_default_write(db)
-        # Another worker may have committed while BEGIN IMMEDIATE waited.
-        cal = db.query(CalendarCal).filter(CalendarCal.owner == owner).first()
-        if cal:
-            return cal
-
-    collision_index = 0
-    while True:
-        default_id = _default_calendar_id(owner, collision_index)
-
-        if dialect == "sqlite":
-            # BEGIN IMMEDIATE above makes this occupancy check authoritative:
-            # another SQLite writer cannot rename, delete, or claim this slot
-            # until the caller commits or rolls back.
-            occupant = db.query(CalendarCal).filter(
-                CalendarCal.id == default_id,
-            ).first()
-            if occupant is not None:
-                if occupant.owner == owner:
-                    return occupant
-                collision_index += 1
-                continue
-
         cal = CalendarCal(
-            id=default_id,
+            id=str(uuid.uuid4()),
             owner=owner,
             name="Personal",
             color="#5b8abf",
             source="local",
         )
-
+        db.add(cal)
-        if dialect == "sqlite":
+        db.commit()
-            db.add(cal)
+        db.refresh(cal)
-            db.flush()
+    return cal
-            return cal
-
-        try:
-            # A uniqueness failure rolls back only this savepoint, not an event
-            # or reminder already staged by the caller's outer transaction.
-            with db.begin_nested():
-                db.add(cal)
-                db.flush()
-            return cal
-        except IntegrityError:
-            # Use a locking/current read so repeatable-read backends can observe
-            # the row that won after our transaction's original empty snapshot.
-            occupant = db.query(CalendarCal).filter(
-                CalendarCal.id == default_id,
-            ).with_for_update().first()
-            if occupant is None:
-                # Do not misclassify an unrelated integrity failure as an ID
-                # collision and loop forever. A concurrently deleted winner is
-                # safe for the caller to retry as a fresh transaction.
-                raise
-            if occupant.owner == owner:
-                return occupant
-            # A renamed calendar owns this deterministic slot. Advance to the
-            # next stable slot; concurrent callers for this owner will still
-            # converge there.
-            collision_index += 1
 
 
 # Per-request user time context. chat_routes sets this from browser timezone
@@ -1119,9 +1015,6 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
         db = SessionLocal()
         try:
             _ensure_default_calendar(db, owner)
-            # Listing calendars intentionally lazily creates a durable default.
-            # Other callers commit it with the event they are creating.
-            db.commit()
             cals = db.query(CalendarCal).filter(CalendarCal.owner == owner).all()
             return {"calendars": [
                 {"name": c.name, "href": c.id, "color": c.color, "source": c.source}
@@ -1130,7 +1023,6 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
         except HTTPException:
             raise
         except Exception as e:
-            db.rollback()
             logger.error("Failed to list calendars: %s", e)
             raise HTTPException(500, "Failed to list calendars")
         finally:

routes/chat_helpers.py

@@ -15,7 +15,7 @@ from core.database import Session as DBSession, ModelEndpoint
 from src.llm_core import normalize_model_id
 from src.endpoint_resolver import normalize_base
 from src.context_compactor import maybe_compact, trim_for_context
-from src.model_context import estimate_tokens, get_context_length
+from src.model_context import estimate_tokens
 from src.auth_helpers import effective_user
 from src.prompt_security import untrusted_context_message
 from src.attachment_refs import attachment_ref
@@ -152,38 +152,10 @@ class ChatContext:
     # Uploads attached to this user turn, resolved and owner-checked for the
     # agent's private context. This is not emitted to the browser.
     uploaded_files: list = field(default_factory=list)
-    # Route-neutral prompt before any model-window compaction/trimming. This is
-    # retained only when explicit foreground fallbacks are enabled so each
-    # concrete candidate can apply its own context budget independently.
-    route_messages: list = field(default_factory=list)
 
 
 # ── Helpers ────────────────────────────────────────────────────────────── #
 
-def _allowed_models_from_privileges(privs: dict) -> Optional[frozenset[str]]:
-    if privs.get("block_all_models"):
-        return frozenset()
-    allowed_raw = privs.get("allowed_models")
-    allowed = allowed_raw if isinstance(allowed_raw, list) else []
-    restricted = bool(privs.get("allowed_models_restricted")) or bool(allowed)
-    return frozenset(model for model in allowed if isinstance(model, str)) if restricted else None
-
-
-def _allowed_models_for_request(request) -> Optional[frozenset[str]]:
-    """Return the caller's model allowlist, or ``None`` when unrestricted."""
-
-    try:
-        user = effective_user(request)
-    except Exception:
-        user = None
-    if not user:
-        return None
-    auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
-    if not auth_manager:
-        return None
-    privs = auth_manager.get_privileges(user) or {}
-    return _allowed_models_from_privileges(privs)
-
 def _enforce_chat_privileges(request, sess) -> None:
     """Apply the per-user privilege gates (allowed_models + max_messages_per_day)
     that both /api/chat and /api/chat_stream must enforce BEFORE any LLM work.
@@ -213,8 +185,10 @@ def _enforce_chat_privileges(request, sess) -> None:
     if privs.get("block_all_models"):
         raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.")
 
-    allowed_models = _allowed_models_from_privileges(privs)
+    allowed_raw = privs.get("allowed_models")
-    if allowed_models is not None and sess.model and sess.model not in allowed_models:
+    allowed = allowed_raw if isinstance(allowed_raw, list) else []
+    restricted = bool(privs.get("allowed_models_restricted")) or bool(allowed)
+    if restricted and sess.model and sess.model not in allowed:
         raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.")
 
     cap = int(privs.get("max_messages_per_day") or 0)
@@ -313,6 +287,96 @@ async def auto_name_session(session_manager, sess):
         logger.error(f"Auto-name failed for {sess.id}: {e}\n{traceback.format_exc()}")
 
 
+def try_fallback_endpoint(sess, session_id: str) -> dict | None:
+    """Find an alternative working endpoint when the current one fails.
+
+    Returns {"model": ..., "endpoint_url": ..., "endpoint_name": ...} or None.
+    """
+    import requests as _req
+    from src.endpoint_resolver import (
+        build_chat_url,
+        build_headers,
+        build_models_url,
+        normalize_base,
+        resolve_endpoint_runtime,
+    )
+    from src.chatgpt_subscription import is_chatgpt_subscription_base
+
+    current_url = sess.endpoint_url or ""
+    owner = getattr(sess, "owner", None)
+    db = SessionLocal()
+    try:
+        q = db.query(ModelEndpoint).filter(
+            ModelEndpoint.is_enabled == True
+        )
+        if owner:
+            from src.auth_helpers import owner_filter
+            q = owner_filter(q, ModelEndpoint, owner)
+        endpoints = q.all()
+    finally:
+        db.close()
+
+    for ep in endpoints:
+        base = normalize_base(ep.base_url)
+        # Skip current endpoint
+        if current_url and base in current_url:
+            continue
+        try:
+            base, api_key = resolve_endpoint_runtime(ep, owner=owner)
+        except Exception:
+            continue
+        ping_url = build_models_url(base)
+        headers = build_headers(api_key, base)
+        try:
+            if ping_url:
+                r = _req.get(ping_url, headers=headers, timeout=5)
+                r.raise_for_status()
+                data = r.json()
+                models = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
+                if not models:
+                    models = [
+                        m.get("name") or m.get("model")
+                        for m in (data.get("models") or [])
+                        if m.get("name") or m.get("model")
+                    ]
+            else:
+                models = json.loads(ep.cached_models or "[]")
+            if not models:
+                continue
+            # Found a working endpoint — update session
+            new_model = models[0]
+            chat_url = build_chat_url(base)
+            new_headers = build_headers(api_key, base)
+            persisted_headers = {} if is_chatgpt_subscription_base(base) else new_headers
+
+            sess.model = new_model
+            sess.endpoint_url = chat_url
+            sess.headers = new_headers
+
+            # Persist
+            _db = SessionLocal()
+            try:
+                _db.query(DBSession).filter(DBSession.id == session_id).update({
+                    "model": new_model,
+                    "endpoint_url": chat_url,
+                    "headers": persisted_headers,
+                })
+                _db.commit()
+            finally:
+                _db.close()
+
+            logger.info(f"Fallback: switched session {session_id} from {current_url} to {ep.name} ({new_model})")
+            return {
+                "model": new_model,
+                "endpoint_url": chat_url,
+                "endpoint_name": ep.name,
+            }
+        except Exception:
+            continue
+
+    return None
+
+
 def extract_preset(chat_handler, preset_id) -> PresetInfo:
     """Extract preset parameters via chat_handler."""
     temperature, max_tokens, system_prompt, char_name = (
@@ -623,9 +687,6 @@ async def build_chat_context(
     use_enhanced_message: bool = False,
     agent_mode: bool = False,
     allow_tool_preprocessing: bool = True,
-    defer_context_shaping: bool = False,
-    continuation_context_message: str | None = None,
-    persist_user_message: bool = True,
 ) -> ChatContext:
     """Build the full context (preface + messages) for an LLM call.
 
@@ -649,14 +710,14 @@ async def build_chat_context(
     # Add user message to history. Nobody/incognito uses a request-local
     # transcript store instead of session history so stale saved chats cannot
     # bleed into context and the turn is not persisted.
-    if persist_user_message and incognito:
+    if incognito:
         user_meta = {"attachments": preprocessed.attachment_meta} if preprocessed.attachment_meta else None
         _append_incognito_message(session_id, "user", preprocessed.user_content, user_meta)
-    elif persist_user_message:
+    else:
         add_user_message(sess, chat_handler, preprocessed, incognito=False)
 
     # Fire events
-    if persist_user_message and not incognito:
+    if not incognito:
         fire_message_event(request, webhook_manager, session_id, sess, message, compare_mode)
 
     # Resolve owner-scoped prefs/context. Browser requests keep the cookie user;
@@ -668,12 +729,7 @@ async def build_chat_context(
         getattr(chat_handler, "upload_handler", None),
         getattr(sess, "owner", None),
     )
-    context_message = (
+    casual_low_signal = _is_casual_low_signal(message)
-        str(continuation_context_message).strip()
-        if continuation_context_message
-        else message
-    )
-    casual_low_signal = _is_casual_low_signal(context_message)
 
     # Memory enabled?
     mem_enabled = not incognito and not no_memory and uprefs.get("memory_enabled", True)
@@ -710,15 +766,7 @@ async def build_chat_context(
     # Build context preface
     # The stream path uses enhanced_message (with CoT/preprocessing applied),
     # the sync path uses text_for_context.
-    _ctx_msg = (
+    _ctx_msg = preprocessed.enhanced_message if use_enhanced_message else preprocessed.text_for_context
-        context_message
-        if continuation_context_message
-        else (
-            preprocessed.enhanced_message
-            if use_enhanced_message
-            else preprocessed.text_for_context
-        )
-    )
     _preface_kwargs = dict(
         message=_ctx_msg,
         session=sess,
@@ -782,22 +830,13 @@ async def build_chat_context(
         except Exception:
             logger.debug("Failed to add current date/time context", exc_info=True)
 
-    route_messages = list(messages)
+    # Auto-compact
-    # Explicit fallback routing must shape from the same route-neutral prompt
+    messages, context_length, was_compacted = await maybe_compact(
-    # for every candidate. Running selected-model compaction here would mutate
+        sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user,
-    # session history before we know which route can answer and would make a
+    )
-    # later larger-context candidate unable to recover discarded history.
-    if defer_context_shaping:
-        context_length = get_context_length(sess.endpoint_url, sess.model)
-        was_compacted = False
-    else:
-        messages, context_length, was_compacted = await maybe_compact(
-            sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user,
-        )
     _before_trim_messages = len(messages)
     _before_trim_tokens = estimate_tokens(messages)
-    if not defer_context_shaping:
+    messages = trim_for_context(messages, context_length)
-        messages = trim_for_context(messages, context_length)
     _after_trim_messages = len(messages)
     _after_trim_tokens = estimate_tokens(messages)
     _context_trimmed = _after_trim_messages < _before_trim_messages or _after_trim_tokens < _before_trim_tokens
@@ -821,7 +860,6 @@ async def build_chat_context(
         context_tokens_after_trim=_after_trim_tokens,
         auto_opened_docs=auto_opened_docs,
         uploaded_files=uploaded_files,
-        route_messages=route_messages,
     )
 
 

routes/cookbook_helpers.py

@@ -1204,41 +1204,6 @@ def _safe_env_prefix(ep: str | None) -> str | None:
     return f'[ -f "{path}" ] && source "{path}" || true'
 
 
-def _local_windows_bash_env_prefix(ep: str | None) -> str | None:
-    """Convert a frontend PowerShell venv prefix for the local Git Bash runner."""
-    if not ep:
-        return ep
-
-    prefix = ep.strip()
-    if not prefix.startswith("&"):
-        return ep
-
-    raw_path = prefix[1:].lstrip()
-    if not raw_path:
-        return ep
-    if raw_path.startswith("'"):
-        if len(raw_path) < 2 or not raw_path.endswith("'"):
-            return ep
-        quoted_path = raw_path[1:-1]
-        if "'" in quoted_path.replace("''", ""):
-            return ep
-        path = quoted_path.replace("''", "'")
-    else:
-        path = raw_path.rstrip()
-        if "'" in path or '"' in path:
-            return ep
-    if any(c in path for c in "\r\n;&|`$<>"):
-        return ep
-    if not path.replace("\\", "/").casefold().endswith("/scripts/activate.ps1"):
-        return ep
-
-    bash_path = _git_bash_path(path)
-    if "\\" in bash_path:
-        return ep
-    bash_path = bash_path[: -len("Activate.ps1")] + "activate"
-    return "source " + shlex.quote(bash_path)
-
-
 def _ssh_ps(host, script_path, port=None):
     """Build SSH command to run a PowerShell script on a Windows remote."""
     pf = f"-p {port} " if port and port != "22" else ""

routes/cookbook_routes.py

@@ -50,7 +50,7 @@ from routes.cookbook_helpers import (
     _SESSION_ID_RE, _validate_repo_id, _validate_serve_model_id, _validate_include, _validate_token,
     _validate_local_dir, _validate_gpus, _shell_path,
     _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, OLLAMA_MISSING_HINT,
-    _safe_env_prefix, _local_windows_bash_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines,
+    _safe_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines,
     _append_serve_exit_code_lines, _append_llama_cpp_linux_accel_build_lines, _cached_model_scan_script,
     load_stored_hf_token,
     _append_vllm_linux_preflight_lines, _ollama_bind_from_cmd, _pip_install_fallback_chain,
@@ -73,30 +73,6 @@ _HF_TOKEN_STATUS_SNIPPET = (
 )
 
 
-def _windows_local_pid_record_line(pid_path: Path, ready_path: Path) -> str:
-    """Build the Git Bash prelude that records a Win32-stoppable PID.
-
-    Python publishes the detached outer process's Win32 PID first, then touches
-    ``ready_path``. The inner Git Bash runner waits for that publication before
-    replacing the fallback with its own Win32 PID from /proc/<msys-pid>/winpid.
-
-    Missing, malformed, or late mappings leave the valid outer PID untouched.
-    """
-    pp = shlex.quote(pid_path.as_posix())
-    rp = shlex.quote(ready_path.as_posix())
-    return (
-        "i=0; "
-        f"while [ ! -e {rp} ] && [ \"$i\" -lt 500 ]; do "
-        "i=$((i+1)); sleep 0.01; done; "
-        f"if [ -e {rp} ]; then "
-        "winpid=\"$(cat /proc/$$/winpid 2>/dev/null || true)\"; "
-        "case \"$winpid\" in ''|*[!0-9]*) ;; "
-        f"*) printf '%s\\n' \"$winpid\" > {pp} ;; esac; "
-        "fi; "
-        f"rm -f {rp}"
-    )
-
-
 def _append_mlx_image_server_script(runner_lines: list[str]) -> None:
     """Write the MLX image API helper next to the tmux runner on remote hosts."""
     script_path = Path(__file__).resolve().parents[1] / "scripts" / "mlx_image_server.py"
@@ -1002,18 +978,15 @@ def setup_cookbook_routes() -> APIRouter:
         directly (simple commands only). Returns the launched job record."""
         log_path = TMUX_LOG_DIR / f"{session_id}.log"
         pid_path = TMUX_LOG_DIR / f"{session_id}.pid"
-        pid_ready_path: Path | None = None
         bash = find_bash()
         if bash:
             # Run the existing bash wrapper verbatim through Git Bash, redirecting
             # all output to the log the poller reads. Paths handed to bash use
             # POSIX form + shell-quoting so drive paths / spaces survive.
             inner = TMUX_LOG_DIR / f"{session_id}_run.sh"
-            pid_ready_path = TMUX_LOG_DIR / f"{session_id}.pid.ready"
+            pp = shlex.quote(pid_path.as_posix())
-            pid_ready_path.unlink(missing_ok=True)
             inner.write_text(
-                _windows_local_pid_record_line(pid_path, pid_ready_path) + "\n"
+                f"printf '%s\\n' \"$$\" > {pp}\n" + "\n".join(bash_lines) + "\n",
-                + "\n".join(bash_lines) + "\n",
                 encoding="utf-8",
             )
             lp = shlex.quote(log_path.as_posix())
@@ -1047,18 +1020,7 @@ def setup_cookbook_routes() -> APIRouter:
             env=env,
             **detached_popen_kwargs(),
         )
-        # Publish a valid Win32 ancestor first. The Git Bash runner may then
-        # replace it with its own Win32 pid, but never before this fallback exists.
         pid_path.write_text(str(proc.pid), encoding="utf-8")
-        if pid_ready_path is not None:
-            try:
-                pid_ready_path.touch()
-            except OSError as e:
-                logger.warning(
-                    "Could not publish Windows local PID handoff for %s: %s",
-                    session_id,
-                    e,
-                )
         return {"pid": proc.pid, "log_path": str(log_path)}
 
     @router.post("/api/model/download")
@@ -1336,7 +1298,7 @@ def setup_cookbook_routes() -> APIRouter:
             # Local: run hf download in the background (tmux on POSIX, a detached
             # process + logfile on Windows where tmux doesn't exist).
             if req.env_prefix:
-                lines.append(_safe_env_prefix(_local_windows_bash_env_prefix(req.env_prefix) if local_windows else req.env_prefix))
+                lines.append(_safe_env_prefix(req.env_prefix))
             else:
                 lines.append("deactivate 2>/dev/null; hash -r")
             # Show whether the HF token reached this run (masked) — tells a gated
@@ -2166,7 +2128,7 @@ def setup_cookbook_routes() -> APIRouter:
             if req.gpus:
                 runner_lines.append(f"export CUDA_VISIBLE_DEVICES='{req.gpus}'")
             if req.env_prefix:
-                runner_lines.append(_safe_env_prefix(_local_windows_bash_env_prefix(req.env_prefix) if local_windows else req.env_prefix))
+                runner_lines.append(_safe_env_prefix(req.env_prefix))
             else:
                 runner_lines.append("deactivate 2>/dev/null; hash -r")
             _append_venv_nvidia_library_path_lines(runner_lines, cmd=req.cmd)

routes/document/__init__.py

@@ -1,6 +0,0 @@
-"""Document route domain package (slice 2m, #4082/#4071).
-
-Contains document_routes.py and document_helpers.py, migrated from the flat
-routes/ directory. Backward-compat shims at routes/document_routes.py and
-routes/document_helpers.py re-export from here.
-"""

routes/document/document_helpers.py

@@ -1,243 +0,0 @@
-"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
-
-"""Document routes — CRUD for living documents with version history."""
-
-import logging
-import os
-import re
-from typing import Any, Dict, Optional
-
-from fastapi import HTTPException, Request
-from pydantic import BaseModel
-
-from core.database import Document, DocumentVersion
-from core.database import Session as DbSession
-from src.auth_helpers import _auth_disabled
-from src.upload_handler import UploadHandler
-
-logger = logging.getLogger(__name__)
-
-
-# ---- Request schemas ----
-
-class DocumentCreate(BaseModel):
-    session_id: Optional[str] = None
-    title: str = "Untitled"
-    language: Optional[str] = None
-    content: str = ""
-
-class DocumentUpdate(BaseModel):
-    content: str
-    summary: Optional[str] = None
-    force_version: bool = False
-
-class DocumentPatch(BaseModel):
-    title: Optional[str] = None
-    language: Optional[str] = None
-    session_id: Optional[str] = None  # link/unlink document to a session
-
-
-# ---- Helpers ----
-
-def _doc_to_dict(doc: Document) -> Dict[str, Any]:
-    return {
-        "id": doc.id,
-        "session_id": doc.session_id,
-        "title": doc.title,
-        "language": doc.language,
-        "current_content": doc.current_content,
-        "version_count": doc.version_count,
-        "is_active": doc.is_active,
-        "archived": bool(getattr(doc, "archived", False)),
-        "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
-        "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
-        # Source-email provenance (set when doc was created from an email
-        # attachment) — drives the "Send signed reply" menu item.
-        "source_email_uid":        getattr(doc, "source_email_uid", None),
-        "source_email_folder":     getattr(doc, "source_email_folder", None),
-        "source_email_account_id": getattr(doc, "source_email_account_id", None),
-        "source_email_message_id": getattr(doc, "source_email_message_id", None),
-    }
-
-def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
-    return {
-        "id": v.id,
-        "document_id": v.document_id,
-        "version_number": v.version_number,
-        "content": v.content,
-        "summary": v.summary,
-        "source": v.source,
-        "created_at": v.created_at.isoformat() if v.created_at else None,
-    }
-
-
-def _verify_doc_owner(db, doc: Document, user: str):
-    """Verify `user` owns this document. Raise 404 if not.
-
-    Documents now carry their own `owner` column, so a doc whose session
-    was deleted (session_id → NULL) can still prove ownership and stay
-    openable / cloneable. We trust that column first and only fall back to
-    the session join for any not-yet-backfilled legacy row.
-    """
-    if user is None:
-        if _auth_disabled():
-            return  # Single-user / no-auth mode: allow access
-        raise HTTPException(403, "Authentication required")
-    if doc.owner is not None:
-        if doc.owner != user:
-            raise HTTPException(404, "Document not found")
-        return
-    # Legacy fallback: derive ownership from the linked session.
-    if not doc.session_id:
-        raise HTTPException(404, "Document not found")
-    session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
-    if not session or session.owner != user:
-        raise HTTPException(404, "Document not found")
-
-
-def _owner_session_filter(q, user):
-    """Restrict a documents query to those owned by `user`.
-
-    Documents now carry their own `owner` column (backfilled at boot from
-    the linked session, or assigned to the admin user for legacy/orphaned
-    docs). We filter on that directly rather than on a session join, so a
-    document whose session was deleted (session_id → NULL) still shows up
-    for its owner instead of silently vanishing from the Library + search.
-
-    The owner backfill runs in init_db before the app serves requests, so
-    by the time this filter is live there are no NULL-owner rows to leak;
-    we therefore match the owner strictly for authenticated callers."""
-    if not user:
-        if user == "" or _auth_disabled():
-            return q
-        return q.filter(False)
-    return q.filter(Document.owner == user)
-
-
-
-def _slug(name: str) -> str:
-    """Filesystem-friendly version of a document title.
-
-    Whitespace becomes underscores; other unsafe punctuation is dropped.
-    Preserves letters, digits, dot, hyphen, underscore. Idempotent.
-    """
-    import re as _re
-    s = (name or "").strip()
-    # Drop the trailing extension if the title happens to include one
-    s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
-    s = _re.sub(r'\s+', '_', s)
-    s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
-    s = _re.sub(r'_+', '_', s).strip('_')
-    return s or "form"
-
-
-# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
-_PDF_RENDER_SCALE = 2.0
-
-
-def _upload_path_inside(upload_dir: str, path: str) -> bool:
-    base = os.path.realpath(upload_dir)
-    p = os.path.realpath(path)
-    try:
-        return os.path.commonpath([base, p]) == base
-    except Exception:
-        return False
-
-
-def _resolve_user_upload_path(
-    upload_handler: Any,
-    upload_id: str,
-    owner: Optional[str],
-    auth_manager=None,
-) -> Optional[str]:
-    """Resolve an upload id to a filesystem path the caller may read."""
-    if upload_handler is None:
-        return None
-    resolved = upload_handler.resolve_upload(
-        upload_id,
-        owner=owner,
-        auth_manager=auth_manager,
-    )
-    if not isinstance(resolved, dict) or not resolved:
-        return None
-    path = resolved.get("path")
-    upload_dir = getattr(upload_handler, "upload_dir", None)
-    if path and upload_dir and not _upload_path_inside(upload_dir, path):
-        logger.warning("Upload path outside upload directory: %s", path)
-        return None
-    return path
-
-
-def _locate_upload(
-    upload_dir: str,
-    file_id: str,
-    owner: Optional[str] = None,
-    auth_manager=None,
-    upload_handler: Any = None,
-):
-    """Find an upload by its filename ID via UploadHandler.resolve_upload."""
-    if upload_handler is None:
-        from src.upload_handler import UploadHandler
-
-        base_dir = os.path.dirname(os.path.abspath(upload_dir))
-        upload_handler = UploadHandler(base_dir, upload_dir)
-    return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
-
-
-def _assert_pdf_marker_upload_owned(
-    request: Request,
-    content: str,
-    user: Optional[str],
-    upload_handler: Any,
-) -> None:
-    """Reject document content whose pdf_source marker points at another user's upload."""
-    if upload_handler is None:
-        return
-    from src.pdf_form_doc import find_source_upload_id
-
-    upload_id = find_source_upload_id(content or "")
-    if not upload_id:
-        return
-    auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
-    if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
-        raise HTTPException(
-            400,
-            "Document PDF marker references an upload you do not own",
-        )
-
-
-def _derive_title(content: str) -> str:
-    """Derive a title from document content."""
-    import re
-    if not isinstance(content, str):
-        return "Untitled"
-    text = content.strip()
-    if not text:
-        return "Untitled"
-
-    # Markdown header
-    md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
-    if md:
-        title = md.group(1).strip()
-        if len(title) > 50:
-            title = title[:48] + "…"
-        return title
-
-    # HTML heading
-    html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
-    if html:
-        title = html.group(1).strip()
-        if len(title) > 50:
-            title = title[:48] + "…"
-        return title
-
-    # First non-empty line (if short enough)
-    for line in text.split('\n'):
-        line = line.strip()
-        if line and 2 <= len(line) <= 60:
-            title = re.sub(r'[:#*`]+$', '', line).strip()
-            if title and len(title) > 50:
-                title = title[:48] + "…"
-            return title or "Untitled"
-
-    return "Untitled"

routes/document_helpers.py

@@ -1,14 +1,243 @@
-"""Backward-compat shim — canonical location is routes/document/document_helpers.py.
+"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
 
-This module is replaced in ``sys.modules`` by the canonical module object so
+"""Document routes — CRUD for living documents with version history."""
-that ``import routes.document_helpers``, ``from routes.document_helpers import
-X``, and the ``sys.modules.pop("routes.document_helpers")`` + re-import
-pattern used by test_security_regressions.py all operate on the *same* object.
-Keeps existing import paths working after slice 2m (#4082/#4071).
-"""
 
-import sys as _sys
+import logging
+import os
+import re
+from typing import Any, Dict, Optional
 
-from routes.document import document_helpers as _canonical  # noqa: F401
+from fastapi import HTTPException, Request
+from pydantic import BaseModel
 
-_sys.modules[__name__] = _canonical
+from core.database import Document, DocumentVersion
+from core.database import Session as DbSession
+from src.auth_helpers import _auth_disabled
+from src.upload_handler import UploadHandler
+
+logger = logging.getLogger(__name__)
+
+
+# ---- Request schemas ----
+
+class DocumentCreate(BaseModel):
+    session_id: Optional[str] = None
+    title: str = "Untitled"
+    language: Optional[str] = None
+    content: str = ""
+
+class DocumentUpdate(BaseModel):
+    content: str
+    summary: Optional[str] = None
+    force_version: bool = False
+
+class DocumentPatch(BaseModel):
+    title: Optional[str] = None
+    language: Optional[str] = None
+    session_id: Optional[str] = None  # link/unlink document to a session
+
+
+# ---- Helpers ----
+
+def _doc_to_dict(doc: Document) -> Dict[str, Any]:
+    return {
+        "id": doc.id,
+        "session_id": doc.session_id,
+        "title": doc.title,
+        "language": doc.language,
+        "current_content": doc.current_content,
+        "version_count": doc.version_count,
+        "is_active": doc.is_active,
+        "archived": bool(getattr(doc, "archived", False)),
+        "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
+        "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
+        # Source-email provenance (set when doc was created from an email
+        # attachment) — drives the "Send signed reply" menu item.
+        "source_email_uid":        getattr(doc, "source_email_uid", None),
+        "source_email_folder":     getattr(doc, "source_email_folder", None),
+        "source_email_account_id": getattr(doc, "source_email_account_id", None),
+        "source_email_message_id": getattr(doc, "source_email_message_id", None),
+    }
+
+def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
+    return {
+        "id": v.id,
+        "document_id": v.document_id,
+        "version_number": v.version_number,
+        "content": v.content,
+        "summary": v.summary,
+        "source": v.source,
+        "created_at": v.created_at.isoformat() if v.created_at else None,
+    }
+
+
+def _verify_doc_owner(db, doc: Document, user: str):
+    """Verify `user` owns this document. Raise 404 if not.
+
+    Documents now carry their own `owner` column, so a doc whose session
+    was deleted (session_id → NULL) can still prove ownership and stay
+    openable / cloneable. We trust that column first and only fall back to
+    the session join for any not-yet-backfilled legacy row.
+    """
+    if user is None:
+        if _auth_disabled():
+            return  # Single-user / no-auth mode: allow access
+        raise HTTPException(403, "Authentication required")
+    if doc.owner is not None:
+        if doc.owner != user:
+            raise HTTPException(404, "Document not found")
+        return
+    # Legacy fallback: derive ownership from the linked session.
+    if not doc.session_id:
+        raise HTTPException(404, "Document not found")
+    session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
+    if not session or session.owner != user:
+        raise HTTPException(404, "Document not found")
+
+
+def _owner_session_filter(q, user):
+    """Restrict a documents query to those owned by `user`.
+
+    Documents now carry their own `owner` column (backfilled at boot from
+    the linked session, or assigned to the admin user for legacy/orphaned
+    docs). We filter on that directly rather than on a session join, so a
+    document whose session was deleted (session_id → NULL) still shows up
+    for its owner instead of silently vanishing from the Library + search.
+
+    The owner backfill runs in init_db before the app serves requests, so
+    by the time this filter is live there are no NULL-owner rows to leak;
+    we therefore match the owner strictly for authenticated callers."""
+    if not user:
+        if user == "" or _auth_disabled():
+            return q
+        return q.filter(False)
+    return q.filter(Document.owner == user)
+
+
+
+def _slug(name: str) -> str:
+    """Filesystem-friendly version of a document title.
+
+    Whitespace becomes underscores; other unsafe punctuation is dropped.
+    Preserves letters, digits, dot, hyphen, underscore. Idempotent.
+    """
+    import re as _re
+    s = (name or "").strip()
+    # Drop the trailing extension if the title happens to include one
+    s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
+    s = _re.sub(r'\s+', '_', s)
+    s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
+    s = _re.sub(r'_+', '_', s).strip('_')
+    return s or "form"
+
+
+# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
+_PDF_RENDER_SCALE = 2.0
+
+
+def _upload_path_inside(upload_dir: str, path: str) -> bool:
+    base = os.path.realpath(upload_dir)
+    p = os.path.realpath(path)
+    try:
+        return os.path.commonpath([base, p]) == base
+    except Exception:
+        return False
+
+
+def _resolve_user_upload_path(
+    upload_handler: Any,
+    upload_id: str,
+    owner: Optional[str],
+    auth_manager=None,
+) -> Optional[str]:
+    """Resolve an upload id to a filesystem path the caller may read."""
+    if upload_handler is None:
+        return None
+    resolved = upload_handler.resolve_upload(
+        upload_id,
+        owner=owner,
+        auth_manager=auth_manager,
+    )
+    if not isinstance(resolved, dict) or not resolved:
+        return None
+    path = resolved.get("path")
+    upload_dir = getattr(upload_handler, "upload_dir", None)
+    if path and upload_dir and not _upload_path_inside(upload_dir, path):
+        logger.warning("Upload path outside upload directory: %s", path)
+        return None
+    return path
+
+
+def _locate_upload(
+    upload_dir: str,
+    file_id: str,
+    owner: Optional[str] = None,
+    auth_manager=None,
+    upload_handler: Any = None,
+):
+    """Find an upload by its filename ID via UploadHandler.resolve_upload."""
+    if upload_handler is None:
+        from src.upload_handler import UploadHandler
+
+        base_dir = os.path.dirname(os.path.abspath(upload_dir))
+        upload_handler = UploadHandler(base_dir, upload_dir)
+    return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
+
+
+def _assert_pdf_marker_upload_owned(
+    request: Request,
+    content: str,
+    user: Optional[str],
+    upload_handler: Any,
+) -> None:
+    """Reject document content whose pdf_source marker points at another user's upload."""
+    if upload_handler is None:
+        return
+    from src.pdf_form_doc import find_source_upload_id
+
+    upload_id = find_source_upload_id(content or "")
+    if not upload_id:
+        return
+    auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
+    if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
+        raise HTTPException(
+            400,
+            "Document PDF marker references an upload you do not own",
+        )
+
+
+def _derive_title(content: str) -> str:
+    """Derive a title from document content."""
+    import re
+    if not isinstance(content, str):
+        return "Untitled"
+    text = content.strip()
+    if not text:
+        return "Untitled"
+
+    # Markdown header
+    md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
+    if md:
+        title = md.group(1).strip()
+        if len(title) > 50:
+            title = title[:48] + "…"
+        return title
+
+    # HTML heading
+    html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
+    if html:
+        title = html.group(1).strip()
+        if len(title) > 50:
+            title = title[:48] + "…"
+        return title
+
+    # First non-empty line (if short enough)
+    for line in text.split('\n'):
+        line = line.strip()
+        if line and 2 <= len(line) <= 60:
+            title = re.sub(r'[:#*`]+$', '', line).strip()
+            if title and len(title) > 50:
+                title = title[:48] + "…"
+            return title or "Untitled"
+
+    return "Untitled"

routes/email_helpers.py

@@ -247,7 +247,6 @@ import re as _re_reply
 _REPLY_OPEN_RE = _re_reply.compile(r"<<<\s*(?:REPLY|SUMMARY|OUTPUT)\s*>>+", _re_reply.I)
 _REPLY_CLOSE_RE = _re_reply.compile(r"<<<\s*END\s*>>+", _re_reply.I)
 _REPLY_ROLE_MARKER_RE = _re_reply.compile(r"</?\|(?:assistant|assistan|user|system|tool)\|>?|</\|end\|>?", _re_reply.I)
-_SUMMARY_BULLET_RE = _re_reply.compile(r"^(?:[-*\u2022]\s+|\d+[.)]\s+)")
 
 
 def _extract_reply(text: str) -> str:
@@ -278,125 +277,6 @@ def _extract_reply(text: str) -> str:
     return _strip_think(t).strip()
 
 
-def _build_email_summary_messages(sender: str, subject: str, body_for_llm: str) -> list[dict[str, str]]:
-    return [
-        {
-            "role": "system",
-            "content": (
-                "You are an email summarizer. Format: 1-3 short bullet points "
-                "(use '- '). Cover: main point, action items, deadlines. If the "
-                "email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR "
-                "CONTENTS - pull invoice totals, deadlines, key clauses, concrete "
-                "numbers/dates from PDFs/docs into the bullets. Be terse.\n\n"
-                "OUTPUT FORMAT: Put ONLY the bullet points between these exact "
-                "markers, each on its own line:\n"
-                "<<<SUMMARY>>>\n"
-                "- ...\n"
-                "<<<END>>>\n"
-                "Any reasoning must come BEFORE <<<SUMMARY>>> (ideally inside "
-                "<think>...</think>). Only the text between the markers is kept."
-            ),
-        },
-        {
-            "role": "user",
-            "content": (
-                f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}"
-                "\n\n---\n\nSummarize the email. Output the bullets between "
-                "<<<SUMMARY>>> and <<<END>>>."
-            ),
-        },
-    ]
-
-
-async def _generate_email_summary(
-    url: str,
-    model: str,
-    sender: str,
-    subject: str,
-    body_for_llm: str,
-    *,
-    headers: dict | None = None,
-    max_tokens: int = 8192,
-    timeout: int = 180,
-) -> str:
-    """Generate an interactive email summary through the shared LLM adapter."""
-    from src.llm_core import llm_call_async
-
-    raw = await llm_call_async(
-        url=url,
-        model=model,
-        messages=_build_email_summary_messages(sender, subject, body_for_llm),
-        temperature=0.3,
-        max_tokens=max_tokens,
-        headers=headers,
-        timeout=timeout,
-        workload="foreground",
-    )
-    return _normalize_email_summary(raw)
-
-
-async def _generate_scheduled_email_summary(
-    url: str,
-    model: str,
-    sender: str,
-    subject: str,
-    body_for_llm: str,
-    *,
-    headers: dict | None = None,
-    owner: str | None = None,
-    max_tokens: int = 8192,
-    timeout: int = 180,
-) -> str:
-    """Generate a scheduled summary through the background task candidate chain."""
-    from src.task_endpoint import task_llm_call_async
-
-    raw = await task_llm_call_async(
-        messages=_build_email_summary_messages(sender, subject, body_for_llm),
-        fallback_url=url,
-        fallback_model=model,
-        fallback_headers=headers,
-        owner=owner,
-        temperature=0.3,
-        max_tokens=max_tokens,
-        timeout=timeout,
-    )
-    return _normalize_email_summary(raw)
-
-
-def _normalize_email_summary(raw) -> str:
-    """Extract a stable cache/UI summary from provider output."""
-    raw_text = raw or ""
-    if _REPLY_OPEN_RE.search(raw_text):
-        summary = _extract_reply(raw_text)
-        if summary:
-            return summary
-
-    cleaned = _strip_think(raw_text).strip()
-    bullets = [
-        line.strip()
-        for line in cleaned.splitlines()
-        if _SUMMARY_BULLET_RE.match(line.strip())
-    ]
-    if bullets:
-        return "\n".join(bullets)
-    return cleaned.strip()
-
-
-EMAIL_SUMMARY_ERROR_CODE = "email_summary_unavailable"
-EMAIL_SUMMARY_ERROR_MESSAGE = "Failed to summarize"
-
-
-def _email_summary_failure_log_detail(exc: BaseException) -> str:
-    """Return useful provider-failure metadata without echoing exception text."""
-    detail = f"type={type(exc).__name__}"
-    status = getattr(exc, "status_code", None)
-    if status is None:
-        status = getattr(getattr(exc, "response", None), "status_code", None)
-    if isinstance(status, int):
-        detail += f" status={status}"
-    return detail
-
-
 def _apply_email_style_mechanics(text: str) -> str:
     """Enforce deterministic writing-style mechanics that models often miss."""
     if not text:

routes/email_pollers.py

@@ -40,7 +40,6 @@ from routes.email_helpers import (
     _pre_retrieve_context,
     _attach_compose_uploads, _cleanup_compose_uploads, _q,
     SCHEDULED_DB, _EMAIL_REPLY_SYS_PROMPT_BASE, _email_cache_owner_clause,
-    _generate_scheduled_email_summary, _email_summary_failure_log_detail,
 )
 
 logger = logging.getLogger(__name__)
@@ -654,7 +653,6 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
         no_msgid = 0
         examined = 0
         _summaries_created = 0
-        _summary_failed = 0
         _events_created = 0
         _replies_drafted = 0
         _reply_failed = 0
@@ -787,17 +785,16 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
 
                 if need_sum:
                     try:
-                        summary = await _generate_scheduled_email_summary(
+                        summary = await task_llm_call_async(
-                            url=url,
+                            messages=[
-                            model=model,
+                                {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
-                            sender=sender,
+                                {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
-                            subject=subject,
+                            ],
-                            body_for_llm=body_for_llm,
+                            fallback_url=url, fallback_model=model, fallback_headers=headers,
-                            headers=req_headers,
                             owner=account_owner or None,
-                            max_tokens=16384,
+                            temperature=0.3, max_tokens=16384, timeout=240,
-                            timeout=240,
                         )
+                        summary = _extract_reply((summary or "").strip())
                         if summary:
                             _c = _sql3.connect(SCHEDULED_DB)
                             _c.execute("""
@@ -811,19 +808,10 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             _summaries_created += 1
                             _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
                             _detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
-                        else:
-                            _summary_failed += 1
-                            _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
-                            _detail_lines.append(f"summary empty · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
                     except Exception as e:
-                        _summary_failed += 1
                         _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
                         _detail_lines.append(f"summary failed · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
-                        logger.warning(
+                        logger.warning(f"Auto-summary {uid} failed: {e}")
-                            "Auto-summary uid=%s failed %s",
-                            _uid_text,
-                            _email_summary_failure_log_detail(e),
-                        )
 
                 if need_reply:
                     await _emit_progress(progress_cb, f"Drafting reply {processed + 1}/{_max_process} · checked {examined}/{len(uid_list)}")
@@ -1332,8 +1320,6 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
             parts.append(f"processed {processed} new")
         if auto_sum:
             parts.append(f"summarized {_summaries_created}")
-            if _summary_failed:
-                parts.append(f"{_summary_failed} summary failed")
         if auto_reply_draft:
             parts.append(f"drafted {_replies_drafted} repl" + ("y" if _replies_drafted == 1 else "ies"))
             if _reply_failed:

routes/email_routes.py

@@ -45,7 +45,6 @@ from src.upload_limits import read_upload_limited, EMAIL_COMPOSE_UPLOAD_MAX_BYTE
 
 from routes.email_helpers import (
     _strip_think, _extract_reply, _apply_email_style_mechanics, require_owner, require_user, _assert_owns_account,
-    _account_visible_to_owner,
     _q, _attach_compose_uploads, _cleanup_compose_uploads,
     _load_settings, _save_settings, _get_email_config,
     _send_smtp_message, _smtp_security_mode,
@@ -58,8 +57,7 @@ from routes.email_helpers import (
     _extract_attachment_to_disk, _extract_html, _extract_text,
     _fetch_sender_thread_context, _pre_retrieve_context,
     _EMAIL_REPLY_SYS_PROMPT_BASE, _POOL_HOOKS,
-    _friendly_email_auth_error, _email_summary_failure_log_detail,
+    _friendly_email_auth_error,
-    _generate_email_summary, EMAIL_SUMMARY_ERROR_CODE, EMAIL_SUMMARY_ERROR_MESSAGE,
     SendEmailRequest, ExtractStyleRequest,
     ATTACHMENTS_DIR, COMPOSE_UPLOADS_DIR, SCHEDULED_DB,
     attachment_extract_dir, _email_cache_owner_clause, email_translation_body_hash,
@@ -196,64 +194,6 @@ def _coerce_port(value, default):
         return None, f"Invalid port {value!r}; must be a whole number"
 
 
-def _lock_email_account_owner_mutation(db, *owners: str) -> None:
-    """Delegate account/default serialization to the shared DB primitive."""
-    from core.database import lock_email_account_owner_mutations
-
-    lock_email_account_owner_mutations(db, *owners)
-
-
-def _email_account_owner_scope(query, owner: str):
-    """Restrict a query to one normalized EmailAccount owner partition."""
-    from core.database import EmailAccount
-    from sqlalchemy import or_
-
-    if owner:
-        return query.filter(EmailAccount.owner == owner)
-    return query.filter(or_(EmailAccount.owner == None, EmailAccount.owner == ""))  # noqa: E711
-
-
-def _discover_email_account_mutation_scope(account_id: str, owner: str) -> str:
-    """Read the initial lock key and fail closed before a mutation session."""
-    from core.database import EmailAccount, SessionLocal
-
-    db = SessionLocal()
-    try:
-        row = db.get(EmailAccount, account_id)
-        if row is None or (owner and not _account_visible_to_owner(row, owner)):
-            raise HTTPException(404, "Account not found")
-        return row.owner or ""
-    except HTTPException:
-        raise
-    except Exception as exc:
-        logger.error("Account-owner mutation check failed: %s", exc)
-        raise HTTPException(503, "Account check failed")
-    finally:
-        db.close()
-
-
-def _lock_and_reload_email_account(db, account_id: str, owner: str, scope: str):
-    """Lock, reload, and revalidate an account, retrying if its owner moved."""
-    from core.database import EmailAccount
-
-    owner_scopes = {scope or ""}
-    while True:
-        _lock_email_account_owner_mutation(db, *owner_scopes)
-        row = db.get(EmailAccount, account_id, populate_existing=True)
-        if row is None or (owner and not _account_visible_to_owner(row, owner)):
-            raise HTTPException(404, "Account not found")
-
-        current_scope = row.owner or ""
-        if current_scope in owner_scopes or db.get_bind().dialect.name == "sqlite":
-            return row
-
-        # The account changed owner after discovery but before lock acquisition.
-        # Release the partial lock set and reacquire all observed scopes in the
-        # shared helper's canonical order, then validate from the database again.
-        db.rollback()
-        owner_scopes.add(current_scope)
-
-
 def _email_tag_owner_aliases(account_id: str | None, owner: str = "") -> list[str]:
     aliases = [owner or ""]
     try:
@@ -2920,22 +2860,13 @@ def setup_email_routes():
                 return indexed_response
             return {"emails": [], "total": 0, "error": "Mail operation failed"}
 
-    def _read_email_sync(uid, folder, account_id, owner, mark_seen=False, full=False):
+    def _read_email_sync(uid, folder, account_id, owner, mark_seen=True, full=False):
         """Sync IMAP read — wrapped in to_thread by the async handler.
 
         The normal reader path fetches the headers plus a bounded body prefix.
         That avoids downloading multi-megabyte attachments just to open a
         message. Full-message fetch remains available for flows that need
         attachment metadata immediately, such as forwarding.
-
-        `mark_seen` defaults to False because it mutates provider state: it
-        selects the mailbox read-write and issues a STORE. Only a foreground
-        open should ask for it, and it has to ask explicitly.
-
-        A failed \\Seen transition is reported as `mark_seen_failed` on an
-        otherwise normal response, never as an error. The body has already been
-        fetched at that point, so refusing to return it would turn a cosmetic
-        flag failure into an unreadable message.
         """
         import time as _t
         _t0 = _t.monotonic()
@@ -2943,28 +2874,9 @@ def setup_email_routes():
         preview_bytes = 384 * 1024
         _t_select = 0.0
         _t_fetch = 0.0
-        mark_seen_failed = False
         try:
             with _imap(account_id, owner=owner) as conn:
-                # A foreground open owns both the body fetch and the \Seen
+                conn.select(_q(folder), readonly=True)
-                # transition. Keep them on one read-write IMAP selection so the
-                # route never schedules a second connection that can race the
-                # response. Prefetch/read-only callers retain BODY.PEEK and a
-                # read-only mailbox selection.
-                try:
-                    conn.select(_q(folder), readonly=not mark_seen)
-                except Exception as select_exc:
-                    if not mark_seen:
-                        raise
-                    # Read-only mailboxes (shared archives, some provider
-                    # folders) reject a read-write SELECT. Serve the message
-                    # read-only and report the flag failure.
-                    logger.warning(
-                        f"read-write SELECT rejected for {folder!r}; "
-                        f"serving read-only without \\Seen: {select_exc}"
-                    )
-                    conn.select(_q(folder), readonly=True)
-                    mark_seen_failed = True
                 _t_select = _t.monotonic() - _t0
                 fetch_query = "(BODY.PEEK[])" if full else f"(BODY.PEEK[HEADER] BODY.PEEK[TEXT]<0.{preview_bytes}>)"
                 status, msg_data = _imap_uid_fetch(conn, uid, fetch_query)
@@ -2990,44 +2902,22 @@ def setup_email_routes():
                         header_part = msg_data[0][1] or b""
                     raw = header_part + b"\r\n" + text_part
 
-                # Parse the fetched payload before mutating provider state. If
+            msg = email_mod.message_from_bytes(raw)
-                # the message is malformed enough that the reader cannot build
-                # a response, the caller gets an error while the message stays
-                # unread instead of receiving a false optimistic rollback.
-                msg = email_mod.message_from_bytes(raw)
 
-                subject = _decode_header(msg.get("Subject", "(no subject)"))
+            subject = _decode_header(msg.get("Subject", "(no subject)"))
-                sender = _decode_header(msg.get("From", "unknown"))
+            sender = _decode_header(msg.get("From", "unknown"))
-                to = _decode_header(msg.get("To", ""))
+            to = _decode_header(msg.get("To", ""))
-                cc = _decode_header(msg.get("Cc", ""))
+            cc = _decode_header(msg.get("Cc", ""))
-                date_str = msg.get("Date", "")
+            date_str = msg.get("Date", "")
-                message_id = msg.get("Message-ID", "")
+            message_id = msg.get("Message-ID", "")
-                in_reply_to = msg.get("In-Reply-To", "")
+            in_reply_to = msg.get("In-Reply-To", "")
-                references = msg.get("References", "")
+            references = msg.get("References", "")
-                body = _extract_text(msg)
+            body = _extract_text(msg)
-                body_html = _extract_html(msg)
+            body_html = _extract_html(msg)
-
-                sender_name, sender_addr = email.utils.parseaddr(sender)
-                parsed_date = email.utils.parsedate_to_datetime(date_str) if date_str else None
-                attachments = _list_attachments_from_msg(msg) if full else (_email_attachment_meta_cache_get(owner, account_id, folder, uid) or [])
-
-                if mark_seen and not mark_seen_failed:
-                    seen_status, _ = conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "(\\Seen)")
-                    if seen_status != "OK":
-                        # Report, don't raise. The parsed body below is still a
-                        # valid response; only the flag claim is untrue.
-                        logger.warning(
-                            f"IMAP STORE \\Seen failed for UID {uid} in {folder!r}: {seen_status}"
-                        )
-                        mark_seen_failed = True
-
-            # Only record the local flag transition when the provider actually
-            # accepted it, so the index and list cache cannot drift ahead of
-            # the mailbox.
-            if mark_seen and not mark_seen_failed:
-                _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
-                _update_list_cache_seen(account_id, folder, uid, True)
 
+            sender_name, sender_addr = email.utils.parseaddr(sender)
+            parsed_date = email.utils.parsedate_to_datetime(date_str) if date_str else None
+            attachments = _list_attachments_from_msg(msg) if full else (_email_attachment_meta_cache_get(owner, account_id, folder, uid) or [])
             related_attachments = []
             if full and not _has_visible_attachments(msg):
                 related_attachments = _related_thread_attachments_sync(
@@ -3148,29 +3038,20 @@ def setup_email_routes():
                 "boundaries": cached_boundaries,
                 "thread_turns": cached_turns,
                 "sender_signature": cached_sender_sig,
-                # Per-request, not part of the message: the route strips this
-                # before caching so a one-off flag failure is never replayed to
-                # later readers.
-                "mark_seen_failed": mark_seen_failed,
             }
         except Exception as e:
             logger.error(f"Failed to read email {uid}: {e}")
             return {"error": "Mail operation failed"}
 
     def _mark_email_seen_sync(uid, folder, account_id, owner):
-        """Synchronously mark a cached email seen and report success."""
         try:
             with _imap(account_id, owner=owner) as conn:
-                conn.select(_q(folder), readonly=False)
+                conn.select(_q(folder))
-                status, _ = conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "(\\Seen)")
+                conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "\\Seen")
-                if status != "OK":
-                    return False
             _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
             _update_list_cache_seen(account_id, folder, uid, True)
-            return True
         except Exception as e:
-            logger.warning(f"mark-seen after cached read failed uid={uid}: {e}")
+            logger.debug(f"mark-seen after cached read failed uid={uid}: {e}")
-            return False
 
     @router.get("/read/{uid}")
     async def read_email_by_uid(
@@ -3196,32 +3077,32 @@ def setup_email_routes():
             if cached.get("attachment_version") != EMAIL_READ_ATTACHMENT_VERSION:
                 cached = None
         if cached is not None:
-            # A cache hit already holds a complete, valid message. Await the
+            if mark_seen:
-            # STORE so the response reports the real flag state, but never let
+                try:
-            # a failed STORE withhold a body we are holding in memory.
+                    _asyncio.create_task(_asyncio.to_thread(_mark_email_seen_sync, uid, folder, account_id, owner))
-            if mark_seen and not await _asyncio.to_thread(
+                except RuntimeError:
-                _mark_email_seen_sync, uid, folder, account_id, owner
+                    pass
-            ):
-                return {**cached, "mark_seen_failed": True}
             return cached
         if not full:
             persisted = _email_preview_cache_get(owner, account_id, folder, uid)
             if persisted and persisted.get("attachment_version") == EMAIL_READ_ATTACHMENT_VERSION:
                 _read_cache_put(ck, persisted)
-                if mark_seen and not await _asyncio.to_thread(
+                if mark_seen:
-                    _mark_email_seen_sync, uid, folder, account_id, owner
+                    try:
-                ):
+                        _asyncio.create_task(_asyncio.to_thread(_mark_email_seen_sync, uid, folder, account_id, owner))
-                    return {**persisted, "mark_seen_failed": True}
+                    except RuntimeError:
+                        pass
                 return persisted
         result = await _asyncio.to_thread(_read_email_sync, uid, folder, account_id, owner, mark_seen, full)
         if result and not result.get("error"):
-            # `mark_seen_failed` describes this request, not the message, so it
+            _read_cache_put(ck, result)
-            # must not enter either cache — a later reader would otherwise be
-            # told a STORE failed that it never issued.
-            cacheable = {k: v for k, v in result.items() if k != "mark_seen_failed"}
-            _read_cache_put(ck, cacheable)
             if not full:
-                _email_preview_cache_put(owner, account_id, folder, uid, cacheable)
+                _email_preview_cache_put(owner, account_id, folder, uid, result)
+            if mark_seen:
+                try:
+                    _asyncio.create_task(_asyncio.to_thread(_mark_email_seen_sync, uid, folder, account_id, owner))
+                except RuntimeError:
+                    pass
         return result
 
     def _schedule_recent_email_warm(emails: list, folder: str, account_id: str | None, owner: str):
@@ -4885,6 +4766,8 @@ def setup_email_routes():
         """Generate a quick AI summary of an email body."""
         try:
             from src.endpoint_resolver import resolve_endpoint
+            from src.llm_core import _uses_max_completion_tokens, _restricts_temperature
+            import requests as _req
 
             body = data.get("body", "")
             subject = data.get("subject", "")
@@ -4895,11 +4778,7 @@ def setup_email_routes():
             if account_id:
                 _assert_owns_account(account_id, owner)
             if not body:
-                return {
+                return {"success": False, "error": "No body provided"}
-                    "success": False,
-                    "error": "No body provided",
-                    "error_code": "email_summary_missing_body",
-                }
 
             # If we know which UID this is, fetch the raw message and pull
             # attachment text so the summary can reference invoice totals,
@@ -4928,43 +4807,53 @@ def setup_email_routes():
             if not url:
                 url, model, headers = resolve_endpoint("default", owner=owner)
             if not url or not model:
-                return {
+                return {"success": False, "error": "No LLM endpoint configured"}
-                    "success": False,
-                    "error": "No model configured for email summaries",
-                    "error_code": "email_summary_not_configured",
-                }
 
             req_headers = {"Content-Type": "application/json"}
             if headers:
                 req_headers.update(headers)
-            try:
+            tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
-                content = await _generate_email_summary(
+            payload = {
-                    url=url,
+                "model": model,
-                    model=model,
+                "messages": [
-                    sender=sender,
+                    {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull invoice totals, deadlines, key clauses, concrete numbers/dates from PDFs/docs into the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
-                    subject=subject,
+                    {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
-                    body_for_llm=body_for_llm,
+                ],
-                    headers=req_headers,
+                tok_key: 8192,
-                    max_tokens=8192,
+                "temperature": 0.3,
-                    timeout=180,
+                "stream": False,
-                )
+            }
-            except Exception as e:
+            # Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature.
-                logger.warning(
+            if _restricts_temperature(model):
-                    "Email summary LLM call failed %s",
+                payload.pop("temperature", None)
-                    _email_summary_failure_log_detail(e),
+            resp = await asyncio.to_thread(
-                )
+                _req.post, url, json=payload, headers=req_headers, timeout=180
-                return {
+            )
-                    "success": False,
+            if not resp.ok:
-                    "error": EMAIL_SUMMARY_ERROR_MESSAGE,
+                return {"success": False, "error": f"LLM HTTP {resp.status_code}"}
-                    "error_code": EMAIL_SUMMARY_ERROR_CODE,
+            rdata = resp.json()
-                }
+            msg = (rdata.get("choices") or [{}])[0].get("message", {})
+            content = (msg.get("content") or "").strip()
+            content = _extract_reply(content)
 
             if not content:
-                return {
+                # Model put everything in reasoning_content — extract bullet points
-                    "success": False,
+                rc = (msg.get("reasoning_content") or "").strip()
-                    "error": "The model returned an empty summary",
+                # Find bullet-point style output (lines starting with -, •, *, or numbered)
-                    "error_code": "email_summary_empty",
+                bullet_lines = []
-                }
+                for line in rc.split("\n"):
+                    stripped = line.strip()
+                    if re.match(r"^[-•*]\s+|^\d+[.)]\s+", stripped):
+                        bullet_lines.append(stripped)
+                if bullet_lines:
+                    content = "\n".join(bullet_lines)
+                else:
+                    # Last resort: take the last paragraph
+                    paragraphs = [p.strip() for p in rc.split("\n\n") if p.strip()]
+                    content = paragraphs[-1] if paragraphs else rc[:500]
+
+            if not content:
+                return {"success": False, "error": "Empty response from model"}
 
             # Cache the summary if we have a message_id
             mid = data.get("message_id", "")
@@ -4987,15 +4876,8 @@ def setup_email_routes():
 
             return {"success": True, "summary": content, "model_used": model}
         except Exception as e:
-            logger.error(
+            logger.error(f"Failed to summarize: {e}")
-                "Email summary route failed %s",
+            return {"success": False, "error": "Mail operation failed"}
-                _email_summary_failure_log_detail(e),
-            )
-            return {
-                "success": False,
-                "error": EMAIL_SUMMARY_ERROR_MESSAGE,
-                "error_code": EMAIL_SUMMARY_ERROR_CODE,
-            }
 
     @router.post("/translate")
     async def translate_email(data: dict, owner: str = Depends(require_owner)):
@@ -5004,6 +4886,7 @@ def setup_email_routes():
             from src.endpoint_resolver import (
                 resolve_endpoint,
                 resolve_utility_fallback_candidates,
+                resolve_chat_fallback_candidates,
             )
             from src.llm_core import llm_call_async_with_fallback
 
@@ -5065,6 +4948,8 @@ def setup_email_routes():
                 pass
             for cand in resolve_utility_fallback_candidates(owner=owner) or []:
                 _add(*cand)
+            for cand in resolve_chat_fallback_candidates(owner=owner) or []:
+                _add(*cand)
             if not candidates:
                 return {"success": False, "error": "No LLM endpoint configured"}
 
@@ -5324,11 +5209,13 @@ def setup_email_routes():
             # Build a candidate chain so a stale session-stored API key
             # (the most common cause of "authentication failed" here)
             # doesn't kill AI Reply outright — fall through to the
-            # user's Utility / Default endpoints and active Utility fallback
+            # user's Utility / Default endpoints AND their configured
-            # chain. Dedupe by url+model so we don't retry the same endpoint.
+            # fallback chains. Dedupe by url+model so we don't retry
+            # the same broken endpoint.
             from src.llm_core import llm_call_async_with_fallback
             from src.endpoint_resolver import (
                 resolve_utility_fallback_candidates,
+                resolve_chat_fallback_candidates,
             )
             _seen = set()
             _candidates = []
@@ -5353,9 +5240,11 @@ def setup_email_routes():
                 _add(_d_url, _d_model, _d_headers)
             except Exception:
                 pass
-            # Active Utility fallbacks last.
+            # Configured fallback chains last.
             for cand in resolve_utility_fallback_candidates(owner=owner) or []:
                 _add(*cand)
+            for cand in resolve_chat_fallback_candidates(owner=owner) or []:
+                _add(*cand)
             _messages = [
                 {"role": "system", "content": system_prompt},
                 {"role": "user", "content": user_msg},
@@ -5539,9 +5428,9 @@ def setup_email_routes():
         import uuid as _uuid
         db = SessionLocal()
         try:
-            _lock_email_account_owner_mutation(db, owner)
             q = db.query(EmailAccount).filter(EmailAccount.is_default == True)  # noqa: E712
-            q = _email_account_owner_scope(q, owner)
+            if owner:
+                q = q.filter(EmailAccount.owner == owner)
             row = q.first()
             if row is None:
                 row = EmailAccount(id=_uuid.uuid4().hex, owner=owner, name="Default", is_default=True, enabled=True)
@@ -5567,7 +5456,8 @@ def setup_email_routes():
             if data.get("smtp_password"):
                 row.smtp_password = _enc(data["smtp_password"])
             clear_q = db.query(EmailAccount).filter(EmailAccount.id != row.id)
-            clear_q = _email_account_owner_scope(clear_q, owner)
+            if owner:
+                clear_q = clear_q.filter(EmailAccount.owner == owner)
             clear_q.update({EmailAccount.is_default: False})
             db.commit()
         finally:
@@ -5662,7 +5552,6 @@ def setup_email_routes():
             return {"ok": False, "error": port_err}
         db = SessionLocal()
         try:
-            _lock_email_account_owner_mutation(db, owner)
             row = EmailAccount(
                 id=_uuid.uuid4().hex,
                 name=name,
@@ -5689,7 +5578,9 @@ def setup_email_routes():
             # the one-default invariant — but scope it to THIS user's accounts,
             # otherwise creating a default would clear every other user's
             # default flag too.
-            scope_q = _email_account_owner_scope(db.query(EmailAccount), owner)
+            scope_q = db.query(EmailAccount)
+            if owner:
+                scope_q = scope_q.filter(EmailAccount.owner == owner)
             existing_count = scope_q.count()
             if row.is_default or existing_count == 0:
                 scope_q.update({EmailAccount.is_default: False})
@@ -5740,39 +5631,28 @@ def setup_email_routes():
 
     @router.delete("/accounts/{account_id}")
     async def delete_email_account(account_id: str, owner: str = Depends(require_user)):
-        initial_scope = _discover_email_account_mutation_scope(account_id, owner)
+        _assert_owns_account(account_id, owner)
         from core.database import SessionLocal, EmailAccount
         db = SessionLocal()
         try:
-            row = _lock_and_reload_email_account(
+            row = db.get(EmailAccount, account_id)
-                db, account_id, owner, initial_scope
+            if not row:
-            )
+                return {"ok": False, "error": "Account not found"}
-            row_scope = row.owner or ""
             was_default = bool(row.is_default)
             db.delete(row)
-            # Flush the removal before staging a replacement default.  The
+            db.commit()
-            # partial unique index is checked statement-by-statement, and the
-            # ORM is otherwise free to UPDATE the promoted row before DELETE.
-            db.flush()
             # If the deleted row was default, promote the next-oldest enabled
             # row owned by THIS user. Without the owner filter we'd promote
             # another user's account and the deleter would silently inherit
             # it as their default.
             if was_default:
-                promote_q = db.query(EmailAccount).filter(
+                promote_q = db.query(EmailAccount).filter(EmailAccount.enabled == True)  # noqa: E712
-                    EmailAccount.id != account_id,
+                if owner:
-                    EmailAccount.enabled == True,  # noqa: E712
+                    promote_q = promote_q.filter(EmailAccount.owner == owner)
-                )
+                promote = promote_q.order_by(EmailAccount.created_at.asc()).first()
-                promote_q = _email_account_owner_scope(promote_q, row_scope)
-                promote = promote_q.order_by(
-                    EmailAccount.created_at.asc(), EmailAccount.id.asc()
-                ).first()
                 if promote:
                     promote.is_default = True
-            # Deletion and any replacement promotion are one durable state
+                    db.commit()
-            # transition, so another worker can never observe or race the old
-            # split-commit gap.
-            db.commit()
             return {"ok": True}
         finally:
             db.close()
@@ -5985,18 +5865,18 @@ def setup_email_routes():
 
     @router.post("/accounts/{account_id}/set-default")
     async def set_default_account(account_id: str, owner: str = Depends(require_user)):
-        initial_scope = _discover_email_account_mutation_scope(account_id, owner)
+        _assert_owns_account(account_id, owner)
         from core.database import SessionLocal, EmailAccount
         db = SessionLocal()
         try:
-            row = _lock_and_reload_email_account(
+            row = db.get(EmailAccount, account_id)
-                db, account_id, owner, initial_scope
+            if not row:
-            )
+                return {"ok": False, "error": "Account not found"}
-            # Scope the sweep to the target row's normalized owner partition;
+            # SECURITY: scope the "clear other defaults" sweep to this user's
-            # this also handles visible legacy NULL/empty-owner accounts.
+            # accounts so we don't unset another user's default flag.
-            clear_q = _email_account_owner_scope(
+            clear_q = db.query(EmailAccount)
-                db.query(EmailAccount), row.owner or ""
+            if owner:
-            )
+                clear_q = clear_q.filter(EmailAccount.owner == owner)
             clear_q.update({EmailAccount.is_default: False})
             row.is_default = True
             db.commit()
@@ -6015,7 +5895,7 @@ def setup_email_routes():
             raise HTTPException(400, "GOOGLE_OAUTH_CLIENT_ID not set — add it to .env")
         redirect_uri = (
             os.environ.get("GOOGLE_OAUTH_REDIRECT_URI")
-            or f"{request.url.scheme}://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback"
+            or f"http://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback"
         )
         state = make_oauth_state(account_id, owner)
         params = urllib.parse.urlencode({
@@ -6052,7 +5932,7 @@ def setup_email_routes():
         client_secret = os.environ.get("GOOGLE_OAUTH_CLIENT_SECRET", "")
         redirect_uri = (
             os.environ.get("GOOGLE_OAUTH_REDIRECT_URI")
-            or f"{request.url.scheme}://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback"
+            or f"http://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback"
         )
         import httpx as _httpx
         try:

routes/gallery/gallery_routes.py

@@ -127,25 +127,6 @@ def _load_grounding_backend():
     return cached
 
 
-def _model_input_to_device(value, device: str, torch):
-    if not hasattr(value, "to"):
-        return value
-    if (
-        device == "mps"
-        and hasattr(torch, "float64")
-        and getattr(value, "dtype", None) == torch.float64
-    ):
-        return value.to(device=device, dtype=torch.float32)
-    return value.to(device)
-
-
-def _model_inputs_to_device(inputs, device: str, torch) -> Dict[str, Any]:
-    return {
-        key: _model_input_to_device(value, device, torch)
-        for key, value in inputs.items()
-    }
-
-
 def _ground_text_to_box(image, text: str, *, threshold: float = 0.05):
     query = (text or "").strip()
     if not query:
@@ -161,7 +142,10 @@ def _ground_text_to_box(image, text: str, *, threshold: float = 0.05):
         labels.append(f"a photo of {query}")
     try:
         inputs = processor(text=[labels], images=image, return_tensors="pt")
-        model_inputs = _model_inputs_to_device(inputs, device, torch)
+        model_inputs = {
+            k: (v.to(device) if hasattr(v, "to") else v)
+            for k, v in inputs.items()
+        }
         with torch.no_grad():
             outputs = model(**model_inputs)
         target_sizes = torch.tensor([[image.height, image.width]])
@@ -1885,7 +1869,10 @@ def setup_gallery_routes() -> APIRouter:
 
         try:
             inputs = processor(image, **kwargs)
-            model_inputs = _model_inputs_to_device(inputs, device, torch)
+            model_inputs = {
+                k: (v.to(device) if hasattr(v, "to") else v)
+                for k, v in inputs.items()
+            }
             with torch.no_grad():
                 outputs = model(**model_inputs)
             masks = processor.image_processor.post_process_masks(

routes/history/history_routes.py

@@ -137,6 +137,44 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             entry["metadata"] = meta
         return entry
 
+    def _db_message_metadata(m: DbChatMessage) -> Dict[str, Any]:
+        meta = {}
+        if m.meta_data:
+            try:
+                meta = json.loads(m.meta_data) or {}
+            except (json.JSONDecodeError, ValueError):
+                meta = {}
+        if m.timestamp and "timestamp" not in meta:
+            meta["timestamp"] = m.timestamp.isoformat() + "Z"
+        return meta
+
+    def _hydrate_session_history_from_db(session_id: str, rows: list[DbChatMessage]) -> None:
+        """Rebuild in-memory context from raw DB rows after a history load.
+
+        The browser history endpoint can return paged/display-trimmed messages,
+        but the next model call reads ``session.history``. After a restart or a
+        stale in-memory session, selecting an old chat through the paged endpoint
+        used to show the transcript while the model only saw fresh context.
+        """
+        if not rows:
+            return
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            return
+        session.history = [
+            ChatMessage(role=m.role, content=m.content, metadata=_db_message_metadata(m) or None)
+            for m in rows
+        ]
+        session.message_count = len(session.history)
+
+    def _session_needs_db_history_hydration(session_id: str, total: int) -> bool:
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            return False
+        return len(session.history or []) < int(total or 0)
+
     @router.get("/api/history/{session_id}")
     async def get_session_history(
         request: Request,
@@ -160,8 +198,6 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 )
                 page_offset = int(offset) if offset is not None else max(total - page_limit, 0)
                 page_offset = max(0, min(page_offset, total))
-                # Keep display pagination page-scoped. ``get_session`` is the
-                # full model-context hydration seam and must not be entered here.
                 rows = (
                     db.query(DbChatMessage)
                     .filter(DbChatMessage.session_id == session_id)
@@ -170,6 +206,14 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                     .limit(page_limit)
                     .all()
                 )
+                if _session_needs_db_history_hydration(session_id, total):
+                    full_rows = (
+                        db.query(DbChatMessage)
+                        .filter(DbChatMessage.session_id == session_id)
+                        .order_by(DbChatMessage.timestamp)
+                        .all()
+                    )
+                    _hydrate_session_history_from_db(session_id, full_rows)
                 history_dict = [
                     entry for entry in (_db_history_entry(m) for m in rows)
                     if not (entry.get("metadata") or {}).get("hidden")
@@ -214,10 +258,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                     entry["metadata"] = msg["metadata"]
                 history_dict.append(entry)
 
-        # Fallback: load from DB if in-memory renders empty. Display only —
+        # Fallback: load from DB if in-memory is empty
-        # get_session above is the hydration seam, so nothing here writes back
-        # into session.history — rebuilding it from raw rows would overwrite
-        # parsed multimodal content and the _db_id edit/delete keys it just set.
         if not history_dict:
             db = SessionLocal()
             try:
@@ -227,10 +268,17 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                     .order_by(DbChatMessage.timestamp)
                     .all()
                 )
+                db_history = []
+                for m in db_messages:
+                    db_history.append(_db_history_entry(m))
+                if db_history:
+                    # Rebuild in-memory history from the full set so hidden
+                    # messages (e.g. compaction summaries) are kept for AI context.
+                    _hydrate_session_history_from_db(session_id, db_messages)
                 # Response excludes hidden messages, matching the in-memory path.
                 history_dict = [
-                    entry for entry in (_db_history_entry(m) for m in db_messages)
+                    m for m in db_history
-                    if not (entry.get("metadata") or {}).get("hidden")
+                    if not (m.get("metadata") or {}).get("hidden")
                 ]
             except Exception as e:
                 logger.error(f"DB fallback failed for {session_id}: {e}")
@@ -597,14 +645,8 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             body = await request.json()
             keep_count = body.get("keep_count", 0)
 
-            # Get the source session. keep_count indexes into source.history,
+            # Get the source session
-            # so this must go through get_session — reading the cache directly
+            source = session_manager.sessions.get(session_id)
-            # forks an empty transcript out of a metadata-only session after a
-            # restart (display pagination no longer hydrates it).
-            try:
-                source = session_manager.get_session(session_id)
-            except KeyError:
-                raise HTTPException(404, "Session not found")
             if not source:
                 raise HTTPException(404, "Session not found")
 

routes/mcp/__init__.py

@@ -1,5 +0,0 @@
-"""MCP route domain package (slice 2o, #4082/#4071).
-
-Contains mcp_routes.py, migrated from the flat routes/ directory.
-Backward-compat shim at routes/mcp_routes.py re-exports from here.
-"""

routes/mcp/mcp_routes.py

@@ -1,703 +0,0 @@
-# routes/mcp_routes.py
-"""MCP (Model Context Protocol) server management routes."""
-import json
-import os
-import uuid
-import urllib.parse
-import html
-from pathlib import Path
-from fastapi import APIRouter, Form, HTTPException, Request
-from fastapi.responses import RedirectResponse, HTMLResponse
-import logging
-import httpx
-
-from core.database import McpServer, SessionLocal
-from core.middleware import require_admin
-from src.constants import DATA_DIR, MCP_OAUTH_DIR
-from src.mcp_manager import McpManager
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/api/mcp", tags=["mcp"])
-
-
-def _mcp_oauth_base_dir() -> Path:
-    """Directory that may contain OAuth files managed by Odysseus."""
-    return Path(MCP_OAUTH_DIR).resolve(strict=False)
-
-
-def _resolve_mcp_oauth_path(raw_path, field_name: str) -> str:
-    """Resolve an MCP OAuth path and keep it under DATA_DIR/mcp_oauth."""
-    raw = str(raw_path or "").strip()
-    if not raw:
-        return ""
-
-    base = _mcp_oauth_base_dir()
-    path = Path(os.path.expanduser(raw))
-    if not path.is_absolute():
-        path = base / path
-    resolved = path.resolve(strict=False)
-
-    try:
-        resolved.relative_to(base)
-    except ValueError as exc:
-        raise HTTPException(
-            400,
-            f"Invalid OAuth {field_name}: path must stay under {base}",
-        ) from exc
-    return str(resolved)
-
-
-def _sanitize_mcp_oauth_config(oauth_cfg):
-    """Return an OAuth config copy with file paths confined to mcp_oauth."""
-    if not oauth_cfg:
-        return oauth_cfg
-    if not isinstance(oauth_cfg, dict):
-        return {}
-    sanitized = dict(oauth_cfg)
-    for field_name in ("keys_file", "token_file"):
-        if sanitized.get(field_name):
-            sanitized[field_name] = _resolve_mcp_oauth_path(
-                sanitized[field_name],
-                field_name,
-            )
-    return sanitized
-
-
-def _mcp_oauth_token_missing(oauth_cfg, *, strict: bool = True) -> bool:
-    """Check token existence without letting legacy bad paths break listing."""
-    if not isinstance(oauth_cfg, dict):
-        return False
-    try:
-        token_file = _resolve_mcp_oauth_path(oauth_cfg.get("token_file", ""), "token_file")
-    except HTTPException:
-        if strict:
-            raise
-        logger.warning("Ignoring MCP OAuth config with unsafe token_file")
-        return True
-    return bool(token_file and not os.path.exists(token_file))
-
-
-def _apply_mcp_oauth_env(env: dict, oauth_cfg) -> None:
-    """Pass sanitized Gmail package paths to MCP servers that honor them."""
-    if not oauth_cfg or not isinstance(env, dict):
-        return
-    keys_file = oauth_cfg.get("keys_file")
-    token_file = oauth_cfg.get("token_file")
-    if keys_file:
-        env["GMAIL_OAUTH_PATH"] = keys_file
-    if token_file:
-        env["GMAIL_CREDENTIALS_PATH"] = token_file
-
-
-def _load_disabled_map():
-    """Load per-server disabled tool sets from DB."""
-    db = SessionLocal()
-    try:
-        disabled_map = {}
-        for srv in db.query(McpServer).all():
-            if srv.disabled_tools:
-                try:
-                    names = json.loads(srv.disabled_tools)
-                    if names:
-                        disabled_map[srv.id] = set(names)
-                except (json.JSONDecodeError, TypeError):
-                    pass
-        return disabled_map
-    finally:
-        db.close()
-
-
-def _mcp_oauth_redirect_uri() -> str:
-    """Shared callback URL for legacy Google and generic MCP OAuth flows."""
-    from src.mcp_oauth import REDIRECT_URI
-    return REDIRECT_URI
-
-
-def setup_mcp_routes(mcp_manager: McpManager):
-    """Setup MCP routes with the provided manager."""
-
-    @router.get("/servers")
-    def list_servers(request: Request):
-        """List all configured MCP servers with connection status."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            servers = db.query(McpServer).all()
-            result = []
-            for srv in servers:
-                status = mcp_manager.get_server_status(srv.id)
-                oauth_cfg = json.loads(srv.oauth_config) if srv.oauth_config else None
-                needs_oauth = False
-                if oauth_cfg:
-                    needs_oauth = _mcp_oauth_token_missing(oauth_cfg, strict=False)
-                disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else []
-                total_tools = status.get("tool_count", 0)
-                result.append({
-                    "id": srv.id,
-                    "name": srv.name,
-                    "transport": srv.transport,
-                    "command": srv.command,
-                    "args": json.loads(srv.args) if srv.args else [],
-                    "env": json.loads(srv.env) if srv.env else {},
-                    "url": srv.url,
-                    "is_enabled": srv.is_enabled,
-                    "status": status.get("status", "disconnected"),
-                    "tool_count": total_tools,
-                    "disabled_tool_count": len(disabled_list),
-                    "enabled_tool_count": max(0, total_tools - len(disabled_list)),
-                    "error": status.get("error"),
-                    "auth_url": status.get("auth_url"),
-                    "has_oauth": oauth_cfg is not None,
-                    "needs_oauth": needs_oauth,
-                })
-            return result
-        finally:
-            db.close()
-
-    @router.post("/servers")
-    async def add_server(
-        request: Request,
-        name: str = Form(...),
-        transport: str = Form("stdio"),
-        command: str = Form(None),
-        args: str = Form("[]"),
-        env: str = Form("{}"),
-        url: str = Form(None),
-        oauth_file: str = Form(None),
-        oauth_config: str = Form(None),
-    ):
-        """Add a new MCP server config and attempt connection. Admin-only:
-        registering a stdio server is equivalent to executing arbitrary
-        binaries on the host."""
-        require_admin(request)
-        server_id = str(uuid.uuid4())[:8]
-
-        # Validate
-        if transport == "stdio" and not command:
-            raise HTTPException(400, "command is required for stdio transport")
-        if transport == "sse" and not url:
-            raise HTTPException(400, "url is required for SSE transport")
-        if transport == "http" and not url:
-            raise HTTPException(400, "url is required for HTTP transport")
-
-        # Parse JSON fields
-        try:
-            parsed_args = json.loads(args) if args else []
-        except json.JSONDecodeError:
-            parsed_args = []
-        try:
-            parsed_env = json.loads(env) if env else {}
-        except json.JSONDecodeError:
-            parsed_env = {}
-        if not isinstance(parsed_env, dict):
-            parsed_env = {}
-
-        # Parse OAuth config
-        parsed_oauth_config = None
-        if oauth_config:
-            try:
-                parsed_oauth_config = _sanitize_mcp_oauth_config(json.loads(oauth_config))
-            except json.JSONDecodeError:
-                pass
-        _apply_mcp_oauth_env(parsed_env, parsed_oauth_config)
-
-        # Write OAuth credentials file if provided (for Google MCP servers)
-        logger.info(f"MCP add_server: oauth_file={oauth_file!r}")
-        if oauth_file:
-            try:
-                oauth_data = json.loads(oauth_file)
-                oauth_dir = _resolve_mcp_oauth_path(oauth_data.get("dir", ""), "dir")
-                oauth_filename = oauth_data.get("filename", "")
-                client_id = oauth_data.get("client_id", "")
-                client_secret = oauth_data.get("client_secret", "")
-                if oauth_dir and oauth_filename and client_id and client_secret:
-                    filepath = _resolve_mcp_oauth_path(
-                        Path(oauth_dir) / str(oauth_filename),
-                        "filename",
-                    )
-                    os.makedirs(os.path.dirname(filepath), exist_ok=True)
-                    creds = {
-                        "installed": {
-                            "client_id": client_id,
-                            "client_secret": client_secret,
-                            "redirect_uris": ["http://localhost"],
-                            "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-                            "token_uri": "https://accounts.google.com/o/oauth2/token",
-                        }
-                    }
-                    with open(filepath, "w", encoding="utf-8") as f:
-                        json.dump(creds, f, indent=2)
-                    logger.info(f"Wrote OAuth credentials to {filepath}")
-                    parsed_env.pop("GOOGLE_CLIENT_ID", None)
-                    parsed_env.pop("GOOGLE_CLIENT_SECRET", None)
-            except (json.JSONDecodeError, OSError) as e:
-                logger.warning(f"Failed to write OAuth file: {e}")
-
-        # Save to DB
-        db = SessionLocal()
-        try:
-            srv = McpServer(
-                id=server_id,
-                name=name,
-                transport=transport,
-                command=command,
-                args=json.dumps(parsed_args),
-                env=json.dumps(parsed_env),
-                url=url,
-                is_enabled=True,
-                oauth_config=json.dumps(parsed_oauth_config) if parsed_oauth_config else None,
-            )
-            db.add(srv)
-            db.commit()
-        finally:
-            db.close()
-
-        # Check if OAuth token already exists — skip connection attempt if not
-        needs_oauth = False
-        if parsed_oauth_config:
-            needs_oauth = _mcp_oauth_token_missing(parsed_oauth_config)
-
-        connected = False
-        if not needs_oauth:
-            connected = await mcp_manager.connect_server(
-                server_id=server_id,
-                name=name,
-                transport=transport,
-                command=command,
-                args=parsed_args,
-                env=parsed_env,
-                url=url,
-            )
-
-        status = mcp_manager.get_server_status(server_id)
-        needs_auth = status.get("status") == "needs_auth"
-        return {
-            "id": server_id,
-            "name": name,
-            "connected": connected,
-            "status": "needs_oauth" if needs_oauth else status.get("status", "disconnected"),
-            "tool_count": status.get("tool_count", 0),
-            "error": "OAuth authorization required" if needs_oauth else status.get("error"),
-            "needs_oauth": needs_oauth,
-            "needs_auth": needs_auth,
-            "auth_url": status.get("auth_url"),
-        }
-
-    @router.post("/servers/{server_id}/reconnect")
-    async def reconnect_server(server_id: str, request: Request):
-        """Reconnect to an MCP server."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-
-            await mcp_manager.disconnect_server(server_id)
-
-            args = json.loads(srv.args) if srv.args else []
-            env = json.loads(srv.env) if srv.env else {}
-            connected = await mcp_manager.connect_server(
-                server_id=server_id,
-                name=srv.name,
-                transport=srv.transport,
-                command=srv.command,
-                args=args,
-                env=env,
-                url=srv.url,
-            )
-
-            status = mcp_manager.get_server_status(server_id)
-            return {
-                "connected": connected,
-                "status": status.get("status", "disconnected"),
-                "tool_count": status.get("tool_count", 0),
-                "error": status.get("error"),
-                "auth_url": status.get("auth_url"),
-                "needs_auth": status.get("status") == "needs_auth",
-            }
-        finally:
-            db.close()
-
-    @router.patch("/servers/{server_id}")
-    async def toggle_server(server_id: str, request: Request, is_enabled: str = Form(...)):
-        """Enable or disable an MCP server."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-
-            enabled = str(is_enabled).lower() == "true"
-            srv.is_enabled = enabled
-            db.commit()
-
-            if enabled:
-                args = json.loads(srv.args) if srv.args else []
-                env = json.loads(srv.env) if srv.env else {}
-                await mcp_manager.connect_server(
-                    server_id=server_id,
-                    name=srv.name,
-                    transport=srv.transport,
-                    command=srv.command,
-                    args=args,
-                    env=env,
-                    url=srv.url,
-                )
-            else:
-                await mcp_manager.disconnect_server(server_id)
-
-            return {"id": server_id, "is_enabled": enabled}
-        finally:
-            db.close()
-
-    @router.delete("/servers/{server_id}")
-    async def delete_server(server_id: str, request: Request):
-        """Remove an MCP server."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-
-            await mcp_manager.disconnect_server(server_id)
-
-            db.delete(srv)
-            db.commit()
-            return {"status": "deleted"}
-        finally:
-            db.close()
-
-    @router.get("/tools")
-    def list_tools(request: Request):
-        """List all discovered MCP tools across all connected servers."""
-        require_admin(request)
-        disabled_map = _load_disabled_map()
-        return mcp_manager.get_all_tools(disabled_map)
-
-    @router.get("/servers/{server_id}/tools")
-    def list_server_tools(server_id: str, request: Request):
-        """List all tools for a specific MCP server with enabled/disabled state."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-            disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else []
-            disabled_set = set(disabled_list)
-        finally:
-            db.close()
-
-        all_tools = mcp_manager.get_all_tools()
-        server_tools = [t for t in all_tools if t["server_id"] == server_id]
-        for t in server_tools:
-            t["is_disabled"] = t["name"] in disabled_set
-        return server_tools
-
-    @router.patch("/servers/{server_id}/tools")
-    async def update_disabled_tools(server_id: str, request: Request):
-        """Bulk update disabled tools list for a server.
-
-        Expects JSON body: {"disabled": ["tool_name_1", "tool_name_2"]}
-        """
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-
-            body = await request.json()
-            disabled = body.get("disabled", [])
-            if not isinstance(disabled, list):
-                raise HTTPException(400, "disabled must be a list of tool names")
-
-            srv.disabled_tools = json.dumps(disabled) if disabled else None
-            db.commit()
-
-            return {"id": server_id, "disabled_count": len(disabled)}
-        finally:
-            db.close()
-
-    # ── OAuth flow for Google MCP servers ──────────────────────────
-
-    @router.get("/oauth/authorize/{server_id}")
-    def oauth_authorize(server_id: str, request: Request):
-        """Show OAuth authorization page with Google sign-in link."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-            if not srv.oauth_config:
-                raise HTTPException(400, "Server has no OAuth config")
-
-            oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config))
-            keys_file = oauth_cfg.get("keys_file", "")
-            if not keys_file or not os.path.exists(keys_file):
-                raise HTTPException(400, "OAuth keys file not found")
-
-            with open(keys_file, encoding="utf-8") as f:
-                keys_data = json.load(f)
-            keys = keys_data.get("installed") or keys_data.get("web")
-            if not keys:
-                raise HTTPException(400, "Invalid OAuth keys file format")
-
-            client_id = keys["client_id"]
-            scopes = oauth_cfg.get("scopes", [])
-
-            # For Desktop App creds, default to localhost — the user will
-            # paste the resulting URL back if they're on a different device.
-            redirect_uri = _mcp_oauth_redirect_uri()
-
-            params = {
-                "client_id": client_id,
-                "redirect_uri": redirect_uri,
-                "response_type": "code",
-                "scope": " ".join(scopes),
-                "access_type": "offline",
-                "prompt": "consent",
-                "state": server_id,
-            }
-            auth_url = "https://accounts.google.com/o/oauth2/v2/auth?" + urllib.parse.urlencode(params)
-
-            # Determine if user is accessing from the same machine
-            host = request.headers.get("host", "")
-            is_local = host.startswith("localhost") or host.startswith("127.0.0.1")
-
-            if is_local:
-                # Same machine — just redirect, callback will work directly
-                return RedirectResponse(auth_url)
-            else:
-                # Remote device — show paste-back page
-                return HTMLResponse(_oauth_authorize_page(auth_url, server_id, redirect_uri))
-        finally:
-            db.close()
-
-    @router.get("/oauth/callback")
-    async def oauth_callback(code: str, state: str, request: Request):
-        """Handle OAuth callback. Generic MCP OAuth flows resolve via the
-        pending-state registry; Google flows fall through to the legacy path."""
-        require_admin(request)
-        from src.mcp_oauth import resolve_pending
-        if resolve_pending(state, code):
-            return HTMLResponse(_oauth_result_page(
-                "Authorization Successful",
-                "The MCP server is connecting. You can close this window and return to Odysseus.",
-                success=True,
-            ))
-        # Legacy Google path: state is the server_id
-        return await _exchange_and_connect(state, code, request)
-
-    @router.post("/oauth/exchange/{server_id}")
-    async def oauth_exchange(server_id: str, request: Request, callback_url: str = Form(...)):
-        """Manual code exchange — user pastes the callback URL from their browser."""
-        require_admin(request)
-        try:
-            parsed = urllib.parse.urlparse(callback_url)
-            params = urllib.parse.parse_qs(parsed.query)
-            code = params.get("code", [None])[0]
-            if not code:
-                return HTMLResponse(_oauth_result_page("Error", "No authorization code found in the URL. Make sure you copied the full URL from your browser."), status_code=400)
-        except Exception:
-            return HTMLResponse(_oauth_result_page("Error", "Invalid URL format."), status_code=400)
-
-        # Generic MCP OAuth: if the pasted URL carries a state we are waiting on,
-        # resolve it directly (the background connect finishes the handshake).
-        state = params.get("state", [None])[0]
-        from src.mcp_oauth import resolve_pending
-        if state and resolve_pending(state, code):
-            return HTMLResponse(_oauth_result_page(
-                "Authorization Successful",
-                "The MCP server is connecting. You can close this window and return to Odysseus.",
-                success=True,
-            ))
-
-        return await _exchange_and_connect(server_id, code, request)
-
-    async def _exchange_and_connect(server_id: str, code: str, request: Request):
-        """Exchange auth code for tokens and connect the MCP server."""
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                return HTMLResponse(_oauth_result_page("Error", "Server not found."), status_code=404)
-            if not srv.oauth_config:
-                return HTMLResponse(_oauth_result_page("Error", "No OAuth config."), status_code=400)
-
-            oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config))
-            keys_file = oauth_cfg.get("keys_file", "")
-            token_file = oauth_cfg.get("token_file", "")
-            if not keys_file or not token_file:
-                raise HTTPException(400, "OAuth keys/token file not configured")
-
-            with open(keys_file, encoding="utf-8") as f:
-                keys_data = json.load(f)
-            keys = keys_data.get("installed") or keys_data.get("web")
-            client_id = keys["client_id"]
-            client_secret = keys["client_secret"]
-
-            redirect_uri = _mcp_oauth_redirect_uri()
-
-            async with httpx.AsyncClient() as client:
-                resp = await client.post(
-                    "https://oauth2.googleapis.com/token",
-                    data={
-                        "code": code,
-                        "client_id": client_id,
-                        "client_secret": client_secret,
-                        "redirect_uri": redirect_uri,
-                        "grant_type": "authorization_code",
-                    },
-                )
-
-            if resp.status_code != 200:
-                err = resp.text
-                logger.error(f"OAuth token exchange failed: {err}")
-                return HTMLResponse(_oauth_result_page("Authorization Failed", f"Google returned an error: {err}"), status_code=400)
-
-            tokens = resp.json()
-            logger.info(f"OAuth tokens received for server {server_id}")
-
-            # Save tokens to the file the MCP package expects
-            os.makedirs(os.path.dirname(token_file), exist_ok=True)
-            with open(token_file, "w", encoding="utf-8") as f:
-                json.dump(tokens, f, indent=2)
-            logger.info(f"Saved OAuth tokens to {token_file}")
-
-            # Attempt to connect the MCP server now
-            args = json.loads(srv.args) if srv.args else []
-            env = json.loads(srv.env) if srv.env else {}
-            connected = await mcp_manager.connect_server(
-                server_id=server_id,
-                name=srv.name,
-                transport=srv.transport,
-                command=srv.command,
-                args=args,
-                env=env,
-                url=srv.url,
-            )
-
-            if connected:
-                status = mcp_manager.get_server_status(server_id)
-                tool_count = status.get("tool_count", 0)
-                return HTMLResponse(_oauth_result_page(
-                    "Authorization Successful",
-                    f"{srv.name} connected with {tool_count} tools. You can close this window.",
-                    success=True,
-                ))
-            else:
-                status = mcp_manager.get_server_status(server_id)
-                return HTMLResponse(_oauth_result_page(
-                    "Authorized but Connection Failed",
-                    f"Tokens saved, but the server failed to connect: {status.get('error', 'unknown error')}. Try reconnecting from Settings.",
-                ))
-        except HTTPException as e:
-            logger.warning(f"OAuth callback rejected: {e.detail}")
-            return HTMLResponse(_oauth_result_page("Error", str(e.detail)), status_code=e.status_code)
-        except Exception as e:
-            logger.exception(f"OAuth callback error: {e}")
-            return HTMLResponse(_oauth_result_page("Error", str(e)), status_code=500)
-        finally:
-            db.close()
-
-    return router
-
-
-def _oauth_authorize_page(
-    auth_url: str,
-    server_id: str,
-    redirect_uri: str,
-) -> str:
-    """Page with Google sign-in link and URL paste-back form for remote access."""
-    # Escape values interpolated into the page: `server_id` comes from the OAuth
-    # state and is not trusted.
-    auth_url = html.escape(auth_url, quote=True)
-    server_id = html.escape(server_id, quote=True)
-    redirect_uri = html.escape(redirect_uri, quote=True)
-    return f"""<!DOCTYPE html>
-<html><head>
-<meta charset="UTF-8"><title>Authorize — Odysseus</title>
-<style>
-  body {{ font-family: 'Fira Code', monospace; background: #0f0f0f; color: #e0e0e0;
-    display: flex; justify-content: center; align-items: center; min-height: 100vh; }}
-  .card {{ background: #1a1a1a; border: 1px solid #333; border-radius: 12px;
-    padding: 2rem; max-width: 480px; text-align: center; }}
-  h2 {{ color: #e06c75; margin-bottom: 0.5rem; font-size: 1.1rem; }}
-  p {{ color: #aaa; font-size: 0.82rem; line-height: 1.6; margin: 0.8rem 0; }}
-  .step {{ text-align: left; color: #ccc; font-size: 0.82rem; line-height: 1.7; margin: 1rem 0; }}
-  .step b {{ color: #e06c75; }}
-  a.auth-link {{
-    display: inline-block; margin: 1rem 0; padding: 0.6rem 1.5rem;
-    background: #e06c75; color: #fff; text-decoration: none; border-radius: 6px;
-    font-weight: 600; font-size: 0.9rem;
-  }}
-  a.auth-link:hover {{ background: #c55; }}
-  input[type=text] {{
-    width: 100%; padding: 0.5rem; margin: 0.5rem 0;
-    background: #0f0f0f; border: 1px solid #333; border-radius: 6px;
-    color: #e0e0e0; font-family: 'Fira Code', monospace; font-size: 0.8rem;
-  }}
-  input:focus {{ outline: none; border-color: #e06c75; }}
-  button {{
-    padding: 0.5rem 1.5rem; border: none; border-radius: 6px;
-    background: #e06c75; color: #fff; font-weight: 600; cursor: pointer;
-    font-family: 'Fira Code', monospace; font-size: 0.85rem; margin-top: 0.3rem;
-  }}
-  button:hover {{ background: #c55; }}
-  .divider {{ border-top: 1px solid #333; margin: 1.2rem 0; }}
-</style></head>
-<body><div class="card">
-  <h2>Authorize Google Account</h2>
-  <div class="step">
-    <b>1.</b> Click the button below to sign in with Google<br>
-    <b>2.</b> After approving, your browser will show an error page — that's normal<br>
-    <b>3.</b> Copy the full URL from your browser's address bar<br>
-    <b>4.</b> Paste it below and click Connect
-  </div>
-  <a class="auth-link" href="{auth_url}" target="_blank" rel="noopener">Sign in with Google</a>
-  <div class="divider"></div>
-  <!-- Relative action: the browser resolves it against the origin this page was
-       served from, so the form follows the user through any proxy without the
-       app having to know the scheme or the host. An absolute http:// action is
-       blocked as mixed content on exactly the HTTPS deployments that need
-       paste-back, and request.url.scheme cannot be trusted to spot them —
-       uvicorn only honours X-Forwarded-Proto from a peer in
-       --forwarded-allow-ips, which defaults to 127.0.0.1 and excludes a proxy
-       arriving over the Docker bridge. -->
-  <form method="POST" action="/api/mcp/oauth/exchange/{server_id}">
-    <p>Paste the URL from your browser after signing in:</p>
-    <input type="text" name="callback_url" placeholder="{redirect_uri}?code=..." required>
-    <br><button type="submit">Connect</button>
-  </form>
-</div></body></html>"""
-
-
-def _oauth_result_page(title: str, message: str, success: bool = False) -> str:
-    """Generate a simple HTML page for the OAuth result."""
-    safe_title = html.escape(title)
-    safe_message = html.escape(message)
-    color = "#00661a" if success else "#e06c75"
-    icon = "&#10003;" if success else "&#10007;"
-    return f"""<!DOCTYPE html>
-<html><head>
-<meta charset="UTF-8"><title>{safe_title}</title>
-<style>
-  body {{ font-family: 'Fira Code', monospace; background: #0f0f0f; color: #e0e0e0;
-    display: flex; justify-content: center; align-items: center; min-height: 100vh; }}
-  .card {{ background: #1a1a1a; border: 1px solid #333; border-radius: 12px;
-    padding: 2rem; max-width: 420px; text-align: center; }}
-  .icon {{ font-size: 3rem; color: {color}; margin-bottom: 1rem; }}
-  h2 {{ color: {color}; margin-bottom: 0.5rem; font-size: 1.1rem; }}
-  p {{ color: #aaa; font-size: 0.85rem; line-height: 1.5; }}
-</style></head>
-<body><div class="card">
-  <div class="icon">{icon}</div>
-  <h2>{safe_title}</h2>
-  <p>{safe_message}</p>
-</div></body></html>"""

routes/mcp_routes.py

@@ -1,18 +1,697 @@
-"""Backward-compat shim — canonical location is routes/mcp/mcp_routes.py.
+# routes/mcp_routes.py
+"""MCP (Model Context Protocol) server management routes."""
+import json
+import os
+import uuid
+import urllib.parse
+import html
+from pathlib import Path
+from fastapi import APIRouter, Form, HTTPException, Request
+from fastapi.responses import RedirectResponse, HTMLResponse
+import logging
+import httpx
 
-This module is replaced in ``sys.modules`` by the canonical module object so
+from core.database import McpServer, SessionLocal
-that ``import routes.mcp_routes``, ``from routes.mcp_routes import X``,
+from core.middleware import require_admin
-``importlib.import_module("routes.mcp_routes")``, the
+from src.constants import DATA_DIR, MCP_OAUTH_DIR
-``sys.modules.pop("routes.mcp_routes")`` + re-import pattern in
+from src.mcp_manager import McpManager
-test_security_regressions.py, and the ``monkeypatch.setattr(mcp_routes,
-"MCP_OAUTH_DIR", ...)`` pattern all operate on the *same* object. This also
-makes ``mcp_routes.__file__`` resolve to the canonical file (which the
-source-introspection at line 839 reads). Keeps existing import paths working
-after slice 2o (#4082/#4071).
-"""
 
-import sys as _sys
+logger = logging.getLogger(__name__)
 
-from routes.mcp import mcp_routes as _canonical  # noqa: F401
+router = APIRouter(prefix="/api/mcp", tags=["mcp"])
 
-_sys.modules[__name__] = _canonical
+
+def _mcp_oauth_base_dir() -> Path:
+    """Directory that may contain OAuth files managed by Odysseus."""
+    return Path(MCP_OAUTH_DIR).resolve(strict=False)
+
+
+def _resolve_mcp_oauth_path(raw_path, field_name: str) -> str:
+    """Resolve an MCP OAuth path and keep it under DATA_DIR/mcp_oauth."""
+    raw = str(raw_path or "").strip()
+    if not raw:
+        return ""
+
+    base = _mcp_oauth_base_dir()
+    path = Path(os.path.expanduser(raw))
+    if not path.is_absolute():
+        path = base / path
+    resolved = path.resolve(strict=False)
+
+    try:
+        resolved.relative_to(base)
+    except ValueError as exc:
+        raise HTTPException(
+            400,
+            f"Invalid OAuth {field_name}: path must stay under {base}",
+        ) from exc
+    return str(resolved)
+
+
+def _sanitize_mcp_oauth_config(oauth_cfg):
+    """Return an OAuth config copy with file paths confined to mcp_oauth."""
+    if not oauth_cfg:
+        return oauth_cfg
+    if not isinstance(oauth_cfg, dict):
+        return {}
+    sanitized = dict(oauth_cfg)
+    for field_name in ("keys_file", "token_file"):
+        if sanitized.get(field_name):
+            sanitized[field_name] = _resolve_mcp_oauth_path(
+                sanitized[field_name],
+                field_name,
+            )
+    return sanitized
+
+
+def _mcp_oauth_token_missing(oauth_cfg, *, strict: bool = True) -> bool:
+    """Check token existence without letting legacy bad paths break listing."""
+    if not isinstance(oauth_cfg, dict):
+        return False
+    try:
+        token_file = _resolve_mcp_oauth_path(oauth_cfg.get("token_file", ""), "token_file")
+    except HTTPException:
+        if strict:
+            raise
+        logger.warning("Ignoring MCP OAuth config with unsafe token_file")
+        return True
+    return bool(token_file and not os.path.exists(token_file))
+
+
+def _apply_mcp_oauth_env(env: dict, oauth_cfg) -> None:
+    """Pass sanitized Gmail package paths to MCP servers that honor them."""
+    if not oauth_cfg or not isinstance(env, dict):
+        return
+    keys_file = oauth_cfg.get("keys_file")
+    token_file = oauth_cfg.get("token_file")
+    if keys_file:
+        env["GMAIL_OAUTH_PATH"] = keys_file
+    if token_file:
+        env["GMAIL_CREDENTIALS_PATH"] = token_file
+
+
+def _load_disabled_map():
+    """Load per-server disabled tool sets from DB."""
+    db = SessionLocal()
+    try:
+        disabled_map = {}
+        for srv in db.query(McpServer).all():
+            if srv.disabled_tools:
+                try:
+                    names = json.loads(srv.disabled_tools)
+                    if names:
+                        disabled_map[srv.id] = set(names)
+                except (json.JSONDecodeError, TypeError):
+                    pass
+        return disabled_map
+    finally:
+        db.close()
+
+
+def _mcp_oauth_redirect_uri() -> str:
+    """Shared callback URL for legacy Google and generic MCP OAuth flows."""
+    from src.mcp_oauth import REDIRECT_URI
+    return REDIRECT_URI
+
+
+def setup_mcp_routes(mcp_manager: McpManager):
+    """Setup MCP routes with the provided manager."""
+
+    @router.get("/servers")
+    def list_servers(request: Request):
+        """List all configured MCP servers with connection status."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            servers = db.query(McpServer).all()
+            result = []
+            for srv in servers:
+                status = mcp_manager.get_server_status(srv.id)
+                oauth_cfg = json.loads(srv.oauth_config) if srv.oauth_config else None
+                needs_oauth = False
+                if oauth_cfg:
+                    needs_oauth = _mcp_oauth_token_missing(oauth_cfg, strict=False)
+                disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else []
+                total_tools = status.get("tool_count", 0)
+                result.append({
+                    "id": srv.id,
+                    "name": srv.name,
+                    "transport": srv.transport,
+                    "command": srv.command,
+                    "args": json.loads(srv.args) if srv.args else [],
+                    "env": json.loads(srv.env) if srv.env else {},
+                    "url": srv.url,
+                    "is_enabled": srv.is_enabled,
+                    "status": status.get("status", "disconnected"),
+                    "tool_count": total_tools,
+                    "disabled_tool_count": len(disabled_list),
+                    "enabled_tool_count": max(0, total_tools - len(disabled_list)),
+                    "error": status.get("error"),
+                    "auth_url": status.get("auth_url"),
+                    "has_oauth": oauth_cfg is not None,
+                    "needs_oauth": needs_oauth,
+                })
+            return result
+        finally:
+            db.close()
+
+    @router.post("/servers")
+    async def add_server(
+        request: Request,
+        name: str = Form(...),
+        transport: str = Form("stdio"),
+        command: str = Form(None),
+        args: str = Form("[]"),
+        env: str = Form("{}"),
+        url: str = Form(None),
+        oauth_file: str = Form(None),
+        oauth_config: str = Form(None),
+    ):
+        """Add a new MCP server config and attempt connection. Admin-only:
+        registering a stdio server is equivalent to executing arbitrary
+        binaries on the host."""
+        require_admin(request)
+        server_id = str(uuid.uuid4())[:8]
+
+        # Validate
+        if transport == "stdio" and not command:
+            raise HTTPException(400, "command is required for stdio transport")
+        if transport == "sse" and not url:
+            raise HTTPException(400, "url is required for SSE transport")
+        if transport == "http" and not url:
+            raise HTTPException(400, "url is required for HTTP transport")
+
+        # Parse JSON fields
+        try:
+            parsed_args = json.loads(args) if args else []
+        except json.JSONDecodeError:
+            parsed_args = []
+        try:
+            parsed_env = json.loads(env) if env else {}
+        except json.JSONDecodeError:
+            parsed_env = {}
+        if not isinstance(parsed_env, dict):
+            parsed_env = {}
+
+        # Parse OAuth config
+        parsed_oauth_config = None
+        if oauth_config:
+            try:
+                parsed_oauth_config = _sanitize_mcp_oauth_config(json.loads(oauth_config))
+            except json.JSONDecodeError:
+                pass
+        _apply_mcp_oauth_env(parsed_env, parsed_oauth_config)
+
+        # Write OAuth credentials file if provided (for Google MCP servers)
+        logger.info(f"MCP add_server: oauth_file={oauth_file!r}")
+        if oauth_file:
+            try:
+                oauth_data = json.loads(oauth_file)
+                oauth_dir = _resolve_mcp_oauth_path(oauth_data.get("dir", ""), "dir")
+                oauth_filename = oauth_data.get("filename", "")
+                client_id = oauth_data.get("client_id", "")
+                client_secret = oauth_data.get("client_secret", "")
+                if oauth_dir and oauth_filename and client_id and client_secret:
+                    filepath = _resolve_mcp_oauth_path(
+                        Path(oauth_dir) / str(oauth_filename),
+                        "filename",
+                    )
+                    os.makedirs(os.path.dirname(filepath), exist_ok=True)
+                    creds = {
+                        "installed": {
+                            "client_id": client_id,
+                            "client_secret": client_secret,
+                            "redirect_uris": ["http://localhost"],
+                            "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                            "token_uri": "https://accounts.google.com/o/oauth2/token",
+                        }
+                    }
+                    with open(filepath, "w", encoding="utf-8") as f:
+                        json.dump(creds, f, indent=2)
+                    logger.info(f"Wrote OAuth credentials to {filepath}")
+                    parsed_env.pop("GOOGLE_CLIENT_ID", None)
+                    parsed_env.pop("GOOGLE_CLIENT_SECRET", None)
+            except (json.JSONDecodeError, OSError) as e:
+                logger.warning(f"Failed to write OAuth file: {e}")
+
+        # Save to DB
+        db = SessionLocal()
+        try:
+            srv = McpServer(
+                id=server_id,
+                name=name,
+                transport=transport,
+                command=command,
+                args=json.dumps(parsed_args),
+                env=json.dumps(parsed_env),
+                url=url,
+                is_enabled=True,
+                oauth_config=json.dumps(parsed_oauth_config) if parsed_oauth_config else None,
+            )
+            db.add(srv)
+            db.commit()
+        finally:
+            db.close()
+
+        # Check if OAuth token already exists — skip connection attempt if not
+        needs_oauth = False
+        if parsed_oauth_config:
+            needs_oauth = _mcp_oauth_token_missing(parsed_oauth_config)
+
+        connected = False
+        if not needs_oauth:
+            connected = await mcp_manager.connect_server(
+                server_id=server_id,
+                name=name,
+                transport=transport,
+                command=command,
+                args=parsed_args,
+                env=parsed_env,
+                url=url,
+            )
+
+        status = mcp_manager.get_server_status(server_id)
+        needs_auth = status.get("status") == "needs_auth"
+        return {
+            "id": server_id,
+            "name": name,
+            "connected": connected,
+            "status": "needs_oauth" if needs_oauth else status.get("status", "disconnected"),
+            "tool_count": status.get("tool_count", 0),
+            "error": "OAuth authorization required" if needs_oauth else status.get("error"),
+            "needs_oauth": needs_oauth,
+            "needs_auth": needs_auth,
+            "auth_url": status.get("auth_url"),
+        }
+
+    @router.post("/servers/{server_id}/reconnect")
+    async def reconnect_server(server_id: str, request: Request):
+        """Reconnect to an MCP server."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+
+            await mcp_manager.disconnect_server(server_id)
+
+            args = json.loads(srv.args) if srv.args else []
+            env = json.loads(srv.env) if srv.env else {}
+            connected = await mcp_manager.connect_server(
+                server_id=server_id,
+                name=srv.name,
+                transport=srv.transport,
+                command=srv.command,
+                args=args,
+                env=env,
+                url=srv.url,
+            )
+
+            status = mcp_manager.get_server_status(server_id)
+            return {
+                "connected": connected,
+                "status": status.get("status", "disconnected"),
+                "tool_count": status.get("tool_count", 0),
+                "error": status.get("error"),
+                "auth_url": status.get("auth_url"),
+                "needs_auth": status.get("status") == "needs_auth",
+            }
+        finally:
+            db.close()
+
+    @router.patch("/servers/{server_id}")
+    async def toggle_server(server_id: str, request: Request, is_enabled: str = Form(...)):
+        """Enable or disable an MCP server."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+
+            enabled = str(is_enabled).lower() == "true"
+            srv.is_enabled = enabled
+            db.commit()
+
+            if enabled:
+                args = json.loads(srv.args) if srv.args else []
+                env = json.loads(srv.env) if srv.env else {}
+                await mcp_manager.connect_server(
+                    server_id=server_id,
+                    name=srv.name,
+                    transport=srv.transport,
+                    command=srv.command,
+                    args=args,
+                    env=env,
+                    url=srv.url,
+                )
+            else:
+                await mcp_manager.disconnect_server(server_id)
+
+            return {"id": server_id, "is_enabled": enabled}
+        finally:
+            db.close()
+
+    @router.delete("/servers/{server_id}")
+    async def delete_server(server_id: str, request: Request):
+        """Remove an MCP server."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+
+            await mcp_manager.disconnect_server(server_id)
+
+            db.delete(srv)
+            db.commit()
+            return {"status": "deleted"}
+        finally:
+            db.close()
+
+    @router.get("/tools")
+    def list_tools(request: Request):
+        """List all discovered MCP tools across all connected servers."""
+        require_admin(request)
+        disabled_map = _load_disabled_map()
+        return mcp_manager.get_all_tools(disabled_map)
+
+    @router.get("/servers/{server_id}/tools")
+    def list_server_tools(server_id: str, request: Request):
+        """List all tools for a specific MCP server with enabled/disabled state."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+            disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else []
+            disabled_set = set(disabled_list)
+        finally:
+            db.close()
+
+        all_tools = mcp_manager.get_all_tools()
+        server_tools = [t for t in all_tools if t["server_id"] == server_id]
+        for t in server_tools:
+            t["is_disabled"] = t["name"] in disabled_set
+        return server_tools
+
+    @router.patch("/servers/{server_id}/tools")
+    async def update_disabled_tools(server_id: str, request: Request):
+        """Bulk update disabled tools list for a server.
+
+        Expects JSON body: {"disabled": ["tool_name_1", "tool_name_2"]}
+        """
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+
+            body = await request.json()
+            disabled = body.get("disabled", [])
+            if not isinstance(disabled, list):
+                raise HTTPException(400, "disabled must be a list of tool names")
+
+            srv.disabled_tools = json.dumps(disabled) if disabled else None
+            db.commit()
+
+            return {"id": server_id, "disabled_count": len(disabled)}
+        finally:
+            db.close()
+
+    # ── OAuth flow for Google MCP servers ──────────────────────────
+
+    @router.get("/oauth/authorize/{server_id}")
+    def oauth_authorize(server_id: str, request: Request):
+        """Show OAuth authorization page with Google sign-in link."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+            if not srv.oauth_config:
+                raise HTTPException(400, "Server has no OAuth config")
+
+            oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config))
+            keys_file = oauth_cfg.get("keys_file", "")
+            if not keys_file or not os.path.exists(keys_file):
+                raise HTTPException(400, "OAuth keys file not found")
+
+            with open(keys_file, encoding="utf-8") as f:
+                keys_data = json.load(f)
+            keys = keys_data.get("installed") or keys_data.get("web")
+            if not keys:
+                raise HTTPException(400, "Invalid OAuth keys file format")
+
+            client_id = keys["client_id"]
+            scopes = oauth_cfg.get("scopes", [])
+
+            # For Desktop App creds, default to localhost — the user will
+            # paste the resulting URL back if they're on a different device.
+            redirect_uri = _mcp_oauth_redirect_uri()
+
+            params = {
+                "client_id": client_id,
+                "redirect_uri": redirect_uri,
+                "response_type": "code",
+                "scope": " ".join(scopes),
+                "access_type": "offline",
+                "prompt": "consent",
+                "state": server_id,
+            }
+            auth_url = "https://accounts.google.com/o/oauth2/v2/auth?" + urllib.parse.urlencode(params)
+
+            # Determine if user is accessing from the same machine
+            host = request.headers.get("host", "")
+            is_local = host.startswith("localhost") or host.startswith("127.0.0.1")
+
+            if is_local:
+                # Same machine — just redirect, callback will work directly
+                return RedirectResponse(auth_url)
+            else:
+                # Remote device — show paste-back page
+                return HTMLResponse(_oauth_authorize_page(auth_url, server_id, host, redirect_uri))
+        finally:
+            db.close()
+
+    @router.get("/oauth/callback")
+    async def oauth_callback(code: str, state: str, request: Request):
+        """Handle OAuth callback. Generic MCP OAuth flows resolve via the
+        pending-state registry; Google flows fall through to the legacy path."""
+        require_admin(request)
+        from src.mcp_oauth import resolve_pending
+        if resolve_pending(state, code):
+            return HTMLResponse(_oauth_result_page(
+                "Authorization Successful",
+                "The MCP server is connecting. You can close this window and return to Odysseus.",
+                success=True,
+            ))
+        # Legacy Google path: state is the server_id
+        return await _exchange_and_connect(state, code, request)
+
+    @router.post("/oauth/exchange/{server_id}")
+    async def oauth_exchange(server_id: str, request: Request, callback_url: str = Form(...)):
+        """Manual code exchange — user pastes the callback URL from their browser."""
+        require_admin(request)
+        try:
+            parsed = urllib.parse.urlparse(callback_url)
+            params = urllib.parse.parse_qs(parsed.query)
+            code = params.get("code", [None])[0]
+            if not code:
+                return HTMLResponse(_oauth_result_page("Error", "No authorization code found in the URL. Make sure you copied the full URL from your browser."), status_code=400)
+        except Exception:
+            return HTMLResponse(_oauth_result_page("Error", "Invalid URL format."), status_code=400)
+
+        # Generic MCP OAuth: if the pasted URL carries a state we are waiting on,
+        # resolve it directly (the background connect finishes the handshake).
+        state = params.get("state", [None])[0]
+        from src.mcp_oauth import resolve_pending
+        if state and resolve_pending(state, code):
+            return HTMLResponse(_oauth_result_page(
+                "Authorization Successful",
+                "The MCP server is connecting. You can close this window and return to Odysseus.",
+                success=True,
+            ))
+
+        return await _exchange_and_connect(server_id, code, request)
+
+    async def _exchange_and_connect(server_id: str, code: str, request: Request):
+        """Exchange auth code for tokens and connect the MCP server."""
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                return HTMLResponse(_oauth_result_page("Error", "Server not found."), status_code=404)
+            if not srv.oauth_config:
+                return HTMLResponse(_oauth_result_page("Error", "No OAuth config."), status_code=400)
+
+            oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config))
+            keys_file = oauth_cfg.get("keys_file", "")
+            token_file = oauth_cfg.get("token_file", "")
+            if not keys_file or not token_file:
+                raise HTTPException(400, "OAuth keys/token file not configured")
+
+            with open(keys_file, encoding="utf-8") as f:
+                keys_data = json.load(f)
+            keys = keys_data.get("installed") or keys_data.get("web")
+            client_id = keys["client_id"]
+            client_secret = keys["client_secret"]
+
+            redirect_uri = _mcp_oauth_redirect_uri()
+
+            async with httpx.AsyncClient() as client:
+                resp = await client.post(
+                    "https://oauth2.googleapis.com/token",
+                    data={
+                        "code": code,
+                        "client_id": client_id,
+                        "client_secret": client_secret,
+                        "redirect_uri": redirect_uri,
+                        "grant_type": "authorization_code",
+                    },
+                )
+
+            if resp.status_code != 200:
+                err = resp.text
+                logger.error(f"OAuth token exchange failed: {err}")
+                return HTMLResponse(_oauth_result_page("Authorization Failed", f"Google returned an error: {err}"), status_code=400)
+
+            tokens = resp.json()
+            logger.info(f"OAuth tokens received for server {server_id}")
+
+            # Save tokens to the file the MCP package expects
+            os.makedirs(os.path.dirname(token_file), exist_ok=True)
+            with open(token_file, "w", encoding="utf-8") as f:
+                json.dump(tokens, f, indent=2)
+            logger.info(f"Saved OAuth tokens to {token_file}")
+
+            # Attempt to connect the MCP server now
+            args = json.loads(srv.args) if srv.args else []
+            env = json.loads(srv.env) if srv.env else {}
+            connected = await mcp_manager.connect_server(
+                server_id=server_id,
+                name=srv.name,
+                transport=srv.transport,
+                command=srv.command,
+                args=args,
+                env=env,
+                url=srv.url,
+            )
+
+            if connected:
+                status = mcp_manager.get_server_status(server_id)
+                tool_count = status.get("tool_count", 0)
+                return HTMLResponse(_oauth_result_page(
+                    "Authorization Successful",
+                    f"{srv.name} connected with {tool_count} tools. You can close this window.",
+                    success=True,
+                ))
+            else:
+                status = mcp_manager.get_server_status(server_id)
+                return HTMLResponse(_oauth_result_page(
+                    "Authorized but Connection Failed",
+                    f"Tokens saved, but the server failed to connect: {status.get('error', 'unknown error')}. Try reconnecting from Settings.",
+                ))
+        except HTTPException as e:
+            logger.warning(f"OAuth callback rejected: {e.detail}")
+            return HTMLResponse(_oauth_result_page("Error", str(e.detail)), status_code=e.status_code)
+        except Exception as e:
+            logger.exception(f"OAuth callback error: {e}")
+            return HTMLResponse(_oauth_result_page("Error", str(e)), status_code=500)
+        finally:
+            db.close()
+
+    return router
+
+
+def _oauth_authorize_page(
+    auth_url: str,
+    server_id: str,
+    host: str,
+    redirect_uri: str = "http://localhost:7000/api/mcp/oauth/callback",
+) -> str:
+    """Page with Google sign-in link and URL paste-back form for remote access."""
+    # Escape values interpolated into the page: `host` comes from the request
+    # Host header and `server_id` from the OAuth state — neither is trusted.
+    auth_url = html.escape(auth_url, quote=True)
+    server_id = html.escape(server_id, quote=True)
+    host = html.escape(host, quote=True)
+    redirect_uri = html.escape(redirect_uri, quote=True)
+    return f"""<!DOCTYPE html>
+<html><head>
+<meta charset="UTF-8"><title>Authorize — Odysseus</title>
+<style>
+  body {{ font-family: 'Fira Code', monospace; background: #0f0f0f; color: #e0e0e0;
+    display: flex; justify-content: center; align-items: center; min-height: 100vh; }}
+  .card {{ background: #1a1a1a; border: 1px solid #333; border-radius: 12px;
+    padding: 2rem; max-width: 480px; text-align: center; }}
+  h2 {{ color: #e06c75; margin-bottom: 0.5rem; font-size: 1.1rem; }}
+  p {{ color: #aaa; font-size: 0.82rem; line-height: 1.6; margin: 0.8rem 0; }}
+  .step {{ text-align: left; color: #ccc; font-size: 0.82rem; line-height: 1.7; margin: 1rem 0; }}
+  .step b {{ color: #e06c75; }}
+  a.auth-link {{
+    display: inline-block; margin: 1rem 0; padding: 0.6rem 1.5rem;
+    background: #e06c75; color: #fff; text-decoration: none; border-radius: 6px;
+    font-weight: 600; font-size: 0.9rem;
+  }}
+  a.auth-link:hover {{ background: #c55; }}
+  input[type=text] {{
+    width: 100%; padding: 0.5rem; margin: 0.5rem 0;
+    background: #0f0f0f; border: 1px solid #333; border-radius: 6px;
+    color: #e0e0e0; font-family: 'Fira Code', monospace; font-size: 0.8rem;
+  }}
+  input:focus {{ outline: none; border-color: #e06c75; }}
+  button {{
+    padding: 0.5rem 1.5rem; border: none; border-radius: 6px;
+    background: #e06c75; color: #fff; font-weight: 600; cursor: pointer;
+    font-family: 'Fira Code', monospace; font-size: 0.85rem; margin-top: 0.3rem;
+  }}
+  button:hover {{ background: #c55; }}
+  .divider {{ border-top: 1px solid #333; margin: 1.2rem 0; }}
+</style></head>
+<body><div class="card">
+  <h2>Authorize Google Account</h2>
+  <div class="step">
+    <b>1.</b> Click the button below to sign in with Google<br>
+    <b>2.</b> After approving, your browser will show an error page — that's normal<br>
+    <b>3.</b> Copy the full URL from your browser's address bar<br>
+    <b>4.</b> Paste it below and click Connect
+  </div>
+  <a class="auth-link" href="{auth_url}" target="_blank" rel="noopener">Sign in with Google</a>
+  <div class="divider"></div>
+  <form method="POST" action="http://{host}/api/mcp/oauth/exchange/{server_id}">
+    <p>Paste the URL from your browser after signing in:</p>
+    <input type="text" name="callback_url" placeholder="{redirect_uri}?code=..." required>
+    <br><button type="submit">Connect</button>
+  </form>
+</div></body></html>"""
+
+
+def _oauth_result_page(title: str, message: str, success: bool = False) -> str:
+    """Generate a simple HTML page for the OAuth result."""
+    safe_title = html.escape(title)
+    safe_message = html.escape(message)
+    color = "#00661a" if success else "#e06c75"
+    icon = "&#10003;" if success else "&#10007;"
+    return f"""<!DOCTYPE html>
+<html><head>
+<meta charset="UTF-8"><title>{safe_title}</title>
+<style>
+  body {{ font-family: 'Fira Code', monospace; background: #0f0f0f; color: #e0e0e0;
+    display: flex; justify-content: center; align-items: center; min-height: 100vh; }}
+  .card {{ background: #1a1a1a; border: 1px solid #333; border-radius: 12px;
+    padding: 2rem; max-width: 420px; text-align: center; }}
+  .icon {{ font-size: 3rem; color: {color}; margin-bottom: 1rem; }}
+  h2 {{ color: {color}; margin-bottom: 0.5rem; font-size: 1.1rem; }}
+  p {{ color: #aaa; font-size: 0.85rem; line-height: 1.5; }}
+</style></head>
+<body><div class="card">
+  <div class="icon">{icon}</div>
+  <h2>{safe_title}</h2>
+  <p>{safe_message}</p>
+</div></body></html>"""

routes/memory/memory_routes.py

@@ -21,7 +21,7 @@ def _strip_list_prefix(text: str) -> str:
         return text
     return _LIST_PREFIX_RE.sub("", text, count=1).strip()
 
-from services.memory import MemoryManager, MemoryStoreUnreadable
+from services.memory import MemoryManager
 from core.session_manager import SessionManager
 from src.request_models import MemoryAddRequest
 from core.database import SessionLocal
@@ -35,22 +35,6 @@ from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
 logger = logging.getLogger(__name__)
 
 
-def _load_for_update(memory_manager) -> List[Dict[str, Any]]:
-    """Load the whole store for a read-modify-write cycle.
-
-    A transient read failure must not look like an empty store: the caller
-    would append to ``[]`` and save that back, atomically destroying every
-    existing memory (issue #5673). Surface it as a 503 and change nothing.
-    """
-    try:
-        return memory_manager.load_all_for_update()
-    except MemoryStoreUnreadable as e:
-        logger.error("Refusing to rewrite the memory store: %s", e)
-        raise HTTPException(
-            503, "Memory store is temporarily unreadable — no changes were made."
-        )
-
-
 def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
     """Set up memory-related routes."""
     router = APIRouter(prefix="/api/memory", tags=["memory"])
@@ -132,7 +116,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
         new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
         if memory_data.session_id:
             new_entry["session_id"] = memory_data.session_id
-        all_mem = _load_for_update(memory_manager)
+        all_mem = memory_manager.load_all()
         all_mem.append(new_entry)
         memory_manager.save(all_mem)
         # Sync vector index
@@ -503,7 +487,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
     def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
         """Pin or unpin a memory. Pinned memories are always included in context."""
         user = _owner(request)
-        all_mem = _load_for_update(memory_manager)
+        all_mem = memory_manager.load_all()
         for i, memory in enumerate(all_mem):
             if memory["id"] == memory_id:
                 _verify_memory_owner(memory, user)
@@ -528,7 +512,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
     def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
         """Update an existing memory item with new text and optional category."""
         user = _owner(request)
-        all_mem = _load_for_update(memory_manager)
+        all_mem = memory_manager.load_all()
         for i, memory in enumerate(all_mem):
             if memory["id"] == memory_id:
                 _verify_memory_owner(memory, user)
@@ -550,7 +534,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
     def delete_memory(request: Request, memory_id: str):
         """Delete a memory item by its ID."""
         user = _owner(request)
-        all_mem = _load_for_update(memory_manager)
+        all_mem = memory_manager.load_all()
 
         # Find and verify ownership before deleting
         target = next((m for m in all_mem if m["id"] == memory_id), None)

routes/model_routes.py

@@ -46,12 +46,10 @@ _ENDPOINT_SETTING_FIELDS = {
 }
 
 _ENDPOINT_FALLBACK_FIELDS = {
-    "foreground_model_fallbacks": "Foreground Model Fallbacks",
+    "default_model_fallbacks": "Default Model Fallbacks",
     "utility_model_fallbacks": "Utility Model Fallbacks",
     "vision_model_fallbacks":  "Vision Model Fallbacks",
 }
-# `default_model_fallbacks` is intentionally absent. The legacy data remains
-# stored as-is even when an endpoint is removed, but no longer affects routing.
 
 
 def _speech_settings_using_endpoint(settings: dict, ep_id: str) -> list:
@@ -181,12 +179,7 @@ def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int:
     if not isinstance(all_prefs, dict):
         return 0
     users = all_prefs.get("_users")
-    # A mixed store can contain auth-disabled foreground policy at the root
+    pref_sets = users.values() if isinstance(users, dict) else [all_prefs]
-    # alongside named-owner preferences. Both are active namespaces; legacy
-    # `default_model_fallbacks` remains untouched by the field allowlist.
-    pref_sets = [all_prefs]
-    if isinstance(users, dict):
-        pref_sets.extend(users.values())
     cleared_users = 0
     for prefs in pref_sets:
         if isinstance(prefs, dict) and _clear_endpoint_settings_for_endpoint(prefs, ep_id):
@@ -1351,14 +1344,14 @@ def _legacy_visible_api_models(ep) -> List[str]:
 def _picker_models_for_endpoint(ep, base_url: str, kind: str):
     """Return model IDs that should appear in the picker for an endpoint.
 
-    API providers expose remote inventory from /v1/models. Default to that
+    API providers expose remote inventory from /v1/models. Treat that cache as
-    visible inventory until an explicit pinned-model allow-list is saved.
+    inventory, not approval: only manually pinned API models should appear in
-    Local/self-hosted endpoints keep the older hide-list behavior.
+    the picker. Local/self-hosted endpoints keep the older hide-list behavior.
     """
     pinned = _normalize_model_ids(getattr(ep, "pinned_models", None))
     if _picker_requires_pinning(base_url, kind):
         if not _has_explicit_pinned_models(ep):
-            pinned = _legacy_visible_api_models(ep)
+            pinned = _legacy_visible_api_models(ep) if _hidden_model_ids(ep) else []
         return pinned, pinned
     return _visible_models(
         _cached_model_ids(ep),
@@ -2342,7 +2335,9 @@ def setup_model_routes(model_discovery):
                 else:
                     response.headers["X-Model-Refresh-Status"] = "failed"
                     response.headers["X-Model-Refresh-Warning"] = "Model refresh failed or returned no models; kept cached models."
-            _, pinned = _picker_models_for_endpoint(ep, base, kind)
+            pinned = _normalize_model_ids(getattr(ep, "pinned_models", None))
+            if picker_requires_pinning and not _has_explicit_pinned_models(ep):
+                pinned = _legacy_visible_api_models(ep)
             pinned_set = set(pinned)
             return [
                 {
@@ -2442,6 +2437,7 @@ def setup_model_routes(model_discovery):
             _user_prefs = _load_for_user(_user) or {}
             ep_id = (_user_prefs.get("default_endpoint_id") or "").strip()
             model = (_user_prefs.get("default_model") or "").strip()
+            _fallbacks = _user_prefs.get("default_model_fallbacks") or []
             # If user has no personal default, fall back to global default
             # But only based on the "share_defaults_with_users" flag
             # (only if share_defaults_with_users is enabled)
@@ -2450,9 +2446,12 @@ def setup_model_routes(model_discovery):
                     ep_id = settings.get("default_endpoint_id", "")
                 if not model:
                     model = settings.get("default_model", "")
+                if not _fallbacks:
+                    _fallbacks = settings.get("default_model_fallbacks") or []
         else:
             ep_id = settings.get("default_endpoint_id", "")
             model = settings.get("default_model", "")
+            _fallbacks = settings.get("default_model_fallbacks") or []
         db = SessionLocal()
         try:
             ep = None
@@ -2467,6 +2466,33 @@ def setup_model_routes(model_discovery):
                 if _user and not _is_admin:
                     ep_q = owner_filter(ep_q, ModelEndpoint, _user)
                 ep = ep_q.first()
+            # Configured fallback chain — when the chosen default endpoint is
+            # gone/disabled, honor the user's configured `default_model_fallbacks`
+            # in order BEFORE arbitrarily grabbing the first enabled endpoint.
+            # (Previously this jumped straight to "first enabled", which is why
+            # deleting/changing the main endpoint silently reassigned the default
+            # chat to some unrelated endpoint instead of the fallback.)
+            if not ep:
+                for entry in _fallbacks:
+                    if not isinstance(entry, dict):
+                        continue
+                    fid = (entry.get("endpoint_id") or "").strip()
+                    if not fid:
+                        continue
+                    cand_q = db.query(ModelEndpoint).filter(
+                        ModelEndpoint.id == fid, ModelEndpoint.is_enabled == True
+                    )
+                    if _user and not _is_admin:
+                        cand_q = owner_filter(cand_q, ModelEndpoint, _user)
+                    cand = cand_q.first()
+                    if cand:
+                        ep = cand
+                        # Use the fallback entry's model. Reset even when empty
+                        # so we don't carry the prior endpoint's stale model onto
+                        # this fallback — the cached-models lookup below then
+                        # fills it from the fallback endpoint.
+                        model = (entry.get("model") or "").strip()
+                        break
             # Last resort: first enabled endpoint owned by THIS user. Do not
             # include null-owner/shared endpoints here: a brand-new user with
             # no explicit default should not auto-open a pending chat using an

routes/personal_routes.py

@@ -1,13 +1,11 @@
 # routes/personal_routes.py
 """Routes for personal documents management."""
-import asyncio
 import os
 import logging
 import shutil
 import uuid
 from typing import Any, Dict, List, Tuple
 from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Depends
-from fastapi.concurrency import run_in_threadpool
 from src.request_models import DirectoryRequest
 from core.constants import BASE_DIR, PERSONAL_DIR, PERSONAL_UPLOADS_DIR
 from src.rag_singleton import get_rag_manager
@@ -20,6 +18,7 @@ UPLOADS_DIR = PERSONAL_UPLOADS_DIR
 
 logger = logging.getLogger(__name__)
 
+
 def _personal_upload_dir_for_owner(owner: str | None, *, create: bool = True) -> str:
     """Return the per-owner upload directory used for direct RAG uploads."""
     owner_segment = secure_filename((owner or "local").strip())[:80] or "local"
@@ -142,22 +141,6 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
     """
     router = APIRouter(prefix="/api/personal")
 
-    # Serializes directory index jobs across requests. Indexing runs in the
-    # threadpool (#5558), so concurrent requests would otherwise run in parallel
-    # and race PersonalDocsManager's unsynchronized list mutations and file
-    # writes; before the threadpool move they serialized on the blocked event
-    # loop, so one-at-a-time is behavior parity.
-    #
-    # An asyncio.Lock acquired in the async handler BEFORE offloading: a waiting
-    # request parks on the event loop instead of pinning a threadpool worker (an
-    # earlier threading.Lock taken INSIDE the worker meant queued jobs held pool
-    # tokens while blocked, starving every other run_in_threadpool caller).
-    # add/remove/reload all take this lock, so their mutations never interleave.
-    # Per-router (not module-global) so each app binds it to its own event loop.
-    # Scope is the single process: multi-worker deployments would need a shared
-    # lock (out of scope for #5558).
-    _index_job_lock = asyncio.Lock()
-
     def _rag():
         """Get the current RAG manager, retrying init if needed."""
         return get_rag_manager()
@@ -189,12 +172,8 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
         return {"files": files, "directories": directories}
     
     @router.post("/reload")
-    async def api_personal_reload(owner: str = Depends(require_user), _admin: None = Depends(require_admin)):
+    def api_personal_reload(owner: str = Depends(require_user), _admin: None = Depends(require_admin)):
-        # refresh_index() re-extracts text across every tracked directory —
+        personal_docs_manager.refresh_index()
-        # blocking work. Take the shared job lock (so it cannot race an add /
-        # remove) and run it off the event loop.
-        async with _index_job_lock:
-            await run_in_threadpool(personal_docs_manager.refresh_index)
         return {"ok": True, "count": len(personal_docs_manager.index)}
     
     @router.post("/add_directory")
@@ -228,26 +207,12 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
             # Use the RAGManager to index the directory
             rag = _rag()
             if rag:
-                def _index_directory():
+                result = rag.index_personal_documents(directory, owner=owner)
-                    result = rag.index_personal_documents(directory, owner=owner)
+                
-                    if result["success"]:
-                        # Also update the personal_docs_manager to track this
-                        # directory. Kept inside the offloaded call: it triggers
-                        # refresh_index(), which re-extracts text across tracked
-                        # directories.
-                        personal_docs_manager.add_directory(directory, index=False)
-                    return result
-
-                # Indexing walks, embeds, and stores the whole tree — minutes
-                # on a real directory. The handler is async, so calling it
-                # inline runs it on the event loop and every other request
-                # queues behind it until it finishes (#5558). Serialize on the
-                # async job lock BEFORE offloading so a queued request parks on
-                # the loop instead of pinning a threadpool worker.
-                async with _index_job_lock:
-                    result = await run_in_threadpool(_index_directory)
-
                 if result["success"]:
+                    # Also update the personal_docs_manager to track this directory
+                    personal_docs_manager.add_directory(directory, index=False)
+                    
                     return {
                         "success": True,
                         "message": f"Successfully indexed {result['indexed_count']} chunks from {directory}",
@@ -286,25 +251,17 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
 
             logger.info(f"Removing directory from RAG: {directory}")
 
+            # Always remove from personal_docs_manager tracking
+            if hasattr(personal_docs_manager, 'remove_directory'):
+                personal_docs_manager.remove_directory(directory)
+
+            # Remove from RAG vector store (best-effort)
             rag = _rag()
-
+            if rag:
-            def _remove_directory():
+                try:
-                # Always remove from personal_docs_manager tracking. This
+                    rag.remove_directory(directory)
-                # mutates the same unsynchronized list/index an add job touches
+                except Exception as e:
-                # and re-extracts text (refresh_index), so it is blocking work.
+                    logger.warning(f"RAG removal failed for directory {directory}: {e}")
-                if hasattr(personal_docs_manager, 'remove_directory'):
-                    personal_docs_manager.remove_directory(directory)
-                # Remove from RAG vector store (best-effort).
-                if rag:
-                    try:
-                        rag.remove_directory(directory)
-                    except Exception as e:
-                        logger.warning(f"RAG removal failed for directory {directory}: {e}")
-
-            # Same job lock as add/reload so remove cannot interleave with an
-            # in-flight add; offloaded off the event loop.
-            async with _index_job_lock:
-                await run_in_threadpool(_remove_directory)
 
             return {
                 "success": True,
@@ -332,73 +289,54 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
         total_failed = 0
         uploaded_files = []
 
-        # Chunking, embedding and the tracking update are blocking work over the
+        for upload in files:
-        # same vector/tracking state add_directory mutates (#5634). Take the
+            try:
-        # shared job lock BEFORE offloading so a queued request parks on the loop
+                file_path, stored_name, safe_name = _unique_personal_upload_path(upload_dir, upload.filename)
-        # instead of pinning a threadpool worker, matching add_directory.
+                content_bytes = await upload.read(PERSONAL_UPLOAD_MAX_BYTES + 1)
-        # Read and process one capped payload at a time so a multi-file request
+                if len(content_bytes) > PERSONAL_UPLOAD_MAX_BYTES:
-        # cannot retain len(files) * PERSONAL_UPLOAD_MAX_BYTES in memory.
+                    logger.warning(f"Rejected oversized personal upload: {upload.filename!r}")
-        async with _index_job_lock:
-            for upload in files:
-                try:
-                    file_path, stored_name, safe_name = _unique_personal_upload_path(
-                        upload_dir, upload.filename
-                    )
-                    content_bytes = await upload.read(PERSONAL_UPLOAD_MAX_BYTES + 1)
-                    if len(content_bytes) > PERSONAL_UPLOAD_MAX_BYTES:
-                        logger.warning(f"Rejected oversized personal upload: {upload.filename!r}")
-                        total_failed += 1
-                        continue
-
-                    def _index_upload():
-                        with open(file_path, "wb") as f:
-                            f.write(content_bytes)
-
-                        ext = os.path.splitext(safe_name)[1].lower()
-                        if ext == ".pdf":
-                            from src.personal_docs import extract_pdf_text
-                            text = extract_pdf_text(file_path)
-                        else:
-                            text = content_bytes.decode("utf-8", errors="replace")
-
-                        if not text or not text.strip():
-                            return 0, 1, None
-
-                        indexed = 0
-                        failed = 0
-                        chunks = rag._split_into_chunks(text, chunk_size=500)
-                        for i, chunk in enumerate(chunks):
-                            metadata = {
-                                "source": file_path,
-                                "filename": safe_name,
-                                "stored_filename": stored_name,
-                                "directory": upload_dir,
-                                "type": ext,
-                                "chunk_id": i,
-                            }
-                            if user:
-                                metadata["owner"] = user
-                            if rag.add_document(chunk, metadata):
-                                indexed += 1
-                            else:
-                                failed += 1
-                        return indexed, failed, safe_name
-
-                    indexed, failed, uploaded_name = await run_in_threadpool(_index_upload)
-                    total_indexed += indexed
-                    total_failed += failed
-                    if uploaded_name:
-                        uploaded_files.append(uploaded_name)
-                except Exception as e:
-                    logger.error(f"Failed to upload/index {upload.filename}: {e}")
                     total_failed += 1
+                    continue
+                with open(file_path, "wb") as f:
+                    f.write(content_bytes)
 
-            # Same transition, same lock: the tracking update must not land
+                ext = os.path.splitext(safe_name)[1].lower()
-            # while another job is mid-write over the same state.
+                if ext == ".pdf":
-            if uploaded_files and hasattr(personal_docs_manager, "add_directory"):
+                    from src.personal_docs import extract_pdf_text
-                await run_in_threadpool(
+                    text = extract_pdf_text(file_path)
-                    personal_docs_manager.add_directory, upload_dir, index=False
+                else:
-                )
+                    text = content_bytes.decode("utf-8", errors="replace")
+
+                if not text or not text.strip():
+                    total_failed += 1
+                    continue
+
+                # Chunk and index
+                chunks = rag._split_into_chunks(text, chunk_size=500)
+                for i, chunk in enumerate(chunks):
+                    metadata = {
+                        "source": file_path,
+                        "filename": safe_name,
+                        "stored_filename": stored_name,
+                        "directory": upload_dir,
+                        "type": ext,
+                        "chunk_id": i,
+                    }
+                    if user:
+                        metadata["owner"] = user
+                    if rag.add_document(chunk, metadata):
+                        total_indexed += 1
+                    else:
+                        total_failed += 1
+
+                uploaded_files.append(safe_name)
+            except Exception as e:
+                logger.error(f"Failed to upload/index {upload.filename}: {e}")
+                total_failed += 1
+
+        # Track uploads directory
+        if uploaded_files and hasattr(personal_docs_manager, "add_directory"):
+            personal_docs_manager.add_directory(upload_dir, index=False)
 
         return {
             "success": True,
@@ -411,47 +349,38 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
     async def delete_file_from_rag(filepath: str = Query(...), owner: str = Depends(require_user), _admin: None = Depends(require_admin)):
         """Delete a specific file from RAG index and optionally from disk."""
         try:
-            def _delete_file():
+            # Remove chunks from RAG vector store (best-effort)
-                # Remove chunks from RAG vector store (best-effort)
+            removed = 0
-                removed = 0
+            rag = _rag()
-                rag = _rag()
+            if rag:
-                if rag:
-                    try:
-                        removed = rag.delete_by_source(filepath)
-                    except Exception as e:
-                        logger.warning(f"RAG removal failed for {filepath}: {e}")
-
-                # Delete file from disk if it's in the caller's own uploads dir.
-                # Scope to the per-owner subdir, not the shared uploads root, so one
-                # admin can't delete another user's personal files by path.
-                deleted_from_disk = False
                 try:
-                    abs_target = os.path.realpath(filepath)
+                    removed = rag.delete_by_source(filepath)
-                    base_abs = os.path.realpath(_personal_upload_dir_for_owner(owner, create=False))
+                except Exception as e:
-                    in_uploads = (
+                    logger.warning(f"RAG removal failed for {filepath}: {e}")
-                        abs_target == base_abs
-                        or os.path.commonpath([abs_target, base_abs]) == base_abs
-                    )
-                except ValueError:
-                    # commonpath raises on mixed drives / non-comparable paths
-                    in_uploads = False
-                if in_uploads and abs_target != base_abs:
-                    try:
-                        os.remove(abs_target)
-                        deleted_from_disk = True
-                    except FileNotFoundError:
-                        pass  # already gone — race with another request or cleanup
 
-                # Exclude the file from the listing (persists across restarts)
+            # Delete file from disk if it's in the caller's own uploads dir.
-                personal_docs_manager.exclude_file(filepath)
+            # Scope to the per-owner subdir, not the shared uploads root, so one
-                return removed, deleted_from_disk
+            # admin can't delete another user's personal files by path.
+            deleted_from_disk = False
+            try:
+                abs_target = os.path.realpath(filepath)
+                base_abs = os.path.realpath(_personal_upload_dir_for_owner(owner, create=False))
+                in_uploads = (
+                    abs_target == base_abs
+                    or os.path.commonpath([abs_target, base_abs]) == base_abs
+                )
+            except ValueError:
+                # commonpath raises on mixed drives / non-comparable paths
+                in_uploads = False
+            if in_uploads and abs_target != base_abs:
+                try:
+                    os.remove(abs_target)
+                    deleted_from_disk = True
+                except FileNotFoundError:
+                    pass  # already gone — race with another request or cleanup
 
-            # Vector removal, the disk unlink and the exclusion write are one
+            # Exclude the file from the listing (persists across restarts)
-            # transition over the same state add_directory mutates (#5634), and
+            personal_docs_manager.exclude_file(filepath)
-            # all three block. Take the shared job lock BEFORE offloading, as
-            # add_directory does.
-            async with _index_job_lock:
-                removed, deleted_from_disk = await run_in_threadpool(_delete_file)
 
             return {
                 "success": True,

routes/prefs_routes.py

@@ -1,16 +1,12 @@
 """User preferences API — per-user key/value store backed by a JSON file."""
 import json
+import os
 from typing import Optional
 from fastapi import APIRouter, Request
-from core.atomic_io import atomic_write_json
 from src.auth_helpers import get_current_user
 from src.constants import USER_PREFS_FILE
 
 PREFS_FILE = USER_PREFS_FILE
-_FOREGROUND_POLICY_KEYS = (
-    "foreground_fallback_enabled",
-    "foreground_model_fallbacks",
-)
 
 
 def _load():
@@ -24,33 +20,26 @@ def _load():
 
 
 def _save(prefs):
-    atomic_write_json(PREFS_FILE, prefs, indent=2)
+    os.makedirs(os.path.dirname(PREFS_FILE) or ".", exist_ok=True)
+    tmp = f"{PREFS_FILE}.tmp.{os.getpid()}"
+    with open(tmp, "w", encoding="utf-8") as f:
+        json.dump(prefs, f, indent=2)
+        f.flush()
+        os.fsync(f.fileno())
+    os.replace(tmp, PREFS_FILE)
 
 
 def _load_for_user(user: Optional[str] = None) -> dict:
     """Load preferences for a specific user."""
     all_prefs = _load()
-    users = all_prefs.get("_users")
+    if "_users" in all_prefs:
-    if isinstance(users, dict):
         if user is None:
             # Auth disabled — return first user's prefs for backward compat
-            prefs = dict(next(iter(users.values()), {}))
+            users = all_prefs["_users"]
-            # Foreground fallback consent is never borrowed from a named
+            return dict(next(iter(users.values()), {}))
-            # owner. Auth-disabled operation has a separate flat/root opt-in
+        return dict(all_prefs["_users"].get(user, {}))
-            # that remains inert when authentication is enabled again.
+    # Legacy flat format — return as-is
-            for key in _FOREGROUND_POLICY_KEYS:
+    return dict(all_prefs)
-                prefs.pop(key, None)
-                if key in all_prefs:
-                    prefs[key] = all_prefs[key]
-            return prefs
-        prefs = users.get(user, {})
-        return dict(prefs) if isinstance(prefs, dict) else {}
-    # A legacy flat store belongs only to auth-disabled single-user mode.
-    # Copying it into the first named user's new `_users` record during an
-    # auth transition would silently transfer another user's preferences and,
-    # critically, foreground fallback consent. Named owners therefore start
-    # with an empty record and must write their own preferences explicitly.
-    return dict(all_prefs) if user is None else {}
 
 
 def _save_for_user(user: Optional[str], prefs: dict):
@@ -62,40 +51,17 @@ def _save_for_user(user: Optional[str], prefs: dict):
         # `prefs` flat would overwrite the whole `_users` map and destroy every
         # other user's preferences. Instead write back into the same (first)
         # slot _load_for_user(None) reads from, preserving the others.
-        users = all_prefs.get("_users")
+        if "_users" in all_prefs:
-        if isinstance(users, dict):
+            users = all_prefs["_users"]
             first_key = next(iter(users), None)
             if first_key is not None:
-                existing_named = users.get(first_key)
+                users[first_key] = prefs
-                existing_named = (
-                    dict(existing_named)
-                    if isinstance(existing_named, dict)
-                    else {}
-                )
-                named_foreground = {
-                    key: existing_named[key]
-                    for key in _FOREGROUND_POLICY_KEYS
-                    if key in existing_named
-                }
-                users[first_key] = {
-                    key: value
-                    for key, value in prefs.items()
-                    if key not in _FOREGROUND_POLICY_KEYS
-                }
-                users[first_key].update(named_foreground)
-                for key in _FOREGROUND_POLICY_KEYS:
-                    if key in prefs:
-                        all_prefs[key] = prefs[key]
                 _save(all_prefs)
                 return
         _save(prefs)
         return
-    if not isinstance(all_prefs.get("_users"), dict):
+    if "_users" not in all_prefs:
-        # Preserve the flat single-user object as inert legacy data while
+        all_prefs = {"_users": {}}
-        # creating the first named-owner namespace. In particular, historical
-        # fallback values must not be deleted or copied into the new owner.
-        all_prefs = dict(all_prefs)
-        all_prefs["_users"] = {}
     all_prefs["_users"][user] = prefs
     _save(all_prefs)
 

routes/research/research_routes.py

@@ -15,7 +15,7 @@ from pydantic import BaseModel, Field
 from core.middleware import INTERNAL_TOOL_USER
 from src.endpoint_resolver import resolve_endpoint
 from src.auth_helpers import _auth_disabled, get_current_user
-from src.owner_identity import REQUEST_SENTINEL_OWNERS
+from core.auth import RESERVED_USERNAMES
 from src.constants import DEEP_RESEARCH_DIR
 
 _SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
@@ -496,7 +496,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         user = require_privilege(request, "can_use_research")
         if user == INTERNAL_TOOL_USER:
             tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip()
-            if tool_owner and tool_owner not in REQUEST_SENTINEL_OWNERS:
+            if tool_owner and tool_owner not in RESERVED_USERNAMES:
                 auth_mgr = getattr(request.app.state, "auth_manager", None)
                 if auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
                     try:

routes/search/__init__.py

@@ -1,5 +0,0 @@
-"""Search route domain package (slice 2j, #4082/#4071).
-
-Contains search_routes.py, migrated from the flat routes/ directory.
-Backward-compat shim at routes/search_routes.py re-exports from here.
-"""

routes/search/search_routes.py

@@ -1,111 +0,0 @@
-"""Search routes — /api/search/config GET, /api/search POST."""
-
-import logging
-from typing import Dict, Any
-
-from fastapi import APIRouter, Request
-
-import time
-
-from services.search import get_search_config, comprehensive_web_search, PROVIDER_INFO
-from services.search.core import _call_provider
-from services.search.providers import _get_provider_key, _get_search_instance
-
-logger = logging.getLogger(__name__)
-
-
-async def _request_values(request: Request) -> Dict[str, Any]:
-    """Accept JSON, form data, or query params for search endpoints.
-
-    The browser UI posts FormData, while the agent's generic app_api tool
-    posts JSON. FastAPI Form(...) rejects JSON with a 422 before our handler
-    runs, which made the model think SearXNG was broken.
-    """
-    values: Dict[str, Any] = dict(request.query_params)
-    content_type = (request.headers.get("content-type") or "").lower()
-    try:
-        if "application/json" in content_type:
-            body = await request.json()
-            if isinstance(body, dict):
-                values.update(body)
-        else:
-            form = await request.form()
-            values.update(dict(form))
-    except Exception:
-        pass
-    return values
-
-
-def setup_search_routes(config) -> APIRouter:
-    router = APIRouter(tags=["search"])
-
-    @router.get("/api/search/config")
-    async def get_search_settings() -> Dict[str, Any]:
-        return get_search_config()
-
-    @router.post("/api/search")
-    async def do_web_search(request: Request) -> Dict[str, Any]:
-        """Standalone web search — returns context string + source list.
-
-        Used by Compare mode to pre-search once and share results across panes.
-        """
-        values = await _request_values(request)
-        query = str(values.get("query") or values.get("q") or "").strip()
-        if not query:
-            return {"context": "", "sources": [], "error": "query is required"}
-        time_filter = values.get("time_filter") or values.get("freshness")
-        if time_filter is not None:
-            time_filter = str(time_filter).strip() or None
-        try:
-            context, sources = comprehensive_web_search(
-                query, return_sources=True, time_filter=time_filter,
-            )
-            return {"context": context, "sources": sources}
-        except Exception as e:
-            logger.error(f"Standalone web search failed: {e}")
-            return {"context": "", "sources": [], "error": str(e)}
-
-    @router.get("/api/search/providers")
-    async def list_search_providers():
-        """Return available search providers with config status."""
-        providers = []
-        for pid, (label, needs_key, needs_url) in PROVIDER_INFO.items():
-            if pid == "disabled":
-                continue
-            available = True
-            if needs_key and not _get_provider_key(pid):
-                available = False
-            if needs_url and pid == "searxng" and not _get_search_instance():
-                available = False
-            providers.append({
-                "id": pid,
-                "label": label,
-                "available": available,
-            })
-        return providers
-
-    @router.post("/api/search/query")
-    async def search_with_provider(request: Request) -> Dict[str, Any]:
-        """Search using a specific provider. Used by compare search mode."""
-        values = await _request_values(request)
-        query = str(values.get("query") or values.get("q") or "").strip()
-        provider = str(values.get("provider") or "").strip()
-        try:
-            count = int(values.get("count") or values.get("limit") or 10)
-        except Exception:
-            count = 10
-        if not query:
-            return {"results": [], "provider": provider, "error": "query is required"}
-        if provider not in PROVIDER_INFO or provider == "disabled":
-            return {"results": [], "provider": provider, "error": "Unknown provider"}
-        t0 = time.time()
-        try:
-            results = _call_provider(provider, query, min(count, 20))
-            elapsed = round(time.time() - t0, 2)
-            return {"results": results, "provider": provider, "time": elapsed}
-        except Exception as e:
-            elapsed = round(time.time() - t0, 2)
-            logger.error(f"Search provider {provider} failed: {e}")
-            return {"results": [], "provider": provider, "time": elapsed, "error": str(e)}
-
-    return router

routes/search_routes.py

@@ -1,13 +1,111 @@
-"""Backward-compat shim — canonical location is routes/search/search_routes.py.
+"""Search routes — /api/search/config GET, /api/search POST."""
 
-This module is replaced in ``sys.modules`` by the canonical module object so
+import logging
-that ``import routes.search_routes`` and ``from routes.search_routes import X``
+from typing import Dict, Any
-keep resolving to the canonical module. Keeps existing import paths working
-after slice 2j (#4082/#4071).
-"""
 
-import sys as _sys
+from fastapi import APIRouter, Request
 
-from routes.search import search_routes as _canonical  # noqa: F401
+import time
 
-_sys.modules[__name__] = _canonical
+from services.search import get_search_config, comprehensive_web_search, PROVIDER_INFO
+from services.search.core import _call_provider
+from services.search.providers import _get_provider_key, _get_search_instance
+
+logger = logging.getLogger(__name__)
+
+
+async def _request_values(request: Request) -> Dict[str, Any]:
+    """Accept JSON, form data, or query params for search endpoints.
+
+    The browser UI posts FormData, while the agent's generic app_api tool
+    posts JSON. FastAPI Form(...) rejects JSON with a 422 before our handler
+    runs, which made the model think SearXNG was broken.
+    """
+    values: Dict[str, Any] = dict(request.query_params)
+    content_type = (request.headers.get("content-type") or "").lower()
+    try:
+        if "application/json" in content_type:
+            body = await request.json()
+            if isinstance(body, dict):
+                values.update(body)
+        else:
+            form = await request.form()
+            values.update(dict(form))
+    except Exception:
+        pass
+    return values
+
+
+def setup_search_routes(config) -> APIRouter:
+    router = APIRouter(tags=["search"])
+
+    @router.get("/api/search/config")
+    async def get_search_settings() -> Dict[str, Any]:
+        return get_search_config()
+
+    @router.post("/api/search")
+    async def do_web_search(request: Request) -> Dict[str, Any]:
+        """Standalone web search — returns context string + source list.
+
+        Used by Compare mode to pre-search once and share results across panes.
+        """
+        values = await _request_values(request)
+        query = str(values.get("query") or values.get("q") or "").strip()
+        if not query:
+            return {"context": "", "sources": [], "error": "query is required"}
+        time_filter = values.get("time_filter") or values.get("freshness")
+        if time_filter is not None:
+            time_filter = str(time_filter).strip() or None
+        try:
+            context, sources = comprehensive_web_search(
+                query, return_sources=True, time_filter=time_filter,
+            )
+            return {"context": context, "sources": sources}
+        except Exception as e:
+            logger.error(f"Standalone web search failed: {e}")
+            return {"context": "", "sources": [], "error": str(e)}
+
+    @router.get("/api/search/providers")
+    async def list_search_providers():
+        """Return available search providers with config status."""
+        providers = []
+        for pid, (label, needs_key, needs_url) in PROVIDER_INFO.items():
+            if pid == "disabled":
+                continue
+            available = True
+            if needs_key and not _get_provider_key(pid):
+                available = False
+            if needs_url and pid == "searxng" and not _get_search_instance():
+                available = False
+            providers.append({
+                "id": pid,
+                "label": label,
+                "available": available,
+            })
+        return providers
+
+    @router.post("/api/search/query")
+    async def search_with_provider(request: Request) -> Dict[str, Any]:
+        """Search using a specific provider. Used by compare search mode."""
+        values = await _request_values(request)
+        query = str(values.get("query") or values.get("q") or "").strip()
+        provider = str(values.get("provider") or "").strip()
+        try:
+            count = int(values.get("count") or values.get("limit") or 10)
+        except Exception:
+            count = 10
+        if not query:
+            return {"results": [], "provider": provider, "error": "query is required"}
+        if provider not in PROVIDER_INFO or provider == "disabled":
+            return {"results": [], "provider": provider, "error": "Unknown provider"}
+        t0 = time.time()
+        try:
+            results = _call_provider(provider, query, min(count, 20))
+            elapsed = round(time.time() - t0, 2)
+            return {"results": results, "provider": provider, "time": elapsed}
+        except Exception as e:
+            elapsed = round(time.time() - t0, 2)
+            logger.error(f"Search provider {provider} failed: {e}")
+            return {"results": [], "provider": provider, "time": elapsed, "error": str(e)}
+
+    return router

routes/session_routes.py

@@ -801,6 +801,15 @@ def setup_session_routes(
         finally:
             db.close()
 
+    @router.get("/history/{sid}")
+    def get_history(request: Request, sid: str):
+        _verify_session_owner(request, sid)
+        try:
+            session = session_manager.get_session(sid)
+        except KeyError:
+            raise HTTPException(404, f"Session {sid} not found")
+        return {"history": [msg.to_dict() for msg in session.history]}
+    
     @router.get("/session/{sid}/export")
     def export_session(request: Request, sid: str, fmt: str = "md", filename: str = ""):
         """Export conversation history as a downloadable file.

routes/skills_routes.py

@@ -18,7 +18,6 @@ from pydantic import BaseModel, Field
 
 from services.memory.skills import SkillsManager
 from src.auth_helpers import get_current_user
-from src.prompt_security import untrusted_context_message
 from core.middleware import require_admin
 
 logger = logging.getLogger(__name__)
@@ -108,23 +107,6 @@ def _skill_test_task(skill: dict) -> str:
     )
 
 
-def _skill_test_messages(md: str, task: str) -> list[dict]:
-    """Keep user-editable skill text out of the trusted system role."""
-    return [
-        {
-            "role": "system",
-            "content": (
-                "You are TESTING a skill. Follow the supplied reusable procedure "
-                "to complete the user's task for real, using available tools step "
-                "by step. If the skill is wrong, unclear, or references tools that "
-                "do not exist, do your best; the problems will be reviewed afterward."
-            ),
-        },
-        untrusted_context_message("skill under test", md),
-        {"role": "user", "content": task},
-    ]
-
-
 async def _eval_skill_run(skill_md: str, task: str, transcript: str,
                           url: str, model: str, headers: Optional[dict]) -> dict:
     """LLM-as-judge: grade a skill test run from its transcript. Advisory only.
@@ -429,21 +411,7 @@ async def _eval_skill_retrieval_precision(skill_md: str, others: list,
 _skill_test_jobs: dict = {}
 
 
-async def _run_skill_test_job(
+async def _run_skill_test_job(key, name, md, task, url, model, headers, owner, skills_manager=None):
-    key,
-    name,
-    md,
-    task,
-    url,
-    model,
-    headers,
-    owner,
-    skills_manager=None,
-    *,
-    messages=None,
-    transcript=None,
-    exact_approval=None,
-):
     """Background coroutine: run the skill in an agent loop, capture a condensed
     log + transcript, then have the judge grade it. Writes into _skill_test_jobs."""
     import json as _json
@@ -453,7 +421,7 @@ async def _run_skill_test_job(
     if job is None:
         return
     log = job["log"]
-    transcript = transcript if isinstance(transcript, list) else []
+    transcript = []
     say_buf = []
 
     def _flush_say():
@@ -461,12 +429,18 @@ async def _run_skill_test_job(
             log.append({"type": "say", "text": "".join(say_buf)})
             say_buf.clear()
 
-    messages = list(messages) if isinstance(messages, list) else _skill_test_messages(md, task)
+    messages = [
+        {"role": "system", "content":
+            "You are TESTING a skill. Below is a reusable skill (a procedure). Follow it "
+            "to complete the user's task for real, using your available tools, step by "
+            "step. If the skill is wrong, unclear, or references tools that don't exist, "
+            "do your best — the problems will be reviewed afterward.\n\n=== SKILL ===\n" + md},
+        {"role": "user", "content": task},
+    ]
     try:
         async for chunk in stream_agent_loop(
             url, model, messages, headers=headers,
             temperature=0.3, max_tokens=0, max_rounds=8, owner=owner,
-            exact_approval=exact_approval,
         ):
             if not chunk.startswith("data: ") or chunk.strip() == "data: [DONE]":
                 continue
@@ -484,25 +458,8 @@ async def _run_skill_test_job(
             elif d.get("type") == "tool_output":
                 _flush_say()
                 out = str(d.get("output") or "")[:600]
-                tool_log = {"type": "tool_output", "output": out}
+                log.append({"type": "tool_output", "output": out})
-                approval = d.get("ask_user")
-                if isinstance(approval, dict):
-                    tool_log["ask_user"] = approval
-                log.append(tool_log)
                 transcript.append(f"[output] {out}\n")
-                if (
-                    isinstance(approval, dict)
-                    and approval.get("kind") == "tool_approval"
-                    and approval.get("approval_id")
-                ):
-                    # Manual skill tests have their own polling UI instead of a
-                    # chat session. Pause the run and retain only server-side
-                    # continuation state until the same owner approves/denies
-                    # this exact sealed action.
-                    job["status"] = "awaiting_approval"
-                    job["approval"] = approval
-                    job["_transcript"] = transcript
-                    return
             elif d.get("type") == "agent_step":
                 _flush_say()
                 log.append({"type": "agent_step", "round": d.get("round")})
@@ -514,9 +471,6 @@ async def _run_skill_test_job(
         _flush_say()
         log.append({"type": "error", "error": str(e)})
 
-    job.pop("approval", None)
-    job.pop("_transcript", None)
-    job.pop("_run", None)
     log.append({"type": "evaluating"})
     try:
         job["verdict"] = await _eval_skill_run(md, task, "".join(transcript), url, model, headers)
@@ -740,8 +694,12 @@ async def _run_skill_test_once(md: str, task: str, url, model, headers, owner) -
     import json as _json
     from src.agent_loop import stream_agent_loop
     transcript = []
-    approval_required = None
+    messages = [
-    messages = _skill_test_messages(md, task)
+        {"role": "system", "content":
+            "You are TESTING a skill. Follow this skill's procedure to complete the task "
+            "for real, using your tools, step by step.\n\n=== SKILL ===\n" + md},
+        {"role": "user", "content": task},
+    ]
     try:
         # max_tokens explicitly set: passing 0 lets some upstreams (Ollama,
         # OpenAI-compat) generate an empty completion, which manifested as
@@ -761,44 +719,11 @@ async def _run_skill_test_once(md: str, task: str, url, model, headers, owner) -
                 transcript.append(f"\n[tool {d.get('tool')}] {str(d.get('command') or d.get('args') or '')[:300]}\n")
             elif d.get("type") == "tool_output":
                 transcript.append(f"[output] {str(d.get('output') or '')[:600]}\n")
-                approval = d.get("ask_user")
-                if (
-                    isinstance(approval, dict)
-                    and approval.get("kind") == "tool_approval"
-                ):
-                    approval_required = approval
-                    break
             elif d.get("type") == "agent_step":
                 transcript.append(f"\n--- round {d.get('round')} ---\n")
     except Exception as e:
         transcript.append(f"\n[run error] {e}\n")
     text = "".join(transcript)
-    if approval_required is not None:
-        # Unattended audits have no authority to approve and no UI that could
-        # resume this record. Destructively deny it now instead of leaving a
-        # reusable opaque grant pending until TTL/cap eviction.
-        try:
-            from src.tool_approvals import tool_approval_store
-            tool_approval_store.consume(
-                approval_required.get("approval_id"),
-                decision="deny",
-                owner=owner,
-                session_id=None,
-            )
-        except Exception:
-            logger.debug("Could not retire unattended skill approval", exc_info=True)
-        return text, {
-            "verdict": "inconclusive",
-            "confidence": 1.0,
-            "summary": (
-                "This automated audit reached an exact action that requires "
-                "a human approval; no action was executed."
-            ),
-            "issues": [
-                "Run this skill's manual test and review the sealed action."
-            ],
-            "approval_required": True,
-        }
     verdict = await _eval_skill_run(md, task, text, url, model, headers)
     return text, verdict
 
@@ -938,26 +863,6 @@ async def _audit_one_skill(skills_manager, skill, url, model, headers,
     transcript, verdict = await _run_skill_test_once(md, task, url, model, headers, owner)
     v = verdict.get("verdict")
     log(f"{name}: verdict = {v} ({verdict.get('summary', '')[:80]})")
-    if verdict.get("approval_required"):
-        # An unattended audit is not authority for an action influenced by the
-        # skill under test. Preserve the skill's current publication/confidence
-        # state and route the exact action to the manual test UI instead of
-        # letting a safety pause demote, rewrite, or auto-publish the skill.
-        skills_manager.set_audit(
-            name,
-            "inconclusive",
-            by_teacher=False,
-            worker_model=model,
-            owner=owner,
-        )
-        status = skill.get("status") or "draft"
-        log(f"{name}: {status} unchanged — exact action needs manual approval")
-        return {
-            "skill": name,
-            "result": "approval_required",
-            "verdict": verdict,
-            "status": status,
-        }
     if v == "pass":
         # Procedure works. If the reviewer still flagged metadata (tags/category/
         # when_to_use/description), do ONE fixer pass to correct the frontmatter
@@ -1504,7 +1409,7 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter:
 
         # Prefer the configured DEFAULT (→ Utility) model — not the current chat
         # session's model. Fall back to the caller's session model only if unset.
-        url, model, headers = resolve_endpoint("utility", owner=user)
+        url, model, headers = resolve_endpoint("default", owner=user)
         if not url or not model:
             url = url or ((body.get("endpoint_url") or "").strip() or None)
             model = model or ((body.get("model") or "").strip() or None)
@@ -1526,19 +1431,6 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter:
             logger.warning(f"Skill-test model resolve failed: {_e}")
 
         key = (user or "", name)
-        previous_job = _skill_test_jobs.get(key) or {}
-        previous_approval = previous_job.get("approval") or {}
-        if previous_approval.get("approval_id"):
-            try:
-                from src.tool_approvals import tool_approval_store
-                tool_approval_store.consume(
-                    previous_approval["approval_id"],
-                    decision="deny",
-                    owner=user,
-                    session_id=None,
-                )
-            except Exception:
-                logger.debug("Could not retire replaced skill approval", exc_info=True)
         _skill_test_jobs[key] = {
             "status": "running",
             "task": task,
@@ -1547,138 +1439,10 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter:
             "started": _time.time(),
             "log": [{"type": "skill_test_start", "task": task, "skill": name, "model": model}],
             "verdict": None,
-            "_run": {
-                "md": md,
-                "url": url,
-                "model": model,
-                "headers": headers,
-                "owner": user,
-            },
         }
         _asyncio.create_task(_run_skill_test_job(key, name, md, task, url, model, headers, user, skills_manager))
         return {"ok": True, "status": "running", "skill": name, "model": model}
 
-    @router.post("/{skill_id}/test-approval")
-    async def approve_skill_test_action(request: Request, skill_id: str):
-        """Resume a manual skill test with one exact server-sealed action."""
-        import asyncio as _asyncio
-        from src.tool_approvals import tool_approval_store
-
-        user = _owner(request)
-        skills = skills_manager.load(owner=user)
-        match = next(
-            (s for s in skills if s.get("name") == skill_id or s.get("id") == skill_id),
-            None,
-        )
-        if not match:
-            raise HTTPException(404, "Skill not found")
-        _verify_owner(match, user)
-        name = match.get("name")
-        key = (user or "", name)
-        job = _skill_test_jobs.get(key)
-        if not job or job.get("status") != "awaiting_approval":
-            raise HTTPException(409, "This skill test is not awaiting an approval.")
-
-        body = await request.json()
-        if not isinstance(body, dict):
-            raise HTTPException(400, "Tool approval body must be a JSON object.")
-        approval_id = str(body.get("approval_id") or "")
-        decision = str(body.get("decision") or "").strip().lower()
-        expected = job.get("approval") or {}
-        if approval_id != str(expected.get("approval_id") or ""):
-            raise HTTPException(409, "This approval does not match the pending skill test action.")
-        if decision not in {"approve", "deny"}:
-            raise HTTPException(400, "Invalid tool approval decision.")
-
-        pending = tool_approval_store.peek(approval_id)
-        normalized_owner = str(user or "").strip().casefold()
-        if (
-            pending is None
-            or pending.owner != normalized_owner
-            or pending.session_id != ""
-        ):
-            raise HTTPException(409, "This tool approval is invalid or expired.")
-        exact_approval = tool_approval_store.consume(
-            approval_id,
-            decision=decision,
-            owner=user,
-            session_id=None,
-            # The button here says "Allow once" and there is no chat to carry a
-            # scope into, so the gate must re-arm behind the sealed action.
-            allow_continuation=False,
-        )
-
-        if decision == "approve" and exact_approval is None:
-            raise HTTPException(409, "This tool approval could not be consumed.")
-        job.pop("approval", None)
-        if decision == "deny":
-            job.pop("_transcript", None)
-            job.pop("_run", None)
-            job["log"].append({
-                "type": "approval_denied",
-                "text": "Exact action denied; the skill test stopped without executing it.",
-            })
-            job["verdict"] = {
-                "verdict": "inconclusive",
-                "confidence": 1.0,
-                "summary": "The test stopped because its exact action was denied.",
-                "issues": [],
-            }
-            job["status"] = "done"
-            return {"ok": True, "status": "done", "decision": "deny"}
-
-        run = job.get("_run") or {}
-        transcript = job.pop("_transcript", [])
-        # stream_agent_loop owns its per-round message list internally. Rebuild
-        # continuation context from the original untrusted skill plus the
-        # accumulated transcript so repeated approvals do not lose earlier
-        # approved results, while keeping every transcript byte tainted.
-        messages = _skill_test_messages(
-            run.get("md", ""),
-            job.get("task", ""),
-        )
-        if transcript:
-            messages.append(untrusted_context_message(
-                "skill test transcript",
-                "".join(str(item) for item in transcript),
-            ))
-        messages.extend([
-            {
-                "role": "assistant",
-                "content": str(expected.get("question") or "Allow this exact action once?"),
-            },
-            {
-                "role": "user",
-                "content": (
-                    f"Approved the exact {exact_approval.pending.tool_name} "
-                    "action shown above once."
-                ),
-            },
-        ])
-        job["status"] = "running"
-        job["log"].append({
-            "type": "approval_granted",
-            "text": (
-                f"Approved exact {exact_approval.pending.tool_name} action once; "
-                "resuming test."
-            ),
-        })
-        _asyncio.create_task(_run_skill_test_job(
-            key,
-            name,
-            run.get("md", ""),
-            job.get("task", ""),
-            run.get("url"),
-            run.get("model"),
-            run.get("headers"),
-            run.get("owner"),
-            skills_manager,
-            messages=messages,
-            transcript=transcript,
-            exact_approval=exact_approval,
-        ))
-        return {"ok": True, "status": "running", "decision": "approve"}
-
     @router.get("/{skill_id}/test-status")
     async def test_skill_status(request: Request, skill_id: str):
         """Current background-test state for a skill (status / log / verdict)."""
@@ -1695,7 +1459,6 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter:
             "model": job.get("model"),
             "log": job.get("log", []),
             "verdict": job.get("verdict"),
-            "approval": job.get("approval"),
         }
 
     @router.post("/audit-all")

routes/task/__init__.py

@@ -1,5 +0,0 @@
-"""Task route domain package (slice 2p, #4082/#4071).
-
-Contains task_routes.py, migrated from the flat routes/ directory.
-Backward-compat shim at routes/task_routes.py re-exports from here.
-"""

routes/vault/__init__.py

@@ -1,5 +0,0 @@
-"""Vault route domain package (slice 2k, #4082/#4071).
-
-Contains vault_routes.py, migrated from the flat routes/ directory.
-Backward-compat shim at routes/vault_routes.py re-exports from here.
-"""

routes/vault/vault_routes.py

@@ -1,242 +0,0 @@
-"""
-vault_routes.py
-
-Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
-Stores the BW_SESSION key in data/vault.json with restrictive permissions.
-"""
-
-import json
-import logging
-import os
-import shutil
-import asyncio
-from pathlib import Path
-from datetime import datetime
-from fastapi import APIRouter, Request
-from pydantic import BaseModel
-
-from core.middleware import require_admin
-from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
-from src.constants import VAULT_FILE as _VAULT_FILE
-
-logger = logging.getLogger(__name__)
-
-VAULT_FILE = Path(_VAULT_FILE)
-
-
-def _find_bw() -> str:
-    """Locate the bw binary, checking PATH and common npm-global locations.
-
-    On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
-    which_tool via PATHEXT.
-    """
-    p = which_tool("bw")
-    if p:
-        return p
-    if IS_WINDOWS:
-        appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
-        for candidate in (
-            os.path.join(appdata, "npm", "bw.cmd"),
-            os.path.join(appdata, "npm", "bw.exe"),
-        ):
-            if os.path.isfile(candidate):
-                return candidate
-        return "bw"
-    home = os.path.expanduser("~")
-    for candidate in (
-        f"{home}/.npm-global/bin/bw",
-        f"{home}/.nvm/versions/node/*/bin/bw",
-        "/usr/local/bin/bw",
-        "/opt/homebrew/bin/bw",
-    ):
-        if "*" in candidate:
-            import glob
-            for m in glob.glob(candidate):
-                if os.path.isfile(m) and os.access(m, os.X_OK):
-                    return m
-        elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
-            return candidate
-    return "bw"  # fall back to PATH lookup (will FileNotFoundError, handled below)
-
-
-def _load_config() -> dict:
-    if VAULT_FILE.exists():
-        try:
-            data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
-            return data if isinstance(data, dict) else {}
-        except Exception:
-            pass
-    return {}
-
-
-def _save_config(cfg: dict):
-    VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
-    VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
-    # POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
-    # is ACL-restricted already).
-    safe_chmod(str(VAULT_FILE), 0o600)
-
-
-async def _run_bw(args: list, session: str = None, input_text: str = None,
-                  bw_password: str = None) -> tuple:
-    env = {}
-    env.update(os.environ)
-    if session:
-        env["BW_SESSION"] = session
-    # Secrets must never be passed as argv — process arguments are world-readable
-    # via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
-    # support for bw commands that need it; unlock/login callers should prefer
-    # stdin so the master password is not left in the child environment either.
-    if bw_password is not None:
-        env["BW_PASSWORD"] = bw_password
-    bw_path = _find_bw()
-    try:
-        proc = await asyncio.create_subprocess_exec(
-            bw_path, *args,
-            stdin=asyncio.subprocess.PIPE if input_text else None,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-            env=env,
-        )
-    except FileNotFoundError:
-        return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
-    except Exception as e:
-        return "", f"Failed to launch bw: {e}", 1
-    try:
-        stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
-    except Exception as e:
-        return "", f"bw subprocess error: {e}", 1
-    return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
-
-
-class VaultConfig(BaseModel):
-    server_url: str = ""
-    email: str = ""
-
-
-class VaultUnlockRequest(BaseModel):
-    master_password: str
-
-
-class VaultLoginRequest(BaseModel):
-    email: str
-    master_password: str
-
-
-def setup_vault_routes():
-    router = APIRouter(prefix="/api/vault", tags=["vault"])
-
-    @router.get("/config")
-    async def get_config(request: Request):
-        """Return vault config (no sensitive fields)."""
-        require_admin(request)
-        cfg = _load_config()
-        return {
-            "server_url": cfg.get("server_url", ""),
-            "email": cfg.get("email", ""),
-            "unlocked": bool(cfg.get("session")),
-            "unlocked_at": cfg.get("unlocked_at", ""),
-            "bw_installed": await _check_bw_installed(),
-        }
-
-    @router.post("/config")
-    async def save_config(req: VaultConfig, request: Request):
-        """Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
-        require_admin(request)
-        cfg = _load_config()
-        cfg["server_url"] = req.server_url.strip().rstrip("/")
-        cfg["email"] = req.email.strip()
-
-        if cfg["server_url"]:
-            _, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
-            if rc != 0:
-                return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
-
-        _save_config(cfg)
-        return {"ok": True}
-
-    @router.post("/login")
-    async def login(req: VaultLoginRequest, request: Request):
-        """Log in to Vaultwarden (required once per account)."""
-        require_admin(request)
-        cfg = _load_config()
-        # Update email
-        cfg["email"] = req.email
-        _save_config(cfg)
-
-        stdout, stderr, rc = await _run_bw(
-            ["login", req.email, "--raw"],
-            input_text=req.master_password + "\n",
-        )
-        if rc != 0:
-            # Already logged in is OK
-            if "already logged in" in stderr.lower():
-                return {"ok": True, "already": True}
-            return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
-        # bw login --raw prints session key on success (when 2FA disabled)
-        if stdout:
-            cfg["session"] = stdout
-            cfg["unlocked_at"] = datetime.utcnow().isoformat()
-            _save_config(cfg)
-        return {"ok": True}
-
-    @router.post("/unlock")
-    async def unlock(req: VaultUnlockRequest, request: Request):
-        """Unlock the vault and save the session key."""
-        require_admin(request)
-        # Pass the master password on stdin, not argv. argv is visible through
-        # `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
-        # the child process environment.
-        stdout, stderr, rc = await _run_bw(
-            ["unlock", "--raw"],
-            input_text=req.master_password + "\n",
-        )
-        if rc != 0:
-            return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
-        session = stdout.strip()
-        if not session:
-            return {"ok": False, "error": "bw returned empty session"}
-        cfg = _load_config()
-        cfg["session"] = session
-        cfg["unlocked_at"] = datetime.utcnow().isoformat()
-        _save_config(cfg)
-        return {"ok": True, "message": "Vault unlocked"}
-
-    @router.post("/lock")
-    async def lock(request: Request):
-        """Lock the vault (clear session from config)."""
-        require_admin(request)
-        cfg = _load_config()
-        cfg.pop("session", None)
-        cfg.pop("unlocked_at", None)
-        _save_config(cfg)
-        # Also tell bw to lock
-        await _run_bw(["lock"])
-        return {"ok": True, "message": "Vault locked"}
-
-    @router.post("/logout")
-    async def logout(request: Request):
-        """Log out of the Bitwarden CLI completely."""
-        require_admin(request)
-        await _run_bw(["logout"])
-        cfg = _load_config()
-        cfg.pop("session", None)
-        cfg.pop("email", None)
-        cfg.pop("unlocked_at", None)
-        _save_config(cfg)
-        return {"ok": True}
-
-    return router
-
-
-async def _check_bw_installed() -> bool:
-    try:
-        proc = await asyncio.create_subprocess_exec(
-            _find_bw(), "--version",
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-        )
-        await proc.communicate()
-        return proc.returncode == 0
-    except Exception:
-        return False

routes/vault_routes.py

@@ -1,14 +1,242 @@
-"""Backward-compat shim — canonical location is routes/vault/vault_routes.py.
+"""
+vault_routes.py
 
-This module is replaced in ``sys.modules`` by the canonical module object so
+Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
-that ``import routes.vault_routes``, ``from routes.vault_routes import X``,
+Stores the BW_SESSION key in data/vault.json with restrictive permissions.
-and the ``import ... as vr`` + ``monkeypatch.setattr(vr, ...)`` pattern used
-by test_vault_password_not_in_argv.py all operate on the *same* object.
-Keeps existing import paths working after slice 2k (#4082/#4071).
 """
 
-import sys as _sys
+import json
+import logging
+import os
+import shutil
+import asyncio
+from pathlib import Path
+from datetime import datetime
+from fastapi import APIRouter, Request
+from pydantic import BaseModel
 
-from routes.vault import vault_routes as _canonical  # noqa: F401
+from core.middleware import require_admin
+from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
+from src.constants import VAULT_FILE as _VAULT_FILE
 
-_sys.modules[__name__] = _canonical
+logger = logging.getLogger(__name__)
+
+VAULT_FILE = Path(_VAULT_FILE)
+
+
+def _find_bw() -> str:
+    """Locate the bw binary, checking PATH and common npm-global locations.
+
+    On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
+    which_tool via PATHEXT.
+    """
+    p = which_tool("bw")
+    if p:
+        return p
+    if IS_WINDOWS:
+        appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
+        for candidate in (
+            os.path.join(appdata, "npm", "bw.cmd"),
+            os.path.join(appdata, "npm", "bw.exe"),
+        ):
+            if os.path.isfile(candidate):
+                return candidate
+        return "bw"
+    home = os.path.expanduser("~")
+    for candidate in (
+        f"{home}/.npm-global/bin/bw",
+        f"{home}/.nvm/versions/node/*/bin/bw",
+        "/usr/local/bin/bw",
+        "/opt/homebrew/bin/bw",
+    ):
+        if "*" in candidate:
+            import glob
+            for m in glob.glob(candidate):
+                if os.path.isfile(m) and os.access(m, os.X_OK):
+                    return m
+        elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
+            return candidate
+    return "bw"  # fall back to PATH lookup (will FileNotFoundError, handled below)
+
+
+def _load_config() -> dict:
+    if VAULT_FILE.exists():
+        try:
+            data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
+            return data if isinstance(data, dict) else {}
+        except Exception:
+            pass
+    return {}
+
+
+def _save_config(cfg: dict):
+    VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
+    VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
+    # POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
+    # is ACL-restricted already).
+    safe_chmod(str(VAULT_FILE), 0o600)
+
+
+async def _run_bw(args: list, session: str = None, input_text: str = None,
+                  bw_password: str = None) -> tuple:
+    env = {}
+    env.update(os.environ)
+    if session:
+        env["BW_SESSION"] = session
+    # Secrets must never be passed as argv — process arguments are world-readable
+    # via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
+    # support for bw commands that need it; unlock/login callers should prefer
+    # stdin so the master password is not left in the child environment either.
+    if bw_password is not None:
+        env["BW_PASSWORD"] = bw_password
+    bw_path = _find_bw()
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            bw_path, *args,
+            stdin=asyncio.subprocess.PIPE if input_text else None,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            env=env,
+        )
+    except FileNotFoundError:
+        return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
+    except Exception as e:
+        return "", f"Failed to launch bw: {e}", 1
+    try:
+        stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
+    except Exception as e:
+        return "", f"bw subprocess error: {e}", 1
+    return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
+
+
+class VaultConfig(BaseModel):
+    server_url: str = ""
+    email: str = ""
+
+
+class VaultUnlockRequest(BaseModel):
+    master_password: str
+
+
+class VaultLoginRequest(BaseModel):
+    email: str
+    master_password: str
+
+
+def setup_vault_routes():
+    router = APIRouter(prefix="/api/vault", tags=["vault"])
+
+    @router.get("/config")
+    async def get_config(request: Request):
+        """Return vault config (no sensitive fields)."""
+        require_admin(request)
+        cfg = _load_config()
+        return {
+            "server_url": cfg.get("server_url", ""),
+            "email": cfg.get("email", ""),
+            "unlocked": bool(cfg.get("session")),
+            "unlocked_at": cfg.get("unlocked_at", ""),
+            "bw_installed": await _check_bw_installed(),
+        }
+
+    @router.post("/config")
+    async def save_config(req: VaultConfig, request: Request):
+        """Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
+        require_admin(request)
+        cfg = _load_config()
+        cfg["server_url"] = req.server_url.strip().rstrip("/")
+        cfg["email"] = req.email.strip()
+
+        if cfg["server_url"]:
+            _, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
+            if rc != 0:
+                return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
+
+        _save_config(cfg)
+        return {"ok": True}
+
+    @router.post("/login")
+    async def login(req: VaultLoginRequest, request: Request):
+        """Log in to Vaultwarden (required once per account)."""
+        require_admin(request)
+        cfg = _load_config()
+        # Update email
+        cfg["email"] = req.email
+        _save_config(cfg)
+
+        stdout, stderr, rc = await _run_bw(
+            ["login", req.email, "--raw"],
+            input_text=req.master_password + "\n",
+        )
+        if rc != 0:
+            # Already logged in is OK
+            if "already logged in" in stderr.lower():
+                return {"ok": True, "already": True}
+            return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
+        # bw login --raw prints session key on success (when 2FA disabled)
+        if stdout:
+            cfg["session"] = stdout
+            cfg["unlocked_at"] = datetime.utcnow().isoformat()
+            _save_config(cfg)
+        return {"ok": True}
+
+    @router.post("/unlock")
+    async def unlock(req: VaultUnlockRequest, request: Request):
+        """Unlock the vault and save the session key."""
+        require_admin(request)
+        # Pass the master password on stdin, not argv. argv is visible through
+        # `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
+        # the child process environment.
+        stdout, stderr, rc = await _run_bw(
+            ["unlock", "--raw"],
+            input_text=req.master_password + "\n",
+        )
+        if rc != 0:
+            return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
+        session = stdout.strip()
+        if not session:
+            return {"ok": False, "error": "bw returned empty session"}
+        cfg = _load_config()
+        cfg["session"] = session
+        cfg["unlocked_at"] = datetime.utcnow().isoformat()
+        _save_config(cfg)
+        return {"ok": True, "message": "Vault unlocked"}
+
+    @router.post("/lock")
+    async def lock(request: Request):
+        """Lock the vault (clear session from config)."""
+        require_admin(request)
+        cfg = _load_config()
+        cfg.pop("session", None)
+        cfg.pop("unlocked_at", None)
+        _save_config(cfg)
+        # Also tell bw to lock
+        await _run_bw(["lock"])
+        return {"ok": True, "message": "Vault locked"}
+
+    @router.post("/logout")
+    async def logout(request: Request):
+        """Log out of the Bitwarden CLI completely."""
+        require_admin(request)
+        await _run_bw(["logout"])
+        cfg = _load_config()
+        cfg.pop("session", None)
+        cfg.pop("email", None)
+        cfg.pop("unlocked_at", None)
+        _save_config(cfg)
+        return {"ok": True}
+
+    return router
+
+
+async def _check_bw_installed() -> bool:
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            _find_bw(), "--version",
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        await proc.communicate()
+        return proc.returncode == 0
+    except Exception:
+        return False

routes/webhook/__init__.py

@@ -1,5 +0,0 @@
-"""Webhook route domain package (slice 2l, #4082/#4071).
-
-Contains webhook_routes.py, migrated from the flat routes/ directory.
-Backward-compat shim at routes/webhook_routes.py re-exports from here.
-"""

routes/webhook/webhook_routes.py

@@ -1,395 +0,0 @@
-"""Webhook, API Token, and sync chat routes."""
-
-import uuid
-import logging
-from typing import Optional
-
-import httpx
-from fastapi import APIRouter, HTTPException, Request, Form
-from pydantic import BaseModel, Field
-
-from core.database import SessionLocal, Webhook, ModelEndpoint
-from src.auth_helpers import owner_filter
-from src.url_security import validate_public_http_url
-from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/api", tags=["webhooks"])
-
-# Input limits
-MAX_NAME_LEN = 100
-MAX_URL_LEN = 2048
-MAX_SECRET_LEN = 256
-MAX_MESSAGE_LEN = 32_000
-
-
-from core.middleware import require_admin as _require_admin
-
-
-def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
-    """First enabled ModelEndpoint visible to token_owner — their own rows plus
-    legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
-    let a chat-scoped token fall back onto another user's private endpoint and
-    silently spend that owner's API key/quota. Prefer owner rows before shared
-    rows. Fails closed to null-owner rows only when token_owner is absent.
-    Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
-    """
-    query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)  # noqa: E712
-    if token_owner:
-        query = owner_filter(query, ModelEndpoint, token_owner)
-        return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
-    return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first()  # noqa: E711
-
-
-def _caller_owns_session(sess_owner, caller) -> bool:
-    """Strict session-ownership gate for the token-authenticated sync-chat
-    endpoint (`POST /api/v1/chat`).
-
-    Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
-    gates in notes/calendar/gallery: a caller may resume a session ONLY when
-    its owner matches them exactly. A null/empty session owner (legacy or
-    migrated rows) is deliberately NOT resumable by an arbitrary token — the
-    old ``sess_owner and sess_owner != caller`` form skipped the check whenever
-    ``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
-    device) could resume such a session, inject a message, and read back its
-    history and reuse the owner's endpoint credentials. Fail closed: an
-    unresolvable caller also returns False.
-    """
-    if not caller:
-        return False
-    return sess_owner == caller
-
-
-def setup_webhook_routes(
-    webhook_manager: WebhookManager,
-    auth_manager,
-    session_manager=None,
-    api_key_manager=None,
-) -> APIRouter:
-
-    @router.get("/webhooks")
-    def list_webhooks(request: Request):
-        _require_admin(request)
-        db = SessionLocal()
-        try:
-            hooks = db.query(Webhook).all()
-            return [
-                {
-                    "id": w.id,
-                    "name": w.name,
-                    "url": w.url,
-                    "has_secret": bool(w.secret),
-                    "events": w.events.split(",") if w.events else [],
-                    "is_active": w.is_active,
-                    "last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
-                    "last_status_code": w.last_status_code,
-                    "last_error": w.last_error,
-                    "created_at": w.created_at.isoformat() if w.created_at else None,
-                }
-                for w in hooks
-            ]
-        finally:
-            db.close()
-
-    @router.post("/webhooks")
-    def create_webhook(
-        request: Request,
-        name: str = Form(""),
-        url: str = Form(""),
-        secret: str = Form(""),
-        events: str = Form(""),
-    ):
-        _require_admin(request)
-        name = name.strip()[:MAX_NAME_LEN]
-        if not name:
-            raise HTTPException(400, "Webhook name is required")
-        try:
-            url = validate_webhook_url(url)
-        except ValueError as e:
-            raise HTTPException(400, str(e))
-        try:
-            events = validate_events(events)
-        except ValueError as e:
-            raise HTTPException(400, str(e))
-
-        secret_val = secret.strip()[:MAX_SECRET_LEN] or None
-        # Encrypt the secret at rest using the same Fernet key as API keys
-        encrypted_secret = None
-        if secret_val and api_key_manager:
-            encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
-        elif secret_val:
-            encrypted_secret = secret_val  # Fallback if no encryption available
-
-        webhook_id = str(uuid.uuid4())[:8]
-        db = SessionLocal()
-        try:
-            db.add(Webhook(
-                id=webhook_id,
-                name=name,
-                url=url,
-                secret=encrypted_secret,
-                events=events,
-                is_active=True,
-            ))
-            db.commit()
-        finally:
-            db.close()
-
-        return {"id": webhook_id, "name": name}
-
-    @router.post("/webhooks/{webhook_id}/test")
-    async def test_webhook(request: Request, webhook_id: str):
-        _require_admin(request)
-        db = SessionLocal()
-        try:
-            wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
-            if not wh:
-                raise HTTPException(404, "Webhook not found")
-            url, secret = wh.url, wh.secret
-        finally:
-            db.close()
-
-        await webhook_manager.deliver_test(webhook_id, url, secret)
-        return {"status": "sent"}
-
-    @router.patch("/webhooks/{webhook_id}")
-    def toggle_webhook(request: Request, webhook_id: str):
-        _require_admin(request)
-        db = SessionLocal()
-        try:
-            wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
-            if not wh:
-                raise HTTPException(404, "Webhook not found")
-            wh.is_active = not wh.is_active
-            db.commit()
-            return {"id": webhook_id, "is_active": wh.is_active}
-        finally:
-            db.close()
-
-    @router.delete("/webhooks/{webhook_id}")
-    def delete_webhook(request: Request, webhook_id: str):
-        _require_admin(request)
-        db = SessionLocal()
-        try:
-            deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
-            db.commit()
-            if not deleted:
-                raise HTTPException(404, "Webhook not found")
-        finally:
-            db.close()
-        return {"status": "deleted"}
-
-    # ================================================================
-    # Sync Chat Endpoint (for n8n / Make / Activepieces)
-    # ================================================================
-
-    # Known provider base URLs — auto-resolved from api_key prefix or model name
-    KNOWN_PROVIDERS = {
-        "deepseek": "https://api.deepseek.com/v1",
-        "openai": "https://api.openai.com/v1",
-        "mistral": "https://api.mistral.ai/v1",
-        "groq": "https://api.groq.com/openai/v1",
-        "together": "https://api.together.xyz/v1",
-        "openrouter": "https://openrouter.ai/api/v1",
-        "ollama": "https://ollama.com/api",
-        "opencode-zen": "https://opencode.ai/zen/v1",
-        "opencode-go": "https://opencode.ai/zen/go/v1",
-        "fireworks": "https://api.fireworks.ai/inference/v1",
-        "venice": "https://api.venice.ai/api/v1",
-        "kimi-code": "https://api.kimi.com/coding/v1",
-        "kimicode": "https://api.kimi.com/coding/v1",
-    }
-
-    # Model prefix → provider mapping for auto-detection
-    MODEL_PROVIDER_MAP = {
-        "deepseek": "deepseek",
-        "gpt-": "openai",
-        "o1": "openai",
-        "o3": "openai",
-        "o4": "openai",
-        "mistral": "mistral",
-        "llama": "groq",
-        "mixtral": "groq",
-        "kimi-for-coding": "kimi-code",
-        "kimi": "kimi-code",
-    }
-
-    def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
-        """Try to auto-resolve a base URL from provider name or model prefix."""
-        if provider and provider.lower() in KNOWN_PROVIDERS:
-            return KNOWN_PROVIDERS[provider.lower()]
-        if model:
-            model_lower = model.lower()
-            for prefix, prov in MODEL_PROVIDER_MAP.items():
-                if model_lower.startswith(prefix):
-                    return KNOWN_PROVIDERS[prov]
-        return None
-
-    class SyncChatRequest(BaseModel):
-        message: str = Field(..., max_length=MAX_MESSAGE_LEN)
-        model: Optional[str] = Field(None, max_length=200)
-        session: Optional[str] = Field(None, max_length=100)
-        api_key: Optional[str] = Field(None, max_length=256)
-        base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
-        provider: Optional[str] = Field(None, max_length=50)
-
-    @router.post("/v1/chat")
-    async def sync_chat(request: Request, body: SyncChatRequest):
-        if not getattr(request.state, "api_token", False):
-            raise HTTPException(403, "This endpoint requires an API token")
-        scopes = set(getattr(request.state, "api_token_scopes", []) or [])
-        if "chat" not in scopes:
-            raise HTTPException(403, "API token is not scoped for chat")
-        token_owner = getattr(request.state, "api_token_owner", None)
-
-        from core.models import ChatMessage
-        from src.llm_core import llm_call_async
-        from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
-
-        message = body.message.strip()
-        if not message:
-            raise HTTPException(400, "Message is required")
-
-        session_id = body.session
-        sess = None
-
-        # --- Case 1: Resume an existing session ---
-        if session_id and session_manager:
-            try:
-                sess = session_manager.get_session(session_id)
-            except (KeyError, Exception):
-                raise HTTPException(404, "Session not found")
-            # SECURITY: verify the API-token's user owns this session — without
-            # this any token holder could resume any user's chat by passing its
-            # ID. The token's user is on request.state.user (set by API-token
-            # middleware); fall back to require_user if not present.
-            try:
-                from src.auth_helpers import get_current_user as _gcu
-                _tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
-            except Exception:
-                _tok_user = None
-            # Strict ownership (see _caller_owns_session): fail closed so a
-            # null-owner / cross-owner session can't be resumed by an arbitrary
-            # chat-scoped token.
-            _sess_owner = getattr(sess, "owner", None)
-            if not _caller_owns_session(_sess_owner, _tok_user):
-                raise HTTPException(404, "Session not found")
-
-        # --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
-        if not sess and body.api_key:
-            api_key = body.api_key.strip()
-            model = body.model or "deepseek-chat"
-
-            # Validate only token-supplied direct base_url; auto-resolved known-provider
-            # URLs are not subject to extra local/LAN blocking beyond existing provider logic.
-            direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
-            if direct_base_url:
-                try:
-                    base_url = validate_public_http_url(direct_base_url)
-                except ValueError as e:
-                    detail = str(e).replace("URL", "base_url", 1)
-                    raise HTTPException(400, detail)
-            else:
-                base_url = _resolve_base_url(model, body.provider)
-            if not base_url:
-                raise HTTPException(400,
-                    "Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
-                    "or provider ('deepseek', 'openai', 'groq', etc.)")
-            base_url = normalize_base(base_url)
-            endpoint_url = build_chat_url(base_url)
-
-            if not session_manager:
-                raise HTTPException(500, "Session manager not available")
-
-            sid = str(uuid.uuid4())
-            sess = session_manager.create_session(
-                session_id=sid, name="API Chat", endpoint_url=endpoint_url,
-                model=model, owner=token_owner,
-            )
-            sess.headers = build_headers(api_key, base_url)
-            session_manager.save_sessions()
-            session_id = sid
-
-        # --- Case 3: Fall back to first configured ModelEndpoint ---
-        if not sess:
-            db = SessionLocal()
-            try:
-                ep = _select_api_chat_fallback_endpoint(db, token_owner)
-            finally:
-                db.close()
-
-            if not ep:
-                raise HTTPException(400,
-                    "No session, api_key, or configured endpoints. "
-                    "Pass api_key + model, or configure an endpoint in Admin.")
-
-            base_url = normalize_base(ep.base_url)
-            endpoint_url = build_chat_url(base_url)
-            model = body.model or "auto"
-            api_key = ep.api_key
-            if getattr(ep, "provider_auth_id", None):
-                try:
-                    from src.endpoint_resolver import resolve_endpoint_runtime
-                    base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
-                    endpoint_url = build_chat_url(base_url)
-                except Exception:
-                    raise HTTPException(500, "Could not resolve endpoint credentials")
-
-            if model == "auto":
-                try:
-                    async with httpx.AsyncClient(timeout=5) as client:
-                        models_url = build_models_url(base_url)
-                        hdrs = build_headers(api_key, base_url)
-                        if models_url:
-                            resp = await client.get(models_url, headers=hdrs)
-                            resp.raise_for_status()
-                            data = resp.json()
-                            items = data if isinstance(data, list) else (data.get("data") or [])
-                            ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
-                            if not ids and isinstance(data, dict):
-                                ids = [
-                                    m.get("name") or m.get("model")
-                                    for m in (data.get("models") or [])
-                                    if m.get("name") or m.get("model")
-                                ]
-                        else:
-                            import json as _json
-                            ids = _json.loads(ep.cached_models or "[]")
-                        model = ids[0] if ids else "auto"
-                except Exception:
-                    raise HTTPException(500, "Could not discover models from endpoint")
-
-            if not session_manager:
-                raise HTTPException(500, "Session manager not available")
-
-            sid = str(uuid.uuid4())
-            sess = session_manager.create_session(
-                session_id=sid, name="API Chat", endpoint_url=endpoint_url,
-                model=model, owner=token_owner,
-            )
-            if api_key:
-                sess.headers = build_headers(api_key, base_url)
-                session_manager.save_sessions()
-            session_id = sid
-
-        # --- Send message and get response ---
-        sess.add_message(ChatMessage("user", message))
-
-        messages = [{"role": m.role, "content": m.content} for m in sess.history]
-
-        reply = await llm_call_async(
-            sess.endpoint_url, sess.model, messages,
-            headers=sess.headers, timeout=120,
-        )
-        sess.add_message(ChatMessage("assistant", reply))
-        session_manager.save_sessions()
-
-        webhook_manager.fire_and_forget("chat.completed", {
-            "session_id": session_id, "model": sess.model,
-            "user_message": message[:2000], "response": reply[:2000],
-        })
-
-        return {"response": reply, "session_id": session_id, "model": sess.model}
-
-    return router

routes/webhook_routes.py

@@ -1,16 +1,395 @@
-"""Backward-compat shim — canonical location is routes/webhook/webhook_routes.py.
+"""Webhook, API Token, and sync chat routes."""
 
-This module is replaced in ``sys.modules`` by the canonical module object so
+import uuid
-that ``import routes.webhook_routes``, ``from routes.webhook_routes import X``,
+import logging
-``importlib.import_module("routes.webhook_routes")``, and the
+from typing import Optional
-``__import__("routes.webhook_routes", fromlist=[...])`` + ``setattr(wh_mod,
-...)`` pattern used by test_null_owner_gates.py all operate on the *same*
-object. Keeps existing import paths working after slice 2l (#4082/#4071).
-Source-introspection tests read the canonical file by path.
-"""
 
-import sys as _sys
+import httpx
+from fastapi import APIRouter, HTTPException, Request, Form
+from pydantic import BaseModel, Field
 
-from routes.webhook import webhook_routes as _canonical  # noqa: F401
+from core.database import SessionLocal, Webhook, ModelEndpoint
+from src.auth_helpers import owner_filter
+from src.url_security import validate_public_http_url
+from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
 
-_sys.modules[__name__] = _canonical
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api", tags=["webhooks"])
+
+# Input limits
+MAX_NAME_LEN = 100
+MAX_URL_LEN = 2048
+MAX_SECRET_LEN = 256
+MAX_MESSAGE_LEN = 32_000
+
+
+from core.middleware import require_admin as _require_admin
+
+
+def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
+    """First enabled ModelEndpoint visible to token_owner — their own rows plus
+    legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
+    let a chat-scoped token fall back onto another user's private endpoint and
+    silently spend that owner's API key/quota. Prefer owner rows before shared
+    rows. Fails closed to null-owner rows only when token_owner is absent.
+    Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
+    """
+    query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)  # noqa: E712
+    if token_owner:
+        query = owner_filter(query, ModelEndpoint, token_owner)
+        return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
+    return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first()  # noqa: E711
+
+
+def _caller_owns_session(sess_owner, caller) -> bool:
+    """Strict session-ownership gate for the token-authenticated sync-chat
+    endpoint (`POST /api/v1/chat`).
+
+    Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
+    gates in notes/calendar/gallery: a caller may resume a session ONLY when
+    its owner matches them exactly. A null/empty session owner (legacy or
+    migrated rows) is deliberately NOT resumable by an arbitrary token — the
+    old ``sess_owner and sess_owner != caller`` form skipped the check whenever
+    ``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
+    device) could resume such a session, inject a message, and read back its
+    history and reuse the owner's endpoint credentials. Fail closed: an
+    unresolvable caller also returns False.
+    """
+    if not caller:
+        return False
+    return sess_owner == caller
+
+
+def setup_webhook_routes(
+    webhook_manager: WebhookManager,
+    auth_manager,
+    session_manager=None,
+    api_key_manager=None,
+) -> APIRouter:
+
+    @router.get("/webhooks")
+    def list_webhooks(request: Request):
+        _require_admin(request)
+        db = SessionLocal()
+        try:
+            hooks = db.query(Webhook).all()
+            return [
+                {
+                    "id": w.id,
+                    "name": w.name,
+                    "url": w.url,
+                    "has_secret": bool(w.secret),
+                    "events": w.events.split(",") if w.events else [],
+                    "is_active": w.is_active,
+                    "last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
+                    "last_status_code": w.last_status_code,
+                    "last_error": w.last_error,
+                    "created_at": w.created_at.isoformat() if w.created_at else None,
+                }
+                for w in hooks
+            ]
+        finally:
+            db.close()
+
+    @router.post("/webhooks")
+    def create_webhook(
+        request: Request,
+        name: str = Form(""),
+        url: str = Form(""),
+        secret: str = Form(""),
+        events: str = Form(""),
+    ):
+        _require_admin(request)
+        name = name.strip()[:MAX_NAME_LEN]
+        if not name:
+            raise HTTPException(400, "Webhook name is required")
+        try:
+            url = validate_webhook_url(url)
+        except ValueError as e:
+            raise HTTPException(400, str(e))
+        try:
+            events = validate_events(events)
+        except ValueError as e:
+            raise HTTPException(400, str(e))
+
+        secret_val = secret.strip()[:MAX_SECRET_LEN] or None
+        # Encrypt the secret at rest using the same Fernet key as API keys
+        encrypted_secret = None
+        if secret_val and api_key_manager:
+            encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
+        elif secret_val:
+            encrypted_secret = secret_val  # Fallback if no encryption available
+
+        webhook_id = str(uuid.uuid4())[:8]
+        db = SessionLocal()
+        try:
+            db.add(Webhook(
+                id=webhook_id,
+                name=name,
+                url=url,
+                secret=encrypted_secret,
+                events=events,
+                is_active=True,
+            ))
+            db.commit()
+        finally:
+            db.close()
+
+        return {"id": webhook_id, "name": name}
+
+    @router.post("/webhooks/{webhook_id}/test")
+    async def test_webhook(request: Request, webhook_id: str):
+        _require_admin(request)
+        db = SessionLocal()
+        try:
+            wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
+            if not wh:
+                raise HTTPException(404, "Webhook not found")
+            url, secret = wh.url, wh.secret
+        finally:
+            db.close()
+
+        await webhook_manager.deliver_test(webhook_id, url, secret)
+        return {"status": "sent"}
+
+    @router.patch("/webhooks/{webhook_id}")
+    def toggle_webhook(request: Request, webhook_id: str):
+        _require_admin(request)
+        db = SessionLocal()
+        try:
+            wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
+            if not wh:
+                raise HTTPException(404, "Webhook not found")
+            wh.is_active = not wh.is_active
+            db.commit()
+            return {"id": webhook_id, "is_active": wh.is_active}
+        finally:
+            db.close()
+
+    @router.delete("/webhooks/{webhook_id}")
+    def delete_webhook(request: Request, webhook_id: str):
+        _require_admin(request)
+        db = SessionLocal()
+        try:
+            deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
+            db.commit()
+            if not deleted:
+                raise HTTPException(404, "Webhook not found")
+        finally:
+            db.close()
+        return {"status": "deleted"}
+
+    # ================================================================
+    # Sync Chat Endpoint (for n8n / Make / Activepieces)
+    # ================================================================
+
+    # Known provider base URLs — auto-resolved from api_key prefix or model name
+    KNOWN_PROVIDERS = {
+        "deepseek": "https://api.deepseek.com/v1",
+        "openai": "https://api.openai.com/v1",
+        "mistral": "https://api.mistral.ai/v1",
+        "groq": "https://api.groq.com/openai/v1",
+        "together": "https://api.together.xyz/v1",
+        "openrouter": "https://openrouter.ai/api/v1",
+        "ollama": "https://ollama.com/api",
+        "opencode-zen": "https://opencode.ai/zen/v1",
+        "opencode-go": "https://opencode.ai/zen/go/v1",
+        "fireworks": "https://api.fireworks.ai/inference/v1",
+        "venice": "https://api.venice.ai/api/v1",
+        "kimi-code": "https://api.kimi.com/coding/v1",
+        "kimicode": "https://api.kimi.com/coding/v1",
+    }
+
+    # Model prefix → provider mapping for auto-detection
+    MODEL_PROVIDER_MAP = {
+        "deepseek": "deepseek",
+        "gpt-": "openai",
+        "o1": "openai",
+        "o3": "openai",
+        "o4": "openai",
+        "mistral": "mistral",
+        "llama": "groq",
+        "mixtral": "groq",
+        "kimi-for-coding": "kimi-code",
+        "kimi": "kimi-code",
+    }
+
+    def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
+        """Try to auto-resolve a base URL from provider name or model prefix."""
+        if provider and provider.lower() in KNOWN_PROVIDERS:
+            return KNOWN_PROVIDERS[provider.lower()]
+        if model:
+            model_lower = model.lower()
+            for prefix, prov in MODEL_PROVIDER_MAP.items():
+                if model_lower.startswith(prefix):
+                    return KNOWN_PROVIDERS[prov]
+        return None
+
+    class SyncChatRequest(BaseModel):
+        message: str = Field(..., max_length=MAX_MESSAGE_LEN)
+        model: Optional[str] = Field(None, max_length=200)
+        session: Optional[str] = Field(None, max_length=100)
+        api_key: Optional[str] = Field(None, max_length=256)
+        base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
+        provider: Optional[str] = Field(None, max_length=50)
+
+    @router.post("/v1/chat")
+    async def sync_chat(request: Request, body: SyncChatRequest):
+        if not getattr(request.state, "api_token", False):
+            raise HTTPException(403, "This endpoint requires an API token")
+        scopes = set(getattr(request.state, "api_token_scopes", []) or [])
+        if "chat" not in scopes:
+            raise HTTPException(403, "API token is not scoped for chat")
+        token_owner = getattr(request.state, "api_token_owner", None)
+
+        from core.models import ChatMessage
+        from src.llm_core import llm_call_async
+        from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
+
+        message = body.message.strip()
+        if not message:
+            raise HTTPException(400, "Message is required")
+
+        session_id = body.session
+        sess = None
+
+        # --- Case 1: Resume an existing session ---
+        if session_id and session_manager:
+            try:
+                sess = session_manager.get_session(session_id)
+            except (KeyError, Exception):
+                raise HTTPException(404, "Session not found")
+            # SECURITY: verify the API-token's user owns this session — without
+            # this any token holder could resume any user's chat by passing its
+            # ID. The token's user is on request.state.user (set by API-token
+            # middleware); fall back to require_user if not present.
+            try:
+                from src.auth_helpers import get_current_user as _gcu
+                _tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
+            except Exception:
+                _tok_user = None
+            # Strict ownership (see _caller_owns_session): fail closed so a
+            # null-owner / cross-owner session can't be resumed by an arbitrary
+            # chat-scoped token.
+            _sess_owner = getattr(sess, "owner", None)
+            if not _caller_owns_session(_sess_owner, _tok_user):
+                raise HTTPException(404, "Session not found")
+
+        # --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
+        if not sess and body.api_key:
+            api_key = body.api_key.strip()
+            model = body.model or "deepseek-chat"
+
+            # Validate only token-supplied direct base_url; auto-resolved known-provider
+            # URLs are not subject to extra local/LAN blocking beyond existing provider logic.
+            direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
+            if direct_base_url:
+                try:
+                    base_url = validate_public_http_url(direct_base_url)
+                except ValueError as e:
+                    detail = str(e).replace("URL", "base_url", 1)
+                    raise HTTPException(400, detail)
+            else:
+                base_url = _resolve_base_url(model, body.provider)
+            if not base_url:
+                raise HTTPException(400,
+                    "Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
+                    "or provider ('deepseek', 'openai', 'groq', etc.)")
+            base_url = normalize_base(base_url)
+            endpoint_url = build_chat_url(base_url)
+
+            if not session_manager:
+                raise HTTPException(500, "Session manager not available")
+
+            sid = str(uuid.uuid4())
+            sess = session_manager.create_session(
+                session_id=sid, name="API Chat", endpoint_url=endpoint_url,
+                model=model, owner=token_owner,
+            )
+            sess.headers = build_headers(api_key, base_url)
+            session_manager.save_sessions()
+            session_id = sid
+
+        # --- Case 3: Fall back to first configured ModelEndpoint ---
+        if not sess:
+            db = SessionLocal()
+            try:
+                ep = _select_api_chat_fallback_endpoint(db, token_owner)
+            finally:
+                db.close()
+
+            if not ep:
+                raise HTTPException(400,
+                    "No session, api_key, or configured endpoints. "
+                    "Pass api_key + model, or configure an endpoint in Admin.")
+
+            base_url = normalize_base(ep.base_url)
+            endpoint_url = build_chat_url(base_url)
+            model = body.model or "auto"
+            api_key = ep.api_key
+            if getattr(ep, "provider_auth_id", None):
+                try:
+                    from src.endpoint_resolver import resolve_endpoint_runtime
+                    base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
+                    endpoint_url = build_chat_url(base_url)
+                except Exception:
+                    raise HTTPException(500, "Could not resolve endpoint credentials")
+
+            if model == "auto":
+                try:
+                    async with httpx.AsyncClient(timeout=5) as client:
+                        models_url = build_models_url(base_url)
+                        hdrs = build_headers(api_key, base_url)
+                        if models_url:
+                            resp = await client.get(models_url, headers=hdrs)
+                            resp.raise_for_status()
+                            data = resp.json()
+                            items = data if isinstance(data, list) else (data.get("data") or [])
+                            ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
+                            if not ids and isinstance(data, dict):
+                                ids = [
+                                    m.get("name") or m.get("model")
+                                    for m in (data.get("models") or [])
+                                    if m.get("name") or m.get("model")
+                                ]
+                        else:
+                            import json as _json
+                            ids = _json.loads(ep.cached_models or "[]")
+                        model = ids[0] if ids else "auto"
+                except Exception:
+                    raise HTTPException(500, "Could not discover models from endpoint")
+
+            if not session_manager:
+                raise HTTPException(500, "Session manager not available")
+
+            sid = str(uuid.uuid4())
+            sess = session_manager.create_session(
+                session_id=sid, name="API Chat", endpoint_url=endpoint_url,
+                model=model, owner=token_owner,
+            )
+            if api_key:
+                sess.headers = build_headers(api_key, base_url)
+                session_manager.save_sessions()
+            session_id = sid
+
+        # --- Send message and get response ---
+        sess.add_message(ChatMessage("user", message))
+
+        messages = [{"role": m.role, "content": m.content} for m in sess.history]
+
+        reply = await llm_call_async(
+            sess.endpoint_url, sess.model, messages,
+            headers=sess.headers, timeout=120,
+        )
+        sess.add_message(ChatMessage("assistant", reply))
+        session_manager.save_sessions()
+
+        webhook_manager.fire_and_forget("chat.completed", {
+            "session_id": session_id, "model": sess.model,
+            "user_message": message[:2000], "response": reply[:2000],
+        })
+
+        return {"response": reply, "session_id": session_id, "model": sess.model}
+
+    return router

scripts/demo_email/demo_account.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Create/remove the switchable 'Demo' EmailAccount in Odysseus.
+"""Create/remove the switchable, non-default 'Demo' EmailAccount in Odysseus.
 
 Mirrors the existing local-Dovecot account (localhost:31143, STARTTLS) but points
 at the throwaway demo@odysseus.local mailbox. Password is stored Fernet-encrypted
@@ -20,14 +20,7 @@ from pathlib import Path
 ROOT = Path(__file__).resolve().parent.parent.parent
 sys.path.insert(0, str(ROOT))
 
-from core.database import (  # noqa: E402
+from core.database import SessionLocal, EmailAccount, Base, engine  # noqa: E402
-    Base,
-    EmailAccount,
-    SessionLocal,
-    engine,
-    lock_email_account_owner_mutations,
-)
-from sqlalchemy import or_  # noqa: E402
 from src.secret_storage import encrypt  # noqa: E402
 
 NAME = "Demo"
@@ -38,98 +31,18 @@ IMAP_PASSWORD = "demodemo"
 OWNER = ""
 
 
-def _owner_scope(query, owner: str):
-    if owner:
-        return query.filter(EmailAccount.owner == owner)
-    return query.filter(or_(EmailAccount.owner == None, EmailAccount.owner == ""))  # noqa: E711
-
-
-def _discover_demo_scopes() -> set[str]:
-    db = SessionLocal()
-    try:
-        return {
-            row.owner or ""
-            for row in db.query(EmailAccount).filter(
-                EmailAccount.name == NAME,
-                EmailAccount.imap_user == IMAP_USER,
-            ).all()
-        }
-    finally:
-        db.close()
-
-
-def _lock_and_load_demo_rows(db, scopes: set[str]):
-    """Reload Demo rows under every observed owner lock."""
-    scopes = set(scopes) or {OWNER}
-    while True:
-        lock_email_account_owner_mutations(db, *scopes)
-        rows = (
-            db.query(EmailAccount)
-            .filter(
-                EmailAccount.name == NAME,
-                EmailAccount.imap_user == IMAP_USER,
-            )
-            .order_by(EmailAccount.created_at.asc(), EmailAccount.id.asc())
-            .all()
-        )
-        current_scopes = {row.owner or "" for row in rows}
-        if current_scopes.issubset(scopes) or db.get_bind().dialect.name == "sqlite":
-            return rows
-        db.rollback()
-        scopes.update(current_scopes)
-
-
-def _promote_oldest_enabled(db, owner: str, excluded_ids: list[str]) -> None:
-    remaining = _owner_scope(
-        db.query(EmailAccount).filter(
-            EmailAccount.enabled == True,  # noqa: E712
-            ~EmailAccount.id.in_(excluded_ids),
-        ),
-        owner,
-    )
-    if remaining.filter(EmailAccount.is_default == True).first() is not None:  # noqa: E712
-        return
-    promote = remaining.order_by(
-        EmailAccount.created_at.asc(), EmailAccount.id.asc()
-    ).first()
-    if promote is not None:
-        promote.is_default = True
-
-
 def setup() -> int:
     Base.metadata.create_all(bind=engine)
-    scopes = _discover_demo_scopes() | {OWNER}
     db = SessionLocal()
     try:
-        rows = _lock_and_load_demo_rows(db, scopes)
+        acct = db.query(EmailAccount).filter(
-        acct = rows[0] if rows else None
+            EmailAccount.name == NAME, EmailAccount.imap_user == IMAP_USER
+        ).first()
         if acct is None:
             acct = EmailAccount(id=uuid.uuid4().hex, name=NAME)
             db.add(acct)
-        old_scope = acct.owner or ""
-        was_default = bool(acct.is_default)
-        if old_scope != OWNER:
-            # Move a non-default row first so the unique index cannot see two
-            # defaults transiently while SQLAlchemy flushes the owner move and
-            # old-scope promotion in separate UPDATE statements.
-            acct.is_default = False
-            acct.owner = OWNER
-            db.flush()
-            if was_default:
-                _promote_oldest_enabled(db, old_scope, [acct.id])
-
-        target_default = _owner_scope(
-            db.query(EmailAccount).filter(
-                EmailAccount.id != acct.id,
-                EmailAccount.is_default == True,  # noqa: E712
-            ),
-            OWNER,
-        ).first()
         acct.owner = OWNER
-        # Keep Demo non-default when a real default exists.  If it is the only
+        acct.is_default = False          # never default — user switches to it
-        # enabled account, it must be default to preserve normal create
-        # semantics and avoid leaving the owner partition without one.
-        acct.is_default = target_default is None
         acct.enabled = True
         acct.imap_host = "localhost"
         acct.imap_port = 31143
@@ -144,27 +57,20 @@ def setup() -> int:
         acct.smtp_password = encrypt(IMAP_PASSWORD)
         acct.from_address = IMAP_USER
         db.commit()
-        state = "default" if acct.is_default else "non-default"
+        print(f"'{NAME}' account ready (id={acct.id}, non-default, switchable).")
-        print(f"'{NAME}' account ready (id={acct.id}, {state}, switchable).")
         return 0
     finally:
         db.close()
 
 
 def teardown() -> int:
-    scopes = _discover_demo_scopes()
     db = SessionLocal()
     try:
-        rows = _lock_and_load_demo_rows(db, scopes)
+        rows = db.query(EmailAccount).filter(
-        deleted_ids = [row.id for row in rows]
+            EmailAccount.name == NAME, EmailAccount.imap_user == IMAP_USER
-        default_scopes = {row.owner or "" for row in rows if row.is_default}
+        ).all()
         for r in rows:
             db.delete(r)
-        # Ensure the old default DELETE reaches the database before a
-        # replacement UPDATE; the unique index is enforced per statement.
-        db.flush()
-        for owner in default_scopes:
-            _promote_oldest_enabled(db, owner, deleted_ids)
         db.commit()
         print(f"removed {len(rows)} '{NAME}' account row(s).")
         return 0

scripts/migrate_searxng_settings.py

@@ -1,166 +0,0 @@
-#!/usr/bin/env python3
-"""Make retained SearXNG settings inherit defaults without replacing them."""
-
-from __future__ import annotations
-
-import os
-import stat
-import sys
-import tempfile
-from pathlib import Path
-
-import yaml
-from yaml.nodes import MappingNode
-from yaml.tokens import BlockMappingStartToken, FlowMappingStartToken
-
-
-_UTF8_BOM = b"\xef\xbb\xbf"
-
-
-def _parse_root_mapping(text: str) -> tuple[MappingNode | None, dict]:
-    """Parse settings with the same safe YAML semantics SearXNG uses."""
-    try:
-        loaded = yaml.safe_load(text)
-        node = yaml.compose(text, Loader=yaml.SafeLoader)
-    except yaml.YAMLError:
-        raise ValueError("settings file is not valid single-document YAML") from None
-
-    if loaded is None and node is None:
-        return None, {}
-    if not isinstance(loaded, dict) or not isinstance(node, MappingNode):
-        raise ValueError("settings root is not a mapping")
-    return node, loaded
-
-
-def _flow_mapping_start(text: str) -> int:
-    """Return the root flow mapping's opening-brace character offset."""
-    try:
-        for token in yaml.scan(text, Loader=yaml.SafeLoader):
-            if isinstance(token, FlowMappingStartToken):
-                return token.start_mark.index
-    except yaml.YAMLError:
-        pass
-    raise ValueError("flow-style settings mapping has no opening brace")
-
-
-def _newline_for(contents: bytes) -> bytes:
-    first_lf = contents.find(b"\n")
-    if first_lf > 0 and contents[first_lf - 1 : first_lf + 1] == b"\r\n":
-        return b"\r\n"
-    return b"\n"
-
-
-def _block_mapping_position(text: str, root: MappingNode | None) -> tuple[int, int]:
-    """Return a safe character offset and indent for a root block mapping key."""
-    if root is None:
-        return len(text), 0
-
-    try:
-        for token in yaml.scan(text, Loader=yaml.SafeLoader):
-            if not isinstance(token, BlockMappingStartToken):
-                continue
-            line_start = token.start_mark.index - token.start_mark.column
-            if not text[line_start : token.start_mark.index].strip():
-                return line_start, token.start_mark.column
-            return root.end_mark.index, token.start_mark.column
-    except yaml.YAMLError:
-        pass
-    return root.end_mark.index, root.start_mark.column
-
-
-def _add_block_default_inheritance(
-    contents: bytes, text: str, root: MappingNode | None
-) -> bytes:
-    newline = _newline_for(contents)
-    character_offset, indent_width = _block_mapping_position(text, root)
-    bom_length = len(_UTF8_BOM) if contents.startswith(_UTF8_BOM) else 0
-    offset = bom_length + len(text[:character_offset].encode("utf-8"))
-    separator = b""
-    if offset not in (0, bom_length) and not contents[:offset].endswith((b"\n", b"\r")):
-        separator = newline
-    addition = (
-        separator
-        + b" " * indent_width
-        + b"use_default_settings: true"
-        + newline
-    )
-    return contents[:offset] + addition + contents[offset:]
-
-
-def migrate_settings(path: Path) -> bool:
-    """Add the missing inheritance key atomically; return whether the file changed."""
-    source_stat = path.lstat()
-    if not stat.S_ISREG(source_stat.st_mode):
-        raise ValueError(f"settings path is not a regular file: {path}")
-
-    contents = path.read_bytes()
-    if not contents:
-        return False
-
-    text = contents.decode("utf-8-sig")
-    root, loaded = _parse_root_mapping(text)
-    if "use_default_settings" in loaded:
-        return False
-
-    if root is not None and root.flow_style:
-        start = _flow_mapping_start(text)
-        bom_length = len(_UTF8_BOM) if contents.startswith(_UTF8_BOM) else 0
-        offset = bom_length + len(text[: start + 1].encode("utf-8"))
-        separator = b", " if root.value else b""
-        updated = (
-            contents[:offset]
-            + b"use_default_settings: true"
-            + separator
-            + contents[offset:]
-        )
-    else:
-        updated = _add_block_default_inheritance(contents, text, root)
-    fd, temporary_name = tempfile.mkstemp(
-        prefix=f".{path.name}.odysseus-", dir=path.parent
-    )
-    temporary = Path(temporary_name)
-    try:
-        # chmod before chown: the Compose cap set is `cap_drop: ALL` plus
-        # CHOWN/SETGID/SETUID/DAC_OVERRIDE, with no FOWNER. Once the temporary
-        # file belongs to searxng:searxng — which every retained settings file
-        # does, because searxng's entrypoint chowns /etc/searxng — root can no
-        # longer chmod it and the migration dies with EPERM.
-        os.fchmod(fd, stat.S_IMODE(source_stat.st_mode))
-        os.fchown(fd, source_stat.st_uid, source_stat.st_gid)
-        with os.fdopen(fd, "wb") as handle:
-            fd = -1
-            handle.write(updated)
-            handle.flush()
-            os.fsync(handle.fileno())
-        os.replace(temporary, path)
-        directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
-        try:
-            os.fsync(directory_fd)
-        finally:
-            os.close(directory_fd)
-    finally:
-        if fd >= 0:
-            os.close(fd)
-        temporary.unlink(missing_ok=True)
-    return True
-
-
-def main(argv: list[str]) -> int:
-    if len(argv) > 2:
-        print(f"usage: {Path(argv[0]).name} [settings.yml]", file=sys.stderr)
-        return 2
-
-    path = Path(argv[1]) if len(argv) == 2 else Path("/etc/searxng/settings.yml")
-    try:
-        changed = migrate_settings(path)
-    except (OSError, UnicodeError, ValueError) as exc:
-        print(f"SearXNG settings migration failed: {exc}", file=sys.stderr)
-        return 1
-
-    if changed:
-        print("Added use_default_settings inheritance to retained SearXNG settings")
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main(sys.argv))

scripts/odysseus-webhook

@@ -2,7 +2,7 @@
 """odysseus-webhook — shell wrapper for scheduled-task webhook tokens.
 
 Tasks in the scheduled-task system can carry a `webhook_token`. Any
-HTTP POST to `/api/tasks/<task-id>/webhook/<token>` fires the task. This CLI lists,
+HTTP POST to `/api/webhook/<token>` fires the task. This CLI lists,
 rotates, and revokes those tokens.
 
     odysseus-webhook list                       # tasks that have a token
@@ -21,7 +21,6 @@ quiet_logs()
 
 import argparse, json, logging, os, secrets, sys
 from pathlib import Path
-from urllib.parse import quote
 
 try:
     from core.database import SessionLocal, ScheduledTask
@@ -54,14 +53,6 @@ def _summary(t: "ScheduledTask", reveal: bool = False) -> dict:
     }
 
 
-def _task_webhook_url(base: str, task_id: str, token: str) -> str:
-    """Build the live task-route URL without leaking ids into path syntax."""
-    root = (base or "http://localhost:7000").rstrip("/")
-    task_part = quote(str(task_id), safe="")
-    token_part = quote(str(token), safe="")
-    return f"{root}/api/tasks/{task_part}/webhook/{token_part}"
-
-
 def cmd_list(args):
     db = SessionLocal()
     try:
@@ -118,7 +109,8 @@ def cmd_url(args):
             fail(f"no task with id {args.id!r}")
         if not t.webhook_token:
             fail(f"task {args.id!r} has no webhook token (rotate one first)")
-        url = _task_webhook_url(args.base, t.id, t.webhook_token)
+        base = (args.base or "http://localhost:7000").rstrip("/")
+        url = f"{base}/api/webhook/{t.webhook_token}"
         emit({
             "task_id": t.id,
             "name": t.name,

services/docs/service.py

@@ -50,46 +50,16 @@ class DocsService:
             List of DocChunk objects
         """
         results = self.rag.search(query, k=top_k)
-        chunks = []
+        return [
-
+            DocChunk(
-        for result in results:
+                text=r.get("text", r.get("content", "")),
-            if not isinstance(result, dict):
+                source=r.get("source", r.get("metadata", {}).get("source", "unknown")),
-                continue
+                score=r.get("score", 0.0),
-
+                metadata=r.get("metadata"),
-            metadata = result.get("metadata")
-            if not isinstance(metadata, dict):
-                metadata = {}
-
-            text = result.get("document")
-            if text is None:
-                text = result.get("text")
-            if text is None:
-                text = result.get("content")
-            if text is None:
-                text = ""
-
-            source = result.get("source")
-            if source is None:
-                source = metadata.get("source")
-            if source is None:
-                source = "unknown"
-
-            score = result.get("similarity")
-            if score is None:
-                score = result.get("score")
-            if score is None:
-                score = 0.0
-
-            chunks.append(
-                DocChunk(
-                    text=text,
-                    source=source,
-                    score=score,
-                    metadata=metadata,
-                )
             )
-
+            for r in results
-        return chunks
+            if isinstance(r, dict)
+        ]
 
     async def index(self, directory: str) -> IndexResult:
         """
@@ -103,8 +73,8 @@ class DocsService:
         """
         result = self.rag.index_personal_documents(directory)
         return IndexResult(
-            indexed=result.get("indexed_count", result.get("indexed", 0)),
+            indexed=result.get("indexed", 0),
-            failed=result.get("failed_count", result.get("failed", 0)),
+            failed=result.get("failed", 0),
             errors=result.get("errors", []),
         )
 

services/memory/__init__.py

@@ -2,7 +2,7 @@
 """Memory service — persistent memory storage and retrieval."""
 
 from .service import MemoryService, Memory, MemorySearchResult
-from .memory import MemoryManager, MemoryStoreUnreadable
+from .memory import MemoryManager
 from .memory_vector import MemoryVectorStore
 
 __all__ = [
@@ -10,6 +10,5 @@ __all__ = [
     "Memory",
     "MemorySearchResult",
     "MemoryManager",
-    "MemoryStoreUnreadable",
     "MemoryVectorStore",
 ]

services/memory/memory.py

@@ -5,16 +5,6 @@ application runtime instantiates ``src.memory.MemoryManager``, so keeping a
 parallel implementation here risks silent drift between import paths.
 """
 
-from src.memory import (
+from src.memory import MemoryManager, get_text_similarity, tokenize
-    MemoryManager,
-    MemoryStoreUnreadable,
-    get_text_similarity,
-    tokenize,
-)
 
-__all__ = [
+__all__ = ["MemoryManager", "get_text_similarity", "tokenize"]
-    "MemoryManager",
-    "MemoryStoreUnreadable",
-    "get_text_similarity",
-    "tokenize",
-]

services/memory/memory_extractor.py

@@ -17,8 +17,6 @@ import os
 import re
 from typing import Optional
 
-from src.memory import MemoryStoreUnreadable
-
 logger = logging.getLogger(__name__)
 
 
@@ -389,13 +387,7 @@ async def extract_and_store(
         # Get owner from session
         _owner = getattr(session, 'owner', None)
 
-        # Strict load: this is a read-modify-write. Degrading to [] here would
+        existing = memory_manager.load_all()
-        # save only the newly extracted facts and drop the entire store.
-        try:
-            existing = memory_manager.load_all_for_update()
-        except MemoryStoreUnreadable as e:
-            logger.error("Skipping auto memory extraction, store unreadable: %s", e)
-            return
         added = 0
 
         for fact in facts:
@@ -634,18 +626,7 @@ async def audit_memories(
 
         # Merge audited entries back with other users' entries
         if owner:
-            # Strict load: the merge below reconstructs the whole file. If this
+            all_entries = memory_manager.load_all()
-            # degraded to [] we would save only this owner's audited slice and
-            # destroy every other tenant's memories.
-            try:
-                all_entries = memory_manager.load_all_for_update()
-            except MemoryStoreUnreadable as e:
-                logger.error("Aborting memory audit save, store unreadable: %s", e)
-                return {
-                    "before": before_count,
-                    "after": before_count,
-                    "error": "store_unreadable",
-                }
             audited_ids = {e["id"] for e in final_entries}
             other_entries = [e for e in all_entries if e.get("owner") != owner and (e.get("owner") is not None)]
             # Also keep legacy entries that weren't part of this audit

services/memory/skill_format.py

@@ -50,7 +50,7 @@ import json
 import logging
 import re
 from dataclasses import dataclass, field
-from datetime import datetime, timezone
+from datetime import datetime
 from typing import Any, Dict, List, Optional
 
 logger = logging.getLogger(__name__)
@@ -100,18 +100,6 @@ def _parse_scalar(raw: str) -> Any:
     if raw.lower() in ("null", "none", "~"):
         return None
     if (raw[0] == raw[-1]) and raw[0] in ("'", '"'):
-        if raw[0] == '"':
-            # _emit_scalar writes double-quoted scalars with json.dumps, so
-            # decode the escapes instead of only stripping the quotes. Without
-            # this, `\"` / `\\` / `\uXXXX` stayed verbatim in the value and the
-            # next save escaped their backslashes again, doubling them on every
-            # load/save cycle (issue #5210).
-            try:
-                return json.loads(raw)
-            except ValueError:
-                # Hand-written file using escapes JSON rejects (e.g. a bare
-                # Windows path). Keep the previous literal reading.
-                pass
         return raw[1:-1]
     # Try number
     try:
@@ -183,26 +171,6 @@ def parse_frontmatter(text: str) -> tuple[Dict[str, Any], str]:
     return fm, body
 
 
-# Characters that force a quoted scalar. The punctuation would otherwise change
-# how the value reads back; the second row is every character str.splitlines()
-# treats as a line break, and parse_frontmatter() reads one scalar per line, so
-# emitting one of those bare would split the value across lines.
-_FM_MUST_QUOTE = (
-    ":", "#", "[", "]", "{", "}", ",", "&", "*", "!", "|", ">", "'", '"', "%", "@",
-    "\n", "\r", "\v", "\f", "\x1c", "\x1d", "\x1e", "\x85", "\u2028", "\u2029",
-)
-
-# json.dumps escapes every C0 control character, but with ensure_ascii=False it
-# passes NEL / LINE SEPARATOR / PARAGRAPH SEPARATOR through literally, and
-# str.splitlines() still breaks on all three. Re-escape exactly those, which
-# json.loads decodes again on the way in, so the pair stays symmetric.
-_FM_POST_DUMPS_ESCAPES = (
-    ("\x85", "\\u0085"),
-    ("\u2028", "\\u2028"),
-    ("\u2029", "\\u2029"),
-)
-
-
 def _emit_scalar(v: Any) -> str:
     if v is None:
         return "null"
@@ -213,15 +181,8 @@ def _emit_scalar(v: Any) -> str:
     if isinstance(v, list):
         return "[" + ", ".join(_emit_scalar(x) for x in v) + "]"
     s = str(v)
-    if any(c in s for c in _FM_MUST_QUOTE):
+    if any(c in s for c in (":", "#", "\n", "[", "]", "{", "}", ",", "&", "*", "!", "|", ">", "'", '"', "%", "@")):
-        # ensure_ascii=False keeps non-ASCII text as itself. SKILL.md is UTF-8 at
+        return json.dumps(s)
-        # both ends (skills.py reads it, atomic_write_text writes it), so the
-        # \uXXXX form bought nothing and leaked into the parsed value (#5210).
-        out = json.dumps(s, ensure_ascii=False)
-        for ch, esc in _FM_POST_DUMPS_ESCAPES:
-            if ch in out:
-                out = out.replace(ch, esc)
-        return out
     return s
 
 
@@ -480,4 +441,4 @@ class Skill:
 
 
 def _now_iso() -> str:
-    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    return datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")

services/memory/skill_importer.py

@@ -1,18 +1,16 @@
 """Import SKILL.md bundles from public GitHub (or skills.sh → GitHub) URLs."""
 from __future__ import annotations
 
-import ipaddress
 import logging
 import os
-import time
+import re
 from dataclasses import dataclass
-from typing import Dict, Iterable, List, Optional, Tuple, cast
+from typing import Dict, List, Optional, Tuple
 from urllib.parse import quote, urljoin, urlparse
 
-import httpcore
 import httpx
 
-from src.url_safety import _default_resolver, check_outbound_url
+from src.url_safety import check_outbound_url
 
 logger = logging.getLogger(__name__)
 
@@ -27,7 +25,6 @@ TEXT_NAMES = {"skill.md", "license", "license.md", "readme.md"}
 _GITHUB_HOSTS = frozenset({
     "github.com", "www.github.com", "api.github.com", "raw.githubusercontent.com",
 })
-_SKILLS_SH_HOSTS = frozenset({"skills.sh", "www.skills.sh"})
 
 
 def _github_host(url: str) -> str:
@@ -75,158 +72,18 @@ def _is_text_file(name: str) -> bool:
 _MAX_FETCH_REDIRECTS = 5
 
 
-def _validated_ips(raw_ips: List[str]) -> List[ipaddress._BaseAddress]:
+def _check_fetch_url(url: str) -> None:
-    """Parse and de-duplicate one resolver snapshot in resolver order."""
+    """SSRF guard for skill-import fetches (defense-in-depth).
-    ips: List[ipaddress._BaseAddress] = []
-    seen = set()
-    for raw in raw_ips:
-        if not isinstance(raw, str):
-            continue
-        try:
-            ip = ipaddress.ip_address(raw.split("%", 1)[0])
-        except ValueError:
-            continue
-        if ip in seen:
-            continue
-        seen.add(ip)
-        ips.append(ip)
-    return ips
 
-
+    Skill bundles only ever come from public GitHub, never an internal
-def _resolve_and_check_url(url: str) -> List[ipaddress._BaseAddress]:
+    address, so block private/loopback/link-local targets on every hop —
-    """Return the exact address snapshot approved for one fetch hop."""
+    matching the hardened web-fetch path in
-    resolved_ips: List[str] = []
+    ``services/search/content.py:_get_public_url`` rather than the lenient
-
+    default used for admin-configured model endpoints.
-    def _recording_resolver(host: str) -> List[str]:
+    """
-        answers = list(_default_resolver(host))
+    ok, reason = check_outbound_url(url, block_private=True)
-        resolved_ips[:] = answers
-        return answers
-
-    ok, reason = check_outbound_url(
-        url,
-        block_private=True,
-        resolver=_recording_resolver,
-    )
     if not ok:
-        raise SkillImportError(f"outbound URL blocked: {reason}")
+        raise SkillImportError(reason)
-
-    pinned_ips = _validated_ips(resolved_ips)
-    if not pinned_ips:
-        raise SkillImportError("outbound URL blocked: host did not resolve to a usable address")
-    return pinned_ips
-
-
-# Backward compatibility alias for tests importing _check_fetch_url directly
-_check_fetch_url = _resolve_and_check_url
-
-
-class _PinnedBackend(httpcore.NetworkBackend):
-    """Connect only to addresses from one validated DNS snapshot."""
-
-    def __init__(self, ips: List[ipaddress._BaseAddress]):
-        self._ips = [str(ip) for ip in ips]
-        self._real = httpcore.SyncBackend()
-
-    def connect_tcp(
-        self,
-        host: str,
-        port: int,
-        timeout: float | None = None,
-        local_address: str | None = None,
-        socket_options=None,
-    ):
-        deadline = None if timeout is None else time.monotonic() + timeout
-        last_exc: Optional[Exception] = None
-        for ip in self._ips:
-            remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
-            try:
-                return self._real.connect_tcp(
-                    ip,
-                    port,
-                    remaining,
-                    local_address,
-                    socket_options,
-                )
-            except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc:
-                last_exc = exc
-                if deadline is not None and time.monotonic() >= deadline:
-                    break
-        if last_exc is not None:
-            raise last_exc
-        raise httpcore.ConnectError("no validated address available")
-
-    def connect_unix_socket(self, path, timeout=None, socket_options=None):
-        return self._real.connect_unix_socket(path, timeout, socket_options)
-
-    def sleep(self, seconds: float) -> None:
-        return self._real.sleep(seconds)
-
-
-_HTTPCORE_TO_HTTPX_EXC = {
-    httpcore.ConnectError: httpx.ConnectError,
-    httpcore.ConnectTimeout: httpx.ConnectTimeout,
-    httpcore.LocalProtocolError: httpx.LocalProtocolError,
-    httpcore.NetworkError: httpx.NetworkError,
-    httpcore.PoolTimeout: httpx.PoolTimeout,
-    httpcore.ProtocolError: httpx.ProtocolError,
-    httpcore.ProxyError: httpx.ProxyError,
-    httpcore.ReadError: httpx.ReadError,
-    httpcore.ReadTimeout: httpx.ReadTimeout,
-    httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
-    httpcore.TimeoutException: httpx.TimeoutException,
-    httpcore.UnsupportedProtocol: httpx.UnsupportedProtocol,
-    httpcore.WriteError: httpx.WriteError,
-    httpcore.WriteTimeout: httpx.WriteTimeout,
-}
-
-
-class _PinnedTransport(httpx.BaseTransport):
-    """Pin socket connects while preserving URL authority, Host, and TLS SNI."""
-
-    def __init__(self, ips: List[ipaddress._BaseAddress]):
-        self._pinned_ips = list(ips)
-        self._pool = httpcore.ConnectionPool(
-            ssl_context=httpx.create_ssl_context(),
-            http1=True,
-            http2=False,
-            network_backend=_PinnedBackend(ips),
-        )
-
-    def handle_request(self, request: httpx.Request) -> httpx.Response:
-        core_request = httpcore.Request(
-            method=request.method,
-            url=httpcore.URL(
-                scheme=request.url.raw_scheme,
-                host=request.url.raw_host,
-                port=request.url.port,
-                target=request.url.raw_path,
-            ),
-            headers=request.headers.raw,
-            content=request.stream,
-            extensions=request.extensions,
-        )
-        core_response = None
-        try:
-            core_response = self._pool.handle_request(core_request)
-            content = b"".join(cast(Iterable[bytes], core_response.stream))
-        except Exception as exc:
-            mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
-            if mapped is not None:
-                raise mapped(str(exc)) from exc
-            raise
-        finally:
-            if core_response is not None:
-                core_response.close()
-
-        return httpx.Response(
-            status_code=core_response.status,
-            headers=core_response.headers,
-            content=content,
-            extensions=core_response.extensions,
-        )
-
-    def close(self) -> None:
-        self._pool.close()
 
 
 def _get_checked(
@@ -243,76 +100,49 @@ def _get_checked(
     hand lets us re-validate every hop, closing that blind-SSRF gap.
     """
     current = url
-    for _ in range(_MAX_FETCH_REDIRECTS + 1):
+    with httpx.Client(follow_redirects=False, timeout=timeout) as client:
-        pinned_ips = _resolve_and_check_url(current)
+        for _ in range(_MAX_FETCH_REDIRECTS + 1):
-        with httpx.Client(
+            _check_fetch_url(current)
-            transport=_PinnedTransport(pinned_ips),
-            follow_redirects=False,
-            timeout=timeout,
-        ) as client:
             r = client.get(current, headers=headers)
-
+            if r.status_code in (301, 302, 303, 307, 308):
-        if r.status_code in (301, 302, 303, 307, 308):
+                location = r.headers.get("location")
-            location = r.headers.get("location")
+                if not location:
-            if not location:
+                    return r
-                return r
+                current = urljoin(str(r.url), location)
-            current = urljoin(str(r.url), location)
+                continue
-            continue
+            return r
-        return r
     raise SkillImportError("too many redirects while fetching skill bundle")
 
 
 def parse_skill_source(url: str) -> ResolvedSource:
     """Normalize skills.sh / GitHub web URLs into owner/repo/ref/path."""
-    url = (url or "").strip()
+    raw = (url or "").strip()
-    if not url:
+    if not raw:
         raise SkillImportError("URL is required")
 
-    # ``urlparse`` only reports an unambiguous scheme when the URL carries the
+    # skills.sh often links to GitHub; try to unwrap ?url= or redirect target later.
-    # ``scheme://`` form. Opaque schemes (``mailto:``, ``javascript:``) and a
+    if "skills.sh" in raw and "github.com" not in raw:
-    # schemeless ``host:port`` both parse a "scheme" that is not one, so they
+        r = _get_checked(raw, timeout=20.0)
-    # fall through to the host check below and are rejected on the host instead.
-    scheme = urlparse(url).scheme.lower()
-    if scheme not in ("http", "https"):
-        if scheme and url.lower().startswith(f"{scheme}://"):
-            raise SkillImportError(f"unsupported URL scheme: {scheme}")
-        # Schemeless "github.com/owner/repo" — accept only a supported host.
-        rough_host = (urlparse("//" + url).hostname or "").lower()
-        if rough_host not in _GITHUB_HOSTS and rough_host not in _SKILLS_SH_HOSTS:
-            raise SkillImportError("Only GitHub or skills.sh URLs are supported")
-        url = "https://" + url
-
-    parsed = urlparse(url)
-    hostname = (parsed.hostname or "").lower()
-    if hostname not in _GITHUB_HOSTS and hostname not in _SKILLS_SH_HOSTS:
-        raise SkillImportError("Only GitHub or skills.sh URLs are supported")
-
-    # A skills.sh link is only usable if it redirects to an exact supported
-    # GitHub host. Scraping the page body for a github.com link cannot work:
-    # skill pages only ever link the repository root, never the skill's
-    # subdirectory, so the scrape resolves every skill in a repo to the same
-    # (wrong) bundle. Fail with an actionable message instead.
-    if hostname in _SKILLS_SH_HOSTS:
-        r = _get_checked(url, timeout=20.0)
         if r.status_code >= 400:
             raise _github_response_error(r)
         final = str(r.url)
-        if _github_host(final) not in _GITHUB_HOSTS:
+        _assert_github_url(final, context="redirect target")
-            raise SkillImportError(
+        # Page may embed a github link; prefer final URL if redirected.
-                "skills.sh did not redirect to GitHub — open the skill's "
+        if "github.com" in final:
-                "repository on GitHub, navigate to the exact skill folder or "
+            raw = final
-                "SKILL.md file, and paste that URL; the repository-root link "
+        else:
-                "alone is not sufficient"
+            m = re.search(r"https?://github\.com/[^\s\"')]+", r.text or "")
-            )
+            if m:
-        url = final
+                raw = m.group(0).rstrip(".,)")
 
-    # Update parsed and hostname to reflect the new GitHub URL
+    parsed = urlparse(raw)
-    parsed = urlparse(url)
+    host = _github_host(raw)
-    hostname = (parsed.hostname or "").lower()
+    if host not in _GITHUB_HOSTS:
+        raise SkillImportError(
+            "Only GitHub URLs are supported (https://github.com/... or raw.githubusercontent.com/...)"
+        )
 
-    _assert_github_url(url)
+    if host == "raw.githubusercontent.com":
-
-    if hostname == "raw.githubusercontent.com":
         # /owner/repo/ref/path/to/file
         bits = [p for p in parsed.path.split("/") if p]
         if len(bits) < 4:

services/search/content.py

@@ -2,18 +2,22 @@
 
 import copy
 import io
+import ipaddress
 import json
 import os
 import re
 import logging
+import socket
+import ssl
 from datetime import datetime, timedelta
-from typing import List
+from typing import Iterable, List, cast
+from urllib.parse import urljoin, urlparse
 
 import httpx
+import httpcore
 from bs4 import BeautifulSoup
 
 from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT
-from src import outbound_fetch as _outbound_fetch
 
 from .analytics import RateLimitError, error_logger
 from .cache import (
@@ -25,40 +29,336 @@ from .cache import (
 
 logger = logging.getLogger(__name__)
 
-def _is_private_address(addr):
+_PRIVATE_NETWORKS = (
-    return _outbound_fetch._is_private_address(addr)
+    ipaddress.ip_network("0.0.0.0/8"),
+    ipaddress.ip_network("10.0.0.0/8"),
+    ipaddress.ip_network("127.0.0.0/8"),
+    ipaddress.ip_network("169.254.0.0/16"),
+    ipaddress.ip_network("172.16.0.0/12"),
+    ipaddress.ip_network("192.168.0.0/16"),
+    ipaddress.ip_network("::1/128"),
+    ipaddress.ip_network("fc00::/7"),
+    ipaddress.ip_network("fe80::/10"),
+)
 
 
-def _resolve_hostname_ips(hostname):
+def _is_private_address(addr: ipaddress._BaseAddress) -> bool:
-    return _outbound_fetch._resolve_hostname_ips(hostname)
+    if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None:
-
+        addr = addr.ipv4_mapped
-
+    return (
-def _public_http_url(url):
+        addr.is_private
-    return _outbound_fetch._public_http_url(url, resolver=_resolve_hostname_ips)
+        or addr.is_loopback
-
+        or addr.is_link_local
-
+        or addr.is_reserved
-def _resolve_public_ips(url):
+        or addr.is_multicast
-    return _outbound_fetch._resolve_public_ips(url, resolver=_resolve_hostname_ips)
+        or addr.is_unspecified
-
+        or any(addr in net for net in _PRIVATE_NETWORKS)
-
-_PinnedBackend = _outbound_fetch._PinnedBackend
-_PinnedTransport = _outbound_fetch._PinnedTransport
-BodyTooLargeError = _outbound_fetch.BodyTooLargeError
-_CappedFetch = _outbound_fetch._CappedFetch
-
-
-def _get_public_url(url, headers, timeout, max_redirects=5, max_bytes=None):
-    return _outbound_fetch._get_public_url(
-        url,
-        headers=headers,
-        timeout=timeout,
-        max_redirects=max_redirects,
-        max_bytes=max_bytes,
-        resolve_public_ips=_resolve_public_ips,
-        transport_factory=_PinnedTransport,
     )
 
 
+def _resolve_hostname_ips(hostname: str) -> list[ipaddress._BaseAddress]:
+    try:
+        infos = socket.getaddrinfo(hostname, None)
+    except Exception:
+        return []
+    out = []
+    for info in infos:
+        try:
+            out.append(ipaddress.ip_address(info[4][0]))
+        except Exception:
+            continue
+    return out
+
+
+def _public_http_url(url: str) -> bool:
+    try:
+        parsed = urlparse(url)
+        if parsed.scheme not in ("http", "https"):
+            return False
+        host = (parsed.hostname or "").strip()
+        if not host:
+            return False
+        lower = host.lower()
+        if lower in ("localhost", "metadata", "metadata.google.internal"):
+            return False
+        if lower.endswith((".local", ".localhost", ".internal", ".lan", ".intranet")):
+            return False
+        try:
+            return not _is_private_address(ipaddress.ip_address(host))
+        except ValueError:
+            pass
+        addrs = _resolve_hostname_ips(host)
+        return bool(addrs) and not any(_is_private_address(a) for a in addrs)
+    except Exception:
+        return False
+
+
+def _resolve_public_ips(url: str) -> list[ipaddress._BaseAddress]:
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https") or not parsed.hostname:
+        raise httpx.RequestError(f"Blocked non-public URL: {url}")
+    host = (parsed.hostname or "").strip().lower()
+    if host in ("localhost", "metadata", "metadata.google.internal"):
+        raise httpx.RequestError(f"Blocked non-public hostname: {host}")
+    try:
+        ip = ipaddress.ip_address(host)
+        if _is_private_address(ip):
+            raise httpx.RequestError(f"Blocked non-public IP literal: {host}")
+        return [ip]
+    except httpx.RequestError:
+        raise
+    except ValueError:
+        pass
+    addrs = _resolve_hostname_ips(host)
+    if not addrs or any(_is_private_address(a) for a in addrs):
+        raise httpx.RequestError(f"Blocked non-public URL: {url}")
+    return addrs
+
+
+class _PinnedBackend(httpcore.NetworkBackend):
+    """Network backend that connects to a pre-resolved IP.
+
+    httpcore derives the TLS SNI and the ``Host`` header from the URL's
+    origin, not from the host argument passed to ``connect_tcp``. So
+    routing the TCP connect to a resolved IP while leaving the URL
+    untouched keeps SNI / vhost behaviour correct and closes the
+    DNS-rebinding TOCTOU between the SSRF check and the connect.
+    """
+
+    def __init__(self, ip: ipaddress._BaseAddress):
+        self._ip = str(ip)
+        self._real = httpcore.SyncBackend()
+
+    def connect_tcp(
+        self,
+        host: str,
+        port: int,
+        timeout: float | None = None,
+        local_address: str | None = None,
+        socket_options=None,
+    ):
+        return self._real.connect_tcp(
+            self._ip, port, timeout, local_address, socket_options
+        )
+
+    def connect_unix_socket(self, path, timeout=None, socket_options=None):
+        return self._real.connect_unix_socket(path, timeout, socket_options)
+
+    def sleep(self, seconds: float) -> None:
+        return self._real.sleep(seconds)
+
+
+# Map httpcore exception classes to their httpx equivalents. Built
+# once at import time from the public exception classes; avoids any
+# import of httpx's private transport machinery. httpcore's
+# ``ConnectionNotAvailable`` is a pool-internal signal (the pool will
+# close and retry on its own) — we never expect to see it surface to
+# a transport caller, so it has no httpx counterpart here.
+_HTTPCORE_TO_HTTPX_EXC = {
+    httpcore.ConnectError: httpx.ConnectError,
+    httpcore.ConnectTimeout: httpx.ConnectTimeout,
+    httpcore.LocalProtocolError: httpx.LocalProtocolError,
+    httpcore.NetworkError: httpx.NetworkError,
+    httpcore.PoolTimeout: httpx.PoolTimeout,
+    httpcore.ProtocolError: httpx.ProtocolError,
+    httpcore.ProxyError: httpx.ProxyError,
+    httpcore.ReadError: httpx.ReadError,
+    httpcore.ReadTimeout: httpx.ReadTimeout,
+    httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
+    httpcore.TimeoutException: httpx.TimeoutException,
+    httpcore.UnsupportedProtocol: httpx.UnsupportedProtocol,
+    httpcore.WriteError: httpx.WriteError,
+    httpcore.WriteTimeout: httpx.WriteTimeout,
+}
+
+
+class _PinnedTransport(httpx.BaseTransport):
+    """Transport that pins every TCP connect to a pre-resolved IP.
+
+    Uses only the public ``httpcore`` and ``httpx`` APIs — no
+    subclassing of ``httpx.HTTPTransport``, no reads of private
+    ``httpcore.ConnectionPool`` attributes, no imports from
+    ``httpx private transport internals``. The URL is passed through unchanged so SNI
+    / vhost work as if httpx had been given the hostname directly;
+    only the TCP destination is pinned, closing the DNS-rebinding
+    TOCTOU between the SSRF check and the connect.
+    """
+
+    def __init__(self, ip: ipaddress._BaseAddress, *, http2: bool = False):
+        self._pool = httpcore.ConnectionPool(
+            ssl_context=ssl.create_default_context(),
+            http1=True,
+            http2=http2,
+            network_backend=_PinnedBackend(ip),
+        )
+
+    def __enter__(self):
+        self._pool.__enter__()
+        return self
+
+    def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None:
+        self._pool.__exit__(exc_type, exc_value, traceback)
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        httpcore_req = httpcore.Request(
+            method=request.method,
+            url=httpcore.URL(
+                scheme=request.url.raw_scheme,
+                host=request.url.raw_host,
+                port=request.url.port,
+                target=request.url.raw_path,
+            ),
+            headers=request.headers.raw,
+            content=request.stream,
+            extensions=request.extensions,
+        )
+        try:
+            httpcore_resp = self._pool.handle_request(httpcore_req)
+            # Eager materialisation matches the original
+            # ``response.text`` usage in fetch_webpage_content. The
+            # sync pool's stream is a plain Iterable[bytes] despite
+            # the httpcore type hint unioning the async variant.
+            content = b"".join(cast(Iterable[bytes], httpcore_resp.stream))
+        except Exception as exc:
+            mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
+            if mapped is not None:
+                raise mapped(str(exc)) from exc
+            raise
+
+        return httpx.Response(
+            status_code=httpcore_resp.status,
+            headers=httpcore_resp.headers,
+            content=content,
+            extensions=httpcore_resp.extensions,
+        )
+
+    def close(self) -> None:
+        self._pool.close()
+
+class BodyTooLargeError(Exception):
+    """The server declared a body larger than the hard fetch ceiling."""
+
+    def __init__(self, url: str, declared_bytes: int):
+        self.url = url
+        self.declared_bytes = declared_bytes
+        super().__init__(
+            f"response body is {declared_bytes:,} bytes, over the "
+            f"{WEB_FETCH_HARD_MAX_BYTES:,}-byte hard cap"
+        )
+
+
+class _CappedFetch:
+    """Result of a size-capped streaming GET.
+
+    Carries just what fetch_webpage_content needs from an httpx.Response,
+    plus the cap bookkeeping: the (possibly truncated) body, whether the
+    cap cut it short, and the size the server declared via Content-Length
+    (wire bytes; None when absent).
+    """
+
+    __slots__ = ("status_code", "headers", "content", "truncated",
+                 "declared_bytes", "encoding", "url")
+
+    def __init__(self, status_code, headers, content, truncated,
+                 declared_bytes, encoding, url):
+        self.status_code = status_code
+        self.headers = headers
+        self.content = content
+        self.truncated = truncated
+        self.declared_bytes = declared_bytes
+        self.encoding = encoding
+        self.url = url
+
+    @property
+    def text(self) -> str:
+        return self.content.decode(self.encoding or "utf-8", errors="replace")
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            request = httpx.Request("GET", self.url)
+            raise httpx.HTTPStatusError(
+                f"HTTP {self.status_code} for {self.url}",
+                request=request,
+                response=httpx.Response(self.status_code, request=request),
+            )
+
+
+def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5,
+                    max_bytes: int = None) -> "_CappedFetch":
+    """Capped streaming GET with SSRF-guarded, DNS-pinned manual redirects.
+
+    Each hop is resolved once, validated as public, and then the actual TCP
+    connection is pinned to that resolved IP. The request URL is left unchanged
+    so Host and TLS SNI keep the original hostname.
+    """
+    cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
+    current = url
+    for _ in range(max_redirects + 1):
+        ips = _resolve_public_ips(current)
+
+        # Force identity transfer-encoding. With gzip/deflate the wire bytes
+        # and Content-Length can be a small fraction of the decoded body, so a
+        # tiny compressed response could pass the hard-cap preflight and then
+        # expand past the ceiling in one decoded chunk before the streamed cap
+        # below can slice it.
+        req_headers = dict(headers or {})
+        req_headers["Accept-Encoding"] = "identity"
+
+        with httpx.Client(
+            headers=req_headers,
+            timeout=timeout,
+            follow_redirects=False,
+            transport=_PinnedTransport(ips[0]),
+        ) as client:
+            with client.stream("GET", current) as response:
+                if response.status_code in (301, 302, 303, 307, 308):
+                    location = response.headers.get("location")
+                    if not location:
+                        return _CappedFetch(response.status_code, response.headers, b"",
+                                            False, None, response.encoding, str(response.url))
+                    current = urljoin(str(response.url), location)
+                    continue
+
+                # A server can ignore the identity request and still return a
+                # compressed body; httpx.iter_bytes would then decode it, and a
+                # tiny gzip can balloon into one decoded chunk far past the cap.
+                # Refuse compressed Content-Encoding so the streamed cap stays
+                # a real memory bound.
+                enc = (response.headers.get("content-encoding") or "").strip().lower()
+                if enc and enc != "identity":
+                    raise httpx.RequestError(
+                        f"Refusing compressed response (Content-Encoding: {enc}) after "
+                        "requesting identity: cannot bound decoded body size",
+                        request=httpx.Request("GET", current),
+                    )
+
+                declared = None
+                raw_len = response.headers.get("content-length")
+                if raw_len and raw_len.isdigit():
+                    declared = int(raw_len)
+
+                if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
+                    raise BodyTooLargeError(current, declared)
+
+                chunks = []
+                read = 0
+                truncated = False
+                for chunk in response.iter_bytes():
+                    read += len(chunk)
+                    if read > cap:
+                        keep = cap - (read - len(chunk))
+                        if keep > 0:
+                            chunks.append(chunk[:keep])
+                        truncated = True
+                        break
+                    chunks.append(chunk)
+
+                return _CappedFetch(response.status_code, response.headers,
+                                    b"".join(chunks), truncated, declared,
+                                    response.encoding, str(response.url))
+
+    raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current))
+
 # PDF extraction (optional dependency)
 try:
     from pdfminer.high_level import extract_text as pdf_extract_text

services/tts/tts_service.py

@@ -2,7 +2,6 @@
 """Multi-provider TTS service — dispatches to local Kokoro, OpenAI-compatible API, or browser."""
 
 import io
-import os
 import wave
 import logging
 import hashlib
@@ -42,11 +41,6 @@ class TTSService:
         self.cache_dir = Path(cache_dir)
         self.cache_dir.mkdir(parents=True, exist_ok=True)
         self._kokoro = None  # lazy-init
-        
-        try:
-            self.max_cache_bytes = int(os.getenv("ODYSSEUS_TTS_CACHE_MAX_BYTES", 500 * 1024 * 1024))
-        except ValueError:
-            self.max_cache_bytes = 500 * 1024 * 1024
 
     # ── Settings ──
 
@@ -95,53 +89,6 @@ class TTSService:
         ext = ".mp3" if (len(data) >= 3 and (data[:3] == b'ID3' or (data[0] == 0xff and (data[1] & 0xe0) == 0xe0))) else ".wav"
         (self.cache_dir / f"{key}{ext}").write_bytes(data)
 
-        self._enforce_cache_limit()
-
-    def _enforce_cache_limit(self):
-            """Evicts oldest files if the cache exceeds the configured byte limit."""
-            if self.max_cache_bytes <= 0:
-                return
-
-            try:
-                files = []
-                total_size = 0
-
-                # Safely scan files and sum sizes, ignoring files deleted mid-scan
-                for f in self.cache_dir.iterdir():
-                    try:
-                        if f.is_file() and f.suffix.lower() in (".mp3", ".wav"):
-                            files.append(f)
-                            total_size += f.stat().st_size
-                    except OSError:
-                        continue
-
-                if total_size > self.max_cache_bytes:
-                    logger.info(
-                        f"TTS cache ({total_size} bytes) exceeded limit ({self.max_cache_bytes} bytes). Evicting oldest files."
-                    )
-
-                    # Sort files by modification time (oldest first)
-                    try:
-                        files.sort(key=lambda f: f.stat().st_mtime)
-                    except OSError as e:
-                        logger.warning(f"Failed to sort cache files by mtime: {e}")
-
-                    # Trim down to 80% of max capacity
-                    target_size = self.max_cache_bytes * 0.8
-
-                    while files and total_size > target_size:
-                        f = files.pop(0)
-                        try:
-                            size = f.stat().st_size
-                            f.unlink()
-                            total_size -= size
-                        except OSError as e:
-                            logger.warning(f"Failed to evict cache file {f}: {e}")
-                            continue
-
-            except Exception as e:
-                logger.warning(f"Error enforcing TTS cache limit: {e}", exc_info=True)
-
     def clear_cache(self):
         count = 0
         for f in self.cache_dir.glob("*.*"):

specs/architecture-runtime-inventory.md

@@ -1,412 +1,31 @@
-# Architecture Runtime Inventory
+# Architecture runtime inventory disposition
-
+
-> **Purpose**: Phase 0 planning baseline for codebase readability improvements (#4071).
+> [!NOTE]
-> **Parent issue**: [#4082](https://github.com/odysseus-dev/odysseus/issues/4082)
+> The dated architecture runtime inventory has been classified and transferred
-> **Last updated**: dev@b58af42 | 2026-06-16
+> to [`discovery/architecture-runtime-inventory.md`](../discovery/architecture-runtime-inventory.md)
-> **Status**: Draft — to be reviewed before follow-up slices open.
+> after verification of its stable facts.
-> **Snapshot basis**: Importer / file / import-line counts are refreshed to `dev@b58af42` (2026-06-16) and are recomputable via the commands in §3.4. **Line counts** in §2.1 / §2.2 are a snapshot from an earlier baseline and drift as `dev` moves — recompute any of them with `wc -l <file>`. This inventory tracks structure and risk, not live metrics.
+>
-
+> Stable runtime structure and subsystem boundaries were transferred to the
-This document maps the current runtime module structure, identifies high-risk boundaries, and recommends safe first refactor slices. It does **not** move files, change imports, or alter runtime behavior.
+> proposed canonical destination,
-
+> [`docs/ARCHITECTURE.md`](../docs/ARCHITECTURE.md), for maintainer review.
----
+
-
+## Classification
-## 1. Current Structure Overview
+
-
+| Content | Disposition |
-### 1.1 Top-Level Layout
+|---|---|
-
+| Stable runtime structure | Proposed canonical architecture document |
-```
+| Stable subsystem boundaries | Proposed canonical architecture document |
-odysseus/
+| Frontend module organization | `static/js/MODULE_SUMMARY.md` |
-├── app.py                    # FastAPI app entrypoint (1,145 lines)
+| Counts, rankings, and generated metrics | Non-canonical discovery inventory |
-├── conf/                     # Configuration (config.py, settings.py, settings_scrub.py)
+| Investigation context and unresolved questions | Non-canonical discovery material |
-├── src/                      # 95 flat .py files + 2 subdirectories
+| Refactor options and prioritization | Issues, Plane, or discovery material |
-│   ├── agent_tools/          # Tool helpers: document, filesystem, subprocess, web
+
-│   └── search/               # Search subsystem
+No dated metrics, prioritization, or investigation findings remain in this
-├── routes/                   # 54 flat .py files — HTTP route handlers
+specification path.
-├── core/                     # 10 files — database models, auth, middleware, session
+
-├── mcp_servers/              # 5 files — MCP server implementations
+## Review status
-├── scripts/                  # CLI tools and one-shot scripts
+
-├── static/                   # Frontend HTML/CSS/JS
+- Findings and scope have been validated.
-├── tests/                    # 583 test files (~54,800 lines)
+- Transfer verification has completed.
-└── services/                 # (exists as needed)
+- The final canonical status of `docs/ARCHITECTURE.md` requires maintainer review.
-```
+- No implementation behaviour, test taxonomy, or runtime files were changed.
-
-### 1.2 Directory Flatness Metric
-
-| Directory | Flat `.py` Files | Subdirectories | Concern |
-|-----------|-----------------|----------------|---------|
-| `src/` | **95** | 2 (`agent_tools/`, `search/`) | No domain grouping; 95 files in one directory |
-| `routes/` | **54** | 0 | All route handlers in one flat directory |
-| `core/` | 10 | 0 | Manageable, but `database.py` is oversized |
-
----
-
-## 2. Largest Runtime Modules
-
-### 2.1 Python Backend
-
-| Rank | File | Lines | Classes | Functions | Risk |
-|------|------|-------|---------|-----------|------|
-| 1 | `src/tool_implementations.py` | **4,032** | 0 | ~48 | **HIGH** |
-| 2 | `routes/email_routes.py` | **3,245** | — | — | **MEDIUM** |
-| 3 | `routes/cookbook_routes.py` | **2,969** | — | — | **MEDIUM** |
-| 4 | `src/agent_loop.py` | **2,961** | 0 | ~24 | **HIGH** |
-| 5 | `src/task_scheduler.py` | **2,330** | — | 5 | MEDIUM |
-| 6 | `routes/model_routes.py` | **2,266** | — | — | MEDIUM |
-| 7 | `core/database.py` | **2,265** | 28 | ~59 helpers | **HIGH** |
-| 8 | `src/builtin_actions.py` | **2,262** | 2 | ~24 | MEDIUM |
-| 9 | `src/llm_core.py` | **2,164** | — | — | MEDIUM |
-| 10 | `mcp_servers/email_server.py` | 2,197 | — | — | LOW (separate process) |
-| 11 | `src/visual_report.py` | 1,918 | — | — | LOW |
-| 12 | `routes/gallery_routes.py` | 1,896 | — | — | LOW |
-| 13 | `src/ai_interaction.py` | 1,846 | — | — | MEDIUM |
-| 14 | `routes/document_routes.py` | 1,717 | — | — | LOW |
-| 15 | `routes/skills_routes.py` | 1,648 | — | — | LOW |
-
-**Heuristic**: Files > 2,000 lines with 20+ public symbols and many importers are the highest-risk splits. Files 1,000–2,000 lines are medium-risk if tightly coupled.
-
-### 2.2 Frontend
-
-| File | Lines | Concern |
-|------|-------|---------|
-| `static/style.css` | **36,653** | Entire app CSS in one file (tracked separately in #2617) |
-| `static/js/document.js` | **9,776** | Single JS file for document functionality |
-| `static/js/slashCommands.js` | 6,498 | |
-| `static/js/settings.js` | 5,266 | |
-| `static/js/emailLibrary.js` | 5,217 | |
-| `static/js/notes.js` | 5,124 | |
-| `static/js/chat.js` | 4,985 | |
-| `static/app.js` | 4,090 | |
-
-**Note**: Frontend modularization is tracked separately in #2617 (CSS) and is not the focus of this Phase 0 inventory. Frontend is listed here for completeness but follow-up slices should target Python backend boundaries first.
-
----
-
-## 3. Import Dependency Graph
-
-### 3.1 Who Depends on `core/database.py`
-
-**102 files** import from `core.database` — this is the most depended-upon module:
-
-- All route handlers (`routes/*.py`)
-- Most `src/*.py` files
-- `core/session_manager.py`, `core/auth.py`
-- Multiple test files
-
-**Implication**: Any split of `core/database.py` is the highest-risk refactor. It should be tackled **last**, never first.
-
-### 3.2 Who Depends on `src/tool_implementations.py`
-
-**17 files** import from `src.tool_implementations`:
-- `src/agent_loop.py`, `src/builtin_actions.py`, `src/tool_index.py`
-- `src/task_scheduler.py`, `src/tool_policy.py`
-- Various tests
-
-### 3.3 Who Depends on `src/agent_loop.py`
-
-**22 files** import from `src.agent_loop`:
-
-- `src/tool_policy.py`, `src/teacher_escalation.py`, `src/bg_monitor.py`
-- `src/task_scheduler.py`
-- Multiple test files
-
-### 3.4 Cross-Layer Import Violations
-
-**`src/` importing from `routes/`** (backwards dependency — domain logic depending on HTTP layer):
-
-```
-src/tool_implementations.py ──→ routes/calendar_routes.py
-src/tool_implementations.py ──→ routes/cookbook_helpers.py
-src/tool_implementations.py ──→ routes/email_helpers.py
-src/tool_implementations.py ──→ routes/email_pollers.py
-src/tool_implementations.py ──→ routes/email_routes.py
-src/tool_implementations.py ──→ routes/model_routes.py
-src/tool_implementations.py ──→ routes/note_routes.py
-src/tool_implementations.py ──→ routes/prefs_routes.py
-```
-
-> These are **runtime imports** (inside function bodies, not at module top), which mitigates circular import risk but indicates fuzzy layer boundaries. Function-level inline imports from the HTTP layer into business logic are a code smell.
-
-**Import counts (top-level)**:
-| Direction | Count | Notes |
-|-----------|-------|-------|
-| `routes/` → `src/` | **374** | Expected: HTTP handlers call domain logic |
-| `routes/` → `core/` | **126** | Expected: handlers access DB models |
-| `src/` → `routes/` | **31** | **Unexpected**: domain logic reaching into HTTP layer (direct grep of import lines referencing `routes/`) |
-| `src/` → `core/` | **106** | Acceptable but could be reduced with a data-access layer |
-
-> **How the metrics in this document are computed** — recompute against current `dev` before treating any count as authoritative (the tree drifts; these numbers are a snapshot, not a live value):
-> - `src/` flat `.py` files: `find src -maxdepth 1 -name '*.py' | wc -l`
-> - `tests/` test files: `find tests -name 'test_*.py' | wc -l`
-> - `core.database` importers: `grep -rlE '(from|import) +core\.database' --include='*.py' . | grep -v core/database.py | wc -l`
-> - `src.agent_loop` importers: `grep -rlE '(from|import) +src\.agent_loop' --include='*.py' . | grep -v src/agent_loop.py | wc -l`
-> - Cross-layer import lines: `grep -rhE '(from|import) +<pkg>' --include='*.py' <dir>/ | wc -l` (e.g. `(from|import) +routes` over `src/`)
-
----
-
-## 4. Route Ownership Map
-
-Routes can be grouped into logical feature domains. Current flat structure obscures these boundaries:
-
-| Domain | Route Files | Total Lines | Review Complexity |
-|--------|-------------|-------------|-------------------|
-| **Email** | `email_routes.py`, `email_helpers.py`, `email_pollers.py` | 5,936 | HIGH — most complex domain |
-| **Chat / Agent** | `chat_routes.py`, `chat_helpers.py`, `shell_routes.py`, `codex_routes.py`, `skills_routes.py` | 6,365 | HIGH — core interaction surface |
-| **Cookbook** | `cookbook_routes.py`, `cookbook_helpers.py`, `cookbook_output.py` | 4,110 | MEDIUM |
-| **Model / LLM** | `model_routes.py`, `assistant_routes.py`, `copilot_routes.py` | 2,764 | MEDIUM |
-| **Calendar / Contacts** | `calendar_routes.py`, `contacts_routes.py` | 2,336 | MEDIUM |
-| **Documents** | `document_routes.py`, `document_helpers.py` | 1,954 | LOW |
-| **Auth** | `auth_routes.py`, `api_token_routes.py`, `device_flow.py` | 1,171 | LOW |
-| **Tasks** | `task_routes.py` (standalone) | 1,157 | LOW |
-| **Session** | `session_routes.py` (standalone) | 1,287 | LOW |
-| **Gallery** | `gallery_routes.py`, `gallery_helpers.py` | 1,896 | LOW |
-| **Memory** | `memory_routes.py` | — | LOW |
-| **Research** | `research_routes.py` | — | LOW |
-| **MCP** | `mcp_routes.py` | — | LOW |
-| **Notes** | `note_routes.py` | — | LOW |
-| **Other** | `prefs_routes.py`, `upload_routes.py`, `vault_routes.py`, `webhook_routes.py`, `workspace_routes.py`, `search_routes.py`, `history_routes.py`, `hwfit_routes.py`, `preset_routes.py`, `signature_routes.py`, `backup_routes.py`, `cleanup_routes.py`, `diagnostics_routes.py`, `embedding_routes.py`, `emoji_routes.py`, `font_routes.py`, `stt_routes.py`, `tts_routes.py`, `compare_routes.py`, `personal_routes.py`, `editor_draft_routes.py`, `admin_wipe_routes.py`, `chatgpt_subscription_routes.py` | 2,000+ | LOW individual, HIGH cumulative |
-
----
-
-## 5. Tool Registry & Implementation Boundaries
-
-### 5.1 Current Tool Architecture
-
-| Component | File | Lines | Role |
-|-----------|------|-------|------|
-| Tool schemas | `src/tool_schemas.py` | 1,392 | JSON Schema tool definitions (Duck-TypedDict) |
-| Tool index | `src/tool_index.py` | 542 | RAG-based tool retrieval from ChromaDB |
-| Tool implementations | `src/tool_implementations.py` | 4,032 | 33 `do_*` functions — all tool execution logic |
-| Tool security | `src/tool_security.py` | — | Owner-scoped tool blocking |
-| Tool policy | `src/tool_policy.py` | — | Guide-only directive, plan-mode disabled tools |
-| Tool utils | `src/tool_utils.py` | — | Shared tool helpers |
-
-### 5.2 Tool Implementation Categories
-
-The 33 `do_*` functions in `tool_implementations.py` fall into natural domain groups — the basis for slice 1's split in §6.2:
-
-| Category | `do_*` functions | Count |
-|----------|------------------|-------|
-| **System / config** | `do_manage_skills`, `do_manage_tasks`, `do_manage_endpoints`, `do_manage_mcp`, `do_manage_webhooks`, `do_manage_tokens`, `do_manage_settings`, `do_api_call`, `do_app_api` | 9 |
-| **Cookbook / model serving** | `do_download_model`, `do_serve_model`, `do_list_served_models`, `do_stop_served_model`, `do_tail_serve_output`, `do_list_downloads`, `do_cancel_download`, `do_search_hf_models`, `do_adopt_served_model`, `do_list_cookbook_servers`, `do_list_serve_presets`, `do_serve_preset`, `do_list_cached_models` | 13 |
-| **Notes** | `do_manage_notes` | 1 |
-| **Calendar** | `do_manage_calendar` | 1 |
-| **Search** | `do_search_chats` | 1 |
-| **Research** | `do_manage_research`, `do_trigger_research` | 2 |
-| **Contacts** | `do_resolve_contact`, `do_manage_contact` | 2 |
-| **Vault** | `do_vault_search`, `do_vault_get`, `do_vault_unlock` | 3 |
-| **Image** | `do_edit_image` | 1 |
-| | **Total** | **33** |
-
-> Low-level tools (filesystem, subprocess, web fetch, document parsing) live in `src/agent_tools/`, **not** in `tool_implementations.py` — out of scope for this split.
-
----
-
-## 6. Risk Assessment & Candidate Slice Ranking
-
-> **Candidate proposals, not a committed plan.** The rankings, package shapes (e.g. `src/pkg/`, `src/domain/`, `src/infra/`, `src/api/`), split ordering, and route-grouping strategy below are **options for maintainer discussion**. Per #4082/#4071, slice ownership and order are settled by maintainers before any follow-up PR. §1–§3 above are the factual current-state inventory.
-
-### 6.1 Risk Scale
-
-| Level | Criteria |
-|-------|----------|
-| **LOW** | File has ≤3 importers AND ≤500 lines, OR is a pure refactor with clear boundaries |
-| **MEDIUM** | File has 4–15 importers OR 500–1,500 lines |
-| **HIGH** | File has 16+ importers OR >2,000 lines, OR has cross-layer import violations |
-
-### 6.2 Ranked Split Candidates
-
-| Priority | Target | Risk | Rationale |
-|----------|--------|------|-----------|
-| **1** | `src/tool_implementations.py` → `src/tools/*.py` | **MEDIUM** | 4,032 lines → ~10 files by tool category. Already has natural boundaries. 17 importers, tracked in #3629. Use `__init__.py` shim to keep existing imports working. |
-| **2** | `routes/` → domain subdirectories (one domain per PR) | **MEDIUM** | 54 flat files. Done **one domain at a time** (e.g. a standalone PR for the email domain, then chat, …), not a broad reorganization — route modules carry helper imports, registration assumptions, and test import paths. |
-| **3** | `src/agent_loop.py` → `src/agent/loop.py` + submodules | **MEDIUM-HIGH** | 2,961 lines, 24 functions. Can extract prompt building, classification, verification, and runaway detection. Tracked in #3266. |
-| **4** | `src/` → `src/pkg/`, `src/domain/`, `src/infra/`, `src/api/` | **MEDIUM** | Structural reorganization. Split flat `src/` into layered packages. Must come after routes and tools are stable. |
-| **5** | `routes/email_*.py` consolidation | **LOW** | Already grouped by filename prefix. Low-risk cleanup within the email domain. |
-| **6** | `core/database.py` → `src/infra/database/models/*.py` | **HIGH** | 28 classes, 102 importers. Highest-risk split. Must be **last** in any sequence. Requires careful import shim strategy. |
-| **7** | Frontend CSS modularization | **MEDIUM** | 36,653 lines. Tracked in #2617. Separate timeline from backend work. |
-| **8** | Frontend JS modularization | **MEDIUM** | 9,776 lines in `document.js`. Introduce ES modules at minimum. |
-
-### 6.3 Candidate First 3 Behavior-Preserving Slices
-
-**Slice 1: Split `tool_implementations.py`** (Lowest-risk high-impact)
-
-- Create `src/tools/` package with one file per tool category
-- Add `src/tools/__init__.py` re-exporting all symbols with current names
-- Update 17 importers to use new paths (can be deferred via shim)
-- Validation: `python -m pytest tests/ -x -q` + manual smoke test of tool execution
-- Reference: #3629
-
-**Slice 2: Group `routes/` by domain** (one domain per PR, not a broad sweep)
-
-Route modules carry helper imports, router registration assumptions, and test import paths, so this must be done **one domain at a time** rather than as a single reorganization PR. Example sequence (each its own PR):
-
-- PR 2a: move the **email** domain (`email_routes.py`, `email_helpers.py`, `email_pollers.py`) → `routes/email/` + shim
-- PR 2b: move the **chat/agent** domain → `routes/chat/` + shim
-- PR 2c: move the **cookbook** domain → `routes/cookbook/` + shim
-- …and so on per domain from §4
-
-Each PR: add `__init__.py` re-exporting old names, update `app.py` router imports, validation `python app.py` starts clean. **No behavior change** — pure file reorganization.
-
-**Slice 3: Extract `agent_loop.py` submodules** (Improve reviewability)
-
-- Move prompt assembly → `src/agent/prompt.py`
-- Move request classification → `src/agent/classifier.py`
-- Move sub-agent verification → `src/agent/verifier.py`
-- Move runaway detection → `src/agent/runaway.py`
-- Move context management → `src/agent/context.py`
-- Keep `src/agent/loop.py` as the main orchestration module
-- Validation: `python -m pytest tests/test_agent_loop.py tests/test_loop_breaker_runaway.py -v`
-
----
-
-## 7. Safety Guardrails for Follow-Up Work
-
-Per maintainer guidance in #4082 and #4071:
-
-- [ ] **One domain/slice per PR** — never mix multiple reorganizations
-- [ ] **No behavior changes** mixed with file moves — pure reorganization only
-- [ ] **Keep compatibility shims** — `__init__.py` re-exports for all existing import paths
-- [ ] **Add or identify focused tests** before risky splits
-- [ ] **Do not start with `core/database.py`** or broad route movement unless this inventory shows a safe boundary
-- [ ] **Prefer small, reviewable slices** over large restructures
-- [ ] **No packaging/runtime/tooling migration** mixed into file moves
-- [ ] **No frontend framework migration** inside this stabilization lane
-- [ ] **Validate with `python -m compileall`** — every PR must pass CI checks
-- [ ] **Validate with `pytest`** — run the full test suite before opening each PR
-
----
-
-## 8. Validation Commands
-
-Each follow-up PR should be verifiable with these commands before submission:
-
-```bash
-# Syntax check — must pass with zero errors
-python -m compileall src/ routes/ core/ conf/
-
-# Full test suite — must match baseline pass rate
-python -m pytest tests/ -x -q
-
-# Import shim verification — existing import paths must still work
-python -c "from src.tool_implementations import do_search_chats; print('OK')"
-
-# App startup smoke test (if backend touched)
-timeout 5 python app.py 2>&1 | head -5 || true
-```
-
----
-
-## 9. Open Questions
-
-1. Is `#2538` (specs ground truth) the canonical behavior map baseline, and should this inventory be kept in sync with those specs once merged?
-2. Should route grouping follow the domain map proposed here, or is there a different taxonomy preferred by maintainers?
-3. For the `tool_implementations.py` split (#3629), is the tool categorization in §5.2 acceptable, or should it follow a different grouping?
-4. Should compatibility shims (`__init__.py`) be temporary (removed in a follow-up wave) or permanent?
-5. Should an ADR (Architecture Decision Record) document be started to track decisions made during this process?
-
----
-
-## 10. Future Direction (NOT current state)
-
-The following are **future refactor targets** (candidate directions **pending maintainer agreement**, not committed), recorded here so this inventory does not imply they exist today. None of them are present in the current `dev` tree:
-
-- `main.py` — proposed rename of the `app.py` entrypoint. Today the app boots via `app.py`.
-- `src/agent/` — proposed package to hold `agent_loop.py` submodules (prompt/classifier/verifier/runaway/context). Today `agent_loop.py` is a single flat file in `src/`.
-- `src/infra/`, `src/domain/`, `src/pkg/`, `src/api/` — proposed layered reorganization of the flat `src/` directory (slice 4 in §6).
-
-These become real only when the corresponding slices land.
-
----
-
-## Appendix A: File Listing
-
-### `src/` (95 files — 61 shown; run `ls src/*.py` for the full list)
-
-```
-agent_loop.py          tool_implementations.py   tool_schemas.py
-tool_index.py          tool_security.py          tool_policy.py
-tool_utils.py          builtin_actions.py        task_scheduler.py
-llm_core.py            model_context.py          model_discovery.py
-session_search.py      context_budget.py         context_compactor.py
-ai_interaction.py      action_intents.py         agent_runs.py
-app_helpers.py         app_initializer.py        config.py
-database.py            memory.py                 memory_provider.py
-secret_storage.py      prompt_security.py        url_security.py
-url_safety.py          rate_limiter.py           cleanup_service.py
-readiness.py           service_health.py         exceptions.py
-request_models.py      assistant_log.py          bg_monitor.py
-builtin_mcp.py         chat_helpers.py           chroma_client.py
-document_processor.py  embedding_lanes.py        deep_research.py
-research_handler.py    research_utils.py         personal_docs.py
-rag_manager.py         rag_singleton.py          topic_analyzer.py
-visual_report.py       youtube_handler.py        pdf_forms.py
-pdf_form_doc.py        pdf_runtime.py            caldav_writeback.py
-email_thread_parser.py text_helpers.py           user_time.py
-teacher_escalation.py  cookbook_serve_lifecycle.py
-chatgpt_subscription.py  mcp_manager.py
-```
-
-### `routes/` (54 files)
-
-```
-__init__.py    _validators.py
-auth_routes.py              api_token_routes.py       device_flow.py
-chat_routes.py              chat_helpers.py           shell_routes.py
-codex_routes.py             skills_routes.py
-email_routes.py             email_helpers.py          email_pollers.py
-cookbook_routes.py          cookbook_helpers.py       cookbook_output.py
-model_routes.py             assistant_routes.py       copilot_routes.py
-calendar_routes.py          contacts_routes.py
-document_routes.py          document_helpers.py
-gallery_routes.py           gallery_helpers.py
-task_routes.py              session_routes.py
-note_routes.py              memory_routes.py          research_routes.py
-mcp_routes.py               search_routes.py          history_routes.py
-webhook_routes.py           workspace_routes.py       upload_routes.py
-vault_routes.py             prefs_routes.py           preset_routes.py
-signature_routes.py         personal_routes.py        hwfit_routes.py
-backup_routes.py            cleanup_routes.py         diagnostics_routes.py
-embedding_routes.py         emoji_routes.py           font_routes.py
-stt_routes.py               tts_routes.py             compare_routes.py
-editor_draft_routes.py      chatgpt_subscription_routes.py    admin_wipe_routes.py
-```
-
-### `core/` (10 files)
-
-```
-__init__.py    constants.py    database.py    models.py
-auth.py        middleware.py   session_manager.py   exceptions.py
-atomic_io.py   platform_compat.py
-```
-
----
-
-## Appendix B: Key Import Relationships
-
-```
-core/database.py  ←── 102 importers (routes/*, src/*, core/*, tests/*)
-    ↑
-    ├── routes/auth_routes.py
-    ├── routes/email_routes.py
-    ├── src/builtin_actions.py
-    ├── src/task_scheduler.py
-    ├── src/tool_implementations.py (inline)
-    └── ...97 more
-
-src/tool_implementations.py  ←── 17 importers
-    ↑
-    ├── src/agent_loop.py
-    ├── src/builtin_actions.py
-    ├── src/tool_index.py
-    ├── src/task_scheduler.py
-    ├── src/tool_policy.py
-    └── ...12 more (mostly tests)
-
-src/agent_loop.py  ←── 22 importers
-    ↑
-    ├── src/tool_policy.py
-    ├── src/teacher_escalation.py
-    ├── src/bg_monitor.py
-    ├── src/task_scheduler.py
-    └── 18 more (incl. tests)
-```

src/agent_runs.py

@@ -17,14 +17,13 @@ close / navigation / refresh). It does NOT survive a server restart.
 import asyncio
 import json
 import logging
-import uuid
 from typing import AsyncGenerator, Dict, Optional
 
 logger = logging.getLogger(__name__)
 
 
 class _Run:
-    __slots__ = ("buffer", "subscribers", "status", "task", "evict_task", "run_id")
+    __slots__ = ("buffer", "subscribers", "status", "task", "evict_task")
 
     def __init__(self) -> None:
         self.buffer: list = []          # ordered SSE event strings (replay log)
@@ -32,9 +31,6 @@ class _Run:
         self.status: str = "running"    # running | done | error | stopped
         self.task: Optional[asyncio.Task] = None
         self.evict_task: Optional[asyncio.Task] = None
-        # Stable across every subscription/replay of this exact detached run.
-        # The browser uses it to make local cost accounting replay-idempotent.
-        self.run_id: str = uuid.uuid4().hex
 
 
 _RUNS: Dict[str, _Run] = {}
@@ -57,24 +53,13 @@ def _publish(run: _Run, ev: str) -> None:
             pass
 
 
-def _wake_run_subscribers(run: _Run) -> None:
+def _schedule_evict(session_id: str) -> None:
-    """Close subscribers even when the drain task never reached its body."""
-    for q in list(run.subscribers):
-        try:
-            q.put_nowait((None, None))
-        except Exception:
-            pass
-
-
-def _schedule_evict(session_id: str, expected_run: Optional[_Run] = None) -> None:
     """(Re)arm a grace-period eviction for a terminal run with no subscribers.
     Identity-checked so a run that gets replaced/reused is never evicted by a
     stale timer."""
     run = _RUNS.get(session_id)
     if run is None:
         return
-    if expected_run is not None and run is not expected_run:
-        return
     if run.evict_task and not run.evict_task.done():
         run.evict_task.cancel()
 
@@ -100,38 +85,25 @@ def get_status(session_id: str) -> Optional[str]:
     return r.status if r else None
 
 
-def get_run_id(session_id: str) -> Optional[str]:
+async def _drain(session_id: str, agen: AsyncGenerator[str, None],
-    """Return the opaque identity of the current detached run, if present."""
-    r = _RUNS.get(session_id)
-    return r.run_id if r else None
-
-
-def get_active_run(session_id: str) -> Optional[_Run]:
-    """Return the exact active run currently registered for a session."""
-    r = _RUNS.get(session_id)
-    return r if r and r.status == "running" else None
-
-
-async def _drain(session_id: str, run: _Run, agen: AsyncGenerator[str, None],
                  prev_task: Optional[asyncio.Task] = None) -> None:
     """Pull every event from the wrapped generator into the run buffer, fanning
     each out to live subscribers. Runs to completion regardless of subscribers."""
-    subscribers_woken = False
+    run = _RUNS.get(session_id)
-
+    if run is None:
-    def _wake_subscribers() -> None:
+        return
-        nonlocal subscribers_woken
-        if subscribers_woken:
-            return
-        subscribers_woken = True
-        _wake_run_subscribers(run)
-
     # If this run replaced an in-flight one (rapid double-send), wait for that
     # one to fully finish first. Its CancelledError handler calls aclose(), which
     # persists its partial response — letting it complete before we start writing
     # keeps the two runs' session saves sequential instead of interleaved.
-    try:
+    if prev_task is not None and not prev_task.done():
-        if prev_task is not None and not prev_task.done():
+        try:
             await asyncio.wait({prev_task})
+        except asyncio.CancelledError:
+            raise            # our own cancellation — propagate
+        except Exception:
+            pass
+    try:
         async for ev in agen:
             _publish(run, ev)
         if run.status == "running":
@@ -144,16 +116,6 @@ async def _drain(session_id: str, run: _Run, agen: AsyncGenerator[str, None],
             await agen.aclose()
         except Exception:
             pass
-        # A rapid third replacement can cancel this task while it is still
-        # waiting for its predecessor. Close this run's subscribers promptly,
-        # but keep the task alive until the predecessor finishes so the next
-        # run still observes the transitive session-save ordering barrier.
-        _wake_subscribers()
-        if prev_task is not None and not prev_task.done():
-            try:
-                await asyncio.shield(prev_task)
-            except (asyncio.CancelledError, Exception):
-                pass
     except Exception as e:
         logger.error("[agent-run] %s failed: %s", session_id, e, exc_info=True)
         run.status = "error"
@@ -165,11 +127,15 @@ async def _drain(session_id: str, run: _Run, agen: AsyncGenerator[str, None],
         _publish(run, "data: [DONE]\n\n")
     finally:
         # Wake every subscriber with the end sentinel so their SSE closes.
-        _wake_subscribers()
+        for q in list(run.subscribers):
+            try:
+                q.put_nowait((None, None))
+            except Exception:
+                pass
         # Run is terminal — arm the grace timer so it (and its buffer) is
         # eventually freed even if nobody ever reconnects. subscribe() cancels
         # this on connect and re-arms on disconnect.
-        _schedule_evict(session_id, run)
+        _schedule_evict(session_id)
 
 
 def start(session_id: str, agen: AsyncGenerator[str, None]) -> _Run:
@@ -179,37 +145,20 @@ def start(session_id: str, agen: AsyncGenerator[str, None]) -> _Run:
     prev_task: Optional[asyncio.Task] = None
     if prev:
         if prev.task and not prev.task.done():
-            # A task cancelled before its first instruction never enters
-            # _drain(), so its except/finally blocks cannot update status or
-            # wake a response already bound to this exact run. Terminalize it
-            # synchronously before cancelling; _drain's cleanup is idempotent
-            # when the task had already started.
-            if prev.status == "running":
-                prev.status = "stopped"
-                _wake_run_subscribers(prev)
             prev.task.cancel()
             prev_task = prev.task   # new run awaits this before it starts writing
         if prev.evict_task and not prev.evict_task.done():
             prev.evict_task.cancel()
     run = _Run()
     _RUNS[session_id] = run
-    run.task = asyncio.create_task(_drain(session_id, run, agen, prev_task))
+    run.task = asyncio.create_task(_drain(session_id, agen, prev_task))
     return run
 
 
-async def subscribe(
+async def subscribe(session_id: str) -> AsyncGenerator[str, None]:
-    session_id: str,
-    expected_run: Optional[_Run] = None,
-) -> AsyncGenerator[str, None]:
     """Replay the run's buffer from the start, then stream live until it ends.
-    Safe to call repeatedly (reconnect) and from multiple clients at once.
+    Safe to call repeatedly (reconnect) and from multiple clients at once."""
-
+    run = _RUNS.get(session_id)
-    ``expected_run`` binds a lazy StreamingResponse body to the same run whose
-    identity was put in its response headers. Without that binding, a rapid
-    replacement between response construction and body iteration could replay
-    the replacement run under the prior run's identity.
-    """
-    run = expected_run or _RUNS.get(session_id)
     if run is None:
         return
     q: asyncio.Queue = asyncio.Queue()
@@ -252,19 +201,12 @@ async def subscribe(
         # Last subscriber gone on a finished run — (re)arm eviction so the
         # buffer doesn't linger indefinitely.
         if not run.subscribers and run.status != "running":
-            _schedule_evict(session_id, run)
+            _schedule_evict(session_id)
 
 
-def stop(session_id: str, expected_run_id: Optional[str] = None) -> bool:
+def stop(session_id: str) -> bool:
-    """Cancel the matching in-flight run (which saves its partial output).
+    """Cancel an in-flight run (the wrapped generator saves its partial)."""
-
-    A stale browser may issue Stop after another tab has replaced the session's
-    run. Once the caller knows its opaque run identity, fail closed rather than
-    cancelling that newer run.
-    """
     run = _RUNS.get(session_id)
-    if not expected_run_id or run is None or run.run_id != expected_run_id:
-        return False
     if run and run.task and not run.task.done():
         run.task.cancel()
         return True

src/agent_tools/admin_tools.py

@@ -510,12 +510,7 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
         # set/get/list/delete operate on the REAL app settings (the same store
         # the Settings panel writes), so changing a model / voice / search
         # engine / reminder channel from chat actually takes effect.
-        from src.settings import (
+        from src.settings import load_settings, save_settings, DEFAULT_SETTINGS
-            DEFAULT_SETTINGS,
-            RETIRED_SETTING_KEYS,
-            load_settings,
-            save_settings,
-        )
 
         # Secrets/credentials the agent must NOT write: kept read-only (masked)
         # so API keys never flow through chat. User sets these in the panel.
@@ -567,9 +562,6 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
                 return k2
             return _ALIASES_SET.get(k2, (k or "").strip())
 
-        def _is_managed_key(key):
-            return key in DEFAULT_SETTINGS and key not in RETIRED_SETTING_KEYS
-
         _ENUMS = {
             "image_quality": ["low", "medium", "high"],
             "reminder_channel": ["browser", "email", "ntfy", "webhook"],
@@ -632,18 +624,14 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
 
         if action == "list":
             s = load_settings()
-            shown = {
+            shown = {k: _mask(k, v) for k, v in s.items() if k in DEFAULT_SETTINGS and not isinstance(v, dict)}
-                k: _mask(k, v)
-                for k, v in s.items()
-                if _is_managed_key(k) and not isinstance(v, dict)
-            }
             return {"response": f"{len(shown)} settings (use get/set with a key)", "settings": shown, "exit_code": 0}
 
         elif action == "get":
             key = _resolve(args.get("key", ""))
             if not key:
                 return {"error": "key is required", "exit_code": 1}
-            if not _is_managed_key(key):
+            if key not in DEFAULT_SETTINGS:
                 return {"error": f"Unknown setting '{args.get('key')}'. Use action='list' to see them.", "exit_code": 1}
             val = load_settings().get(key, DEFAULT_SETTINGS.get(key))
             return {"response": f"{key} = {_mask(key, val)}", "value": _mask(key, val), "exit_code": 0}
@@ -654,11 +642,11 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
             if not raw:
                 return {"error": "key is required", "exit_code": 1}
             key = _resolve(raw)
-            if not _is_managed_key(key):
+            if key not in DEFAULT_SETTINGS:
                 return {"error": f"Unknown setting '{raw}'. Use action='list' to see available settings.", "exit_code": 1}
             if _is_secret(key):
                 return {"response": f"'{key}' is a credential/secret. For security I can't set it from chat. Open Settings and set it there.", "exit_code": 0}
-            # Structured settings (dicts/lists like keybinds or vision fallbacks)
+            # Structured settings (dicts/lists like keybinds, default_model_fallbacks)
             # have no safe scalar coercion; _coerce would pass a bare string
             # straight through and clobber the structure. Refuse them here; they're
             # edited in their dedicated panels. (reset/delete still restore the
@@ -687,7 +675,7 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
 
         elif action == "delete" or action == "reset":
             key = _resolve(args.get("key", ""))
-            if not _is_managed_key(key):
+            if key not in DEFAULT_SETTINGS:
                 return {"error": f"Unknown setting '{args.get('key')}'.", "exit_code": 1}
             if _is_secret(key):
                 return {"response": f"'{key}' is a credential. Reset it in the panel.", "exit_code": 0}

src/agent_tools/document_tools.py

@@ -2,7 +2,6 @@ from typing import Any, Dict, List, Optional
 import logging
 import re
 from src.constants import MAX_READ_CHARS
-from src.tool_approvals import document_content_digest
 from src.tool_utils import _parse_tool_args, get_upload_handler
 from src.upload_handler import reserve_upload_references
 
@@ -81,40 +80,6 @@ def _most_recent_owned_document(db, Document, owner: Optional[str], active_only:
     return q.order_by(Document.updated_at.desc()).first()
 
 
-def _approved_document_version_error(doc: Any, ctx: dict) -> Optional[Dict]:
-    """Reject a sealed document action when its target changed meanwhile."""
-    expected_version = ctx.get("expected_document_version")
-    expected_digest = (
-        str(ctx.get("expected_document_digest") or "").strip().lower()
-    )
-    if expected_version is None and not expected_digest:
-        return None
-    try:
-        version_unchanged = (
-            expected_version is None
-            or int(getattr(doc, "version_count", -1)) == int(expected_version)
-        )
-    except (TypeError, ValueError):
-        version_unchanged = False
-    content_unchanged = True
-    if expected_digest:
-        content_unchanged = (
-            doc is not None
-            and document_content_digest(getattr(doc, "current_content", ""))
-            == expected_digest
-        )
-    if version_unchanged and content_unchanged:
-        return None
-    return {
-        "error": (
-            "The target document changed after this action was proposed. "
-            "Review the latest version and request the edit again."
-        ),
-        "exit_code": 1,
-        "document_changed": True,
-    }
-
-
 # ---------------------------------------------------------------------------
 # Document tools — create/update/edit/suggest living documents
 # ---------------------------------------------------------------------------
@@ -489,12 +454,6 @@ class UpdateDocumentTool:
             doc = None
             if target_id:
                 doc = _get_owned_document(db, Document, target_id, owner)
-            if (
-                not doc
-                and target_id
-                and ctx.get("expected_document_version") is not None
-            ):
-                return _approved_document_version_error(None, ctx)
             if not doc:
                 doc = _most_recent_owned_document(db, Document, owner)
                 if doc:
@@ -504,10 +463,6 @@ class UpdateDocumentTool:
             if not doc:
                 return {"error": "No documents exist to update"}
 
-            version_error = _approved_document_version_error(doc, ctx)
-            if version_error:
-                return version_error
-
             is_email_doc = doc.language == "email" or _looks_like_email_document(doc.current_content or "", doc.title or "")
             new_content = _coerce_email_document_content(doc.current_content or "", content) if is_email_doc else content.strip()
             if is_email_doc:
@@ -575,12 +530,6 @@ class EditDocumentTool:
             doc = None
             if target_id:
                 doc = _get_owned_document(db, Document, target_id, owner)
-            if (
-                not doc
-                and target_id
-                and ctx.get("expected_document_version") is not None
-            ):
-                return _approved_document_version_error(None, ctx)
             if not doc:
                 # Fallback: most recently updated document. Avoids "no active doc" errors
                 # after server restart or when the agent loses track of which doc to edit.
@@ -592,10 +541,6 @@ class EditDocumentTool:
             if not doc:
                 return {"error": "No documents exist to edit"}
 
-            version_error = _approved_document_version_error(doc, ctx)
-            if version_error:
-                return version_error
-
             is_email_doc = doc.language == "email" or _looks_like_email_document(doc.current_content or "", doc.title or "")
             blank_find_edits = [e for e in edits if not (e.get("find") or "").strip()]
             if blank_find_edits:
@@ -732,10 +677,6 @@ class SuggestDocumentTool:
             if not doc:
                 return {"error": f"Document {target_id} not found"}
 
-            version_error = _approved_document_version_error(doc, ctx)
-            if version_error:
-                return version_error
-
             # Validate that FIND text exists in document
             valid = []
             for s in suggestions: