mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-08-27 11:20:42 +02:00
fix(email): serialize default-account mutations (#5805)
* fix(email): serialize default account mutations * fix(email): enforce default account invariant --------- Co-authored-by: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com>
This commit is contained in:
RaresKeY
GitHub
parent
3f9633c44f
commit
93eb10d4f0
6 changed files with 1002 additions and 96 deletions
278
core/database.py
278
core/database.py
|
|
@ -5,7 +5,7 @@ from datetime import datetime, timezone
|
|||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.parse import unquote, urlparse
|
||||
from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text
|
||||
from sqlalchemy import DDL, event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, inspect, text
|
||||
from sqlalchemy.engine import Engine, make_url
|
||||
from sqlalchemy.types import TypeDecorator
|
||||
from sqlalchemy.ext.declarative import declarative_base, declared_attr
|
||||
|
|
@ -430,6 +430,93 @@ class EmailAccount(TimestampMixin, Base):
|
|||
)
|
||||
|
||||
|
||||
class EmailAccountOwnerLock(Base):
|
||||
"""Durable per-owner mutex for email-account default mutations.
|
||||
|
||||
Row-locking databases serialize mutations by locking this row before they
|
||||
inspect or stage EmailAccount changes. SQLite uses ``BEGIN IMMEDIATE``
|
||||
instead, because it ignores ``SELECT ... FOR UPDATE``; keeping the table in
|
||||
the shared metadata still makes the non-SQLite path available without a
|
||||
separate migration. The empty key represents the normalized legacy /
|
||||
unconfigured scope shared by ``owner IS NULL`` and ``owner = ''`` rows.
|
||||
"""
|
||||
__tablename__ = "email_account_owner_locks"
|
||||
|
||||
owner_key = Column(String, primary_key=True)
|
||||
|
||||
|
||||
_EMAIL_ACCOUNT_DEFAULT_INDEX = "ux_email_accounts_one_default_per_owner"
|
||||
_EMAIL_ACCOUNT_DEFAULT_INDEX_DDL = {
|
||||
"sqlite": (
|
||||
f"CREATE UNIQUE INDEX IF NOT EXISTS {_EMAIL_ACCOUNT_DEFAULT_INDEX} "
|
||||
"ON email_accounts (COALESCE(owner, '')) WHERE is_default = 1"
|
||||
),
|
||||
"postgresql": (
|
||||
f"CREATE UNIQUE INDEX IF NOT EXISTS {_EMAIL_ACCOUNT_DEFAULT_INDEX} "
|
||||
"ON email_accounts ((COALESCE(owner, ''))) WHERE is_default IS TRUE"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# SQLAlchemy cannot express one portable partial, functional index across the
|
||||
# two supported database families. Register dialect-specific DDL so fresh
|
||||
# databases get the invariant as part of create_all(); the startup migration
|
||||
# below installs the same index on existing databases after normalizing legacy
|
||||
# duplicate rows.
|
||||
for _dialect_name, _index_ddl in _EMAIL_ACCOUNT_DEFAULT_INDEX_DDL.items():
|
||||
event.listen(
|
||||
EmailAccount.__table__,
|
||||
"after_create",
|
||||
DDL(_index_ddl).execute_if(dialect=_dialect_name),
|
||||
)
|
||||
|
||||
|
||||
def lock_email_account_owner_mutations(db, *owners: str) -> None:
|
||||
"""Lock normalized email-account owner scopes in canonical order.
|
||||
|
||||
``NULL`` and the empty string are one legacy/single-user owner partition,
|
||||
matching the unique default-account index. SQLite has only a database
|
||||
writer reservation, while row-locking databases use durable mutex rows.
|
||||
Sorting all requested owner keys keeps multi-owner operations such as user
|
||||
rename from deadlocking with another mutation that requests the same keys
|
||||
in the opposite order.
|
||||
"""
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
owner_keys = sorted({owner or "" for owner in owners} or {""})
|
||||
if db.get_bind().dialect.name == "sqlite":
|
||||
db.execute(text("BEGIN IMMEDIATE"))
|
||||
return
|
||||
|
||||
for owner_key in owner_keys:
|
||||
lock_row = db.get(
|
||||
EmailAccountOwnerLock,
|
||||
owner_key,
|
||||
with_for_update=True,
|
||||
)
|
||||
if lock_row is not None:
|
||||
continue
|
||||
|
||||
inserted = False
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.add(EmailAccountOwnerLock(owner_key=owner_key))
|
||||
db.flush()
|
||||
inserted = True
|
||||
except IntegrityError:
|
||||
# A competing transaction created the mutex row first. Once its
|
||||
# insert commits, lock that durable row before touching accounts.
|
||||
pass
|
||||
|
||||
if not inserted:
|
||||
(
|
||||
db.query(EmailAccountOwnerLock)
|
||||
.filter(EmailAccountOwnerLock.owner_key == owner_key)
|
||||
.with_for_update()
|
||||
.one()
|
||||
)
|
||||
|
||||
|
||||
class ModelEndpoint(TimestampMixin, Base):
|
||||
"""Admin-configured model endpoints. Models are auto-discovered via /v1/models."""
|
||||
__tablename__ = "model_endpoints"
|
||||
|
|
@ -1812,72 +1899,142 @@ class Integration(TimestampMixin, Base):
|
|||
|
||||
|
||||
|
||||
def _migrate_seed_email_account():
|
||||
"""If email_accounts is empty and settings.json has legacy flat imap_host/smtp_host
|
||||
keys, create a single default account from them so nothing breaks for users who
|
||||
upgraded. Safe to run repeatedly — it short-circuits once any row exists."""
|
||||
def _migrate_email_account_default_invariant():
|
||||
"""Normalize legacy duplicates and install durable at-most-one enforcement.
|
||||
|
||||
Older databases only had a non-unique ``(owner, is_default)`` lookup index.
|
||||
Keep the oldest default deterministically in each normalized owner scope,
|
||||
then add the same partial functional unique index used for fresh schemas.
|
||||
"""
|
||||
dialect_name = engine.dialect.name
|
||||
index_ddl = _EMAIL_ACCOUNT_DEFAULT_INDEX_DDL.get(dialect_name)
|
||||
if index_ddl is None:
|
||||
logger.warning(
|
||||
"Email-account default uniqueness is not available for database "
|
||||
"dialect %s; mutations remain serialized but are not protected by "
|
||||
"a database constraint",
|
||||
dialect_name,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
tables = [r[0] for r in conn.execute(text(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='email_accounts'"
|
||||
))]
|
||||
if "email_accounts" not in tables:
|
||||
return
|
||||
existing = conn.execute(text("SELECT COUNT(*) FROM email_accounts")).scalar() or 0
|
||||
if existing > 0:
|
||||
with engine.begin() as conn:
|
||||
if not inspect(conn).has_table(EmailAccount.__tablename__):
|
||||
return
|
||||
default_rows = conn.execute(text("""
|
||||
SELECT id, owner
|
||||
FROM email_accounts
|
||||
WHERE is_default IS TRUE
|
||||
ORDER BY
|
||||
COALESCE(owner, ''),
|
||||
CASE WHEN created_at IS NULL THEN 1 ELSE 0 END,
|
||||
created_at,
|
||||
id
|
||||
""")).mappings()
|
||||
seen_owner_keys = set()
|
||||
duplicate_ids = []
|
||||
for row in default_rows:
|
||||
owner_key = row["owner"] or ""
|
||||
if owner_key in seen_owner_keys:
|
||||
duplicate_ids.append(row["id"])
|
||||
else:
|
||||
seen_owner_keys.add(owner_key)
|
||||
|
||||
import json as _json
|
||||
import uuid as _uuid
|
||||
from pathlib import Path
|
||||
settings_file = Path(SETTINGS_FILE)
|
||||
if not settings_file.exists():
|
||||
return
|
||||
try:
|
||||
s = _json.loads(settings_file.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return
|
||||
for account_id in duplicate_ids:
|
||||
conn.execute(
|
||||
text("UPDATE email_accounts SET is_default = :value WHERE id = :id"),
|
||||
{"value": False, "id": account_id},
|
||||
)
|
||||
conn.execute(text(index_ddl))
|
||||
|
||||
imap_host = (s.get("imap_host") or "").strip()
|
||||
smtp_host = (s.get("smtp_host") or "").strip()
|
||||
if not imap_host and not smtp_host:
|
||||
return # nothing to migrate
|
||||
if duplicate_ids:
|
||||
logger.warning(
|
||||
"Normalized %d duplicate default email account(s) before "
|
||||
"installing %s",
|
||||
len(duplicate_ids),
|
||||
_EMAIL_ACCOUNT_DEFAULT_INDEX,
|
||||
)
|
||||
except Exception:
|
||||
# Starting without the constraint would silently retain the race this
|
||||
# migration is intended to close. Fail startup so an operator sees and
|
||||
# can repair an incompatible schema instead of accepting unsafe writes.
|
||||
logger.exception("Failed to enforce the email-account default invariant")
|
||||
raise
|
||||
|
||||
|
||||
def _migrate_seed_email_account():
|
||||
"""Atomically seed one legacy default account when no account exists.
|
||||
|
||||
Reading settings is intentionally done before taking the owner mutex. The
|
||||
decisive emptiness check and insert share one locked transaction, so two
|
||||
application workers starting together cannot both seed a default row.
|
||||
"""
|
||||
import json as _json
|
||||
import uuid as _uuid
|
||||
|
||||
settings_file = Path(SETTINGS_FILE)
|
||||
if not settings_file.exists():
|
||||
return
|
||||
try:
|
||||
s = _json.loads(settings_file.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return
|
||||
|
||||
imap_host = (s.get("imap_host") or "").strip()
|
||||
smtp_host = (s.get("smtp_host") or "").strip()
|
||||
if not imap_host and not smtp_host:
|
||||
return
|
||||
|
||||
db = None
|
||||
try:
|
||||
if not inspect(engine).has_table(EmailAccount.__tablename__):
|
||||
return
|
||||
db = SessionLocal()
|
||||
lock_email_account_owner_mutations(db, "")
|
||||
existing = db.execute(text("SELECT COUNT(*) FROM email_accounts")).scalar() or 0
|
||||
if existing > 0:
|
||||
return
|
||||
|
||||
now = utcnow_naive()
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("""
|
||||
INSERT INTO email_accounts
|
||||
(id, owner, name, is_default, enabled,
|
||||
imap_host, imap_port, imap_user, imap_password, imap_starttls,
|
||||
smtp_host, smtp_port, smtp_user, smtp_password,
|
||||
from_address, created_at, updated_at)
|
||||
VALUES
|
||||
(:id, :owner, :name, :is_default, :enabled,
|
||||
:imap_host, :imap_port, :imap_user, :imap_password, :imap_starttls,
|
||||
:smtp_host, :smtp_port, :smtp_user, :smtp_password,
|
||||
:from_address, :created_at, :updated_at)
|
||||
"""), {
|
||||
"id": _uuid.uuid4().hex,
|
||||
"owner": None,
|
||||
"name": "Default",
|
||||
"is_default": True,
|
||||
"enabled": True,
|
||||
"imap_host": imap_host,
|
||||
"imap_port": int(s.get("imap_port") or 993),
|
||||
"imap_user": s.get("imap_user") or "",
|
||||
"imap_password": s.get("imap_password") or "",
|
||||
"imap_starttls": bool(s.get("imap_starttls", True)),
|
||||
"smtp_host": smtp_host,
|
||||
"smtp_port": int(s.get("smtp_port") or 465),
|
||||
"smtp_user": s.get("smtp_user") or "",
|
||||
"smtp_password": s.get("smtp_password") or "",
|
||||
"from_address": s.get("email_from") or "",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
})
|
||||
logging.getLogger(__name__).info("Seeded email_accounts 'Default' from settings.json")
|
||||
db.execute(text("""
|
||||
INSERT INTO email_accounts
|
||||
(id, owner, name, is_default, enabled,
|
||||
imap_host, imap_port, imap_user, imap_password, imap_starttls,
|
||||
smtp_host, smtp_port, smtp_user, smtp_password,
|
||||
from_address, created_at, updated_at)
|
||||
VALUES
|
||||
(:id, :owner, :name, :is_default, :enabled,
|
||||
:imap_host, :imap_port, :imap_user, :imap_password, :imap_starttls,
|
||||
:smtp_host, :smtp_port, :smtp_user, :smtp_password,
|
||||
:from_address, :created_at, :updated_at)
|
||||
"""), {
|
||||
"id": _uuid.uuid4().hex,
|
||||
"owner": None,
|
||||
"name": "Default",
|
||||
"is_default": True,
|
||||
"enabled": True,
|
||||
"imap_host": imap_host,
|
||||
"imap_port": int(s.get("imap_port") or 993),
|
||||
"imap_user": s.get("imap_user") or "",
|
||||
"imap_password": s.get("imap_password") or "",
|
||||
"imap_starttls": bool(s.get("imap_starttls", True)),
|
||||
"smtp_host": smtp_host,
|
||||
"smtp_port": int(s.get("smtp_port") or 465),
|
||||
"smtp_user": s.get("smtp_user") or "",
|
||||
"smtp_password": s.get("smtp_password") or "",
|
||||
"from_address": s.get("email_from") or "",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
})
|
||||
db.commit()
|
||||
logger.info("Seeded email_accounts 'Default' from settings.json")
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).warning(f"seed email account migration: {e}")
|
||||
if db is not None:
|
||||
db.rollback()
|
||||
logger.warning("seed email account migration: %s", e)
|
||||
finally:
|
||||
if db is not None:
|
||||
db.close()
|
||||
|
||||
|
||||
# WARNING: Foreign-key enforcement is enabled globally for all SQLite connections.
|
||||
|
|
@ -1960,6 +2117,7 @@ def init_db():
|
|||
_migrate_add_crew_member_id()
|
||||
_migrate_add_assistant_columns()
|
||||
_migrate_add_email_smtp_security()
|
||||
_migrate_email_account_default_invariant()
|
||||
_migrate_seed_email_account()
|
||||
_migrate_add_calendar_metadata()
|
||||
_migrate_add_calendar_is_utc()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue