yilsem/aiturk-hermes-ide
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 3 Wiki Activity
Files
aiturk-ide-1.0.0-beta.1
aiturk-hermes-ide/hermes_state.py
T

17371 lines
775 KiB
Python
Raw Permalink Blame History

#!/usr/bin/env python3
"""
SQLite State Store for Hermes Agent.
Provides persistent session storage with FTS5 full-text search, replacing
the per-session JSONL file approach. Stores session metadata, full message
history, and model configuration for CLI and gateway sessions.
Key design decisions:
- WAL mode for concurrent readers + one writer (gateway multi-platform)
- FTS5 virtual table for fast text search across all session messages
- Compression-triggered session splitting via parent_session_id chains
- Batch runner and RL trajectories are NOT stored here (separate systems)
- Session source tagging ('cli', 'telegram', 'discord', etc.) for filtering
"""
import asyncio
import atexit
import contextlib
import errno
import hashlib
import json
import logging
import os
import queue
import random
import re
import sqlite3
import struct
import sys
import threading
import time
import uuid
import weakref
from collections import deque
from contextlib import contextmanager
from pathlib import Path
from agent.memory_manager import sanitize_context
from agent.session_activity import ActivityProvenance
from agent.message_sanitization import _sanitize_surrogates
# Intrinsic persistence marker stamped on message dicts that are known-durable
# (#92231). One shared constant with agent.context_compressor (this module
# already imports agent.* at module level, and context_compressor is a
# transitive dependency via hermes_state_common). run_agent keeps its own
# predating copy — hermes_state cannot import run_agent (circular) — guarded
# by test_marker_constant_in_sync.
from agent.context_compressor import (
_DB_PERSISTED_MARKER as _DB_PERSISTED_MARKER_KEY,
)
from agent.skill_commands import (
SKILL_EXCERPT_JOINT,
SKILL_SCAFFOLD_SQL_LIKE,
describe_skill_invocation,
)
from hermes_constants import get_hermes_home
from hermes_startup_watchdog import report_startup_progress
from hermes_cli.sqlite_runtime import (
is_sqlite_wal_reset_vulnerable as _is_sqlite_wal_reset_vulnerable,
)
from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, TypeVar, cast
import hermes_state_holders as _state_holders
from hermes_state_common import ( # noqa: F401 (re-exported for back-compat)
_BRANCH_CHILD_SQL,
_COMPRESSION_CHILD_SQL,
_FTS_CJK_TRIGGERS,
_FTS_TRIGGERS,
_LISTABLE_CHILD_SQL,
_PREVIEW_ELIGIBLE_SQL,
_PREVIEW_RAW_SELECT,
_RECOVERABLE_END_REASONS,
_RECOVERABLE_END_REASONS_SQL,
is_automatic_end_reason,
_RESET_END_REASONS,
_RESET_END_REASONS_SQL,
_ephemeral_child_sql,
_legacy_reset_child_sql,
_shape_preview,
_sql_session_last_active,
_sql_session_last_active_by_id,
escape_like as _escape_like,
DEFERRED_INDEX_SQL,
FTS_CJK_STALE_KEY,
FTS_REBUILD_DEFERRAL_KEY,
FTS_SQL,
FTS_STALE_KEY,
FTS_STORAGE_VERSION,
FTS_TRIGRAM_SQL,
LEGACY_FTS_SQL,
LEGACY_FTS_TRIGRAM_SQL,
MAX_FTS5_QUERY_CHARS,
SCHEMA_SQL,
SCHEMA_VERSION,
_PREVIEW_CONTENT_SQL,
_PREVIEW_HEAD_CHARS,
_PREVIEW_MAX_CHARS,
_PREVIEW_SCAFFOLD_WINDOW,
_PREVIEW_SCAFFOLDED_SQL,
_acquire_db_flock,
_clear_lock_holder_record,
_describe_lock_holder,
_read_lock_holder_record,
is_advisory_lock_contention,
)
from hermes_state_portability import SessionPortabilityMixin
from hermes_state_schema import SessionSchemaMixin
from hermes_state_search import SessionSearchMixin
try: # Hard dependency, but tolerate scaffold-phase imports before pip install.
import psutil
except ImportError: # pragma: no cover - stripped/scaffold installs only
psutil = None # type: ignore[assignment]
logger = logging.getLogger(__name__)
MAX_SAFE_RESUME_MESSAGES = 20_000
MAX_SAFE_EXPORT_MESSAGES = 20_000
# Auto-maintenance only VACUUMs when at least this fraction of the database
# file is reclaimable (``PRAGMA freelist_count / PRAGMA page_count``). Below
# it a full rewrite costs more I/O than it returns — pruning a handful of small
# sessions on a dense multi-GB state.db should never rewrite the whole file to
# reclaim a few MB (#54189). Composes with ``min_vacuum_interval_days``.
AUTO_VACUUM_MIN_FREELIST_RATIO = 0.25
def _configured_transcript_limit(key: str, fallback: int) -> int:
"""Resolve a transcript safety limit from config at call time.
Reads ``sessions.<key>`` from config.yaml lazily (avoiding a circular
import at module load) and falls back to the module constant when the
config subsystem is unavailable (scaffold installs, stripped test
environments). A value of 0 disables the guard entirely. No caching:
``load_config_readonly`` is already mtime-cached, and resolving fresh
keeps tests that monkeypatch config or the module constants working.
"""
try:
from hermes_cli.config import load_config_readonly
sessions_cfg = load_config_readonly().get("sessions") or {}
value = sessions_cfg.get(key)
if value is None:
return fallback
limit = int(value)
return limit if limit >= 0 else fallback
except Exception:
return fallback
def resolved_max_resume_messages() -> int:
"""Config-resolved resume guard limit (0 disables the guard)."""
return _configured_transcript_limit(
"max_resume_messages", MAX_SAFE_RESUME_MESSAGES
)
def resolved_max_export_messages() -> int:
"""Config-resolved in-memory export guard limit (0 disables the guard)."""
return _configured_transcript_limit(
"max_export_messages", MAX_SAFE_EXPORT_MESSAGES
)
class SessionResumeTooLargeError(ValueError):
def __init__(
self,
message_count: int,
limit: int = MAX_SAFE_RESUME_MESSAGES,
scope: str = "across its lineage",
):
self.message_count = message_count
self.limit = limit
super().__init__(
f"session has at least {message_count} active messages {scope}; "
f"safe resume limit is {limit}. Export the session instead, or set "
"sessions.max_resume_messages: 0 in config.yaml to disable the guard."
)
class SessionExportTooLargeError(ValueError):
def __init__(
self,
session_id: str,
message_count: int,
limit: int = MAX_SAFE_EXPORT_MESSAGES,
):
self.session_id = session_id
self.message_count = message_count
self.limit = limit
super().__init__(
f"session '{session_id}' has at least {message_count} active messages; "
f"safe in-memory export limit is {limit}"
)
_COMPRESSION_LOCK_HOLDER_PID_RE = re.compile(r"(?:^|:)pid=(\d+)(?::|$)")
def _system_prompt_hash(system_prompt: str) -> str:
return hashlib.sha256(system_prompt.encode("utf-8")).hexdigest()
def _compression_lock_holder_process_is_dead(holder: str) -> bool:
"""Return True only when a structured lock holder's local PID is gone.
Compression locks are stored in a host-local SQLite database and holder
IDs created by ``conversation_compression`` start with ``pid=<n>``. A
process killed during gateway shutdown cannot release its lease, so waiting
for the full TTL makes every new turn repeatedly attempt compaction. Reclaim
only when the kernel proves that PID no longer exists; legacy/unstructured
holders, same-process holders, permission errors, and any probe doubt
remain protected until normal TTL expiry (conservative: PID reuse must
never steal a live lease, and a wrongly-kept lease self-heals via TTL).
"""
match = _COMPRESSION_LOCK_HOLDER_PID_RE.search(holder or "")
if match is None:
return False
try:
pid = int(match.group(1))
except (TypeError, ValueError):
return False
if pid <= 0:
return False
if pid == os.getpid():
# Same-process holder (e.g. another thread's live lease): never
# self-reclaim — the lease refresher and release path own it.
return False
if psutil is not None:
try:
# psutil is the canonical cross-platform liveness answer
# (CONTRIBUTING.md "Critical rules" #1). pid_exists() reports
# recycled PIDs as alive — conservative, the TTL still applies.
return not psutil.pid_exists(pid)
except Exception:
return False # any doubt → keep the lease until TTL expiry
# Scaffold-phase fallback only (psutil missing), and POSIX-only: stdlib
# os.kill(pid, 0) is NOT a no-op probe on Windows (bpo-14484 — sig=0 maps
# to CTRL_C_EVENT and can kill the target's console group). Without psutil
# a Windows host stays TTL-only; the lease TTL remains the recovery path.
if os.name == "nt":
return False
try:
os.kill(pid, 0) # windows-footgun: ok — nt early-returns just above
except ProcessLookupError:
return True
except (PermissionError, OSError, OverflowError):
return False
return False
def _scrub_surrogates(value: Any) -> Any:
"""Replace lone surrogates when *value* is text; pass anything else through.
sqlite3 encodes bound ``str`` parameters as UTF-8 and raises
``UnicodeEncodeError`` on lone surrogates (U+D800..U+DFFF), so a single
such code point anywhere in a message aborts the whole write. No-op for
well-formed text.
"""
return _sanitize_surrogates(value) if isinstance(value, str) else value
def workspace_key(row: Dict[str, Any]) -> Optional[str]:
"""A session's workspace grouping key: its git repo root when known, else
its cwd.
Branch is deliberately excluded so checking out a new branch doesn't
fragment a workspace's session history. Returns None for cwd-less (unbound)
sessions. Both fields are already recorded on ``sessions`` — this just picks
the coarser identity for grouping/filtering.
"""
root = (row.get("git_repo_root") or "").strip()
if root:
return root
cwd = (row.get("cwd") or "").strip()
return cwd or None
def _delegate_from_json(col: str = "model_config") -> str:
return f"json_extract(COALESCE({col}, '{{}}'), '$._delegate_from')"
# Sentinel returned by SessionDB._merge_model_config_json when the session row
# doesn't exist and on_missing="skip" — distinguishes "no row" from the legal
# None result ("merged config is empty → store NULL").
_MODEL_CONFIG_ROW_MISSING = object()
# Billing-bucket classes that aren't a routable provider identity on their
# own — used by session_gateway_runtime's billing_provider fallback and by
# tui_gateway.server._stored_session_runtime_overrides. A session that
# persisted only one of these (never ran /model) must fall back to the
# ambient config default rather than restore a bare bucket. Shared here so
# both consumers stay in sync (previously duplicated as a set in
# tui_gateway/server.py).
_BARE_BILLING_PROVIDERS = frozenset({"auto", "custom"})
def _cwd_prefix_clause(cwd_prefix: str) -> Tuple[str, List[str]]:
prefix = cwd_prefix.rstrip("/\\") or cwd_prefix
# ``_`` and ``%`` are LIKE wildcards but ordinary characters in a path
# (``my_project``), so an unescaped prefix also matches sibling directories.
# Escape the needle and pair it with ESCAPE; the literal separator
# backslash in the Windows pattern needs escaping for the same reason. The
# ``=`` arm is an exact compare and keeps the raw prefix.
esc = _escape_like(prefix)
return (
"(s.cwd = ? OR s.cwd LIKE ? ESCAPE '\\' OR s.cwd LIKE ? ESCAPE '\\')",
[prefix, f"{esc}/%", f"{esc}\\\\%"],
)
def _workspace_key_clause(key: str) -> Tuple[str, List[str]]:
"""Match sessions whose ``workspace_key(row)`` equals ``key``.
Mirrors :func:`workspace_key`: a session belongs to workspace ``key``
when its recorded ``git_repo_root`` equals ``key``, or — for rows that
predate per-session git metadata — when its ``cwd`` is at or under
``key`` (so a session started in ``repo/src`` still groups with ``repo``).
Used by ``hermes -c``/``--resume`` to continue the most recent session in
the *current* workspace rather than the global MRU.
"""
prefix = key.rstrip("/\\") or key
cwd_clause, cwd_params = _cwd_prefix_clause(prefix)
return (
f"(s.git_repo_root = ? OR (COALESCE(s.git_repo_root, '') = '' AND {cwd_clause}))",
[prefix, *cwd_params],
)
def _collect_delegate_child_ids(conn, parent_ids: List[str]) -> List[str]:
"""Delegate-subagent ids to cascade-delete with *parent_ids*.
Only rows carrying the ``_delegate_from`` marker (set at creation, and
backfilled by the v16 migration) — generic untagged children keep the
orphan-don't-delete contract. Walks marker chains recursively so an
orchestrator subagent's own delegate children go too (FK safety).
"""
df = _delegate_from_json()
seeds = {sid for sid in parent_ids if sid}
# Seed the visited set with the parents themselves. A delegation marker
# chain can loop back onto a parent — a cycle, or a parent that is also
# another parent's delegate child when several ids are deleted at once —
# and without this guard that parent would be collected as one of its own
# descendants and cascade-deleted along with all of its messages. Callers
# delete the parents separately, so parents must never appear in the
# returned child set. (#49148)
found: set[str] = set(seeds)
frontier = list(seeds)
while frontier:
ph = ",".join("?" * len(frontier))
cursor = conn.execute(
f"SELECT id FROM sessions WHERE {df} IN ({ph}) "
f"OR (parent_session_id IN ({ph}) AND {df} IS NOT NULL)",
frontier + frontier,
)
frontier = [row["id"] for row in cursor.fetchall() if row["id"] not in found]
found.update(frontier)
# Return only the discovered children — never the parents themselves.
return [sid for sid in found if sid not in seeds]
def _delete_delegate_children(conn, parent_ids: List[str]) -> List[str]:
ids = _collect_delegate_child_ids(conn, parent_ids)
if ids:
ph = ",".join("?" * len(ids))
conn.execute(f"DELETE FROM messages WHERE session_id IN ({ph})", ids)
# FK safety: orphan any untagged stragglers pointing at a doomed row.
conn.execute(
f"UPDATE sessions SET parent_session_id = NULL "
f"WHERE parent_session_id IN ({ph})",
ids,
)
conn.execute(f"DELETE FROM sessions WHERE id IN ({ph})", ids)
return ids
T = TypeVar("T")
DEFAULT_DB_PATH = get_hermes_home() / "state.db"
# How long SessionDB stops attempting read-only opens after one fails, before
# probing again. Long enough that a genuinely unreadable file isn't retried per
# query; short enough that transient fd pressure doesn't strand the read pool.
_READ_OPEN_RETRY_SECONDS = 60.0
# Transient SQLITE_IOERR retry budget for READ-ONLY opens (#100436). A WAL
# database being actively written (checkpoint, WAL reset/truncate, frame
# flush) can surface "disk I/O error" to a concurrent ``mode=ro`` reader in
# a millisecond-wide transition window: the read-only connection cannot
# perform the WAL recovery a read through a stale or mid-update -shm file
# needs, because recovery requires writing the -shm index, which mode=ro
# refuses. The window closes on its own (the writer finishes the transition),
# so a bounded number of short retries makes the open succeed instead of
# 500-ing the whole /api/sessions poll (or any other read-only opener).
# Deliberately NOT attempted on writable opens: a writer owns the
# transition, so an IOERR there means a real storage/fd problem.
_READ_ONLY_IOERR_RETRY_ATTEMPTS = 3
_READ_ONLY_IOERR_RETRY_BACKOFF_S = 0.05
# Hard ceiling on read-only connections ALIVE at once against one database
# FILE — pooled idle ones and checked-out ones together, summed over every
# SessionDB in this process that points at that file. See _PathReadBudget.
#
# Deliberately one constant for both the pool's maxsize and the permit count,
# because bounding only the pool bounds the wrong thing. A LifoQueue caps how
# many connections are *returned*; it says nothing about how many are *open*.
# With an open-on-miss checkout, N readers arriving on an empty pool all miss,
# all open, and peak at N — the surplus is closed on release, so nothing
# accumulates forever, but EMFILE is a peak-instant condition and the burst
# that empties the pool is exactly the burst that exhausts the fd table.
#
# So a connection holds a permit for its whole lifetime: acquired in
# _get_read_conn() before the open, released in _close_read_conn() after the
# close. Once permits are gone the read path degrades to the locked writer
# connection instead of opening more descriptors — slower under load, which is
# the correct trade against a process-wide wedge the supervisor cannot see.
_READ_POOL_MAX = 8
# Hard ceiling on read-only connections ALIVE at once in this PROCESS, across
# every state.db it has open.
#
# _READ_POOL_MAX bounds one file. A multiplexed gateway serves N profiles from
# one process and each profile has its OWN state.db, so a per-file ceiling
# still lets the descriptor cost grow with the profile count — the same shape
# as the per-instance bug, one level out (#98573).
#
# Three profiles' worth. Past it, readers on the (N+1)th file degrade to their
# writer connection instead of opening descriptors, which is the same trade
# _READ_POOL_MAX makes and for the same reason: a slow read path is
# recoverable, a process-wide EMFILE is not.
_READ_POOL_PROCESS_MAX = 24
# Warn when one process accumulates more than this many SessionDB handles on a
# single file. Not a limit — writer connections cannot be rationed the way read
# connections can — a diagnostic for the duplicate-handle class of bug.
_HANDLES_PER_PATH_WARN = 4
# Descriptors kept in reserve for everything that is NOT this module: httpx
# sockets, terminal subprocess pipes, log files.
#
# The ceilings above bound Hermes's SQLite descriptors, which is only ever part
# of the fd table. The #98573 report is exactly that case: ~20 state.db
# descriptors were not the whole 256, they were the share that pushed httpx and
# terminal pipes over, and the EMFILE surfaced in tools/terminal_tool.py rather
# than here. So the read pool also yields when the PROCESS is close to its
# limit, whatever is consuming it.
_FD_HEADROOM_RESERVE = 64
# The fd count is a directory listing; cache it briefly so a burst of reads
# does not turn one syscall per query. Stale by at most this long, which can
# let through at most the ceiling's worth of opens — already bounded above.
_FD_USAGE_CACHE_SECONDS = 0.25
_process_read_permits = threading.BoundedSemaphore(_READ_POOL_PROCESS_MAX)
# Count of read opens refused because the process was low on descriptors. The
# only externally visible signal that the guard is firing; guarded by
# _read_budgets_lock.
_read_open_denied_fd_headroom = 0
_fd_usage_lock = threading.Lock()
_fd_usage_cache: "tuple[float, Optional[int]]" = (0.0, None)
def _open_fd_count() -> Optional[int]:
"""Descriptors open in THIS process, or None when it cannot be measured.
``/proc/self/fd`` on Linux, ``/dev/fd`` on macOS and the BSDs. Windows has
neither, and no RLIMIT_NOFILE to compare against, so the guard is inert
there — which is correct: the CRT limit is thousands of handles, not 256.
"""
for fd_dir in ("/proc/self/fd", "/dev/fd"):
try:
return len(os.listdir(fd_dir))
except OSError as exc:
if exc.errno in (errno.EMFILE, errno.ENFILE):
# The probe itself could not get a descriptor. That IS the
# answer: there is no headroom.
return -1
continue
return None
def _fd_soft_limit() -> Optional[int]:
"""The process's soft RLIMIT_NOFILE, or None when there is no usable one."""
try:
import resource
except ImportError:
return None
try:
soft, _hard = resource.getrlimit(resource.RLIMIT_NOFILE)
except (OSError, ValueError):
return None
if soft in (resource.RLIM_INFINITY, -1):
return None
return int(soft)
def _fd_headroom_ok() -> bool:
"""Whether the process can spare a descriptor for a new read connection.
Fails OPEN when the platform cannot be measured (Windows, no fd directory,
unlimited RLIMIT_NOFILE): an unmeasurable platform is not a tight one, and
refusing every read there would be a self-inflicted convoy. Fails CLOSED
only on evidence — a measured shortfall, or a probe that could not get a
descriptor of its own.
"""
soft = _fd_soft_limit()
if soft is None:
return True
global _fd_usage_cache
now = time.monotonic()
with _fd_usage_lock:
stamp, cached = _fd_usage_cache
fresh = cached is not None and (now - stamp) < _FD_USAGE_CACHE_SECONDS
if not fresh:
cached = _open_fd_count()
with _fd_usage_lock:
_fd_usage_cache = (now, cached)
if cached is None:
return True
if cached < 0:
return False
return (soft - cached) > _FD_HEADROOM_RESERVE
def _reclaim_idle_read_conn_anywhere() -> bool:
"""Close one idle read connection on ANY path in this process.
The process ceiling is shared across files, so the connection that has to
go to make room may belong to a different database entirely — a profile
that has been quiet for an hour should not hold descriptors the profile
being served right now needs.
"""
with _read_budgets_lock:
budgets = list(_read_budgets.values())
for budget in budgets:
if budget.reclaim_idle():
return True
return False
class _PathReadBudget:
"""The read-connection permits for ONE database file, shared process-wide.
``_READ_POOL_MAX`` used to be enforced by a ``BoundedSemaphore`` owned by
each SessionDB, which bounded the wrong noun: the descriptors are spent on
a *file*, so N SessionDB objects on one state.db each got their own
allowance and peak scaled as ``N x (1 + _READ_POOL_MAX)``. A long-lived
gateway holds at least two (``SessionStore`` and ``GatewayRunner`` open
independent handles per profile path) and the count grows with the profile
count, which is how a healthy process walked into EMFILE — #98573.
Holding the permits here instead makes the ceiling mean what its docstring
always claimed: read connections ALIVE at once against this path.
One consequence has to be handled rather than documented away. A pooled
idle connection keeps its permit, so the first instance to warm up would
otherwise pin all eight and every later instance — a cron job's transient
handle, a second profile's store — would be permanently demoted to the
locked writer connection. That is why a permit miss first reclaims an
IDLE connection from a peer instance on the same path: idle descriptors
are transferable, in-use ones are not.
"""
def __init__(self) -> None:
self.permits = threading.BoundedSemaphore(_READ_POOL_MAX)
self._lock = threading.Lock()
# Weak so a SessionDB that is dropped without close() cannot pin its
# peers' budget object; __del__ still runs close() and returns the
# permits.
self._members: "weakref.WeakSet[SessionDB]" = weakref.WeakSet()
self._duplicate_handles_warned = False
def register(self, db: "SessionDB") -> None:
with self._lock:
self._members.add(db)
handles = len(self._members)
warn = (
handles > _HANDLES_PER_PATH_WARN
and not self._duplicate_handles_warned
)
if warn:
self._duplicate_handles_warned = True
if warn:
# The read connections are capped; the WRITER connection each
# handle holds is not, and cannot be — a SessionDB without one
# cannot write. The only real bound on writers is not opening
# redundant handles in the first place (which is what
# GatewayRunner borrowing SessionStore's handle does, #98573), so
# the next duplicate should be visible before it becomes an
# incident rather than inferred from an lsof after one.
logger.warning(
"%d live SessionDB handles on %s in this process; each holds "
"its own writer connection (read connections are capped at %d "
"for the file). A long-lived process should share one handle "
"per path.",
handles,
db.db_path,
_READ_POOL_MAX,
)
def acquire(self, requester: "SessionDB") -> bool:
"""Take a permit for a new read connection, or refuse.
Three gates, broadest first: the process's descriptor headroom, the
process-wide read ceiling, then this file's ceiling. Refusing means
the caller serves the read from the locked writer connection — slower,
never an error.
"""
if not _fd_headroom_ok():
global _read_open_denied_fd_headroom
with _read_budgets_lock:
_read_open_denied_fd_headroom += 1
return False
if not self._acquire_process_permit():
return False
if self._acquire_path_permit(requester):
return True
_process_read_permits.release()
return False
def release(self) -> None:
"""Return one connection's permits. Pairs with a successful acquire()."""
self.permits.release()
_process_read_permits.release()
def _acquire_process_permit(self) -> bool:
if _process_read_permits.acquire(blocking=False):
return True
if not _reclaim_idle_read_conn_anywhere():
return False
# Another thread may take the freed permit first; that is a legitimate
# loss, and the caller degrades to the writer lock rather than looping.
return _process_read_permits.acquire(blocking=False)
def _acquire_path_permit(self, requester: "SessionDB") -> bool:
if self.permits.acquire(blocking=False):
return True
if not self.reclaim_idle(exclude=requester):
return False
return self.permits.acquire(blocking=False)
def reclaim_idle(self, exclude: "Optional[SessionDB]" = None) -> bool:
"""Close one idle pooled connection held by a member. True if one went.
Closing it runs release(), which returns both the path permit and the
process permit, so this is the single reclaim primitive both ceilings
use.
"""
with self._lock:
members = [db for db in self._members if db is not exclude]
for member in members:
if member._evict_one_idle_read_conn():
return True
return False
# canonical db path -> the permits for that file. Weak values: the budget
# lives exactly as long as some SessionDB on that path holds a strong
# reference to it, so a test that churns tmp_path databases does not grow
# this map for the life of the process.
_read_budgets: "weakref.WeakValueDictionary[str, _PathReadBudget]" = (
weakref.WeakValueDictionary()
)
_read_budgets_lock = threading.Lock()
def _read_budget_key(db_path) -> str:
"""Canonicalise a db path so two spellings share one budget."""
try:
return str(Path(db_path).resolve())
except OSError:
return str(db_path)
def _read_budget_for(db_path) -> _PathReadBudget:
key = _read_budget_key(db_path)
with _read_budgets_lock:
budget = _read_budgets.get(key)
if budget is None:
budget = _PathReadBudget()
_read_budgets[key] = budget
return budget
# Import-time snapshot used by _default_db_path() to detect a deliberately
# re-pointed DEFAULT_DB_PATH (tests monkeypatch the constant directly).
_IMPORT_DEFAULT_DB_PATH = DEFAULT_DB_PATH
def _default_db_path() -> Path:
"""Resolve the default state DB path at call time.
``DEFAULT_DB_PATH`` is computed when this module is first imported, which
freezes the developer's real ``~/.hermes`` even when a test fixture later
redirects ``HERMES_HOME`` — importing this module during collection was
enough to point every default ``SessionDB()`` at the real state.db.
Precedence:
1. A deliberately re-pointed ``DEFAULT_DB_PATH`` (differs from the
import-time snapshot — the established test escape hatch) wins.
2. Otherwise resolve ``get_hermes_home()`` fresh so a runtime
``HERMES_HOME`` redirect takes effect regardless of import order.
"""
if DEFAULT_DB_PATH != _IMPORT_DEFAULT_DB_PATH:
return DEFAULT_DB_PATH
return get_hermes_home() / "state.db"
# ---------------------------------------------------------------------------
# Live-DB test-isolation guard
# ---------------------------------------------------------------------------
# Forensic evidence (Aug 2026, live developer machine): the production
# ~/.hermes/state.db accumulated pytest fixture rows — sessions with
# chat_id='chat-1'/'123'/'wx-chat' and gateway_routing scopes literally under
# /tmp/pytest-of-*/ — and a pytest-spawned process flipped the journal mode
# out from under the WAL-mode gateway writer, destroying committed
# transcripts ("Persisted transcript lagged live cached history ... possible
# FTS write corruption"). The hermetic conftest redirects HERMES_HOME per
# test, but any escape (a session-scoped fixture running before the autouse
# fixture, a subprocess child launched without HERMES_HOME, a stale worktree
# without the re-pin, or a developer shell that exports HERMES_HOME to the
# real home so the conftest session sandbox is skipped) silently fell
# through to the real database.
#
# This guard is the single choke point: EVERY ``SessionDB`` construction
# resolves its path here, so under pytest a resolution that lands on a
# production state.db fails hard instead of corrupting live data. It is
# env-based (``PYTEST_CURRENT_TEST`` / ``PYTEST_VERSION`` are set by pytest
# and inherited by subprocess children), so it also protects children that
# never import the test conftest.
#: Escape hatch for the rare legitimate case (a test that genuinely needs
#: the real DB). The in-tree conftest sets this for tests marked
#: ``@pytest.mark.live_system_guard_bypass``; scripts may set it explicitly.
_STATE_DB_GUARD_BYPASS = False
#: Env-carried twin of ``_STATE_DB_GUARD_BYPASS``. A module global cannot
#: cross a process boundary, so a test that deliberately points a *child* at
#: the live DB has no way to opt out once ancestry arms the guard there.
#: Export this in the child's env instead.
_STATE_DB_GUARD_BYPASS_ENV = "HERMES_STATE_DB_GUARD_BYPASS"
#: Additional production roots to refuse (beyond the platform default
#: ``~/.hermes``). The test conftest injects the pre-sandbox production
#: root here so custom-``HERMES_HOME`` deployments are covered too.
_STATE_DB_GUARD_EXTRA_DENY_ROOTS: Tuple[Path, ...] = ()
def _real_platform_state_root() -> Optional[Path]:
"""Resolve the REAL platform-default Hermes root for the guard.
Deliberately avoids ``Path.home()`` / ``hermes_constants``: tests
routinely monkeypatch ``Path.home`` to a tempdir, and ``hermes_state``
is often imported lazily *while* such a patch is active — resolving
through the patched callable would misidentify the test's own hermetic
home as "production" (false positive) or, worse, miss the real one
(false negative). ``os.path.expanduser`` reads the HOME environment
variable / passwd entry, which the hermetic conftest never rewrites.
"""
try:
if sys.platform == "win32":
base = os.environ.get("LOCALAPPDATA", "").strip()
root = (
Path(base) / "hermes"
if base
else Path(os.path.expanduser("~")) / "AppData" / "Local" / "hermes"
)
else:
root = Path(os.path.expanduser("~")) / ".hermes"
return root.resolve()
except Exception:
return None
#: Env marker exported by the hermetic test conftest at the same moment it
#: redirects ``HERMES_HOME`` to the per-session tmp isolation root. Its
#: value is that isolation root. Unlike ``PYTEST_*`` (owned by pytest, and
#: routinely scrubbed by tests that rebuild a child environment), this marker
#: is OURS: it declares "this process tree is running under Hermes test
#: isolation", and it inherits into subprocess children by default — so a
#: child that received the patched ``HERMES_HOME`` also received the marker,
#: and a child that resolves a production DB while carrying it is, by
#: definition, an isolation escape (#82770).
_TEST_ISOLATION_MARKER_ENV = "HERMES_TEST_ISOLATION"
def _running_under_pytest() -> bool:
"""True when this process (or a parent test process) is a pytest run."""
return bool(
os.environ.get("PYTEST_CURRENT_TEST")
or os.environ.get("PYTEST_VERSION")
or os.environ.get(_TEST_ISOLATION_MARKER_ENV)
)
#: Names that identify a pytest launcher in a process command line. Matched
#: against the *basename* of each argv token so ``/tmp/pytest-of-dev/...``
#: paths — which do show up in real argv — cannot false-positive.
_PYTEST_LAUNCHER_NAMES = frozenset(
{"pytest", "py.test", "pytest.exe", "py.test.exe"}
)
#: Memoised ancestry answer. The process tree above us does not change in a
#: way that matters here, and the walk must not cost anything on the hot path.
_PYTEST_ANCESTOR: Optional[bool] = None
def _process_looks_like_pytest(proc: Any) -> bool:
"""True when *proc*'s command line is a pytest invocation.
Covers both ``pytest ...`` (launcher on argv[0]) and ``python -m pytest``
(launcher as a bare ``pytest`` token). A process whose command line we
cannot read is treated as "not pytest": guessing the other way would
refuse production opens for unrelated reasons.
"""
try:
cmdline = proc.cmdline() or []
except Exception:
return False
for arg in cmdline:
try:
token = str(arg).strip('"').strip("'")
# Split on both separators on every host: os.path.basename is
# POSIX-only under Linux and would leave a Windows-style path
# intact, making the matcher's answer depend on the platform.
name = token.replace("\\", "/").rsplit("/", 1)[-1].lower()
except Exception:
continue
if name in _PYTEST_LAUNCHER_NAMES:
return True
return False
def _has_pytest_ancestor() -> bool:
"""True when some ancestor process of this one is a pytest run.
``_running_under_pytest`` reads ``PYTEST_*`` env vars, which a child
spawned with a rebuilt environment loses at the same moment it loses the
``HERMES_HOME`` redirect: that child aims at the production DB *and*
disarms the guard in one step (#82770). Ancestry is the one test-context
signal that survives an env rebuild, so it backs the env check up.
Fails open (``False``) when ``psutil`` is unavailable or the walk errors —
that restores the previous env-only behaviour rather than blocking real
user runs on a psutil hiccup.
"""
global _PYTEST_ANCESTOR
if _PYTEST_ANCESTOR is not None:
return _PYTEST_ANCESTOR
found = False
if psutil is not None:
try:
for parent in psutil.Process().parents():
if _process_looks_like_pytest(parent):
found = True
break
except Exception:
found = False
_PYTEST_ANCESTOR = found
return found
def _in_test_context() -> bool:
"""True when this process is a test run, by environment or by ancestry.
Order matters for cost: the env probe is two dict lookups and covers the
common in-process case, so the ancestry walk only runs for processes the
environment claims are ordinary user runs — and its answer is memoised,
so a real ``hermes`` invocation pays for at most one walk.
"""
if _running_under_pytest():
return True
return _has_pytest_ancestor()
def _production_state_roots() -> List[Path]:
roots: List[Path] = []
real_root = _real_platform_state_root()
if real_root is not None:
roots.append(real_root)
for extra in _STATE_DB_GUARD_EXTRA_DENY_ROOTS:
try:
roots.append(Path(extra).expanduser().resolve())
except Exception:
continue
return roots
def _is_production_state_db(resolved: Path, root: Path) -> bool:
"""True when *resolved* is a DB file of the real Hermes home *root*.
Matches files directly in the root (``<root>/state.db``) and profile
homes (``<root>/profiles/<name>/state.db``). Deliberately does NOT
match deeper scratch paths (e.g. repo worktrees that happen to live
under ``~/.hermes/hermes-agent/...``) so hermetic tests using unusual
tempdirs cannot false-positive.
"""
if resolved.parent == root:
return True
try:
rel = resolved.relative_to(root)
except ValueError:
return False
parts = rel.parts
return len(parts) == 3 and parts[0] == "profiles"
def _ensure_test_isolation(db_path: Path) -> None:
"""Fail hard when a pytest-context process resolves a production DB.
Raises ``RuntimeError`` before any connection, mkdir, journal-mode
pragma, or byte probe can touch the live database. No-op outside
pytest and for hermetic (tmp ``HERMES_HOME``) paths.
"pytest context" means environment *or* process ancestry — see
:func:`_in_test_context`. Env alone is not enough: a child spawned with
a rebuilt environment loses ``PYTEST_*`` and ``HERMES_HOME`` together,
which is precisely the state in which it writes to production (#82770).
"""
if _STATE_DB_GUARD_BYPASS or os.environ.get(_STATE_DB_GUARD_BYPASS_ENV):
return
if not _in_test_context():
return
try:
resolved = Path(db_path).expanduser().resolve()
except Exception:
return
for root in _production_state_roots():
if _is_production_state_db(resolved, root):
raise RuntimeError(
"live-system guard: test attempted to open production "
f"state.db at {resolved} (under real Hermes root {root}). "
"Tests must run against a temporary HERMES_HOME — pass an "
"explicit tmp db_path or let the hermetic conftest redirect "
"HERMES_HOME. If this test genuinely needs the live "
"database, mark it with "
"@pytest.mark.live_system_guard_bypass — or, for a spawned "
f"child process, export {_STATE_DB_GUARD_BYPASS_ENV}=1 in "
"its environment."
)
# ---------------------------------------------------------------------------
# WAL-compatibility fallback
# ---------------------------------------------------------------------------
# SQLite's WAL mode requires shared-memory (mmap) coordination and fcntl
# byte-range locks that don't reliably work on network filesystems (NFS,
# SMB/CIFS, some FUSE mounts, WSL1). Upstream documents this explicitly:
# https://www.sqlite.org/wal.html#sometimes_queries_return_sqlite_busy_in_wal_mode
#
# On those filesystems ``PRAGMA journal_mode=WAL`` raises
# ``sqlite3.OperationalError: locking protocol`` (SQLITE_PROTOCOL). If we
# propagate that, every feature backed by state.db / kanban.db breaks
# silently — /resume, /title, /history, /branch, kanban dispatcher, etc.
#
# ZFS is a separate case: its COW + mmap semantics can corrupt the WAL
# shared-memory (-shm) file under concurrent connection bursts, presenting
# as ``disk I/O error`` rather than ``locking protocol``.
#
# Instead, fall back to ``journal_mode=DELETE`` (the pre-WAL default) which
# works on NFS and ZFS. Concurrency drops — concurrent readers are blocked
# during a write — but the feature works.
#
# Separately, SQLite's WAL-reset bug can corrupt multi-process WAL databases
# on unfixed library builds (issue #69784). See:
# https://sqlite.org/wal.html#walresetbug
# Fixed in 3.51.3+ with backports 3.50.7 and 3.44.6. On vulnerable builds we
# refuse to *enable* WAL for fresh / non-WAL databases (prefer DELETE). We do
# NOT live-downgrade an on-disk WAL database — other gateway/cron/worker
# connections may still hold it open, and flipping journal_mode under them is
# unsafe (same invariant as the NFS path below).
_WAL_INCOMPAT_MARKERS = (
"locking protocol", # SQLITE_PROTOCOL on NFS/SMB
"not authorized", # Some FUSE mounts block WAL pragma outright
"disk i/o error", # ZFS SHM corruption under concurrent connections
)
# Upper bound for the write-ahead log. SQLite defaults to -1 (unlimited),
# which lets state.db-wal keep the high-water mark of the largest-ever
# transaction forever. See _apply_wal_size_limit().
_WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 # 64 MiB
# Last SessionDB() init error, per-process. Surfaced in /resume and
# related slash-command error strings so users know WHY the DB is
# unavailable instead of getting a bare "Session database not available."
# Only SessionDB.__init__ writes to this; kanban_db.connect() failures
# do not update it (by design — kanban failures are reported via their
# own caller's error handling, not via /resume-style slash commands).
_last_init_error: Optional[str] = None
_last_init_error_lock = threading.Lock()
# Paths for which we've already logged a WAL-fallback WARNING. Without
# this, kanban_db.connect() (called on every kanban operation — see
# hermes_cli/kanban_db.py for ~30 call sites) would re-log the same
# filesystem-incompat warning on every connection, filling errors.log.
_wal_fallback_warned_paths: set[str] = set()
_wal_fallback_warned_lock = threading.Lock()
# Dedup WARNING for the WAL-reset vulnerability fallback (issue #69784).
_wal_reset_bug_warned_paths: set[str] = set()
_wal_reset_bug_warned_lock = threading.Lock()
# Dedup ERROR for the "configured delete overridden by on-disk WAL" warning.
_delete_overridden_warned_paths: set[str] = set()
_delete_overridden_warned_lock = threading.Lock()
def _set_last_init_error(msg: Optional[str]) -> None:
"""Record (or clear) the most recent state.db init failure.
Thread-safe via _last_init_error_lock. Callers pass a message to
record a failure or None to clear. SessionDB.__init__ only calls
this to SET on failure — it deliberately does NOT clear on success,
because in a multi-threaded caller (e.g. gateway / web_server per-
request SessionDB() instantiation), a concurrent successful open
racing past a different thread's failure would erase the cause
string that thread's /resume handler is about to format. Explicit
clears (e.g. test fixtures) are still supported by passing None.
"""
global _last_init_error
with _last_init_error_lock:
_last_init_error = msg
def get_last_init_error() -> Optional[str]:
"""Return the most recent state.db init failure, if any.
Slash-command handlers (``/resume``, ``/title``, ``/history``, ``/branch``)
call this to surface the underlying cause in their error messages when
``_session_db is None``. Returns ``None`` if SessionDB initialized
successfully (or hasn't been attempted).
"""
return _last_init_error
# Distinctive opening shared by both background-review harness prompts
# (_SKILL_REVIEW_PROMPT and _MEMORY_REVIEW_PROMPT in agent/background_review.py).
# Matched case-sensitively against the leading content of a user/system message.
_REVIEW_HARNESS_PREFIXES = (
"Review the conversation above and update the skill library",
"Review the conversation above and consider saving to memory",
)
def _is_background_review_harness_message(msg: Dict[str, Any]) -> bool:
"""True when ``msg`` is a persisted background-review harness prompt.
These are user/system turns the forked skill/memory review agent wrote into
a real session in older builds (before the ``_persist_disabled`` isolation
fix). They instruct the agent to act as the curator under a hard tool
restriction, so replaying them as live history hijacks the session.
"""
if not isinstance(msg, dict):
return False
if msg.get("role") not in {"user", "system"}:
return False
content = msg.get("content")
if not isinstance(content, str):
return False
head = content.lstrip()
return any(head.startswith(p) for p in _REVIEW_HARNESS_PREFIXES)
def _strip_background_review_harness(
messages: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
"""Drop background-review harness messages and the curator-mode assistant
reply that immediately followed each one.
Walk the list once; when a harness user/system message is found, skip it and
also skip the next message if it is the assistant turn that answered it.
Everything else passes through untouched and in order.
"""
if not messages:
return messages
out: List[Dict[str, Any]] = []
skip_next_assistant = False
for msg in messages:
if _is_background_review_harness_message(msg):
skip_next_assistant = True
continue
if skip_next_assistant:
skip_next_assistant = False
if isinstance(msg, dict) and msg.get("role") == "assistant":
# The curator-mode reply to the harness prompt — drop it.
continue
out.append(msg)
return out
# Matches a bare protocol/tool-name marker such as "[memory]" or "[skill_manage]".
_STALE_TOOL_CALL_MARKER_RE = re.compile(r"^\[[A-Za-z_][A-Za-z0-9_.-]*\]$")
def _is_stale_tool_call_marker_message(msg: Dict[str, Any]) -> bool:
"""True when ``msg`` is a persisted assistant turn whose content is a bare
bracketed marker (e.g. ``[memory]``) left over from a tool-call turn.
Before the #78148 fix in ``agent.conversation_loop``, a local tool-call
template could emit a bare marker as assistant content alongside a real
tool call. The loop cached that marker as a fallback and later replayed
it as the "final response", persisting it into the session. Sessions
written before the fix can still carry these rows.
"""
if not isinstance(msg, dict):
return False
if msg.get("role") != "assistant":
return False
if not msg.get("tool_calls"):
return False
content = msg.get("content")
if not isinstance(content, str):
return False
return bool(_STALE_TOOL_CALL_MARKER_RE.fullmatch(content.strip()))
def _strip_stale_tool_call_markers(
messages: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
"""Clear bare protocol-marker content persisted before the #78148 fix.
Replaying "[memory]" as if the model had actually answered teaches the
model, by example, to keep emitting the same marker in later turns — the
exact symptom the issue reported. Only the stray ``content`` field is
blanked; the tool call and its result are left untouched so provider
tool_call/tool_result pairing stays intact. Sessions with no affected
rows pass through unchanged.
"""
repaired = 0
for msg in messages:
if _is_stale_tool_call_marker_message(msg):
msg["content"] = ""
repaired += 1
if repaired:
logger.info(
"Cleared %d stale tool-call marker message(s) while restoring session (#78148)",
repaired,
)
return messages
def _normalize_telegram_topic_profile_name(profile_name: Optional[str] = None) -> str:
"""Normalize profile namespace for Telegram topic-mode tables.
Empty / missing values map to ``\"default\"`` so non-multiplexed gateways
keep a single namespace. Multiplexed callers must pass the *routed*
profile (``source.profile``), never the process-global active profile.
"""
name = str(profile_name or "").strip()
return name if name else "default"
def format_session_db_unavailable(prefix: str = "Session database not available") -> str:
"""Format a user-facing 'session DB unavailable' message with cause.
When ``SessionDB()`` init fails, callers set ``_session_db = None`` and
several slash commands (/resume, /title, /history, /branch) previously
responded with a bare ``"Session database not available."`` — no
indication of WHY. This helper includes the captured cause (typically
``"locking protocol"`` from NFS/SMB) and points users at the known
culprit so they can fix it themselves.
Example output:
Session database not available: locking protocol (state.db may be
on NFS/SMB — see https://www.sqlite.org/wal.html).
"""
cause = get_last_init_error()
if not cause:
return f"{prefix}."
hint = ""
if any(marker in cause.lower() for marker in _WAL_INCOMPAT_MARKERS):
hint = " (state.db may be on NFS/SMB/FUSE/ZFS — see https://www.sqlite.org/wal.html)"
return f"{prefix}: {cause}{hint}."
def _on_disk_journal_mode(conn: sqlite3.Connection) -> Optional[str]:
"""Read the journal mode from the SQLite DB header on disk.
Returns the mode string (e.g. ``"wal"``, ``"delete"``), or ``None``
if the value cannot be determined (new DB, or PRAGMA read failed).
A PRAGMA read can fail transiently with ``disk i/o error`` on
virtualized block devices (XFS on cloud hosts). Treating that as
"mode unknown" pushes callers onto their fail-closed unknown-mode
branch even though the on-disk mode is perfectly readable a few
milliseconds later. Retry the read a few times before giving up:
transient EIO clears, deterministic unsupported-filesystem errors do
not. ``None`` is still returned on final failure so the caller's
existing "unknown → refuse to downgrade" logic applies.
"""
last_exc: Optional[Exception] = None
for _ in range(4):
try:
row = conn.execute("PRAGMA journal_mode").fetchone()
except sqlite3.OperationalError as exc:
last_exc = exc
if "disk i/o error" not in str(exc).lower():
return None
time.sleep(0.05)
continue
if row is None:
return None
mode = row[0]
if isinstance(mode, bytes): # defensive: sqlite3 occasionally returns bytes
try:
mode = mode.decode("ascii")
except UnicodeDecodeError:
return None
return str(mode).strip().lower() if mode is not None else None
if last_exc is not None:
logger.debug(
"_on_disk_journal_mode: retries exhausted on disk read (%s)", last_exc
)
return None
def _apply_wal_size_limit(conn: sqlite3.Connection) -> None:
"""Bound the WAL so it returns space to the OS after big transactions.
SQLite's default ``journal_size_limit`` is -1 (unlimited): after a
checkpoint the WAL file is *reused in place* and never truncated, so
``state.db-wal`` permanently retains the high-water mark of the largest
transaction ever run against it.
A single bulk operation is enough to strand gigabytes. Observed on a
3.0 GB ``state.db``: ``hermes sessions optimize`` (FTS merge + VACUUM)
rewrites every page through the WAL, leaving a **3.07 GB**
``state.db-wal`` sitting next to the database indefinitely — the host
went from 6.9 GB free to 772 MB (100% full) and stayed there, because
nothing shrinks the WAL back down. An explicit
``PRAGMA wal_checkpoint(TRUNCATE)`` reclaimed the full 3.07 GB, which
confirms the space was pure slack rather than live data.
That also makes the maintenance command self-defeating on exactly the
databases that need it most: the larger the DB, the larger the WAL it
strands, so ``optimize`` can consume more disk than it frees.
``journal_size_limit`` makes SQLite truncate the WAL back to the limit
at each checkpoint. 64 MiB is comfortably above normal transaction
sizes (so steady-state commits never pay a truncate) while capping the
stranded slack at a bounded, predictable figure.
``hermes_cli/kanban_db.py`` already bounds its WAL growth with
``wal_autocheckpoint=100``; the session store — by far the larger
database — had no equivalent.
Best-effort: never raises. A failure here only costs disk slack, and
must not prevent the database from opening.
"""
try:
conn.execute(f"PRAGMA journal_size_limit={_WAL_SIZE_LIMIT_BYTES}")
except sqlite3.OperationalError as exc: # pragma: no cover - defensive
logger.debug("journal_size_limit not applied: %s", exc)
def _apply_macos_checkpoint_barrier(conn: sqlite3.Connection) -> None:
"""Enable ``PRAGMA checkpoint_fullfsync`` on macOS (no-op elsewhere).
On Darwin, ``synchronous=FULL`` (the WAL default) issues a plain
``fsync()``, which Apple documents does *not* guarantee that data
has reached stable storage or that writes are not reordered — see
the ``fsync(2)`` man page. SQLite's WAL corruption-safety guarantee
assumes the OS honors the fsync write barrier; macOS does not unless
the app uses ``F_FULLFSYNC``.
During a launchd *system* shutdown/reboot the OS page cache is
dropped (effectively a power-loss event for in-flight pages), so a
WAL checkpoint whose ``fsync()`` "reported" durable may never have
hit the platter — corrupting ``state.db`` with a malformed image.
This is the trigger in issue #30636 ("SIGTERM during launchd
shutdown under high load"), distinct from a plain in-session kill
(which the page cache survives and SQLite recovers from).
``checkpoint_fullfsync=1`` forces an ``F_FULLFSYNC`` barrier only at
checkpoint boundaries — where WAL frames land in the main DB — so the
cost amortizes to roughly +0.1 ms/commit (vs ~+4 ms for the broader
``fullfsync=1`` that flushes on every commit's WAL sync). Guarded by
``sys.platform == "darwin"`` because ``F_FULLFSYNC`` is macOS-only;
on other platforms the PRAGMA is a no-op, so we skip it entirely.
Best-effort: never raises.
"""
if sys.platform != "darwin":
return
try:
conn.execute("PRAGMA checkpoint_fullfsync=1")
except sqlite3.OperationalError:
pass
def _enforce_macos_synchronous_full(conn: sqlite3.Connection) -> None:
"""Enforce ``PRAGMA synchronous=FULL`` on macOS to prevent btree corruption.
On Darwin, the default ``synchronous=NORMAL`` only calls ``fsync()``,
which Apple's fsync(2) man page explicitly states does *not* guarantee
data-on-platter or write-ordering. During a WAL checkpoint race with
process termination (e.g., launchd shutdown), this can leave the main
DB with half-written btree pages → ``btreeInitPage error 11``.
WAL mode's durability guarantee assumes the OS honors fsync barriers;
macOS does not unless we explicitly set ``synchronous=FULL``, which issues
a real ``fsync()`` on every transaction commit. The ``F_FULLFSYNC``
barrier at checkpoint boundaries is handled separately by
:func:`_apply_macos_checkpoint_barrier`.
This function is called after any successful WAL activation (either
from ``apply_wal_with_fallback()`` setting a fresh WAL or when probing
an existing WAL mode). It ensures macOS connections always use FULL
synchronous mode, even if a prior connection set ``synchronous=NORMAL``.
Best-effort: never raises.
"""
if sys.platform != "darwin":
return
try:
conn.execute("PRAGMA synchronous=FULL")
except sqlite3.OperationalError:
pass
def is_sqlite_wal_reset_vulnerable(
version_info: Optional[tuple] = None,
) -> bool:
"""Return True when the linked SQLite library has the WAL-reset bug.
Upstream documents the bug in versions 3.7.0 through 3.51.2, fixed in
3.51.3+, with backports 3.50.7 and 3.44.6:
https://sqlite.org/wal.html#walresetbug
Pre-WAL libraries (< 3.7.0) cannot hit the race and are treated as safe.
"""
info = version_info if version_info is not None else sqlite3.sqlite_version_info
return _is_sqlite_wal_reset_vulnerable(info)
def sqlite_source_id() -> str:
"""Return ``sqlite_source_id()``, or an empty string when unavailable."""
try:
conn = sqlite3.connect(":memory:")
try:
row = conn.execute("SELECT sqlite_source_id()").fetchone()
finally:
conn.close()
except sqlite3.Error:
return ""
if not row or row[0] is None:
return ""
return str(row[0])
def _database_has_content(conn: sqlite3.Connection) -> bool:
"""Return whether the database file already holds pages.
Used to tell an EXISTING database apart from a brand-new one before
rewriting its journal mode. ``PRAGMA page_count`` is a header read, so
this costs nothing and takes no lock.
Fail-quiet: any error, or a database we cannot measure, answers False.
The only caller uses this to decide whether to emit a warning, and a
warning that fires when the answer is unknown would fire on every fresh
database -- precisely the case where there is provably no operator choice
being overwritten.
"""
try:
row = conn.execute("PRAGMA page_count").fetchone()
except sqlite3.Error:
return False
if not row or row[0] is None:
return False
try:
return int(row[0]) > 0
except (TypeError, ValueError):
return False
def resolve_journal_mode() -> str:
"""Return the configured journal mode (``wal`` or ``delete``).
``database.journal_mode`` in config.yaml is the canonical operator
setting. ``wal`` remains the default; use ``delete`` when the backing
filesystem does not provide WAL-safe durability (for example macOS
virtiofs, NFS, or SMB). Invalid or malformed values fail safely to the
existing default.
"""
try:
from hermes_cli.config import load_config_readonly
config = load_config_readonly() or {}
database = config.get("database", {})
if not isinstance(database, dict):
return "wal"
raw = database.get("journal_mode", "wal")
except Exception:
return "wal"
if not isinstance(raw, str):
return "wal"
mode = raw.strip().lower()
return mode if mode in ("wal", "delete") else "wal"
class WalUnsupportedError(sqlite3.OperationalError):
"""Raised by :func:`apply_wal_with_fallback` when ``require_wal=True`` and
the filesystem cannot provide WAL journal mode.
Covers both shapes of WAL refusal on network filesystems (NFS / SMB / FUSE
/ the AgentFS NFS overlay): SQLite *raising* ``SQLITE_PROTOCOL`` ("locking
protocol"), and the quieter macOS-NFS case where ``PRAGMA journal_mode=WAL``
silently returns the still-effective mode without raising. Subclasses
``sqlite3.OperationalError`` so existing ``except sqlite3.OperationalError``
DB-init handling still catches it, while callers that specifically mandate
WAL can catch this narrower type.
"""
def apply_wal_with_fallback(
conn: sqlite3.Connection,
*,
db_label: str = "state.db",
require_wal: bool = False,
) -> str:
"""Set ``journal_mode=WAL`` on ``conn``, falling back to DELETE on failure.
Returns the journal mode actually set (``"wal"`` or ``"delete"``).
On WAL-incompatible filesystems (NFS, SMB, some FUSE, ZFS), SQLite either
raises ``OperationalError("locking protocol")`` /
``OperationalError("disk I/O error")`` or — on macOS NFS / SMB /
the AgentFS NFS overlay — silently refuses the switch and leaves the DB in
DELETE. Either way the degradation is logged at ERROR level (it is a real
loss of concurrency — a write blocks concurrent readers — not a cosmetic
warning) and, by default, the function falls back to DELETE (the pre-WAL
default, which works on NFS and ZFS) so the feature keeps working.
On SQLite builds that still contain the WAL-reset corruption bug
(issue #69784), refuse to enable WAL on fresh / non-WAL databases
(prefer DELETE). If the on-disk DB is already WAL, keep WAL and warn
— never live-downgrade under possible concurrent openers.
This gate (#70055) is deliberately RETAINED. An earlier revision of the
lock-cancellation fix (#71724) reverted it on the theory that DELETE was
"the mode that corrupts", but that comparison was confounded: the clean
WAL result came from SQLite 3.53.1, which carries BOTH the WAL-reset fix
AND 3.51.0's defenses against close()-broken POSIX locks, so it says
nothing about 3.50.4. Re-measured on the actually-bundled 3.50.4 with
the lock fix in place, WAL and DELETE are both clean (0/3 each) — i.e.
there is no evidence that WAL is safer here, and upstream still documents
the WAL-reset bug as real through 3.51.2 with serious consequences. Until
a fixed runtime is delivered, keep new databases out of WAL.
Callers that genuinely require WAL concurrency (and would rather fail loudly
than run silently degraded) pass ``require_wal=True``; the function then
raises :class:`WalUnsupportedError` instead of returning ``"delete"``. All
current callers deliberately keep the default ``require_wal=False`` so
NFS-homed installs keep working.
The ERROR is deduplicated per ``db_label``: repeated connections to the
same underlying DB (e.g. kanban_db.connect() which is called on every
kanban operation) log once per process, not once per call. Different
db_labels log independently, so state.db and kanban.db each get one error
on the same NFS mount.
Shared by :class:`SessionDB` and ``hermes_cli.kanban_db.connect`` so
both databases get identical fallback behavior.
Never downgrades to DELETE if the on-disk DB header reports WAL — see
_on_disk_journal_mode. That holds for both the NFS path and the
WAL-reset vulnerability path.
"""
configured = resolve_journal_mode()
# Vulnerable SQLite: do not enable WAL on new/non-WAL files. Resolve the
# operator setting first so an explicit DELETE request still verifies that
# SQLite actually accepted DELETE rather than silently returning MEMORY or
# another connection-specific mode.
if is_sqlite_wal_reset_vulnerable():
return _apply_delete_for_wal_reset_bug(
conn,
db_label=db_label,
require_delete=configured == "delete",
)
# Read-only probe — no flock, no checkpoint, no WAL/SHM unlink.
# Skipping the set-pragma prevents WAL-init from unlinking files other connections hold open.
current_mode = _on_disk_journal_mode(conn)
if current_mode == "wal":
if configured == "delete":
# Never-live-downgrade keeps this WAL; tell the operator their
# configured delete did not apply (see _log_configured_delete_overridden_once).
_log_configured_delete_overridden_once(db_label)
_apply_wal_size_limit(conn)
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return "wal"
# #68545: honor the canonical database.journal_mode setting. Existing
# on-disk WAL databases were returned above and are never live-downgraded.
if configured == "delete":
if current_mode is None:
# The mode probe failed (database locked / busy): another
# process may hold this DB open in WAL. Ownership is not
# provably exclusive, so flipping journal modes here could
# destroy committed-but-uncheckpointed WAL transactions of a
# concurrent writer. Fail loudly instead of downgrading — the
# operator explicitly requested DELETE and we cannot verify it.
raise sqlite3.OperationalError(
"could not verify journal mode before applying configured "
"journal_mode=delete (database is locked — possible "
"concurrent openers); refusing to downgrade a database "
"this process does not exclusively own"
)
actual = _set_journal_mode_no_wait(conn, "DELETE")
if actual != "delete":
raise sqlite3.OperationalError(
f"could not set configured journal_mode=delete (got {actual or 'no result'})"
)
return actual
# Decide BEFORE the flip whether it would silently overwrite a mode
# somebody chose. Both inputs are only readable while the file is still
# in its original state: `current_mode` is the probe above, and
# page_count distinguishes an existing database from a fresh one.
#
# A 0-page database has no prior choice to overwrite, and every caller
# reaches this before creating any schema (SessionDB._connect_and_init
# applies WAL, then _init_schema), so brand-new databases land here empty
# and stay quiet.
_upgrading_existing_db = (
current_mode is not None
and current_mode != "wal"
and _database_has_content(conn)
)
try:
# ``PRAGMA journal_mode=WAL`` is a query-that-sets: it RETURNS the
# resulting journal mode. Network filesystems that refuse WAL by
# *raising* SQLITE_PROTOCOL ("locking protocol") are handled in the
# except branch below. But macOS NFS — and SMB/CIFS, and the AgentFS
# NFS overlay — refuse the switch WITHOUT raising: the pragma simply
# returns the still-effective mode (e.g. ``delete``). Trust the
# returned row, not the mere absence of an exception; otherwise we
# report a false ``"wal"`` AND skip the fallback WARNING, leaving the
# DB silently in DELETE (reader-blocks-writer) with no signal.
row = conn.execute("PRAGMA journal_mode=WAL").fetchone()
mode = str(row[0]).strip().lower() if row and row[0] is not None else ""
if mode == "wal":
if _upgrading_existing_db:
_log_journal_mode_upgrade_once(db_label, current_mode)
_apply_wal_size_limit(conn)
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return "wal"
# Silent refusal (macOS NFS / SMB / AgentFS overlay): WAL was not
# honored, but nothing raised.
silent_exc = WalUnsupportedError(
f"journal_mode=WAL refused without raising (still {mode!r})"
)
if require_wal:
raise silent_exc
_log_wal_fallback_once(db_label, silent_exc)
return mode or "delete"
except sqlite3.OperationalError as exc:
# The require_wal silent-refusal raise above is a WalUnsupportedError
# (an OperationalError subclass) and lands here — propagate it
# unchanged rather than re-running it through the marker logic.
if isinstance(exc, WalUnsupportedError):
raise
msg = str(exc).lower()
if not any(marker in msg for marker in _WAL_INCOMPAT_MARKERS):
# Unrelated OperationalError — don't silently swallow.
raise
# ``disk i/o error`` is ambiguous: on ZFS / APFS-CoW it is a
# deterministic WAL-incompatibility (SHM corruption under concurrent
# connection bursts — #55305, #71498), but it can also be a one-shot
# transient EIO (page-cache pressure, brief lock contention).
# Treating a transient EIO as a permanent downgrade signal produced
# the mixed-journal-mode corruption pattern fixed in 5c49cd0ed0
# (process A downgrades to DELETE while sibling processes set WAL).
# Disambiguate by retrying the pragma a couple of times: transient
# EIO clears and we return "wal"; the deterministic filesystem cases
# keep failing and fall through to the guarded DELETE fallback.
if "disk i/o error" in msg:
for _ in range(2):
time.sleep(0.05)
try:
row = conn.execute("PRAGMA journal_mode=WAL").fetchone()
except sqlite3.OperationalError as retry_exc:
if "disk i/o error" not in str(retry_exc).lower():
raise
exc = retry_exc
continue
mode = (
str(row[0]).strip().lower()
if row and row[0] is not None
else ""
)
if mode == "wal":
# Same flip, later door: a transient EIO cleared and the
# switch went through. The header rewrite is identical, so
# the signal must be too.
if _upgrading_existing_db:
_log_journal_mode_upgrade_once(db_label, current_mode)
_apply_wal_size_limit(conn)
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return "wal"
break
# Don't downgrade if another process already set WAL on disk, or if
# the mode cannot be verified at all (probe blocked by a concurrent
# opener's locks) — ownership is not provably exclusive either way.
existing = _on_disk_journal_mode(conn)
if existing == "wal" or existing is None:
raise
if require_wal:
# Caller mandates WAL — fail loudly instead of degrading to DELETE.
raise WalUnsupportedError(str(exc)) from exc
_log_wal_fallback_once(db_label, exc)
_set_journal_mode_no_wait(conn, "DELETE")
return "delete"
def _set_journal_mode_no_wait(conn: sqlite3.Connection, mode: str) -> str:
"""Execute ``PRAGMA journal_mode=<mode>`` without waiting on other openers.
This is the ONLY place a journal-mode switch pragma may be issued for a
non-WAL target. It temporarily forces ``busy_timeout=0`` so SQLite's own
exclusivity requirement becomes a concurrent-opener detector: leaving WAL
mode requires exclusive access to the database, so if ANY other connection
(this process or another) holds the DB open, the pragma fails immediately
with ``database is locked`` instead of waiting out a busy timeout and
sneaking the flip in between a concurrent writer's transactions — which is
exactly how committed-but-uncheckpointed WAL transactions get destroyed.
Callers must treat a raised ``OperationalError`` as "not exclusively
owned: leave the journal mode alone", never as a retryable condition.
Returns the resulting journal mode as reported by SQLite (lowercase), or
``""`` when SQLite returned no row.
"""
previous_timeout = 0
try:
row = conn.execute("PRAGMA busy_timeout").fetchone()
if row and row[0] is not None:
previous_timeout = int(row[0])
except (sqlite3.OperationalError, TypeError, ValueError):
previous_timeout = 0
conn.execute("PRAGMA busy_timeout=0")
try:
row = conn.execute(f"PRAGMA journal_mode={mode}").fetchone()
return str(row[0]).strip().lower() if row and row[0] is not None else ""
finally:
try:
conn.execute(f"PRAGMA busy_timeout={previous_timeout}")
except sqlite3.OperationalError:
pass
def _apply_delete_for_wal_reset_bug(
conn: sqlite3.Connection,
*,
db_label: str,
require_delete: bool = False,
) -> str:
"""Avoid enabling WAL when the linked SQLite has the WAL-reset bug.
- Already-WAL on disk: leave WAL alone (no live downgrade) and warn.
- Mode unreadable (probe blocked by a concurrent opener's locks):
ownership is not provably exclusive — leave the journal mode alone
and warn. Never treat "could not read the mode" as "not WAL": that
exact confusion let a vulnerable-SQLite process flip a live WAL
state.db to DELETE under a concurrent WAL writer, destroying its
committed-but-uncheckpointed transactions.
- Otherwise: set DELETE (refusing to wait out concurrent openers) and
warn.
- For an explicit operator request, verify SQLite accepted DELETE.
"""
current = _on_disk_journal_mode(conn)
if current == "wal":
_log_wal_reset_bug_once(db_label, kept_wal=True)
if require_delete:
# The vulnerability warning above suggests upgrading SQLite, which
# does not help on a WAL-incompatible filesystem; surface that the
# configured delete is not in effect (see _log_configured_delete_overridden_once).
# Emitted last so the actionable message is the final one in the log.
_log_configured_delete_overridden_once(db_label)
# Do not TRUNCATE / journal_mode=DELETE while other processes may
# still hold this WAL DB open; same safety rule as the NFS path.
_apply_wal_size_limit(conn)
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return "wal"
if current is None:
# The mode probe itself failed — another opener's locks are the
# most likely cause, and the DB may well be in WAL under a live
# writer. Never flip a journal mode we cannot even read.
if require_delete:
raise sqlite3.OperationalError(
"could not verify journal mode before applying configured "
"journal_mode=delete (database is locked — possible "
"concurrent openers); refusing to downgrade a database "
"this process does not exclusively own"
)
_log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True)
return "wal"
actual = ""
try:
actual = _set_journal_mode_no_wait(conn, "DELETE")
except sqlite3.OperationalError as exc:
if require_delete:
raise
lowered = str(exc).lower()
if "locked" in lowered or "busy" in lowered:
# A concurrent opener appeared between the probe and the flip
# (or already held the DB): SQLite refused the exclusive lock.
# Leave the journal mode exactly as it is.
_log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True)
return current or "delete"
# Best-effort for the automatic vulnerable-runtime fallback: DELETE is
# normally already the default for new file-backed databases.
if require_delete and actual != "delete":
raise sqlite3.OperationalError(
"could not set configured journal_mode=delete "
f"(got {actual or 'no result'})"
)
_log_wal_reset_bug_once(db_label, kept_wal=False)
return "delete"
def _wal_reset_repair_hint() -> str:
"""Return a context-appropriate hint for repairing the SQLite runtime.
Uses the codebase's install-type detection so the hint matches what
``hermes update`` can actually do for this install (#75153).
"""
try:
from hermes_cli.config import (
detect_install_method,
recommended_update_command_for_method,
get_project_root,
)
method = detect_install_method(get_project_root())
cmd = recommended_update_command_for_method(method)
if method in {"git", "unknown"}:
return f"Hermes-managed installs can repair the embedded runtime with `{cmd}`"
if method == "docker":
return f"update the container image with `{cmd}`"
# nix/nixos
return cmd
except Exception:
pass
return (
"install a Python build bundled with SQLite 3.51.3+ "
"(or backports 3.50.7 / 3.44.6) and restart Hermes"
)
# Dedup state for _log_journal_mode_upgrade_once, mirroring the
# _wal_fallback_warned_* pair below it.
_journal_upgrade_warned_paths: set = set()
_journal_upgrade_warned_lock = threading.Lock()
def _log_wal_reset_bug_once(
db_label: str,
*,
kept_wal: bool,
indeterminate: bool = False,
) -> None:
"""Log once per (process, db_label) about the WAL-reset vulnerability path."""
with _wal_reset_bug_warned_lock:
if db_label in _wal_reset_bug_warned_paths:
return
_wal_reset_bug_warned_paths.add(db_label)
if indeterminate:
action = (
"journal mode could not be verified or exclusively switched "
"(database is locked — possible concurrent openers); leaving the "
"journal mode untouched (no live downgrade under concurrent "
"openers)"
)
elif kept_wal:
action = (
"is already in WAL mode — leaving WAL in place (no live "
"downgrade under concurrent openers)"
)
else:
action = "using journal_mode=DELETE instead of enabling WAL"
# Check whether this is a Hermes-managed install (uv-managed venv)
# so the warning doesn't promise a repair path that doesn't exist
# for git/pip/system Python installs (#75153).
repair_hint = _wal_reset_repair_hint()
logger.warning(
"%s: linked SQLite %s (interpreter %s) is vulnerable to the WAL-reset "
"corruption bug (https://sqlite.org/wal.html#walresetbug) — %s. "
"Upgrade to SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6); "
"%s. See `hermes doctor`. This warning fires once per "
"process per database.",
db_label,
sqlite3.sqlite_version,
sys.executable,
action,
repair_hint,
)
def _log_journal_mode_upgrade_once(db_label: str, previous_mode: str) -> None:
"""Log a single WARNING per (process, db_label) about a non-WAL -> WAL flip.
``PRAGMA journal_mode`` is a property of the FILE, not of the connection:
switching an existing database to WAL rewrites its header and outlives the
process that did it. Operators do set it directly on the file -- that was
the documented mitigation for the SQLite 3.50.4 WAL-reset bug -- and
nothing here told them the next open would silently put it back.
WARNING, not ERROR, and deliberately so. The reverse move is logged at
ERROR by ``_log_wal_fallback_once`` because dropping to DELETE is a real
loss of concurrency; this direction is normally the desirable one (see
``hermes_cli/managed_uv._default_live_venv``, which treats a database
stuck on DELETE as a bug worth repairing on update). The problem is not
the change, it is that the change was invisible: an operator who chose
DELETE deliberately had no way to learn their choice had been overwritten,
or which lever makes it stick. So this says what happened and names the
durable setting, without claiming a degradation that is not there.
Deduped per process per ``db_label`` like its siblings: kanban opens a
fresh connection per operation, so an undeduped line here would be a log
flood rather than a signal.
"""
with _journal_upgrade_warned_lock:
if db_label in _journal_upgrade_warned_paths:
return
_journal_upgrade_warned_paths.add(db_label)
logger.warning(
"%s: on-disk journal_mode was %s and has been switched to WAL. This "
"rewrites the database header and persists after this process exits. "
"If %s was a deliberate choice (for example the mitigation for the "
"SQLite WAL-reset bug, or a WAL-unsafe filesystem), setting it with "
"PRAGMA on the file will not survive -- every open re-applies the "
"configured mode. Set `database.journal_mode: delete` in config.yaml "
"to make it stick. This message fires once per process per database.",
db_label,
previous_mode,
previous_mode,
)
def _log_wal_fallback_once(db_label: str, exc: Exception) -> None:
"""Log a single ERROR per (process, db_label) about WAL fallback.
ERROR (not WARNING): a DB silently dropped to DELETE means a real loss of
concurrency — under the kanban dispatcher + workers a write blocks readers,
surfacing as SQLITE_BUSY/lock contention — so it must be loud, not cosmetic.
Without this dedup, NFS users running kanban (which opens a fresh
connection on every operation — see hermes_cli/kanban_db.py) would
fill errors.log with hundreds of identical errors per hour.
"""
with _wal_fallback_warned_lock:
if db_label in _wal_fallback_warned_paths:
return
_wal_fallback_warned_paths.add(db_label)
logger.error(
"%s: WAL journal_mode unsupported on this filesystem (%s) — "
"falling back to journal_mode=DELETE (slower rollback-journal "
"mode; reduces concurrency but works on NFS/SMB/FUSE/ZFS). See "
"https://www.sqlite.org/wal.html for details. This message "
"fires once per process per database.",
db_label,
exc,
)
def _log_configured_delete_overridden_once(db_label: str) -> None:
"""Log a single ERROR per (process, db_label) when the operator configured
``journal_mode=delete`` but the on-disk DB is already WAL, so the configured
mode is not in effect.
Counterpart to :func:`_log_wal_fallback_once` for the opposite direction:
there WAL was refused by the filesystem and we silently fell back to DELETE;
here the operator asked for DELETE but an inherited on-disk WAL header means
we keep WAL (the never-live-downgrade rule prevents a live downgrade, which
causes mixed-mode corruption). The signal matters because otherwise the
operator has no indication that ``database.journal_mode: delete`` had no
effect and the DB still requires a one-time offline ``PRAGMA
journal_mode=DELETE`` (with no open connections) to apply.
Fires once per process per database.
"""
with _delete_overridden_warned_lock:
if db_label in _delete_overridden_warned_paths:
return
_delete_overridden_warned_paths.add(db_label)
logger.error(
"%s: database.journal_mode=delete is configured but the on-disk "
"database is already WAL; keeping WAL (a live downgrade under open "
"connections can corrupt the DB). To apply journal_mode=DELETE, stop "
"all connections to this DB and run a one-time offline "
"'PRAGMA journal_mode=DELETE' on the file. This message fires once "
"per process per database.",
db_label,
)
# ---------------------------------------------------------------------------
# Config-driven database pragmas
# ---------------------------------------------------------------------------
# PRAGMA synchronous accepts either an integer or a symbolic name, and operators
# write the names. Mapped here rather than passed through so a typo becomes a
# warning instead of a silently different durability level.
_SYNCHRONOUS_LEVELS: Dict[str, int] = {
"OFF": 0,
"NORMAL": 1,
"FULL": 2,
"EXTRA": 3,
}
_SYNCHRONOUS_NAMES: Dict[int, str] = {v: k for k, v in _SYNCHRONOUS_LEVELS.items()}
_SYNCHRONOUS_FULL = 2
def resolve_synchronous_level(raw_value: Any) -> Optional[int]:
"""Map a configured ``database.synchronous`` value to its PRAGMA integer.
Accepts the symbolic names SQLite documents (``OFF``/``NORMAL``/``FULL``/
``EXTRA``, any case) and the equivalent integers ``0``-``3``. Returns None
for anything else so the caller can warn and leave the level untouched —
guessing at a malformed durability setting is worse than ignoring it.
"""
if isinstance(raw_value, bool):
# bool is an int subclass, and YAML turns a bare `on`/`off` into one.
# "off" as a durability level is a real choice; True is meaningless.
return 0 if raw_value is False else None
if isinstance(raw_value, int):
return raw_value if raw_value in _SYNCHRONOUS_NAMES else None
text = str(raw_value).strip()
if not text:
return None
upper = text.upper()
if upper in _SYNCHRONOUS_LEVELS:
return _SYNCHRONOUS_LEVELS[upper]
try:
value = int(text)
except (TypeError, ValueError):
return None
return value if value in _SYNCHRONOUS_NAMES else None
def _apply_synchronous_pragma(
conn: sqlite3.Connection,
raw_value: Any,
*,
db_label: str,
) -> None:
"""Set ``PRAGMA synchronous`` from config, never below FULL on macOS.
Split out of the integer loop in :func:`apply_database_pragmas` because
this PRAGMA is not interchangeable with the sizing ones around it. Those
trade memory or disk for speed; this one decides whether a commit is on
the platter, so an unrecognised value must not fall through to "SQLite
default" the way a bad ``cache_size`` harmlessly can.
The Darwin floor exists because :func:`_enforce_macos_synchronous_full`
runs during ``apply_wal_with_fallback()`` and this function runs after it,
so a configured ``NORMAL`` would otherwise silently undo the macOS btree
protection that fix put there deliberately. Raising the level on macOS is
allowed; lowering it is refused out loud.
"""
level = resolve_synchronous_level(raw_value)
if level is None:
logger.warning(
"%s: ignoring unrecognized database.synchronous=%r "
"(expected OFF, NORMAL, FULL, EXTRA, or 0-3)",
db_label,
raw_value,
)
return
if sys.platform == "darwin" and level < _SYNCHRONOUS_FULL:
logger.warning(
"%s: refusing database.synchronous=%s on macOS; keeping FULL. "
"Darwin's fsync() does not guarantee write ordering, so a lower "
"level readmits the half-written btree pages FULL exists to "
"prevent.",
db_label,
_SYNCHRONOUS_NAMES[level],
)
return
try:
conn.execute(f"PRAGMA synchronous={level}")
except sqlite3.OperationalError:
pass
def apply_database_pragmas(
conn: sqlite3.Connection,
*,
db_label: str = "state.db",
) -> None:
"""Apply optional performance and WAL-sizing PRAGMAs from ``config.yaml``.
Reads the ``database:`` section and applies configurable PRAGMAs when set
to integer values. The journal mode itself is NOT handled here —
``database.journal_mode`` is owned by :func:`resolve_journal_mode` inside
:func:`apply_wal_with_fallback`, which layers the operator setting under
all the safety guards (never live-downgrading an on-disk WAL DB,
filesystem fallback, WAL-reset-bug gating).
Supported keys under ``database:`` in config.yaml:
* ``cache_size`` — negative value = KiB, positive = pages
(e.g. ``-262144`` = 256 MB page cache)
* ``mmap_size`` — max bytes for memory-mapped I/O (0 = disabled)
* ``temp_store`` — 0=DEFAULT(file), 1=FILE, 2=MEMORY, 3=ALWAYS
* ``wal_autocheckpoint`` — WAL auto-checkpoint threshold in pages
* ``journal_size_limit`` — max journal/WAL size in bytes
* ``synchronous`` — durability level: ``OFF``/``NORMAL``/``FULL``/``EXTRA``
or ``0``-``3``. Unset leaves SQLite's own default, which is a
*compile-time* constant (``SQLITE_DEFAULT_WAL_SYNCHRONOUS``) and so
differs between the bundled, distro and Homebrew builds an operator
might be running. Setting it explicitly is the only way to know.
Best-effort: config load or pragma failures are ignored so DB init
never breaks on a malformed ``database:`` section.
"""
try:
# Local import avoids a circular import with hermes_cli.config.
from hermes_cli.config import cfg_get, load_config_readonly
cfg = load_config_readonly()
except Exception:
return
# Performance PRAGMAs (applied to ALL connection types: writer, read_only,
# and WAL per-thread readers).
for pragma_name in (
"cache_size",
"mmap_size",
"temp_store",
"wal_autocheckpoint",
"journal_size_limit",
):
raw_value = cfg_get(cfg, "database", pragma_name, default=None)
if raw_value is None:
continue
try:
value = int(str(raw_value).strip())
except (TypeError, ValueError):
logger.warning(
"%s: ignoring non-integer database.%s=%r",
db_label,
pragma_name,
raw_value,
)
continue
try:
conn.execute(f"PRAGMA {pragma_name}={value}")
except sqlite3.OperationalError:
pass
# Last, so it wins over nothing and loses to nothing: the sizing pragmas
# above cannot change durability, and the macOS enforcement ran earlier
# during WAL activation (see _apply_synchronous_pragma for why that
# ordering needs an explicit floor rather than an explicit override).
raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None)
if raw_synchronous is not None:
_apply_synchronous_pragma(conn, raw_synchronous, db_label=db_label)
# ---------------------------------------------------------------------------
# Malformed-schema recovery
# ---------------------------------------------------------------------------
# A distinct, nastier failure class than a malformed FTS *inverted index*:
# the ``sqlite_master`` schema table itself becomes inconsistent — most
# commonly a DUPLICATE object definition, e.g. two ``CREATE VIRTUAL TABLE
# messages_fts`` rows. SQLite parses the entire schema while preparing the
# FIRST statement on a connection, so on this class *every* statement raises
# before it runs — including ``PRAGMA journal_mode`` (which is why this trips
# in ``apply_wal_with_fallback`` during ``SessionDB.__init__``, long before
# ``_init_schema`` is reached) and even ``PRAGMA integrity_check`` and a plain
# ``DROP TABLE``. The only operations that still work are
# ``PRAGMA writable_schema=ON`` plus direct ``sqlite_master`` surgery.
#
# Symptom users hit (Desktop/Dashboard show "no sessions" while 200+ JSON
# files sit on disk):
# sqlite3.DatabaseError: malformed database schema (messages_fts) -
# table messages_fts already exists
#
# The canonical ``sessions`` / ``messages`` data is intact in these cases —
# only the derived schema is broken — so recovery preserves all transcripts
# and merely rebuilds the FTS layer.
_MALFORMED_SCHEMA_MARKERS = ("malformed database schema",)
_MALFORMED_DB_MARKERS = (
*_MALFORMED_SCHEMA_MARKERS,
"database disk image is malformed",
)
# Process-global guard so auto-repair is attempted at most once per DB path
# per process (prevents repair loops and serialises concurrent web_server /
# gateway opens against the same malformed file).
_repair_attempted_paths: set[str] = set()
_repair_attempt_lock = threading.Lock()
def is_malformed_db_error(exc: BaseException) -> bool:
"""True for explicit malformed-schema or generic corrupt-image errors.
This broad classifier is for diagnostics and explicit offline recovery
dispatch. Runtime repair must use :func:`is_malformed_schema_error`, since
a generic corrupt-image error does not identify the damaged object.
"""
if not isinstance(exc, sqlite3.DatabaseError):
return False
return any(marker in str(exc).lower() for marker in _MALFORMED_DB_MARKERS)
# SQLITE_IOERR, matched as a plain substring so wrapped error strings still
# classify. Shared by the read-only open retry and the write-path BEGIN retry.
_DISK_IO_ERROR_MARKER = "disk i/o error"
# Broader set for HTTP classification: a read that failed for one of these
# reasons found the store BUSY, not gone. Callers map it to 503 (retry, the
# list was not cleared) instead of 500. Corruption is deliberately absent —
# a malformed store must surface, not be retried into a timeout.
_TRANSIENT_SQLITE_MARKERS = (
_DISK_IO_ERROR_MARKER,
"database is locked",
"database table is locked",
"busy",
)
def is_transient_sqlite_error(exc: BaseException) -> bool:
"""True when a SQLite failure means "busy right now", not "damaged".
One predicate so the read paths cannot drift apart on what counts as
recoverable: the read-only open retry, and the HTTP 503-vs-500 split on
the session-list endpoints, classify the same way.
"""
if not isinstance(exc, sqlite3.OperationalError):
return False
message = str(exc).lower()
return any(marker in message for marker in _TRANSIENT_SQLITE_MARKERS)
def _is_transient_read_only_ioerr(exc: sqlite3.OperationalError, *, attempt: int) -> bool:
"""True when a read-only open should be retried rather than raised.
A ``mode=ro`` connection cannot perform WAL recovery (recovery needs to
write the -shm index, which read-only mode refuses), so a concurrent WAL
checkpoint / reset / frame-flush can surface ``SQLITE_IOERR`` ("disk I/O
error") to a reader on an otherwise healthy database (#100436). The
transition is millisecond-scale, so a bounded number of short retries
clears it without changing classification for genuine storage failures —
a persistent IOERR still exhausts the budget and propagates.
"""
return (
attempt < _READ_ONLY_IOERR_RETRY_ATTEMPTS
and _DISK_IO_ERROR_MARKER in str(exc).lower()
)
def is_malformed_schema_error(exc: BaseException) -> bool:
"""True only when SQLite explicitly reports malformed schema text.
A generic ``database disk image is malformed`` error is SQLITE_CORRUPT
and may come from any B-tree or freelist page. It does not prove that
canonical rows are intact, so runtime schema/FTS repair must fail closed.
"""
if not isinstance(exc, sqlite3.DatabaseError):
return False
return any(marker in str(exc).lower() for marker in _MALFORMED_SCHEMA_MARKERS)
# Markers that mean the host filesystem cannot accept another write. Kept as
# plain substrings so OSError, sqlite3.OperationalError, and wrapped RPC
# error strings all match the same helper.
_DISK_FULL_MARKERS = (
"no space left on device",
"not enough space",
"database or disk is full", # SQLITE_FULL
"disk full",
"full disk",
"enospc",
)
def is_disk_full_error(exc: BaseException | str | None) -> bool:
"""True when *exc* (or a stringified error) is a disk-full / ENOSPC failure.
Covers:
* ``OSError`` with ``errno.ENOSPC``
* SQLite ``OperationalError: database or disk is full`` (SQLITE_FULL)
* Plain English / errno strings that survive RPC wrapping
"""
if exc is None:
return False
if isinstance(exc, OSError) and getattr(exc, "errno", None) == errno.ENOSPC:
return True
text = exc if isinstance(exc, str) else str(exc)
lowered = text.lower()
return any(marker in lowered for marker in _DISK_FULL_MARKERS)
# Every cause bucket classify_persistence_error can return. Consumers that
# enumerate causes (e.g. the cron scheduler's explainer-variant suppression)
# must iterate this tuple instead of hardcoding the list, so adding a bucket
# can never silently desynchronize them.
PERSISTENCE_ERROR_CAUSES = (
"locked",
"compression",
"compression_closed",
"turn_lease",
"corrupt",
"replaced",
"disk",
"unknown",
)
# Markers that mean the database FILE itself is structurally damaged. Kept
# as plain substrings so sqlite3.DatabaseError, wrapped RPC strings, and
# logged message text all match the same helper. NOTE: "database disk image
# is malformed" contains the word "disk", so this check MUST run before the
# disk-full/readonly bucket in classify_persistence_error — otherwise real
# B-tree corruption gets reported to the user as "free some disk space"
# (the misdiagnosis documented on #77386).
_DB_CORRUPTION_MARKERS = (
"malformed", # "database disk image is malformed" (SQLITE_CORRUPT)
"file is not a database", # SQLITE_NOTADB (also connection-level poisoning)
"not a database",
"database corruption",
)
def classify_persistence_error(exc_or_str) -> str:
"""Classify a session-persistence failure into a coarse cause bucket.
Fast-failing a turn on a SessionDB write error is deliberate (the
transcript would otherwise be lost on restart), but the *guidance* the
user gets must match the cause: sustained SQLite write-lock contention
("database is locked" on a shared state.db) needs "storage was busy,
send it again", while a full disk or read-only database needs the
disk-space/permissions advice. Returns one of PERSISTENCE_ERROR_CAUSES:
* ``"locked"`` — SQLite lock/busy contention (another process holds the
database write lock); transient, retry-later guidance applies.
* ``"compression"`` — a live compression lease refused the transcript
write; the database itself is healthy and unlocked.
* ``"compression_closed"`` — the write targeted a session already
rotated (closed) by compression and no live continuation was adopted;
the store is healthy — the client must refresh/adopt the new session
id, so disk-space advice would be a misdiagnosis.
* ``"turn_lease"`` — a presented session-turn-lease holder no longer
owns the conversation (expired, released, or reclaimed); fail-fast
fencing, not a storage fault.
* ``"corrupt"`` — the database file itself is structurally damaged
(``database disk image is malformed`` / SQLITE_NOTADB). Distinct from
``"disk"``: freeing space cannot help, the user needs the repair path
(``hermes doctor`` / automatic schema surgery).
* ``"replaced"`` — the ``state.db`` path no longer names the file this
process opened (out-of-band ``cp``/``mv``/restore). In-file FTS repair
cannot help; writes to the live handle must stop.
* ``"disk"`` — disk full / read-only / permission-shaped failures
(delegates the disk-full patterns to :func:`is_disk_full_error` so the
two classifiers can never drift apart — e.g. ENOSPC).
* ``"unknown"`` — anything else (or no visible exception at all).
"""
if exc_or_str is None:
return "unknown"
# A refused write during a live compression lease is contention, not
# storage damage — but its message ("is being compressed by another
# writer" / "Compression lease lost") contains neither "locked" nor
# "busy", so it must be matched by type and by phrase (for strings that
# survived RPC wrapping).
if isinstance(exc_or_str, SessionTurnLeaseLostError):
return "turn_lease"
if isinstance(exc_or_str, CompressionSessionClosedError):
return "compression_closed"
if isinstance(exc_or_str, CompressionSessionBusyError):
return "compression"
if isinstance(exc_or_str, StateDbReplacedError):
# Includes DeletedWalGenerationError (subclass).
return "replaced"
if isinstance(exc_or_str, StateDbCorruptError):
return "corrupt"
text = str(exc_or_str).lower()
if "turn lease" in text:
return "turn_lease"
if "closed by compression" in text:
return "compression_closed"
if "being compressed" in text or "compression lease" in text:
return "compression"
if "was replaced underneath" in text:
return "replaced"
if "deleted state.db-wal" in text or "deleted state.db-shm" in text:
return "replaced"
# Structural corruption BEFORE the lock and disk buckets: "database disk
# image is malformed" contains "disk" (and some wrapped corruption
# strings mention "locked" recovery attempts), so later buckets would
# steal it and misdiagnose damage as space/contention.
if any(marker in text for marker in _DB_CORRUPTION_MARKERS):
return "corrupt"
if (
"locked" in text
or "busy" in text
):
return "locked"
if (
is_disk_full_error(exc_or_str)
or "disk" in text
or "readonly" in text
or "read-only" in text
):
return "disk"
return "unknown"
def _claim_repair_attempt(db_path: Path) -> bool:
"""Claim the one-shot repair attempt for *db_path* in this process.
Returns True for the first caller, False afterwards. Keeps a malformed
DB from triggering an unbounded repair/reopen loop and stops concurrent
callers from racing surgery on the same file.
"""
key = str(db_path)
with _repair_attempt_lock:
if key in _repair_attempted_paths:
return False
_repair_attempted_paths.add(key)
return True
# Cross-process serialisation for the schema-surgery paths below. The
# ``_repair_attempt_lock`` above is a ``threading.Lock`` — it only covers
# threads inside ONE interpreter, yet a normal Hermes host runs several
# independent processes against the same ``state.db``: the gateway service,
# the Desktop app's own ``hermes serve`` backend, interactive CLI sessions,
# and the TUI slash worker. Two of those hitting a malformed DB at once each
# ran the full ``writable_schema`` surgery + ``VACUUM`` on their own private
# connection, with nothing serialising them.
#
# The timeout is sized for the slowest legitimate holder — a ``VACUUM`` over a
# multi-GB DB in strategy 2. Waiting that long is not a new stall: before this
# lock the losing caller spent the same minutes running its own surgery, it
# just did so on top of the winner's.
_REPAIR_LOCK_TIMEOUT_SECONDS = 120.0
_REPAIR_LOCK_POLL_SECONDS = 0.1
# Copying a multi-GB database through SQLite's online-backup API is a data
# transfer, not an inter-process locking operation. Keep the repair-lock
# bound separate and give a conservative 10 MiB/s budget to each snapshot or
# promotion, with the historical two-minute floor for ordinary state.db files.
_REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND = 10 * 1024 * 1024
_IS_WINDOWS = sys.platform == "win32"
@contextlib.contextmanager
def _cross_process_repair_lock(db_path: Path):
"""Serialize state.db schema surgery across processes.
Yields True when this process holds the repair lock for *db_path*, False
when the bounded acquire timed out or the lock file could not be opened at
all. Unlike the kanban init lock — whose critical section is idempotent,
so proceeding without the lock is merely redundant work — proceeding here
would be exactly the unsafe interleaving we are trying to prevent, so a
caller that gets False must NOT do surgery.
``flock`` is the right primitive for this: the kernel drops the lock when
the holding process dies, so a crashed repairer cannot leave a stale lock
that wedges every future repair (a pidfile would). One exception exists
(issue #100108): a forked child that inherited the lock fd keeps the
flock alive after the acquirer dies, so the acquire path records the
holder's pid + start time and breaks the lock when that holder is
provably dead (see ``_acquire_db_flock``). The acquire is still
bounded because a *live* repairer can legitimately sit in ``VACUUM`` for
minutes on a large DB, and an unbounded wait would hang the caller's open
with no traceback (the failure shape of #36644).
"""
lock_path = db_path.with_name(db_path.name + ".repair.lock")
try:
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = lock_path.open("a+b")
except OSError as exc:
# Fail closed, exactly as a timed-out acquire does. A lock file we
# cannot even open means the filesystem is out of space, inodes or
# descriptors — and a sibling that opened ITS handle before the disk
# filled is still inside writable_schema surgery or VACUUM. Yielding
# True here let two processes run schema surgery on the same live
# state.db concurrently, which is itself the corruption source this
# lock exists to remove (#100368: the disk-full trigger, then a fresh
# corruption on every boot with other writers alive). Callers already
# handle False by re-probing and reporting, and on a read-only
# directory no repair strategy could have written anyway.
logger.warning(
"Could not open state.db repair lock %s (%s) — skipping schema "
"surgery rather than running it without cross-process authority.",
lock_path, exc,
)
yield False
return
acquired = False
try:
if _IS_WINDOWS:
deadline = time.monotonic() + _REPAIR_LOCK_TIMEOUT_SECONDS
while True:
try:
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
acquired = True
break
except (BlockingIOError, OSError) as exc:
if not is_advisory_lock_contention(exc):
logger.warning(
"Could not acquire state.db repair lock %s (%s) — "
"skipping schema surgery on a non-contention error.",
lock_path, exc,
)
acquired = None
break
if time.monotonic() >= deadline:
break
time.sleep(_REPAIR_LOCK_POLL_SECONDS)
else:
acquired, handle = _acquire_db_flock(
str(lock_path),
handle,
_REPAIR_LOCK_TIMEOUT_SECONDS,
_REPAIR_LOCK_POLL_SECONDS,
"state.db repair lock",
)
if acquired is None:
# Non-contention failure already logged with its errno.
acquired = False
elif not acquired:
record = None if _IS_WINDOWS else _read_lock_holder_record(handle)
logger.warning(
"state.db repair lock %s held by another process for more "
"than %.0fs — skipping schema surgery in this process to "
"avoid racing the repairer. Recorded holder: %s.",
lock_path, _REPAIR_LOCK_TIMEOUT_SECONDS,
_describe_lock_holder(record),
)
yield acquired
finally:
try:
if acquired:
if _IS_WINDOWS:
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
_clear_lock_holder_record(handle)
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
except OSError: # pragma: no cover - best effort release
pass
finally:
handle.close()
def _try_acquire_auto_maintenance_lock(db_path: Path) -> Optional[Any]:
"""Non-blocking cross-process lock for one auto-maintenance pass.
The kernel releases this advisory lock if the holder exits, unlike a
durable pid/meta marker. A caller that cannot acquire it must skip the
pass: otherwise two startups can both pass the interval check and the
second can prune a row the first has only just closed recoverably.
"""
lock_path = db_path.with_name(db_path.name + ".auto-maintenance.lock")
try:
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = lock_path.open("a+b")
except OSError as exc:
logger.warning(
"Could not open state.db auto-maintenance lock %s (%s) — skipping "
"automatic maintenance.",
lock_path,
exc,
)
return None
try:
if _IS_WINDOWS:
import msvcrt
handle.seek(0)
msvcrt.locking( # type: ignore[attr-defined]
handle.fileno(), msvcrt.LK_NBLCK, 1 # type: ignore[attr-defined]
)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except (BlockingIOError, OSError):
handle.close()
return None
return handle
def _release_auto_maintenance_lock(handle: Any) -> None:
"""Release a handle returned by :func:`_try_acquire_auto_maintenance_lock`."""
try:
if _IS_WINDOWS:
import msvcrt
handle.seek(0)
msvcrt.locking( # type: ignore[attr-defined]
handle.fileno(), msvcrt.LK_UNLCK, 1 # type: ignore[attr-defined]
)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
except OSError: # pragma: no cover - best effort release
pass
finally:
handle.close()
def _bump_schema_cookie(conn: sqlite3.Connection) -> None:
"""Increment the schema cookie after direct ``sqlite_master`` surgery.
Ordinary DDL bumps this counter for free, and every other connection
compares it before running a prepared statement — that is how they learn
to discard a cached schema. Editing ``sqlite_master`` under
``PRAGMA writable_schema=ON`` does NOT bump it, so live connections in
other processes keep compiling statements against the schema we just
deleted objects from — e.g. writing ``messages`` rows through triggers
into ``messages_fts*`` shadow tables that no longer exist. SQLite's
writable_schema documentation calls out incrementing ``schema_version``
as the required companion to such an edit.
Best-effort and never raises: a failed bump leaves exactly the
pre-existing behaviour, and the repair itself is still worth completing.
"""
try:
current = conn.execute("PRAGMA schema_version").fetchone()[0]
# Wraps within the 32-bit signed range SQLite stores this in; the
# comparison other connections make is equality, not ordering.
conn.execute(f"PRAGMA schema_version={(int(current) + 1) & 0x7FFFFFFF}")
except (sqlite3.DatabaseError, TypeError, IndexError) as exc:
logger.warning("Could not bump state.db schema cookie: %s", exc)
# ── Repair-loop bounding + dead-backup hygiene (#86747) ─────────────────────
#
# ``_claim_repair_attempt`` above is an in-memory set: it bounds the loop
# only WITHIN one process. A corruption class the strategies cannot heal
# (b-tree page damage) failed repair on EVERY process start, and each pass
# took a fresh ~900MB forensic backup — 105 attempts / 89GB of identical
# dead copies in the reporting install. Two persistent bounds fix the class:
#
# * a sidecar attempt ledger (``<db>.repair-attempts.json``) that refuses
# further surgery after ``_MAX_PERSISTENT_REPAIR_ATTEMPTS`` failures on
# the SAME damaged file (fingerprint = size + a bounded content sample; any
# successful repair or replacement changes it and resets the count);
# * backup dedupe + a retention cap in ``_backup_db_file`` — an identical
# damaged file is never copied twice, and only the newest
# ``_MAX_MALFORMED_BACKUPS`` forensic copies are kept.
_MAX_PERSISTENT_REPAIR_ATTEMPTS = 3
_MAX_MALFORMED_BACKUPS = 3
# Sidecars copied alongside a damaged DB and pruned with it. ``-journal`` is
# included because rollback-journal (DELETE) mode — Hermes's fallback on
# NFS/SMB/FUSE/ZFS and on WAL-reset-vulnerable SQLite builds — leaves a hot
# journal on disk whenever a transaction was open, and that file is what
# interprets the damaged bytes. Omitting it from the forensic copy means the
# backup cannot be rolled back to a consistent state by hand.
_DB_SIDECAR_SUFFIXES = ("-wal", "-shm", "-journal")
# Head/tail bytes sampled by ``_db_fingerprint``. Enough to change whenever
# the DB is genuinely repaired, truncated or restored (SQLite rewrites the
# header on any real recovery), while staying O(1) on a multi-GB file.
_FINGERPRINT_SAMPLE_BYTES = 65536
# Byte ranges inside SQLite's 100-byte database header that move on ordinary
# commits rather than on repair, and are therefore masked out of the content
# sample. In rollback-journal (DELETE) mode a commit writes the main file
# directly, bumping the file change counter (24-27) and version-valid-for
# (92-95); a malformed-SCHEMA DB still accepts those writes, so without the
# mask any live session write re-keys the ledger and the repair budget resets
# to 1 forever — the exact unbounded loop this ledger exists to stop. (WAL mode
# routes commits to the -wal sidecar, so the main file's header only moves on
# checkpoint; masking is harmless there and correct for both.) Everything that
# matters for repair identity — the page-1 sqlite_master b-tree — sits after
# byte 100 and stays in the sample.
_FINGERPRINT_VOLATILE_HEADER_RANGES = ((24, 28), (92, 96))
def _mask_volatile_header(head: bytes) -> bytes:
"""Zero the commit-counter fields so ordinary writes don't re-key the ledger."""
if len(head) < 96:
return head
buf = bytearray(head)
for start, end in _FINGERPRINT_VOLATILE_HEADER_RANGES:
buf[start:end] = b"\x00" * (end - start)
return bytes(buf)
# Free-space headroom for the pre-repair forensic backup. The backup is a
# full raw copy of the damaged DB (plus its -wal/-shm sidecars), so a repair
# loop on a large state.db is a disk amplifier: the reporting incident wrote
# ~98MB every ~10s until the volume was nearly full, which would have taken
# down every agent on the host.
#
# Proportional, not a flat floor: an absolute multi-GB reserve would refuse
# backups that fit comfortably on small container/VM volumes, and because a
# refused backup is a HARD STOP (#69603) that would silently convert "repair
# loops" into "repair never runs" for those deployments. Require the copy
# itself plus a small slice of the volume, clamped to a modest floor.
_REPAIR_BACKUP_MIN_FREE_BYTES = 256 * 1024 * 1024 # 256 MiB absolute floor
_REPAIR_BACKUP_FREE_FRACTION = 0.02 # plus 2% of the volume
def _repair_backup_headroom_bytes(total_bytes: int) -> int:
"""Free space required *beyond* the copy itself, for a volume of *total_bytes*."""
return max(
_REPAIR_BACKUP_MIN_FREE_BYTES,
int(total_bytes * _REPAIR_BACKUP_FREE_FRACTION),
)
def _repair_scratch_space_error(db_path: Path) -> Optional[str]:
"""Return an error unless snapshot, VACUUM and promotion can fit safely."""
import shutil
try:
main_bytes = db_path.stat().st_size
snapshot_bytes = main_bytes
for suffix in _DB_SIDECAR_SUFFIXES:
sidecar = db_path.with_name(db_path.name + suffix)
if sidecar.exists():
snapshot_bytes += sidecar.stat().st_size
usage = shutil.disk_usage(db_path.parent)
headroom = _repair_backup_headroom_bytes(usage.total)
# Strategy 2 runs VACUUM on the staged database. SQLite documents that
# VACUUM may need up to twice the database size in additional free
# space while it builds the replacement and journals the overwrite.
# Reserve that beyond the snapshot itself; after VACUUM releases its
# temporary files, the same reserve also covers transactional
# promotion into the live database.
required = snapshot_bytes + (2 * snapshot_bytes) + headroom
if usage.free >= required:
return None
return (
f"only {usage.free / 1e9:.2f}GB free on {db_path.parent}; the "
f"repair snapshot needs up to {snapshot_bytes / 1e9:.2f}GB, "
f"VACUUM may need another {(2 * snapshot_bytes) / 1e9:.2f}GB, and "
f"{headroom / 1e9:.2f}GB must remain as headroom. Free disk space, "
"then retry."
)
except OSError as exc:
return (
f"could not determine free space on {db_path.parent} ({exc}); "
"refusing the repair snapshot rather than risk filling the volume"
)
def _repair_snapshot_timeout_seconds(source_path: Path) -> float:
"""Bound one SQLite snapshot by source size, including live sidecars.
A WAL can contain committed canonical rows which are not yet present in
the main database file. Count it (and the rollback journal where
present), both to describe the work honestly and to avoid applying the
repair-lock timeout to an otherwise healthy large-database copy.
"""
source_bytes = 0
for suffix in ("", *_DB_SIDECAR_SUFFIXES):
candidate = (
source_path
if not suffix
else source_path.with_name(source_path.name + suffix)
)
try:
source_bytes += candidate.stat().st_size
except FileNotFoundError:
continue
return max(
_REPAIR_LOCK_TIMEOUT_SECONDS,
source_bytes / _REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND,
)
def _repair_failure_consumes_attempt(exc: BaseException) -> bool:
"""Whether a pre-strategy SQLite failure proves deterministic corruption.
Lock contention, timeouts, disk-full, I/O and filesystem failures are
environmental aborts: retrying later may succeed and must not exhaust the
repair ledger. Only SQLite's corruption/image result codes prove the
deterministic damage the bounded ledger exists to stop from retrying
forever, even when SQLite cannot stage a snapshot far enough to run a
named strategy.
"""
if not isinstance(exc, sqlite3.DatabaseError):
return False
error_code = getattr(exc, "sqlite_errorcode", None)
if isinstance(error_code, int):
# Extended result codes retain the primary code in the low byte.
primary_code = error_code & 0xFF
return primary_code in (sqlite3.SQLITE_CORRUPT, sqlite3.SQLITE_NOTADB)
# sqlite3 versions before exception result-code attributes need a narrow,
# conservative compatibility path. Do not turn generic DatabaseError
# messages such as "disk is full" or "readonly" into permanent failures.
message = str(exc).lower()
return (
"file is not a database" in message
or "database disk image is malformed" in message
)
def _repair_ledger_path(db_path: Path) -> Path:
return db_path.with_name(db_path.name + ".repair-attempts.json")
def _db_fingerprint(db_path: Path) -> "Optional[str]":
"""Cheap identity for a damaged DB file: size + a bounded content sample.
Deliberately EXCLUDES mtime. The original ledger keyed on
``size:mtime_ns`` on the assumption that "nothing can successfully write
to a damaged file", but that does not hold for the malformed-schema
class: the DB still opens and accepts writes (only ``sqlite_master`` is
unreadable), so live writers, WAL checkpoints and the in-place repair
strategies themselves all move mtime between passes. Every pass then
looked like a NEW file — the attempt counter reset to 1 forever, never
reaching ``_MAX_PERSISTENT_REPAIR_ATTEMPTS``, and the ``_backup_db_file``
dedupe (which compares mtime too) never matched, so each pass wrote
another full-size forensic copy. Observed: a repair every ~10s, a fresh
98MB copy each time, 2.3GB in 20 minutes, disk heading to zero.
Hashing a multi-GB corrupt file on every open is the repeated cost this
ledger exists to avoid, so sample instead of digesting the whole file:
size plus the head/tail slices that any real repair, truncation or
restore necessarily changes. Stable across passes that merely touch
mtime; still resets the attempt count after genuine recovery.
The content read runs under ``offline_file_access`` because it takes a raw
descriptor, and ``close()`` on ANY descriptor cancels every POSIX advisory
lock this process holds on the file — including a peer connection's
RESERVED lock (see ``hermes_cli.sqlite_safe_read`` rule 1). This function
is reached from ``repair_state_db_schema``'s exhaustion probe BEFORE
``_backup_db_file``'s ``has_live_connection`` guard, and the repair path is
entered by one SessionDB while the gateway holds others, so a live peer is
the expected case rather than a theoretical one.
Returns ``None`` when a live connection makes the read unsafe. Callers MUST
NOT substitute a differently-shaped key (an earlier revision fell back to
``size:mtime_ns``): the ledger compares keys for equality, so alternating
between a content key and an mtime key across passes never matches, the
counter resets to 1 every time and the unbounded repair loop this ledger
exists to stop comes straight back. ``None`` means "identity unavailable",
and the ledger helpers below keep using the key already on record.
"""
try:
st = db_path.stat()
try:
from hermes_cli.sqlite_safe_read import (
LiveConnectionError,
offline_file_access,
)
except ImportError:
# Scaffold/embed installs ship hermes_state without hermes_cli. No
# tracked connections exist there, so the raw read is safe.
@contextmanager
def offline_file_access(_path, **_kw):
yield
class LiveConnectionError(Exception):
pass
try:
with offline_file_access(db_path, what="fingerprint"):
with open(db_path, "rb") as fh:
head = fh.read(_FINGERPRINT_SAMPLE_BYTES)
if st.st_size > _FINGERPRINT_SAMPLE_BYTES:
fh.seek(max(0, st.st_size - _FINGERPRINT_SAMPLE_BYTES))
tail = fh.read(_FINGERPRINT_SAMPLE_BYTES)
else:
tail = b""
except LiveConnectionError:
return None
digest = hashlib.sha256(_mask_volatile_header(head) + tail).hexdigest()[:32]
return f"{st.st_size}:{digest}"
except OSError:
return None
def _backup_content_identity(db_path: Path) -> "Optional[str]":
"""Recovery-image identity for forensic-backup dedupe: whole-file + sidecars.
This is a DIFFERENT equivalence relation from :func:`_db_fingerprint`, and
the two MUST NOT be conflated. ``_db_fingerprint`` answers "same repair
epoch?" — it masks SQLite's commit counters and samples only the head/tail
so an ordinary write does not mint a fresh repair budget. That is exactly
the wrong predicate for "may I reuse an existing forensic copy?": a live
writer can commit new transcript/session rows into an *interior* page while
preserving file size and leaving the first/last 64 KiB untouched, so two
materially different recovery images share one ``_db_fingerprint``. Reusing
a backup on that basis hands the operator a snapshot that predates real
user data (and #87409 shows a failed in-place repair can still VACUUM
canonical tables away), so the forensic copy must claim byte identity, not
epoch identity.
So this digests the ENTIRE main file plus every present sidecar
(``-wal``/``-shm``/``-journal``) — the WAL can hold committed frames not yet
checkpointed, so it is part of the recovery image. The cost is an O(n) read;
on a miss the caller is about to do an O(n) *write* (the full raw copy), so
the read is the cheaper half and never the dominant cost. Runs under
``offline_file_access`` for the same POSIX-advisory-lock reason as
``_db_fingerprint``; returns ``None`` when a live connection makes the read
unsafe (caller then declines to dedupe and takes a fresh backup — the safe
side, never a false reuse).
"""
try:
from hermes_cli.sqlite_safe_read import (
LiveConnectionError,
offline_file_access,
)
except ImportError:
@contextmanager
def offline_file_access(_path, **_kw):
yield
class LiveConnectionError(Exception):
pass
def _hash_whole(path: Path, hasher: "Any") -> None:
with open(path, "rb") as fh:
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
hasher.update(chunk)
try:
hasher = hashlib.sha256()
with offline_file_access(db_path, what="backup-identity"):
# Length-delimit every member (main file included) so the
# concatenation is prefix-free — otherwise a main-file tail could
# coincide with a main+sidecar split and dedupe two different
# recovery images together.
hasher.update(f"\0main:{db_path.stat().st_size}\0".encode())
_hash_whole(db_path, hasher)
for suffix in _DB_SIDECAR_SUFFIXES:
sidecar = db_path.with_name(db_path.name + suffix)
if sidecar.exists():
hasher.update(f"\0{suffix}:{sidecar.stat().st_size}\0".encode())
_hash_whole(sidecar, hasher)
return hasher.hexdigest()
except LiveConnectionError:
return None
except OSError:
return None
def _read_repair_ledger(db_path: Path) -> "Dict[str, Any]":
try:
raw = json.loads(_repair_ledger_path(db_path).read_text(encoding="utf-8"))
if isinstance(raw, dict):
return raw
except (OSError, ValueError):
pass
return {}
def _persistent_repair_attempts_exhausted(db_path: Path) -> bool:
"""Whether *db_path* has already burned its cross-restart repair budget.
True only when the ledger records ``_MAX_PERSISTENT_REPAIR_ATTEMPTS``
failed attempts against the CURRENT file fingerprint. Never raises; a
missing/corrupt ledger or unstatable DB reads as "not exhausted" (the
in-process claim and cross-process lock still bound a single run).
When the fingerprint is unavailable because a live connection makes the
content read unsafe, fall back to the SIZE the ledger recorded rather than
reading as "not exhausted". Otherwise a peer connection is enough to hide
an exhausted budget on every pass, which is the unbounded loop again.
"""
ledger = _read_repair_ledger(db_path)
recorded = ledger.get("fingerprint")
fp = _db_fingerprint(db_path)
if fp is None:
# Size is the one component both key shapes share and that a raw read
# is not needed for; an unchanged size means the damaged file is very
# likely the same one the budget was burned on.
try:
size_prefix = f"{db_path.stat().st_size}:"
except OSError:
return False
if not isinstance(recorded, str) or not recorded.startswith(size_prefix):
return False
elif recorded != fp:
return False
return int(ledger.get("failed_attempts", 0)) >= _MAX_PERSISTENT_REPAIR_ATTEMPTS
def _persistent_repair_exhausted_error(db_path: Path) -> str:
"""The stable operator-facing diagnostic for an exhausted repair budget."""
return (
f"automatic repair has already failed "
f"{_MAX_PERSISTENT_REPAIR_ATTEMPTS} times on this exact file — "
"the corruption is beyond the schema/FTS repair strategies "
"(likely b-tree page damage). Manual recovery required: restore "
"a backup, or salvage with `hermes sessions recover --source "
f"{db_path} --inspect-only`, then (if it reports recoverable) "
f"`hermes sessions recover --source {db_path} --output "
"recovered-state.db` (recovery snapshots the damaged file first, "
"then runs the page-level `.recover` lane on the copy; do NOT "
"point a raw `sqlite3` shell at the live database). "
f"Delete {_repair_ledger_path(db_path).name} to force another "
"automatic attempt."
)
def _record_repair_outcome(
db_path: Path, *, repaired: bool, fingerprint: "Optional[str]" = None
) -> None:
"""Update the persistent attempt ledger after a repair pass. Never raises.
Defaults to the post-attempt fingerprint — the file state the NEXT
attempt's exhaustion probe will observe.
When the fingerprint is unavailable (a live connection makes the content
read unsafe), keep the key already on record and still increment: dropping
the pass would let a peer connection reset the budget every time, which is
the unbounded loop this ledger exists to stop. Never write a differently
shaped key — the probe compares for equality, so mixing key shapes across
passes never matches.
"""
ledger_path = _repair_ledger_path(db_path)
try:
if repaired:
ledger_path.unlink(missing_ok=True)
return
ledger = _read_repair_ledger(db_path)
recorded = ledger.get("fingerprint")
fp = fingerprint if fingerprint is not None else _db_fingerprint(db_path)
if fp is None:
if not isinstance(recorded, str):
# No prior key to extend and no way to mint one safely: the
# in-process claim and cross-process lock still bound this run.
return
fp = recorded
attempts = (
int(ledger.get("failed_attempts", 0)) + 1 if recorded == fp else 1
)
import datetime
ledger_path.write_text(
json.dumps(
{
"fingerprint": fp,
"failed_attempts": attempts,
"last_attempt": datetime.datetime.now().isoformat(
timespec="seconds"
),
}
),
encoding="utf-8",
)
except Exception as exc: # pragma: no cover - best effort
logger.warning("Could not update state.db repair ledger: %s", exc)
def _existing_malformed_backups(db_path: Path) -> "List[Path]":
"""Timestamped forensic backups of *db_path*, newest first."""
prefix = f"{db_path.name}.malformed-backup-"
try:
found = [
p
for p in db_path.parent.iterdir()
if p.name.startswith(prefix)
and not p.name.endswith(_DB_SIDECAR_SUFFIXES)
]
except OSError:
return []
return sorted(found, key=lambda p: p.name, reverse=True)
def _prune_malformed_backups(db_path: Path, keep: int = _MAX_MALFORMED_BACKUPS) -> None:
"""Delete all but the *keep* newest forensic backups (and sidecars)."""
for stale in _existing_malformed_backups(db_path)[keep:]:
for victim in (
stale,
*(stale.with_name(stale.name + suffix) for suffix in _DB_SIDECAR_SUFFIXES),
):
try:
victim.unlink(missing_ok=True)
except OSError as exc: # pragma: no cover - best effort
logger.warning("Could not prune stale DB backup %s: %s", victim, exc)
def _backup_db_file(db_path: Path) -> "Tuple[Optional[Path], Optional[str]]":
"""Copy a (possibly malformed) DB file to a timestamped backup beside it.
Raw file copy on purpose: the DB won't open cleanly, so we preserve the
bytes exactly for forensics / manual restore. WAL, SHM and rollback-journal
sidecars are copied too when present. Returns ``(backup_path, None)`` on success or
``(None, reason)`` on failure — callers on the repair path treat a
refused backup as a HARD STOP (see #69603). Repair strategies run on a
scratch snapshot, but the forensic bundle remains the recovery path when
corruption defeats them.
Refuses when a connection to this database is still live in the process:
reading the file would ``close()`` a descriptor for it and cancel that
connection's POSIX advisory locks (see ``hermes_cli.sqlite_safe_read``).
The repair path can be entered by one SessionDB while the gateway holds
others, so this is a real possibility rather than a theoretical one.
"""
import datetime
import shutil
try:
from hermes_cli.sqlite_safe_read import has_live_connection
except ImportError:
has_live_connection = None # type: ignore[assignment]
if has_live_connection is not None and has_live_connection(db_path):
reason = (
f"a connection to {db_path} is still open in this process; "
"raw-copying it would cancel that connection's POSIX advisory "
"locks. Close all SessionDB handles first."
)
logger.error("Refusing to raw-copy %s for backup: %s", db_path, reason)
return None, reason
stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
backup_path = db_path.with_name(f"{db_path.name}.malformed-backup-{stamp}")
# Same-second collision (two distinct damaged states within one second)
# must not silently overwrite the earlier forensic copy.
seq = 1
while backup_path.exists():
backup_path = db_path.with_name(
f"{db_path.name}.malformed-backup-{stamp}_{seq}"
)
seq += 1
try:
# Sweep staging debris from an earlier interrupted pass (kill mid-copy)
# BEFORE the dedupe below. A leftover staging file is a byte-identical
# copy of the damaged DB, so its fingerprint MATCHES and the dedupe
# would otherwise hand it back as a legitimate forensic backup.
# Matches sidecar staging names (``.backup-staging-<stamp>-wal``) too.
# The second pattern is the pre-merge ``.incomplete`` spelling, swept so
# a host that ran that build does not keep prefix-matching debris that
# sorts NEWEST and survives prune forever.
for pattern in (
f"{db_path.name}.backup-staging-*",
f"{db_path.name}.malformed-backup-*.incomplete*",
):
for old in db_path.parent.glob(pattern):
try:
old.unlink(missing_ok=True)
except OSError: # pragma: no cover - best effort
pass
# Dedupe (#86747): a repair loop used to copy the SAME damaged bytes
# on every restart — ~900MB a pass, 89GB over 11 days in the
# reporting install. If the newest existing backup is byte-identical to
# the current recovery image, reuse it.
#
# Matching on mtime made this dedupe miss exactly when it mattered
# most: the malformed-SCHEMA class still accepts writes, so live
# writers and the in-place repair strategies move mtime between
# passes and every pass wrote another full-size copy (2.3GB in 20
# minutes).
#
# Use ``_backup_content_identity`` (whole file + sidecars), NOT the
# repair-epoch ``_db_fingerprint``. They are different equivalence
# relations: the fingerprint masks commit counters and samples only
# head/tail so an ordinary interior-page write does not re-key the
# repair budget — but that same write DOES change the recovery image,
# and deduping on the fingerprint would hand back a stale backup that
# predates the write. A forensic copy must prove byte identity, so it
# pays the O(n) read (cheaper than the O(n) write it avoids on a hit).
try:
# Only hash the source when there is actually a candidate to dedupe
# against — on the common first-corruption pass there is no prior
# backup, and hashing the (possibly multi-GB) source then would be
# pure waste right before the copy reads it again anyway.
existing_backups = _existing_malformed_backups(db_path)[:1]
if existing_backups:
src_id = _backup_content_identity(db_path)
for existing in existing_backups:
if src_id is not None and _backup_content_identity(existing) == src_id:
logger.info(
"Reusing existing forensic backup %s (identical to the "
"damaged DB).", existing,
)
return existing, None
except OSError:
pass
# Disk guard: this is a full raw copy of a possibly multi-GB DB plus
# its sidecars. On a host whose volume is already nearly full — which
# a preceding repair loop may itself have caused — taking it can
# finish off the disk and take down every process on the machine.
# Refuse while there is still room to refuse in.
try:
need = db_path.stat().st_size
for suffix in _DB_SIDECAR_SUFFIXES:
sidecar = db_path.with_name(db_path.name + suffix)
if sidecar.exists():
need += sidecar.stat().st_size
usage = shutil.disk_usage(db_path.parent)
headroom = _repair_backup_headroom_bytes(usage.total)
if usage.free - need < headroom:
reason = (
f"only {usage.free / 1e9:.2f}GB free on {db_path.parent}; "
f"copying the damaged DB needs {need / 1e9:.2f}GB and must "
f"leave {headroom / 1e9:.2f}GB headroom. Free disk space, "
"then retry (or recover manually with "
f"`hermes sessions recover --source {db_path} "
"--inspect-only` first)."
)
logger.error("Refusing forensic backup of %s: %s", db_path, reason)
return None, reason
except OSError as exc:
# Fail CLOSED. This guard exists for the nearly-full volume, which
# is exactly where stat()/disk_usage() is most likely to fail — and
# proceeding would take the multi-GB copy that finishes off the
# disk. A refused backup is a HARD STOP (#69603), so repair simply
# does not run until a human frees space, which is the safe side.
reason = (
f"could not determine free space on {db_path.parent} ({exc}); "
"refusing the forensic copy rather than risk filling the "
f"volume. Free disk space, then retry (or recover manually "
f"with `hermes sessions recover --source {db_path} "
"--inspect-only` first)."
)
logger.error("Refusing forensic backup of %s: %s", db_path, reason)
return None, reason
# Copy to a staging name OUTSIDE the ``.malformed-backup-`` prefix, then
# rename into place only once every copy has succeeded. The prefix
# matters: ``_existing_malformed_backups`` matches on
# ``startswith(f"{db}.malformed-backup-")`` and excludes only ``-wal``/
# ``-shm`` suffixes, so a staging name derived from the backup name (e.g.
# ``…malformed-backup-<stamp>.incomplete``) still counts as a backup —
# it sorts NEWEST (``.incomplete`` > the bare stamp), so prune's
# keep-3-newest slice retained partials and deleted intact copies, and
# the dedupe could hand a partial back as the official ``backup_path``,
# passing the #69603 hard-stop gate with no real forensic copy on disk.
staging = db_path.with_name(f"{db_path.name}.backup-staging-{stamp}")
# (staging_src, final_dst) pairs. ORDER MATTERS for publication: the
# main-DB backup name is the bundle's commit marker —
# ``_existing_malformed_backups`` matches ``{db}.malformed-backup-*``
# and excludes only the ``-wal``/``-shm``/``-journal`` suffixes, so the
# main file appearing is what makes the bundle "count". Sidecars are
# therefore staged/published FIRST and the main DB LAST, so a failure
# partway through never leaves a countable main backup standing over a
# missing sidecar (an incomplete recovery image that would pass the
# #69603 hard stop and dedupe as legitimate on the next pass).
staged_sidecars: "List[Tuple[Path, Path, Path]]" = []
for suffix in _DB_SIDECAR_SUFFIXES:
sidecar = db_path.with_name(db_path.name + suffix)
if sidecar.exists():
side_staging = staging.with_name(staging.name + suffix)
side_dst = backup_path.with_name(backup_path.name + suffix)
staged_sidecars.append((sidecar, side_staging, side_dst))
main_pair = (staging, backup_path)
published: "List[Path]" = []
all_staging_srcs = [staging] + [s for _src, s, _d in staged_sidecars]
try:
shutil.copy2(db_path, staging)
for sidecar, side_staging, _side_dst in staged_sidecars:
shutil.copy2(sidecar, side_staging)
# Publish sidecars first, main DB LAST (the commit marker), so a
# mid-publish failure never leaves a countable-but-incomplete bundle.
publish_order = [
(s, d) for _src, s, d in staged_sidecars
] + [main_pair]
for src, dst in publish_order:
os.replace(src, dst)
published.append(dst)
except Exception:
# Roll back BOTH unpublished staging files AND anything already
# promoted — the old code unlinked only staging srcs, so a failure
# after the main os.replace left the official backup_path on disk.
for src in all_staging_srcs:
try:
src.unlink(missing_ok=True)
except OSError:
pass
for dst in published:
try:
dst.unlink(missing_ok=True)
except OSError:
pass
try:
staging.unlink(missing_ok=True)
except OSError:
pass
raise
# Retention cap (#86747): keep only the newest few forensic copies.
_prune_malformed_backups(db_path)
return backup_path, None
except Exception as exc: # pragma: no cover - best effort
logger.warning("Could not back up malformed DB %s: %s", db_path, exc)
return None, f"backup copy failed: {exc}"
def preflight_db_writability(
db_path: Path,
*,
db_label: str = "state.db",
) -> None:
"""Refuse-or-repair read-only DB files BEFORE the first connection opens.
Port of Kilo-Org/kilocode#12508's startup preflight. A stray read-only
``state.db`` / ``-wal`` / ``-shm`` (sudo run, restored backup, copied
dotfiles) previously surfaced as an opaque
``sqlite3.OperationalError: attempt to write a readonly database`` raised
from deep inside ``_init_schema`` — naming no file and no fix — and the
obvious wrong "fix" (deleting the ``-wal``) silently loses committed
transactions. This preflight:
- **Repairs** permissions with ``chmod u+rw`` when the file lives inside
the Hermes home tree (``get_hermes_home()``) — the safe repair scope:
Hermes owns those files, and the OS makes ``chmod`` fail on files the
user doesn't own, which bounds the repair exactly.
- **Fails fast with an actionable error** naming the exact file and the
exact ``chmod`` command for anything else (root-owned files, read-only
mounts, custom paths outside the home tree).
- Never deletes or truncates a WAL sidecar — once writable, the normal
open path checkpoints its committed frames into the DB as intended.
``:memory:`` and ``file:`` URI paths are skipped (no plain on-disk files
to check). Shared by :class:`SessionDB` and ``hermes_cli.kanban_db``.
"""
raw = str(db_path)
if raw == ":memory:" or raw.startswith("file:"):
return
try:
home: Optional[Path] = Path(get_hermes_home()).resolve()
except Exception: # pragma: no cover - defensive
home = None
def _in_repair_scope(p: Path) -> bool:
if home is None:
return False
try:
return p.resolve().is_relative_to(home)
except (OSError, ValueError):
return False
def _ensure_writable(p: Path, *, is_dir: bool = False) -> None:
import stat as _stat
if os.access(p, os.R_OK | os.W_OK):
return
if _in_repair_scope(p):
try:
add = _stat.S_IRUSR | _stat.S_IWUSR | (_stat.S_IXUSR if is_dir else 0)
os.chmod(p, p.stat().st_mode | add)
except OSError:
pass
if os.access(p, os.R_OK | os.W_OK):
logger.info(
"%s preflight: repaired read-only %s (chmod u+rw%s)",
db_label,
p,
"x" if is_dir else "",
)
return
kind = "directory" if is_dir else "file"
wal_note = (
" Do NOT delete the -wal file — it contains committed data that "
"will be merged into the database once it is writable."
if p.name.endswith("-wal")
else ""
)
raise sqlite3.OperationalError(
f"{db_label} is not writable: {kind} {p} is read-only for this "
f"user. Hermes needs read-write access to open the database. "
f"Fix with: chmod u+rw{'x' if is_dir else ''} '{p}'"
f" (files owned by another user may need sudo/chown).{wal_note}"
)
parent = db_path.parent
if parent.is_dir():
# SQLite needs a writable directory in every journal mode (WAL and
# SHM sidecars in WAL mode; the rollback journal in DELETE mode).
_ensure_writable(parent, is_dir=True)
for suffix in ("", "-wal", "-shm"):
p = db_path.with_name(db_path.name + suffix) if suffix else db_path
if p.is_file():
_ensure_writable(p)
def _connect_repair_durable(
db_path: Path, *, timeout: float = 5.0
) -> sqlite3.Connection:
"""``sqlite3.connect`` for the repair/probe paths, with macOS write barriers.
These paths open ``state.db`` directly rather than through ``SessionDB``
(which routes via :func:`apply_wal_with_fallback`), so they inherited
SQLite's ``synchronous=NORMAL`` default and no ``checkpoint_fullfsync``.
On Darwin that is exactly the combination :func:`_enforce_macos_synchronous_full`
exists to prevent: ``fsync()`` there guarantees neither data-on-platter nor
write ordering, so a rewrite interrupted by process or OS termination can
leave half-written b-tree pages behind.
That matters more here than anywhere else in the module, because what runs
through these connections is ``REINDEX``, ``VACUUM`` and ``writable_schema``
surgery — the operations that rewrite nearly every page of the file. The
2026-08-19 recurrence tore ``messages`` (root page 5) and
``idx_messages_session``, reporting the unmistakable signature: repeated
"2nd reference to page", a rowid out of order, and long runs of leaked
"never used" pages.
Autocommit (``isolation_level=None``) is preserved: callers run DDL and
``VACUUM``, which are illegal inside an implicit transaction.
Applying the barriers is best-effort *by necessity*: SQLite loads the
schema before it runs any statement, so on a malformed schema even
``PRAGMA synchronous=FULL`` raises ``DatabaseError`` ("malformed database
schema (messages_fts) - table messages_fts already exists"). A malformed
database is precisely this helper's input, so raising there would leave
repair unable to open the file it exists to fix. Strategies that go on to
rewrite the whole file call :func:`_reapply_durability_barriers` once the
schema parses again, which is the point at which the pragmas can stick.
"""
conn = sqlite3.connect(str(db_path), timeout=timeout, isolation_level=None)
_reapply_durability_barriers(conn)
return conn
def _reapply_durability_barriers(conn: sqlite3.Connection) -> bool:
"""Best-effort (re)application of the macOS write barriers. Never raises.
Returns True when the pragmas were accepted. Callers about to rewrite the
file wholesale (``VACUUM``, ``REINDEX``) should call this after the schema
becomes parseable, because a connection opened against a malformed schema
could not take them at open time.
"""
try:
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return True
except sqlite3.DatabaseError:
# Schema still unparseable — the pragmas cannot be set yet.
return False
except Exception:
return False
def apply_durability_barriers(conn: sqlite3.Connection) -> bool:
"""Apply state-store durability barriers without changing journal mode.
This is the public entry point for secondary users of ``state.db`` that
must inherit its owner's journal mode while retaining per-connection
durability settings. Also applies the configured ``database.synchronous``
level (a per-connection pragma that would otherwise only ride on the
journal-mode setup path guest connections must not run).
"""
ok = _reapply_durability_barriers(conn)
try:
# Local import avoids a circular import with hermes_cli.config.
from hermes_cli.config import cfg_get, load_config_readonly
cfg = load_config_readonly()
raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None)
if raw_synchronous is not None:
_apply_synchronous_pragma(
conn, raw_synchronous, db_label="state.db (guest)"
)
except Exception:
pass
return ok
@contextmanager
def _exclusive_repair_db_guard(db_path: Path):
"""Yield one live connection that excludes writers for repair surgery.
``locking_mode=EXCLUSIVE`` retains SQLite's file-level exclusion after the
short ``BEGIN EXCLUSIVE`` transaction is rolled back. That rollback is
essential: ``Connection.backup`` may use the guarded connection as a
*source* while it is transaction-free, and it must be transaction-free
when it is later the promotion *destination*. The connection itself
remains open for the entire snapshot -> strategies -> promotion window,
so another writer cannot commit a change that promotion could overwrite.
Existing WAL readers make exclusive acquisition fail rather than being
disturbed. In DELETE mode an existing reader similarly prevents
``BEGIN EXCLUSIVE``; a future reader/writer waits behind the guard. A
repair therefore fails closed whenever this process cannot own that whole
window.
"""
guard: Optional[sqlite3.Connection] = None
try:
# The cross-process repair lock already serializes repairers. Do not
# wait behind an ordinary application connection: a partial repair is
# less safe than an explicit "stop the gateway and retry" result.
guard = _connect_repair_durable(db_path, timeout=0.0)
guard.execute("PRAGMA locking_mode=EXCLUSIVE")
guard.execute("BEGIN EXCLUSIVE")
guard.execute("ROLLBACK")
except (sqlite3.Error, OSError) as exc:
if guard is not None:
try:
guard.execute("PRAGMA locking_mode=NORMAL")
except Exception:
pass
guard.close()
yield None, exc
return
try:
yield guard, None
finally:
try:
# Let SQLite release the exclusive locks before close; this also
# avoids a connection-close checkpoint being mistaken for a
# repair write in callers that immediately reopen state.db.
guard.execute("PRAGMA locking_mode=NORMAL")
except Exception:
pass
guard.close()
def _copy_database_snapshot(
source_path: Path,
destination_path: Path,
*,
source_connection: Optional[sqlite3.Connection] = None,
destination_connection: Optional[sqlite3.Connection] = None,
) -> None:
"""Copy one complete SQLite snapshot without replacing either file inode.
SQLite's online backup API incorporates committed WAL frames into the
source snapshot and writes the destination inside one transaction. This
avoids both the main-file-only staging gap and replacing ``state.db`` from
under handles that already refer to it. If backup is interrupted, SQLite
rolls the destination transaction back.
"""
# Work out the deadline before opening an owned source connection. A
# sidecar disappearing while we stat it is an ordinary staging failure,
# but it must not leak a just-opened SQLite descriptor.
deadline_seconds = _repair_snapshot_timeout_seconds(source_path)
deadline = time.monotonic() + deadline_seconds
source = source_connection or _connect_repair_durable(source_path)
destination = destination_connection
own_source = source_connection is None
own_destination = destination_connection is None
def _check_deadline(_status: int, _remaining: int, _total: int) -> None:
if time.monotonic() >= deadline:
raise TimeoutError(
"timed out copying SQLite repair snapshot after "
f"{deadline_seconds:.0f}s"
)
try:
if destination is None:
destination = _connect_repair_durable(destination_path)
elif destination.in_transaction:
# sqlite3_backup requires a transaction-free destination. The
# exclusive repair guard deliberately retains file exclusion via
# locking_mode, not an active transaction, so it satisfies this.
raise sqlite3.ProgrammingError(
"SQLite repair backup destination has an active transaction"
)
source.backup(
destination,
pages=256,
progress=_check_deadline,
sleep=_REPAIR_LOCK_POLL_SECONDS,
)
finally:
if own_destination and destination is not None:
destination.close()
if own_source:
source.close()
def _db_opens_cleanly(db_path: Path) -> Optional[str]:
"""Probe a DB on a fresh connection. Returns None if healthy, else a reason.
Runs the same first-statement (``PRAGMA journal_mode``) that trips the
malformed-schema parse, then ``PRAGMA integrity_check`` and a canonical
``sessions`` read, and finally a rolled-back ``messages`` write so that
FTS5 index corruption — which leaves base-table reads and
``integrity_check`` passing while every ``INSERT INTO messages`` fails
through the FTS triggers — is reported as unhealthy rather than slipping
past as a false "ok" (#50502).
"""
conn = _connect_repair_durable(db_path)
try:
# Best-effort tokenizer load: a DB carrying the messages_fts_cjk
# index needs the cjk_unicode61 extension before any statement can
# touch that table — including the trigger-driven write probe below.
# Without it, this probe sees the DB exactly as a tokenizer-less
# SessionDB open would (which drops the cjk triggers to keep writes
# working), so tokenizer absence must never classify as corruption.
load_fts5_cjk_extension(conn)
conn.execute("PRAGMA journal_mode").fetchone()
rows = conn.execute("PRAGMA integrity_check").fetchall()
problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != "ok"]
if problems:
return "; ".join(problems[:3])
conn.execute("SELECT COUNT(*) FROM sessions").fetchone()
# FTS5 read probe: run a representative MATCH query against the
# messages_fts* virtual tables. The FTS *write* probe below catches
# the corruption class where base tables read fine but writes fail
# through the triggers (#50502). It does NOT catch partial FTS5
# index corruption — bad shadow-table segments where reads still
# parse but MATCH / snippet / rank queries error out with
# "database disk image is malformed" (a `sqlite3.DatabaseError`,
# not `OperationalError`). session_search, /resume title resolution,
# and any feature relying on FTS5 discovery then break silently
# because the official repair tool's check-only path reports the
# DB as healthy. #66724.
# Catch the full sqlite3 exception hierarchy (not just
# OperationalError) so the malformed-shadow-table class is reported
# rather than letting it crash the caller.
for fts_table in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"):
try:
# No-op queries against the actual FTS5 APIs the search
# tools use. The trigram table is included because it backs
# the title-resolution path; either corruption mode would
# break session recall without this probe. MATCH '""' is
# the empty phrase-token probe — FTS5 rejects MATCH ''
# outright ("fts5: syntax error"), but a quoted empty
# phrase parses, scans zero rows, and exercises the same
# shadow-table read path the search tools use.
conn.execute(
f"SELECT 1 FROM {fts_table} WHERE {fts_table} MATCH '\"\"' LIMIT 1"
).fetchone()
except sqlite3.OperationalError as exc:
# Use the canonical capability classifier instead of a
# hand-rolled substring check. On SQLite builds without the
# fts5 module, the legacy messages_fts table may exist on
# disk (from a prior build that had FTS5) and MATCH queries
# against it raise OperationalError("no such module: fts5");
# the substring check below would misclassify that as
# corruption and send the DB into the repair path, whose
# final fallback deletes the messages_fts% schema
# (hermes_state.py:645-723). The supported degraded-runtime
# path (SessionDB._is_fts5_unavailable_error + the
# regression suite in tests/test_hermes_state.py:600-632)
# treats both "no such module: fts5" and
# "no such tokenizer: trigram" as the capability error.
if SessionDB._is_fts5_unavailable_error(exc):
# Degraded runtime — not the corruption class we probe.
continue
msg = str(exc).lower()
if "no such table" in msg or "no such column" in msg:
# FTS5 not built yet (brand new file mid-init) — not the
# corruption class we probe.
continue
return f"fts5 read probe failed on {fts_table}: {exc}"
except sqlite3.DatabaseError as exc:
# This is the corruption class #66724 actually wants caught:
# partial shadow-table damage where MATCH / snippet / rank
# queries raise DatabaseError("database disk image is malformed")
# while reads of the FTS5 table itself parse fine.
return f"fts5 read probe failed on {fts_table}: {exc}"
# FTS write probe: drive a row through the messages_fts* triggers in a
# transaction that is always rolled back, so a corrupt FTS index that
# rejects writes is caught even though reads look healthy. The probe is
# best-effort — if the messages/sessions tables don't exist yet (brand
# new file mid-init) the OperationalError is treated as "not yet a
# populated DB", not corruption.
probe_session_id = f"_hermes_fts_health_probe_{time.time_ns()}"
try:
conn.execute("BEGIN IMMEDIATE")
conn.execute(
"INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)",
(probe_session_id, "_health_probe", time.time()),
)
conn.execute(
"INSERT INTO messages (session_id, role, content, timestamp) "
"VALUES (?, ?, ?, ?)",
(probe_session_id, "user", "_fts_health_probe", time.time()),
)
conn.execute("ROLLBACK")
except sqlite3.OperationalError as exc:
# Missing tables / FTS disabled — not the corruption class we probe.
try:
conn.execute("ROLLBACK")
except sqlite3.Error:
pass
msg = str(exc).lower()
if "no such table" in msg or "no such column" in msg:
return None
if "no such tokenizer: cjk_unicode61" in msg:
# This probe process couldn't load the cjk extension while
# the DB carries the cjk index — capability gap, not
# corruption. A tokenizer-capable SessionDB serves it fine;
# a tokenizer-less one self-heals by dropping the triggers.
return None
return str(exc)
return None
except sqlite3.DatabaseError as exc:
return str(exc)
finally:
conn.close()
def _foreign_state_db_holders(db_path: Path) -> List[Tuple[int, str]]:
"""Compatibility delegate to the state-holder authority."""
return _state_holders.foreign_state_db_holders(db_path)
def _live_writer_holds_db(db_path: Path) -> bool:
"""Compatibility delegate to the repair-admission authority."""
return _state_holders.live_writer_holds_db(
db_path,
connect_repair_durable=_connect_repair_durable,
)
def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, Any]:
"""Repair a state.db whose ``sqlite_master`` schema is malformed or whose
FTS indexes reject writes.
Handles two corruption classes: the "duplicate object definition" /
malformed-schema class where even ``PRAGMA`` statements fail, and the FTS
write-corruption class (#50502) where base tables read fine and
``integrity_check`` passes but writes fail through the ``messages_fts*``
triggers. Tries least-destructive recovery first and escalates:
1. **Rebuild FTS indexes in place** via the FTS5 ``'rebuild'`` command,
which rewrites the internal b-tree segments from the canonical
``messages`` rows without dropping or recreating anything. Fixes the
FTS write-corruption class while preserving the schema intact.
2. **De-duplicate** ``sqlite_master`` (keep the lowest rowid per
``type``/``name``). Fixes the canonical "table X already exists"
case and PRESERVES the existing FTS index intact.
3. **Drop the FTS schema** (every ``messages_fts*`` object) + ``VACUUM``.
The next ``SessionDB()`` open rebuilds the FTS indexes from the
canonical ``messages`` table.
Canonical ``sessions`` / ``messages`` rows are never modified by a failed
attempt. Mutating strategies run against a complete SQLite snapshot and a
successful result is copied back transactionally. A timestamped raw backup
is taken first unless ``backup=False``.
The surgery below is serialised across processes (see
:func:`_cross_process_repair_lock`): the gateway service, the Desktop
app's backend and interactive CLI sessions all open the same file, and
two of them running ``writable_schema`` surgery concurrently is itself a
corruption source.
Returns a report dict: ``{repaired: bool, strategy: str|None,
backup_path: str|None, error: str|None}``.
"""
report: Dict[str, Any] = {
"repaired": False,
"strategy": None,
"backup_path": None,
"error": None,
}
# Startup-watchdog progress lease: repair (raw backup copy + surgery +
# VACUUM) is I/O-bound — near-zero CPU on a multi-GB file — which the
# watchdog's CPU fallback would misread as a parked deadlock (OOF-298).
# Single lease is deliberate (clamped to _MAX_LEASE_S=900): honest worst
# case is up to the lease duration of zombie time on a wedged repair,
# accepted over per-chunk renewal complexity in the repair loop.
report_startup_progress(900.0, phase="state_db_repair")
db_path = Path(db_path)
if not db_path.exists():
report["error"] = f"{db_path} does not exist"
return report
# Cross-restart attempt cap (#86747): the in-memory claim bounds one
# process, but a corruption class the strategies below cannot heal
# (b-tree page damage) previously re-ran the whole surgery — and took a
# fresh multi-hundred-MB forensic backup — on EVERY restart, forever.
# After _MAX_PERSISTENT_REPAIR_ATTEMPTS failures against the same
# damaged file, stop retrying and surface a terminal, actionable error.
if _persistent_repair_attempts_exhausted(db_path):
report["error"] = _persistent_repair_exhausted_error(db_path)
logger.error("state.db repair skipped: %s", report["error"])
return report
result = report
with _cross_process_repair_lock(db_path) as holding_lock:
if not holding_lock:
# Another process is still inside its critical section, or the
# lock file itself could not be opened (full disk / no fds). It
# may nonetheless have healed the file already (long VACUUM after
# a successful strategy), so re-probe before reporting failure.
if _db_opens_cleanly(db_path) is None:
report["repaired"] = True
report["strategy"] = "repaired_by_other_process"
else:
report["error"] = (
"could not obtain the state.db repair lock (held by "
"another process, or the lock file was unopenable); "
"skipped schema surgery to avoid racing a concurrent "
"repairer"
)
else:
# The fast check above avoids taking the lock for a known-exhausted
# image. Recheck after acquisition: a queued repairer can have
# recorded the final failure while this process waited, and this
# process must not start a fourth attempt.
if _persistent_repair_attempts_exhausted(db_path):
report["error"] = _persistent_repair_exhausted_error(db_path)
logger.error("state.db repair skipped: %s", report["error"])
# Keep the existing WAL-holder preflight: it preserves the
# established fail-closed behaviour for active readers before we
# create a forensic backup. It is not the race defence; the
# retained exclusive guard inside the locked routine is what
# excludes writers continuously through promotion. DELETE-mode
# readers which this probe cannot see are still rejected by the
# later BEGIN EXCLUSIVE acquisition.
elif _live_writer_holds_db(db_path):
report["error"] = (
"a live writer still holds state.db; skipped schema surgery "
"to avoid tearing b-tree pages under a concurrent writer. "
"Stop the gateway (hermes gateway stop) and retry."
)
logger.error("state.db repair skipped: %s", report["error"])
else:
# Probe the mode BEFORE surgery (#89674): every repair
# strategy rewrites the file, and a rebuilt SQLite file comes
# back in the default journal mode (delete) — silently moving
# a WAL store out of WAL with nothing in the logs recording
# the flip. The open-time WAL-reset gate never sees this flip
# because it happens inside the repair path (distinct from
# the open-time flip #89393 warns about). A probe of the
# damaged file may fail, in which case the canonical
# database.journal_mode setting is the restore target.
before_mode = _probe_journal_mode_for_repair(db_path)
result = _repair_state_db_schema_locked(
db_path,
backup=backup,
report=report,
journal_mode_before=before_mode,
)
if result.get("repaired"):
result["journal_mode_before"] = before_mode
# Environmental aborts happen before a strategy gets to mutate the
# isolated snapshot. They are retriable operating conditions, not
# proof that the damaged database exhausted a repair strategy.
# Keep that private signal out of the public report while
# successful health checks still clear a stale persistent failure
# record. This ledger update stays under the same cross-process
# lock as surgery, so two repairers cannot lose each other's
# attempt updates. A queued loser must not record at all: its
# owner is responsible for its outcome.
attempted = bool(result.pop("_repair_attempted", False))
if attempted or result.get("repaired"):
_record_repair_outcome(
db_path, repaired=bool(result.get("repaired"))
)
return result
def _probe_journal_mode_for_repair(db_path: Path) -> Optional[str]:
"""Best-effort journal-mode probe for a (possibly malformed) DB file.
Returns the on-disk mode (``wal``/``delete``), or ``None`` when the file
cannot be opened or probed — a malformed header or a concurrent opener's
locks are both expected on the repair path. Callers fall back to the
configured ``database.journal_mode`` for ``None``.
"""
try:
conn = _connect_repair_durable(db_path)
try:
return _on_disk_journal_mode(conn)
finally:
conn.close()
except (sqlite3.Error, OSError):
return None
def _restore_journal_mode_after_repair(
db_path: Path, before_mode: Optional[str], *, conn=None
) -> None:
"""Re-apply the journal mode after schema surgery (#89674).
A repaired/rebuilt SQLite file comes back in the default journal mode
(delete). Without this restore, a corruption event deterministically
moves a WAL store out of WAL and nothing records the change — the
WAL-reset gate at open time never sees the flip because it happened
inside the repair path, not at open (the open-time flip #89393 warns
about is a different door).
``conn`` must be the exclusive repair guard connection when called from
the repair path (#101064): opening a fresh connection AFTER the guard
released let a writer still holding the unlinked old ``-wal`` inode
coexist with a brand-new ``state.db-wal`` this connection created — two
generations of one store. The transactional promotion already leaves the
destination in its pre-repair mode, so on that path this is mostly the
WAL-companion re-assertion; the reopen is the hazard, not the mode.
The restore runs through :func:`apply_wal_with_fallback` — the canonical
journal-mode path — rather than issuing a switch pragma directly, so it
inherits the vulnerable-SQLite WAL-reset gate (a rebuilt file IS a new
database: on a vulnerable runtime the gate deliberately keeps it in
DELETE, and "restore could not reach WAL" there is the expected outcome,
not a failure), the macOS-NFS silent-refusal handling, and the WAL
companions (size limit, checkpoint barrier, synchronous=FULL) that the
front door applies. ``before_mode`` is the pre-surgery probe (None when
the damaged file could not be probed) and is only used for the log
comparison — the restore target itself is whatever the canonical path
resolves from ``database.journal_mode``.
Best-effort by design: the repair itself already succeeded, so failures
to re-apply are logged at WARNING, never raised.
"""
owned_conn = conn is None
try:
if owned_conn:
conn = _connect_repair_durable(db_path)
try:
after = apply_wal_with_fallback(conn, db_label=db_path.name)
finally:
if owned_conn:
conn.close()
if before_mode and after != before_mode:
logger.warning(
"state.db repair changed journal_mode %r -> %r "
"(pre-surgery probe %r; restore resolved through "
"apply_wal_with_fallback per database.journal_mode and the "
"WAL-reset gate)",
before_mode, after, before_mode,
)
except (sqlite3.Error, OSError) as exc:
logger.warning(
"state.db repair at %s: post-surgery journal-mode restore "
"failed (%s); verify with PRAGMA journal_mode on the next open",
db_path, exc,
)
def _repair_state_db_schema_locked(
db_path: Path,
*,
backup: bool,
report: Dict[str, Any],
journal_mode_before: Optional[str] = None,
) -> Dict[str, Any]:
"""Repair strategies for :func:`repair_state_db_schema`.
Caller must hold the cross-process repair lock for *db_path*.
The strategies run on a SCRATCH COPY and the result is copied back through
SQLite's transactional backup API only once it is proven to open cleanly.
A repair that does not succeed therefore cannot modify or lose committed
canonical data. In WAL mode SQLite may checkpoint already-committed WAL
frames into the main file while the exclusive guard is released; that is
not a repair mutation and does not change the committed database image.
They used to run in place, and Strategy 2 ends in ``VACUUM``. VACUUM does
not preserve what it cannot parse: it rebuilds the file from the schema
SQLite can still read, so when the damage IS in the schema b-tree — page
1's child pointers resolving to data pages, which is exactly the
``malformed database schema ()`` class this function exists to handle —
every table hanging off the unreadable part is silently dropped. The probe
afterwards then correctly reports the file is STILL malformed, so the
function returns ``repaired=False`` and advises a manual restore, having
already destroyed the thing it was asked to save. Destroying the data and
reporting the repair failed are not mutually exclusive outcomes, and
nothing here treated them as a contradiction.
The pre-repair backup (#69603) does not close this: it is a forensic
artefact that nothing reads back, so recovery still depends on a human
noticing a ``.malformed-backup-*`` file and knowing what to do with it.
Not mutating the original in the first place is the property that holds
without a human in the loop.
"""
scratch = db_path.with_name(f"{db_path.name}.repair-scratch")
cleanup_error = _unlink_db_triple(scratch)
if cleanup_error is not None:
report["error"] = (
"could not remove a stale repair snapshot before probing state.db: "
f"{cleanup_error}"
)
logger.error("state.db repair aborted: %s", report["error"])
return report
# Re-probe under the lock: a process we queued behind may have just
# repaired the file, in which case redoing the surgery would undo its
# work on a now-healthy DB (the repair/re-corrupt cascade this lock
# exists to break).
if _db_opens_cleanly(db_path) is None:
report["repaired"] = True
report["strategy"] = "already_healthy"
return report
if backup:
bpath, backup_error = _backup_db_file(db_path)
report["backup_path"] = str(bpath) if bpath else None
if bpath is None:
# HARD STOP (#69603). The forensic recovery image remains required
# when corruption defeats every strategy, even though the
# strategies themselves now run against an isolated snapshot.
report["error"] = (
"pre-repair backup refused; aborting schema repair to avoid "
f"mutating the only copy of the damaged DB: {backup_error}"
)
logger.error("state.db repair aborted: %s", report["error"])
return report
# The forensic copy intentionally happens before this guard: its raw-copy
# safety checks inspect real live holders and would be poisoned by our
# exclusive connection. Everything that can affect the repair image or
# live promotion happens only after writer exclusion is held.
with _exclusive_repair_db_guard(db_path) as (live_guard, guard_error):
if live_guard is None:
report["error"] = (
"could not acquire exclusive state.db repair ownership; "
"skipped schema surgery to avoid overwriting a concurrent "
f"writer. Stop the gateway and retry: {guard_error}"
)
if guard_error is not None and _repair_failure_consumes_attempt(
guard_error
):
report["_repair_attempted"] = True
logger.error("state.db repair skipped: %s", report["error"])
return report
space_error = _repair_scratch_space_error(db_path)
if space_error is not None:
report["error"] = space_error
logger.error("state.db repair aborted: %s", report["error"])
return report
try:
# Reuse live_guard rather than opening a second source connection:
# the guard owns the exclusion, so a second connection could be
# blocked by our own EXCLUSIVE lock on some SQLite builds.
_copy_database_snapshot(
db_path, scratch, source_connection=live_guard
)
except (OSError, sqlite3.Error, TimeoutError) as exc:
report["error"] = (
f"could not stage a complete SQLite repair snapshot of {db_path}: {exc}"
)
if _repair_failure_consumes_attempt(exc):
report["_repair_attempted"] = True
logger.error("state.db repair aborted: %s", report["error"])
_unlink_db_triple(scratch)
return report
try:
# This private marker is consumed by the outer wrapper. A strategy
# failure is a genuine repair outcome and consumes the persistent
# budget. A later promotion failure is classified separately:
# full disks, I/O, permission and lock failures are environmental
# aborts, not evidence that the strategy cannot repair this image.
report["_repair_attempted"] = True
_run_repair_strategies(scratch, report)
if report.get("repaired"):
try:
# Do not os.replace the live DB: Windows rejects
# replacement under open handles, while POSIX would leave
# those handles on the old inode. The same transaction-
# free guard that staged the live image receives the
# promotion, retaining writer exclusion throughout.
_copy_database_snapshot(
scratch,
db_path,
destination_connection=live_guard,
)
except (OSError, sqlite3.Error, TimeoutError) as exc:
report["repaired"] = False
report["strategy"] = None
report["_repair_attempted"] = _repair_failure_consumes_attempt(
exc
)
report["error"] = (
"repaired snapshot could not be promoted transactionally: "
f"{exc}"
)
logger.error("state.db repair promotion failed: %s", exc)
else:
logger.warning(
"state.db repaired via '%s' and promoted transactionally: %s",
report.get("strategy"),
db_path,
)
_restore_journal_mode_after_repair(
db_path,
journal_mode_before,
conn=live_guard,
)
if not report.get("repaired"):
# Logged HERE, not inside the strategies: they run against the
# scratch copy, and naming that throwaway path in the one
# message a human is meant to act on would send them to a file
# that no longer exists by the time they read it.
logger.error(
"state.db schema repair could not recover %s automatically "
"(no committed canonical data was modified or lost; backup: %s); "
"manual restore from backup may be required.",
db_path,
report["backup_path"],
)
return report
finally:
# Never leave a half-repaired file beside the DB for a later probe
# — or a later human — to mistake for the real thing.
cleanup_error = _unlink_db_triple(scratch)
if cleanup_error is not None:
logger.warning(
"Could not remove state.db repair snapshot after repair: %s",
cleanup_error,
)
def _unlink_db_triple(path: Path) -> Optional[str]:
"""Remove *path* and every SQLite sidecar; return any cleanup failure."""
failures: List[str] = []
for suffix in ("", *_DB_SIDECAR_SUFFIXES):
victim = path if not suffix else path.with_name(path.name + suffix)
for attempt in range(10):
try:
victim.unlink()
break
except FileNotFoundError:
break
except PermissionError as exc:
# Windows may retain a just-closed SQLite handle for a few
# scheduler ticks. Bound the retry; a later backup open still
# fails safely if the handle truly remains live.
if _IS_WINDOWS and attempt < 9:
time.sleep(0.05)
continue
failures.append(f"{victim}: {exc}")
break
except OSError as exc:
failures.append(f"{victim}: {exc}")
break
return "; ".join(failures) or None
def _run_repair_strategies(
db_path: Path, report: Dict[str, Any]
) -> Dict[str, Any]:
"""Escalating repair attempts, applied to *db_path* IN PLACE.
Every strategy here mutates its argument — FTS rebuild, REINDEX,
``writable_schema`` surgery, ``VACUUM``. It is therefore only ever called
by :func:`_repair_state_db_schema_locked` on a scratch copy that nothing
else holds open, never on the user's database.
"""
# ── Strategy 0: rebuild FTS indexes in place (FTS write-corruption) ──
# The FTS5 'rebuild' command rewrites the internal index from the canonical
# content table. This is the recommended, least-destructive recovery for a
# corrupt FTS index that rejects message writes while reads still succeed.
try:
conn = _connect_repair_durable(db_path)
try:
# The cjk index can only be rebuilt with its tokenizer loaded;
# best-effort (a tokenizer-less host skips it at the probe below).
load_fts5_cjk_extension(conn)
for table_name in (
"messages_fts", "messages_fts_trigram", "messages_fts_cjk"
):
try:
conn.execute(
f"INSERT INTO {table_name}({table_name}) VALUES('rebuild')"
)
except sqlite3.OperationalError:
# Table absent (FTS disabled / trigram off / cjk not
# present or tokenizer unavailable) — skip it.
continue
finally:
conn.close()
if _db_opens_cleanly(db_path) is None:
report["repaired"] = True
report["strategy"] = "rebuild_fts"
logger.warning(
"state.db FTS indexes rebuilt in place (schema preserved): %s",
db_path,
)
return report
except sqlite3.DatabaseError as exc:
logger.warning("state.db FTS in-place rebuild pass failed: %s", exc)
# ── Strategy 0.5: rebuild stale B-tree indexes (#63386) ──
# PRAGMA integrity_check can report "wrong # of entries in index" when a
# B-tree index (e.g. idx_sessions_handoff_state) falls out of sync with its
# base table. REINDEX rewrites the index b-tree from the canonical table
# rows using the existing index definition, fixing the mismatch without
# touching data or FTS schema.
try:
conn = _connect_repair_durable(db_path)
try:
# REINDEX rewrites every index b-tree; take the barriers now that
# the schema parses, in case the open-time attempt was refused.
_reapply_durability_barriers(conn)
conn.execute("REINDEX")
conn.commit()
finally:
conn.close()
if _db_opens_cleanly(db_path) is None:
report["repaired"] = True
report["strategy"] = "reindex_btree"
logger.warning(
"state.db B-tree indexes rebuilt via REINDEX: %s", db_path
)
return report
except sqlite3.DatabaseError as exc:
logger.warning("state.db REINDEX pass failed: %s", exc)
# ── Strategy 1: de-duplicate sqlite_master (keeps FTS index) ──
try:
conn = _connect_repair_durable(db_path)
try:
conn.execute("PRAGMA writable_schema=ON")
dupes = conn.execute(
"SELECT type, name, COUNT(*) AS c, MIN(rowid) AS keep "
"FROM sqlite_master GROUP BY type, name HAVING c > 1"
).fetchall()
for type_, name, _count, keep in dupes:
conn.execute(
"DELETE FROM sqlite_master "
"WHERE type IS ? AND name IS ? AND rowid <> ?",
(type_, name, keep),
)
if dupes:
_bump_schema_cookie(conn)
conn.execute("PRAGMA writable_schema=OFF")
conn.commit()
finally:
conn.close()
if _db_opens_cleanly(db_path) is None:
report["repaired"] = True
report["strategy"] = "dedup_schema"
logger.warning(
"state.db schema repaired by de-duplicating sqlite_master "
"(FTS index preserved): %s", db_path
)
return report
except sqlite3.DatabaseError as exc:
logger.warning("state.db dedup repair pass failed: %s", exc)
# ── Strategy 2: drop all FTS schema, VACUUM, rebuild on next open ──
#
# The destructive one, and the reason this whole path now runs on a
# scratch copy. VACUUM rebuilds the file from the schema SQLite can still
# parse, so on a damaged schema b-tree it silently drops every table
# hanging off the unreadable part — and the probe below then correctly
# reports the result is still malformed. On a scratch copy that is merely
# a discarded attempt; on the live file it was data loss.
try:
conn = _connect_repair_durable(db_path)
try:
conn.execute("PRAGMA writable_schema=ON")
conn.execute("DELETE FROM sqlite_master WHERE name LIKE 'messages_fts%'")
_bump_schema_cookie(conn)
conn.execute("PRAGMA writable_schema=OFF")
conn.commit()
# The schema is repaired and parseable now, so the barriers can
# finally stick — and VACUUM, which rewrites the entire file, is
# the single most damaging operation to lose halfway.
_reapply_durability_barriers(conn)
conn.execute("VACUUM")
finally:
conn.close()
reason = _db_opens_cleanly(db_path)
if reason is None:
report["repaired"] = True
report["strategy"] = "drop_fts_rebuild"
logger.warning(
"state.db schema repaired by dropping FTS schema; indexes "
"will rebuild from messages on next open: %s", db_path
)
return report
report["error"] = reason
except sqlite3.DatabaseError as exc:
report["error"] = str(exc)
# The "could not recover" log lives in the caller: it must name the user's
# database, not the scratch copy these strategies were handed.
return report
# ── CJK-bigram FTS index (replaces the trigram index when available) ────
#
# The trigram tokenizer needs >=3 chars per query term, so 1-2 char CJK
# terms (ubiquitous in Korean/Chinese: 일본, 구글, 项目, ...) fall through
# to a LIKE full-table scan — measured 3-6s CPU per query on multi-GB
# installs and the dominant base cost of session_search on CJK workloads.
#
# ``cjk_unicode61`` (native/fts5_cjk/, a ~250-line loadable FTS5 tokenizer
# with no dependencies) wraps unicode61: maximal CJK runs are re-emitted as
# overlapping character bigrams (Lucene CJKAnalyzer semantics), everything
# else passes through unchanged. FTS5 phrase semantics turn a query term's
# consecutive bigrams into exact substring matching down to 2 chars at
# index speed. Contributed by Soju06 (PR #65544).
#
# Same v23 storage discipline as the trigram table it replaces:
# external-content over a tool-row-excluding view (zero inline text
# copies; tool rows stay searchable via ``messages_fts``), triggers gated
# on a DEDICATED marker pair (``fts_cjk_rebuild_high_water`` /
# ``fts_cjk_rebuild_progress``) so a cjk-only backfill — e.g. the
# trigram→cjk upgrade on an already-optimized DB — never gates the
# complete ``messages_fts`` index's triggers.
#
# The table exists ONLY when the loadable tokenizer is available
# (``~/.hermes/lib/libfts5_cjk.so``, built by ``native/fts5_cjk/build.sh``).
# A process that cannot load it self-heals by dropping the cjk triggers
# (message writes keep working; the index goes stale and is rebuilt by the
# next ``hermes sessions optimize-storage`` on a capable host).
#
# Split DDL: the table/view part is safe to ensure any time; the triggers
# are created ONLY while the index is complete-or-marker-gated. A stale
# index (trigger gap of unknown extent) must keep its triggers DROPPED —
# an external-content 'delete' op for a rowid the index never held is the
# canonical FTS5 index-corruption hazard the v23 marker gating exists to
# prevent.
FTS_CJK_TABLE_SQL = """
CREATE VIEW IF NOT EXISTS messages_fts_cjk_src AS
SELECT id, role, content, tool_name, tool_calls
FROM messages
WHERE role <> 'tool';
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_cjk USING fts5(
content,
tool_name,
tool_calls,
content='messages_fts_cjk_src',
content_rowid='id',
tokenize='cjk_unicode61'
);
"""
FTS_CJK_TRIGGER_SQL = """
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_insert AFTER INSERT ON messages
WHEN new.role <> 'tool'
AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
WHERE key = 'fts_cjk_rebuild_progress'), -1))
BEGIN
INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls)
VALUES (new.id, new.content, new.tool_name, new.tool_calls);
END;
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_delete AFTER DELETE ON messages
WHEN old.role <> 'tool'
AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
WHERE key = 'fts_cjk_rebuild_progress'), -1))
BEGIN
INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls)
VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls);
END;
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_update
AFTER UPDATE OF content, tool_name, tool_calls, role ON messages
WHEN (old.content IS NOT new.content
OR old.tool_name IS NOT new.tool_name
OR old.tool_calls IS NOT new.tool_calls
OR old.role IS NOT new.role)
AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
WHERE key = 'fts_cjk_rebuild_progress'), -1))
BEGIN
INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls)
SELECT 'delete', old.id, old.content, old.tool_name, old.tool_calls
WHERE old.role <> 'tool';
INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls)
SELECT new.id, new.content, new.tool_name, new.tool_calls
WHERE new.role <> 'tool';
END;
"""
def fts5_cjk_so_path() -> Path:
"""Location of the cjk_unicode61 loadable extension."""
env = os.getenv("HERMES_FTS5_CJK_SO")
if env:
return Path(env).expanduser()
return get_hermes_home() / "lib" / "libfts5_cjk.so"
def _cjk_fts_config_enabled() -> bool:
"""config.yaml ``sessions.cjk_fts`` (default on), via its env bridge."""
return os.getenv("HERMES_CJK_FTS", "1").strip().lower() not in (
"0", "false", "off", "no",
)
def load_fts5_cjk_extension(conn: sqlite3.Connection) -> bool:
"""Best-effort load of the cjk_unicode61 tokenizer into ``conn``.
Returns False (never raises) when the .so is absent, the feature is
disabled via ``sessions.cjk_fts``, or this Python build has extension
loading compiled out — every caller treats False as "behave exactly as
before the cjk index existed".
"""
if not _cjk_fts_config_enabled():
return False
path = fts5_cjk_so_path()
if not path.exists():
return False
try:
conn.enable_load_extension(True)
try:
conn.load_extension(str(path))
finally:
conn.enable_load_extension(False)
return True
except Exception:
logger.warning("fts5_cjk extension load failed (%s)", path, exc_info=True)
return False
class CompressionSessionClosedError(RuntimeError):
"""A durable write targeted a parent already closed by compression."""
def __init__(self, session_id: str):
self.session_id = session_id
super().__init__(
f"Session {session_id!r} is closed by compression; "
"adopt its live continuation before appending messages"
)
class CompressionSessionBusyError(RuntimeError):
"""A non-owner tried to write while compression owns the session."""
class SessionCompressionInProgressError(CompressionSessionBusyError):
"""A concurrent writer collided with a *live* compression lock.
Split out from :class:`CompressionSessionBusyError` because the two
conditions that class covers need opposite handling. This one is
transient: a healthy compressor holds the session for a few seconds and
the lock row carries its own ``expires_at``, so the write can simply wait
(see ``_execute_write``'s patience loop). The other case, a compressor
discovering its own lease is gone, is permanent and must fail fast rather
than spin out the whole patience budget.
Subclassing keeps every existing ``except CompressionSessionBusyError``
handler working unchanged.
"""
class SessionTurnLeaseLostError(RuntimeError):
"""A transcript write presented a turn-lease holder that no longer owns it.
Fail-fast fencing: do not retry inside ``_execute_write``. The caller
either still thinks it owns the conversation after expiry/reclaim, or
the lease row is gone. A later writer may already be persisting a
newer turn; landing this write would interleave a stale reply.
"""
class StateDbReplacedError(RuntimeError):
"""The state.db path no longer names the file this SessionDB opened.
Raised when an out-of-band ``cp``/``mv``/restore replaces the database
under a live gateway. In-place FTS repair and fail-open trigger
dropping cannot fix a generation mismatch; they amplify it.
"""
class DeletedWalGenerationError(StateDbReplacedError):
"""A live process holds a deleted state.db-wal / -shm generation.
Opening or writing through this handle would mint a second WAL inode
(or keep committing on the orphan) — the split-brain that produces
intermittent SQLITE_CORRUPT / SQLITE_IOERR. Stop the writers; do not
unlink the WAL yourself. ``database.journal_mode: delete`` is operator
containment, not a default change.
Subclasses :class:`StateDbReplacedError` so every downstream consumer
that already stops SQLite writes and diverts pending transcripts on a
replaced store (gateway retry queue, run_agent flush) handles the split
WAL generation identically — the correct response is the same: stop
writing, preserve the transcript tail on disk.
"""
# SQLite header: 4-byte big-endian application_id at offset 68. Distinct from
# inode: ``cp`` onto the same path keeps st_ino and truncates+rewrites.
_STATE_DB_APPLICATION_ID_OFFSET = 68
_STATE_DB_GENERATION_KEY = "db_file_generation"
_STATE_DB_REPLACED_MSG = (
"FATAL: state.db was replaced underneath the gateway; refusing further "
"writes to this file. Divert transcripts to sessions/<id>.jsonl (and the "
"gateway pending_messages spool) and restore or reopen after operator "
"intervention."
)
_DELETED_WAL_GENERATION_MSG = (
"FATAL: a live process holds a deleted state.db-wal or state.db-shm "
"inode while the path names a different (or missing) generation. "
"Refusing to open or write so a second WAL cannot be minted. "
"Stop the gateway, dashboard, and cron writers that hold the deleted "
"sidecar, then reopen. Do not delete the WAL yourself. "
"database.journal_mode: delete is operator containment, not a new default."
)
class StateDbCorruptError(sqlite3.DatabaseError):
"""A live SessionDB observed structural (non-FTS) corruption and is quarantined.
Raised once a write on this handle reports bare ``SQLITE_CORRUPT`` /
``SQLITE_NOTADB`` that is neither FTS-scoped (``_is_fts_write_corruption_error``)
nor a replaced-file case (``StateDbReplacedError``). Subclasses
``sqlite3.DatabaseError`` so every existing ``except sqlite3.Error``
degrade path keeps working; ``sqlite_errorcode``/``sqlite_errorname``
are copied from the originating error.
The quarantine is sticky for the life of the handle: later writes fail
fast, the handle never reopens after ``close()``, and ``close()`` skips
its own WAL checkpoint. Field evidence (the #90837 lost/reordered-page
signature, the #90950 page-1 clobber): a handle that kept writing for ~50
minutes after the first structural error checkpointed 15 pages under the
wrong page numbers on shutdown, turning a still-readable file into
``file is not a database``. Stopping the writes is what prevents that;
skipping the explicit checkpoint is the second line of defence. SQLite
still runs its own last-connection checkpoint inside ``close()`` (and
deletes the ``-wal`` sidecar) unless ``SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE``
is set — Python exposes it via ``Connection.setconfig()`` on 3.12+, so
quarantine disables the close-time checkpoint there and the WAL survives
on disk for forensics; on 3.11 the internal checkpoint is unavoidable
(post-quarantine it can only carry pre-corruption committed frames, since
no further writes are accepted). The
recovery boundary is a process restart on a repaired or restored file.
"""
_STATE_DB_CORRUPT_MSG = (
"FATAL: state.db reported structural corruption (database disk image is "
"malformed outside the FTS shadow tables) on a live handle; refusing further "
"writes, automatic reopen, and the close-time WAL checkpoint on this file. "
"Stop the gateway, then run `hermes sessions recover --source <state.db> "
"--inspect-only` or restore a snapshot. Unwritten transcripts are diverted to "
"sessions/<id>.jsonl (and the gateway pending_messages spool)."
)
def divert_session_transcript_jsonl(session_id: str, messages) -> "Optional[Path]":
"""Append pending messages as JSON lines under HERMES_HOME/sessions.
Used when state.db is replaced under a live process so the current
turn is not only in RAM. Returns the jsonl path, or None when there
is nothing to write.
"""
sid = str(session_id or "").strip()
if not sid or not messages:
return None
sessions_dir = get_hermes_home() / "sessions"
sessions_dir.mkdir(parents=True, exist_ok=True)
path = sessions_dir / f"{sid}.jsonl"
with path.open("a", encoding="utf-8") as handle:
for msg in messages:
if isinstance(msg, dict):
handle.write(json.dumps(msg, ensure_ascii=False, default=str) + "\n")
elif msg is not None:
handle.write(json.dumps({"content": str(msg)}, ensure_ascii=False) + "\n")
return path
# _read_sqlite_application_id runs on EVERY write via _raise_if_db_replaced,
# against the LIVE state.db. A bare open()/read()/close() there is the
# howtocorrupt §2.2 bug: close() cancels every POSIX advisory lock this
# process holds on the file — measured on Linux/SQLite 3.53.1, one probe call
# drops the WAL-mode DMS shared lock the writer connection holds on state.db
# (see hermes_cli/sqlite_safe_read.py for the module built around this rule).
# With the DMS lock gone, a fresh opener in another process can treat this
# writer as dead and rerun WAL-index recovery underneath it.
#
# The probe therefore reads through a per-path fd cached for the life of the
# process: opening an fd never cancels locks (only close() does), and
# os.pread takes no shared file position. When the path is re-pointed at a
# new inode (the very replacement this probe exists to detect), the stale fd
# is RETIRED, never closed — closing it would cancel the live connection's
# locks on the old file, the exact bug being avoided. Replacement events are
# rare and halt writes anyway, so the leak is bounded.
_HEADER_PROBE_LOCK = threading.Lock()
_HEADER_PROBE_FDS: "dict[str, tuple[int, int, int]]" = {} # key -> (fd, dev, ino)
_RETIRED_HEADER_PROBE_FDS: "list[int]" = [] # intentionally never closed
def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]":
"""Lock-safe raw header read of a possibly-live SQLite database.
POSIX: pread from a cached, never-closed fd (rebound when the path names
a new inode). Windows: plain read — advisory-lock cancellation is a
POSIX-only hazard and msvcrt locks do not share the failure mode.
"""
if _IS_WINDOWS:
try:
with db_path.open("rb") as handle:
return handle.read(length)
except OSError:
return None
key = str(db_path)
try:
st = os.stat(db_path)
except OSError:
return None
with _HEADER_PROBE_LOCK:
cached = _HEADER_PROBE_FDS.get(key)
if cached is not None and (cached[1], cached[2]) != (st.st_dev, st.st_ino):
# Path re-pointed at a new file. Retire (never close) the old fd.
_RETIRED_HEADER_PROBE_FDS.append(cached[0])
cached = None
del _HEADER_PROBE_FDS[key]
if cached is None:
try:
fd = os.open(db_path, os.O_RDONLY)
except OSError:
return None
try:
fst = os.fstat(fd)
except OSError:
_RETIRED_HEADER_PROBE_FDS.append(fd)
return None
cached = (fd, fst.st_dev, fst.st_ino)
_HEADER_PROBE_FDS[key] = cached
try:
return os.pread(cached[0], length, 0)
except OSError:
return None
def _read_sqlite_application_id(db_path: Path) -> "Optional[int]":
"""Read application_id from the SQLite header without opening a connection.
Safe against live databases: routed through :func:`_pread_db_header`,
which never issues a ``close()`` that would cancel this process's POSIX
locks on the file (howtocorrupt §2.2).
"""
header = _pread_db_header(db_path, _STATE_DB_APPLICATION_ID_OFFSET + 4)
if header is None:
return None
if len(header) < _STATE_DB_APPLICATION_ID_OFFSET + 4:
return None
if header[:16] != b"SQLite format 3\x00":
return None
return int(
struct.unpack(
">I",
header[_STATE_DB_APPLICATION_ID_OFFSET:_STATE_DB_APPLICATION_ID_OFFSET + 4],
)[0]
)
def _stat_db_file_identity(path: Path) -> "Optional[tuple]":
"""Return ``(st_dev, st_ino)`` for *path*, or None when identity is unavailable."""
try:
st = os.stat(path)
except OSError:
return None
# Windows volumes (and some network FS) report st_ino=0; a (0, 0)
# identity would false-positive every check. Skip the inode half of
# the guard there; generation stamp still applies.
if not st.st_dev or not st.st_ino:
return None
return (st.st_dev, st.st_ino)
def _stat_sqlite_sidecar_identity(db_path: Path) -> Dict[str, tuple]:
"""Snapshot ``(st_dev, st_ino)`` for existing WAL/SHM sidecars."""
identities: Dict[str, tuple] = {}
base = os.fspath(db_path)
for suffix in ("-wal", "-shm"):
ident = _stat_db_file_identity(Path(base + suffix))
if ident is not None:
identities[suffix] = ident
return identities
_canonical_sqlite_path = _state_holders.canonical_sqlite_path
def _watched_sqlite_sidecar_paths(db_path) -> Set[str]:
base = os.path.abspath(os.fspath(db_path))
return {
_canonical_sqlite_path(base + "-wal"),
_canonical_sqlite_path(base + "-shm"),
}
def iter_deleted_sqlite_sidecar_holders(db_path) -> List[Tuple[int, str]]:
"""Return processes holding an unlinked ``state.db-wal`` / ``-shm``.
Linux-only (``/proc/<pid>/fd`` readlink). Windows and other hosts
return ``[]`` — Windows cannot unlink a sidecar another process still
holds, and macOS does not use the `` (deleted)`` suffix.
The scan includes this process: on the SessionDB open/write refuse
path, the in-process writer that still holds the orphan inode is the
one that must not mint a replacement WAL (and must stop committing).
``_foreign_state_db_holders`` keeps skipping this PID for FTS
maintenance so a process does not block its own optional repair.
"""
if not sys.platform.startswith("linux"):
return []
holders: List[Tuple[int, str]] = []
watched = _watched_sqlite_sidecar_paths(db_path)
try:
for pid_str in os.listdir("/proc"):
if not pid_str.isdigit():
continue
pid = int(pid_str)
fd_dir = f"/proc/{pid}/fd"
try:
fds = os.listdir(fd_dir)
except OSError:
continue
for fd in fds:
try:
target = os.readlink(f"{fd_dir}/{fd}")
except OSError:
continue
if " (deleted)" not in target:
continue
if _canonical_sqlite_path(target) in watched:
holders.append((pid, target))
except Exception as exc:
logger.debug("deleted-WAL holder scan failed for %s: %s", db_path, exc)
return holders
return holders
def refuse_deleted_wal_generation(db_path) -> None:
"""Raise if any process holds a deleted WAL/SHM generation for *db_path*.
Called *before* ``sqlite3.connect`` so a second opener cannot mint a
replacement WAL inode while a live writer still holds the orphan.
"""
holders = iter_deleted_sqlite_sidecar_holders(db_path)
if not holders:
return
logger.error(_DELETED_WAL_GENERATION_MSG)
raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG)
# ── Process-wide shared SessionDB registry (#90837) ──
#
# The registry itself lives in hermes_state_registry.py — a bounded
# module owning acquisition, generation identity, refcounting,
# retirement, and teardown. These re-exports keep the historical
# import path (``from hermes_state import get_shared_session_db``)
# working for every call site and test that imports from here.
#
# Routing rules (see hermes_state_registry for the full lifecycle):
# - Long-lived in-process callers (gateway, tui_gateway, cron,
# in-process tools) share ONE writer connection per resolved path
# via get_shared_session_db().
# - CLI one-shots, recovery flows, and read-only cross-profile opens
# keep using SessionDB() directly with their own close().
from hermes_state_registry import ( # noqa: F401 (re-export)
close_shared_session_dbs,
get_shared_session_db,
release_or_close,
release_shared_session_db,
)
def _connect_tracked_db(path, tracking_path=None, **kwargs):
"""``sqlite3.connect`` that registers the open fd for lock-safety.
While a connection is live, byte-level probes of the same file are
refused: an ``open()``/``close()`` cancels every POSIX advisory lock this
process holds on it -- including a running VACUUM's EXCLUSIVE lock.
Released automatically on ``close()``.
The ONLY tolerated fallback is the helper being absent entirely
(scaffold/embed installs that ship hermes_state without hermes_cli). A
real connection failure must propagate: silently retrying an *untracked*
connect would disable the guard for the lifetime of that connection,
which is precisely the failure mode this module exists to prevent.
"""
try:
from hermes_cli.sqlite_safe_read import connect_tracked
except ImportError:
logger.debug(
"hermes_cli.sqlite_safe_read unavailable; opening %s untracked "
"(byte-probe guard inactive in this install)",
path,
)
return sqlite3.connect(str(path), **kwargs)
# Open through THIS module's sqlite3.connect so callers (and tests) that
# patch hermes_state.sqlite3.connect keep control of connection creation;
# the helper still owns tracking.
return connect_tracked(
path,
tracking_path=tracking_path,
connect_fn=sqlite3.connect,
**kwargs,
)
def is_zeroed_state_db(
path: Path, *, probe_bytes: int = 100, force: bool = False
) -> bool:
"""Detect the #68474/#97568 zeroed state.db signature (0-byte or NUL header).
Byte-level probe, so it is only safe BEFORE any connection to *path*
exists in this process: ``close()`` cancels every POSIX advisory lock the
process holds on the file, which can pull the EXCLUSIVE lock out from
under a running VACUUM and corrupt the database. The read is routed
through ``read_header_bytes_preopen``, which refuses (returning False
here) once a connection is live. Pass ``force=True`` only for offline
files -- quarantined copies, snapshots, archives.
Prefer ``hermes_cli.backup.is_zeroed_sqlite_file`` when available; this
local copy keeps SessionDB openable without importing the CLI package
in constrained embed paths.
"""
try:
from hermes_cli.backup import is_zeroed_sqlite_file
return is_zeroed_sqlite_file(path, probe_bytes=probe_bytes, force=force)
except Exception:
pass
try:
if not path.is_file():
# Special files (FIFO, device, socket) are never "zeroed", and
# probing a FIFO would block until a writer appears.
return False
size = path.stat().st_size
except OSError:
return False
if size < 0:
return False
from hermes_cli.sqlite_safe_read import has_live_connection, read_header_bytes_preopen
if not force and has_live_connection(path):
return False
head = read_header_bytes_preopen(
path, length=max(16, probe_bytes), force=force
)
if head is None:
return False
if len(head) == 0:
return True
if head.startswith(b"SQLite format 3"):
return False
return all(byte == 0 for byte in head)
@contextlib.contextmanager
def quarantine_cross_process_lock(path: Path, timeout: float = 5.0):
"""Acquire the cross-process lock for path.quarantine.lock."""
import platform
lock_path = path.with_name(path.name + ".quarantine.lock")
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = lock_path.open("a+b")
acquired = False
try:
deadline = time.monotonic() + timeout
if platform.system() == "Windows":
import msvcrt
while True:
try:
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
acquired = True
break
except OSError:
if time.monotonic() >= deadline:
break
time.sleep(0.020)
else:
import fcntl
while True:
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
acquired = True
break
except (BlockingIOError, OSError):
if time.monotonic() >= deadline:
break
time.sleep(0.020)
yield acquired
finally:
try:
if acquired:
if platform.system() == "Windows":
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
except (OSError, AttributeError):
pass
finally:
handle.close()
def quarantine_zeroed_state_db(
path: Path, *, already_locked: bool = False
) -> Optional[Path]:
"""Move a zeroed state.db aside (preserve bytes) and return quarantine path.
Uses a cross-process lock (``#68805``) so two concurrent startups cannot
race: the first process moves the zeroed file and the second re-checks
under the lock, finding the file already gone (or a fresh DB in its place)
instead of clobbering the quarantine.
"""
def _do_quarantine():
if not path.exists():
logger.info(
"quarantine_zeroed_state_db: %s already moved by another process",
path,
)
return None
if not is_zeroed_state_db(path):
logger.info(
"quarantine_zeroed_state_db: %s is no longer zeroed (another "
"process quarantined it and a fresh DB was created)",
path,
)
return None
try:
ts = time.strftime("%Y%m%d-%H%M%S")
except Exception:
ts = "unknown"
dest = path.with_name(
f"{path.name}.zeroed-{ts}-{os.getpid()}.bak"
)
n = 0
while dest.exists():
n += 1
dest = path.with_name(
f"{path.name}.zeroed-{ts}-{os.getpid()}-{n}.bak"
)
try:
path.rename(dest)
except OSError as exc:
logger.error("Failed to quarantine zeroed %s: %s", path, exc)
return None
for suffix in ("-wal", "-shm"):
side = Path(str(path) + suffix)
if side.exists():
try:
side.rename(Path(str(dest) + suffix))
except OSError:
pass
return dest
if already_locked:
return _do_quarantine()
with quarantine_cross_process_lock(path) as acquired:
if not acquired:
logger.error(
"quarantine lock for %s not acquired within 5s — refusing to "
"quarantine without the cross-process lock. The zeroed file "
"is left in place. If sessions fail to load, restore from "
"state-snapshots via `hermes snapshot list` / "
"`hermes snapshot restore <id>`.",
path,
)
return None
return _do_quarantine()
# ── Read-only health/stats probes (hermes doctor, dashboards) ──────────
def collect_state_db_stats(db_path: Path) -> Dict[str, Any]:
"""Best-effort, strictly read-only stats snapshot of a state.db file.
Opens the database with ``mode=ro`` (URI) and a short timeout so it can
run against a *live* database held by a gateway without ever taking a
write lock or mutating the file. Every field is collected independently:
a failed pragma/SELECT yields ``None`` for that field, and the helper
itself never raises.
Deliberately does NOT instantiate :class:`SessionDB` — its constructor
runs schema DDL (migrations, FTS table creation), which is exactly the
kind of write a diagnostics probe must never perform.
Returned keys (all present, any may be None on failure):
- ``page_count``, ``page_size``, ``freelist_count`` — PRAGMA values
- ``logical_size_bytes`` — page_count * page_size (post-checkpoint size)
- ``wal_size_bytes`` — stat() of ``<db>-wal`` (0 when absent)
- ``journal_mode`` — PRAGMA journal_mode string
- ``messages`` / ``sessions`` — row counts
- ``fts_tables`` — dict of {table_name: bool} presence for
messages_fts / messages_fts_trigram / messages_fts_cjk
- ``fts_storage_version`` — int from state_meta, None when the marker is
absent (legacy pre-v23 inline layout)
- ``fts_rebuild_pending`` — True when the deferred v23 backfill has not
finished (high_water present and progress < high_water)
- ``fts_rebuild_high_water`` / ``fts_rebuild_progress`` — raw ints
- ``fts_rebuild_deferral`` — durable blocked-repair diagnostic, when present
"""
stats: Dict[str, Any] = {
"page_count": None,
"page_size": None,
"freelist_count": None,
"logical_size_bytes": None,
"wal_size_bytes": None,
"journal_mode": None,
"messages": None,
"sessions": None,
"fts_tables": None,
"fts_storage_version": None,
"fts_rebuild_pending": None,
"fts_rebuild_high_water": None,
"fts_rebuild_progress": None,
"fts_rebuild_deferral": None,
}
# WAL sidecar size needs no connection at all.
try:
wal_path = Path(str(db_path) + "-wal")
stats["wal_size_bytes"] = wal_path.stat().st_size if wal_path.exists() else 0
except OSError:
pass
conn = None
try:
# mode=ro refuses to create the file and refuses every write; a
# short timeout keeps doctor snappy when a writer holds the lock.
# Route through the tracked connect so byte-probe helpers
# (read_header_bytes_preopen) see this connection and refuse raw
# opens that could cancel our POSIX locks mid-read.
conn = _connect_tracked_db(
f"file:{Path(db_path)}?mode=ro",
tracking_path=Path(db_path),
uri=True,
timeout=2.0,
)
except Exception as exc:
logger.debug("collect_state_db_stats: cannot open %s read-only: %s",
db_path, exc)
return stats
def _scalar(sql: str) -> Any:
try:
row = conn.execute(sql).fetchone()
return row[0] if row else None
except Exception:
return None
try:
pc = _scalar("PRAGMA page_count")
ps = _scalar("PRAGMA page_size")
stats["page_count"] = int(pc) if pc is not None else None
stats["page_size"] = int(ps) if ps is not None else None
if stats["page_count"] is not None and stats["page_size"] is not None:
stats["logical_size_bytes"] = stats["page_count"] * stats["page_size"]
fl = _scalar("PRAGMA freelist_count")
stats["freelist_count"] = int(fl) if fl is not None else None
jm = _scalar("PRAGMA journal_mode")
stats["journal_mode"] = str(jm) if jm is not None else None
msgs = _scalar("SELECT COUNT(*) FROM messages")
stats["messages"] = int(msgs) if msgs is not None else None
sess = _scalar("SELECT COUNT(*) FROM sessions")
stats["sessions"] = int(sess) if sess is not None else None
# FTS table presence via sqlite_master (never SELECTs from the
# virtual tables themselves — a corrupt index must not fail stats).
try:
names = {
row[0]
for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type = 'table' "
"AND name IN (?, ?, ?)",
("messages_fts", "messages_fts_trigram", "messages_fts_cjk"),
).fetchall()
}
stats["fts_tables"] = {
t: (t in names)
for t in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
}
except Exception:
pass
# Raw state_meta reads — cheap, and independent of SessionDB.
def _meta_int(key: str) -> Optional[int]:
try:
row = conn.execute(
"SELECT value FROM state_meta WHERE key = ?", (key,)
).fetchone()
return int(row[0]) if row and row[0] is not None else None
except Exception:
return None
stats["fts_storage_version"] = _meta_int("fts_storage_version")
high_water = _meta_int("fts_rebuild_high_water")
progress = _meta_int("fts_rebuild_progress")
stats["fts_rebuild_high_water"] = high_water
stats["fts_rebuild_progress"] = progress
if high_water is None:
stats["fts_rebuild_pending"] = False
else:
stats["fts_rebuild_pending"] = (progress or 0) < high_water
try:
row = conn.execute(
"SELECT value FROM state_meta WHERE key = ? LIMIT 1",
(FTS_REBUILD_DEFERRAL_KEY,),
).fetchone()
if row:
parsed = json.loads(row[0])
if isinstance(parsed, dict):
stats["fts_rebuild_deferral"] = parsed
except Exception:
pass
finally:
try:
conn.close()
except Exception:
pass
return stats
def count_db_holders(db_path: Path) -> Optional[int]:
"""Best-effort count of processes holding ``db_path`` open (Linux only).
Scans ``/proc/*/fd`` symlinks for the resolved database path. Returns
the number of distinct PIDs with the file open, or ``None`` on any
error or on non-Linux platforms. Never raises; no lsof dependency.
Unreadable per-process fd dirs (other users' processes without root)
are silently skipped, so the count is a lower bound.
"""
try:
if not sys.platform.startswith("linux"):
return None
target = os.path.realpath(str(db_path))
holders = 0
for pid in os.listdir("/proc"):
if not pid.isdigit():
continue
fd_dir = f"/proc/{pid}/fd"
try:
fds = os.listdir(fd_dir)
except OSError:
continue # process gone or not ours
for fd in fds:
try:
if os.readlink(f"{fd_dir}/{fd}") == target:
holders += 1
break # one hit per PID
except OSError:
continue
return holders
except Exception:
return None
def _is_inactive_orphan_desktop_holder(
*,
ppid: int,
age_seconds: float,
min_age_seconds: float,
ephemeral_backend: bool,
connection_statuses: List[str],
) -> bool:
"""Pure safety predicate for the narrow Desktop holder reap."""
return (
ppid in (0, 1)
and age_seconds >= min_age_seconds
and ephemeral_backend
and "ESTABLISHED" not in connection_statuses
)
def _concrete_state_db_holder_pids(
db_path: Path, holders: List[Tuple[int, str]]
) -> List[int]:
"""Return unique PIDs proven to hold this DB or one of its sidecars."""
canonical_db = os.path.normcase(os.path.abspath(os.fspath(db_path)))
watched = {
canonical_db,
canonical_db + "-wal",
canonical_db + "-shm",
}
pids: List[int] = []
seen = set()
for pid, path in holders:
canonical_path = os.path.normcase(
os.path.abspath(path.removesuffix(" (deleted)"))
)
if pid <= 0 or pid in seen or canonical_path not in watched:
continue
seen.add(pid)
pids.append(pid)
return pids
# Lifecycle statuses surfaced by session pickers. Classification looks ONLY at
# a session's final message row — role, whether it carries tool_calls, and its
# finish_reason — so it stays O(1) per session (see
# SessionDB.session_lifecycle_statuses).
SESSION_STATUS_COMPLETE = "complete"
SESSION_STATUS_INTERRUPTED = "interrupted"
SESSION_STATUS_ERROR = "error"
SESSION_STATUS_EMPTY = "empty"
# finish_reason values that mark the turn as having ended in a provider or
# agent error (vs. a normal 'stop'/'length'/'tool_calls' completion).
_ERROR_FINISH_REASONS = frozenset({"error", "agent_error", "content_filter"})
def classify_session_status(
role: Optional[str],
has_tool_calls: bool,
finish_reason: Optional[str],
) -> str:
"""Classify a session's lifecycle from the shape of its final message.
- assistant with a normal finish → ``complete``
- assistant that still has pending tool_calls (no tool result row ever
followed, or it would be the last row instead) → ``interrupted``
- user or tool as the last row → ``interrupted`` (the agent never got to
answer / never consumed the tool result)
- an error finish_reason on the last row → ``error``
- anything unrecognized → ``complete`` (benign default; pickers must not
alarm on unknown shapes)
"""
if (finish_reason or "").strip().lower() in _ERROR_FINISH_REASONS:
return SESSION_STATUS_ERROR
r = (role or "").strip().lower()
if r == "assistant":
# The last row being an assistant message WITH tool_calls means the
# matching tool result never landed — an interrupted tool turn.
return SESSION_STATUS_INTERRUPTED if has_tool_calls else SESSION_STATUS_COMPLETE
if r in {"user", "tool"}:
return SESSION_STATUS_INTERRUPTED
return SESSION_STATUS_COMPLETE
# Parent→child ``profile_name`` inheritance fence (#88381). ``agent:<ns>:...``
# gateway keys encode the profile namespace; a keyless row (CLI / subagent
# lineage) carries none and inherits freely. Two keyed rows must agree on
# ``agent:<ns>:`` — a default child (``agent:main:``) forked from a sibling
# profile's row must not be durably mislabelled as that profile's.
_SAME_KEY_NAMESPACE_SQL = (
"p.session_key IS NULL OR sessions.session_key IS NULL"
" OR substr(p.session_key, 1, instr(substr(p.session_key, 7), ':') + 6)"
" = substr(sessions.session_key, 1, instr(substr(sessions.session_key, 7), ':') + 6)"
)
class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin):
"""
SQLite-backed session storage with FTS5 search.
Thread-safe for the common gateway pattern (multiple reader threads,
single writer via WAL mode). Each method opens its own cursor.
"""
# Only these state-owned producers participate in automatic stale-open
# reconciliation. Messaging-platform and UI/desktop sources have separate
# lifecycle owners; unknown/future sources fail closed (#60609).
_AUTO_PRUNE_STALE_OPEN_SOURCES: Tuple[str, ...] = (
"cli",
"cron",
"kanban",
"acp",
"api_server",
"subagent",
"tool",
)
# ── Write-contention tuning ──
# With multiple hermes processes (gateway + CLI sessions + worktree agents)
# all sharing one state.db, WAL write-lock contention causes visible TUI
# freezes. SQLite's built-in busy handler uses a deterministic sleep
# schedule that causes convoy effects under high concurrency.
#
# Instead, we keep the SQLite timeout short (1s) and handle retries at the
# application level with random jitter, which naturally staggers competing
# writers and avoids the convoy.
#
# Patience is TIME-based, not attempt-based. A shared state.db is
# legitimately held for multi-second stretches by sibling Hermes
# processes: a TRUNCATE checkpoint at close on a large WAL, VACUUM after
# an auto-prune, offline recovery, or an older still-running process
# whose FTS maintenance predates the bounded-merge protocol (every
# `hermes update` leaves mixed-version processes sharing the DB until
# the old ones exit). An attempt-counted budget (~15s incidental worst
# case) silently loses that race and surfaces as
# session_persistence_failed — a destroyed turn — even though the store
# is healthy and merely busy (#74478).
#
# Two budgets: routine writes give up after _WRITE_PATIENCE_S so
# background/UI callers don't stall excessively, while transcript
# writes (append_message / session-row creation — the ones whose
# failure aborts the user's turn) ride out anything shorter than
# _TRANSCRIPT_WRITE_PATIENCE_S. Jitter stays small for the first
# _WRITE_RETRY_SLOW_AFTER_S (fast reclaim on millisecond contention),
# then backs off so a long hold isn't hammered with BEGIN IMMEDIATE
# attempts.
_WRITE_PATIENCE_S = 20.0
_TRANSCRIPT_WRITE_PATIENCE_S = 60.0
# Observation-only activity heartbeat/label writes (#76354 review S1):
# these run on (or adjacent to) the response-critical path and must never
# wait out the full routine patience under contention. Sub-second budget;
# a skipped write is retried naturally at the next heartbeat window.
_ACTIVITY_WRITE_PATIENCE_S = 0.5
# A live compression lock gets its own, much shorter budget than the write
# lock. Compression publishes in a couple of seconds, so a brief wait saves
# the overwhelming majority of concurrent turns (#75083). It deliberately
# stays short: the lease is a correctness boundary, not just a busy signal
# (see test_compression_lease_blocks_non_owner_but_allows_owner_flush), so
# a writer that is still locked out after this budget must still be
# refused rather than allowed to land a stale turn in a session whose
# compression is genuinely long-running or wedged.
_COMPRESSION_BUSY_WAIT_S = 5.0
_WRITE_RETRY_MIN_S = 0.020 # 20ms
_WRITE_RETRY_MAX_S = 0.150 # 150ms
_WRITE_RETRY_SLOW_AFTER_S = 2.0
_WRITE_RETRY_SLOW_MIN_S = 0.250 # 250ms
_WRITE_RETRY_SLOW_MAX_S = 1.000 # 1s
# Attempt a WAL checkpoint every N successful writes (PASSIVE mode).
_CHECKPOINT_EVERY_N_WRITES = 50
# Retain the existing coarse 1000-write maintenance cadence, but replace
# the unbounded FTS5 ``'optimize'`` (measured holding the write lock for
# 9-18 s per index on a 10 GB production DB — longer than a competing
# writer's full retry patience, surfacing as "database is locked" /
# session_persistence_failed) with bounded ``'merge'`` commands. A
# positive merge rank is an approximate output-page budget, so each
# command holds the write lock for milliseconds; up to
# ``_FTS_MERGE_COMMANDS_PER_PASS`` commands run per index per cadence,
# stopping early on the documented no-progress signal. ``usermerge`` is
# lowered to 2 so positive merges act on any level with >= 2 segments —
# without that, levels below the default threshold of 4 are skipped and
# a fragmented index never converges (SQLite FTS5 §6.8-6.9).
_FTS_MERGE_EVERY_N_WRITES = 1000
_FTS_MERGE_MAX_PAGES_PER_INDEX = 500
_FTS_MERGE_COMMANDS_PER_PASS = 4
# Session imports intentionally use a lower cap than exports: import holds
# one BEGIN IMMEDIATE transaction, so bounded batches avoid starving live
# gateway/CLI writers. The dashboard accepts one exported JSON/JSONL file
# at a time, so these still cover normal history restores.
_IMPORT_MAX_SESSIONS = 500
_IMPORT_MAX_MESSAGES_PER_SESSION = 10_000
_IMPORT_MAX_TOTAL_MESSAGES = 50_000
_IMPORT_MAX_SESSION_BYTES = 5 * 1024 * 1024
_IMPORT_MAX_TOTAL_BYTES = 25 * 1024 * 1024
# Demand-started accounting workers retire after an idle window so their
# bound targets do not keep abandoned SessionDB instances (and SQLite
# descriptors) alive forever. A later enqueue starts a fresh worker.
_TOKEN_WRITER_IDLE_SECONDS = 30.0
@staticmethod
def _store_system_prompt(conn, system_prompt: Optional[str]) -> Optional[str]:
if system_prompt is None:
return None
prompt_hash = _system_prompt_hash(system_prompt)
conn.execute(
"INSERT OR IGNORE INTO system_prompts (hash, prompt) VALUES (?, ?)",
(prompt_hash, system_prompt),
)
return prompt_hash
@staticmethod
def _delete_unreferenced_system_prompts(conn) -> None:
conn.execute(
"DELETE FROM system_prompts "
"WHERE NOT EXISTS ("
"SELECT 1 FROM sessions "
"WHERE sessions.system_prompt_hash = system_prompts.hash"
")"
)
@staticmethod
def _session_row_dict(row: sqlite3.Row) -> Dict[str, Any]:
data = dict(row)
if "_system_prompt_resolved" in data:
resolved = data.pop("_system_prompt_resolved")
if "system_prompt" in data:
data["system_prompt"] = resolved
return data
@staticmethod
def _close_connection_quietly(conn: Optional[sqlite3.Connection]) -> None:
"""Close a partially initialized connection without masking its error."""
if conn is None:
return
try:
conn.close()
except Exception:
logger.debug("Could not close a SessionDB connection", exc_info=True)
def __init__(self, db_path: Path = None, read_only: bool = False):
self.db_path = db_path or _default_db_path()
# Fail hard (before any connection/pragma/mkdir) if a pytest-context
# process resolved the developer's production state.db — see the
# live-DB test-isolation guard block near _default_db_path().
_ensure_test_isolation(self.db_path)
self.read_only = read_only
self._lock = threading.Lock()
# Read-path split (WAL only): recall/browse queries borrow a
# read-only connection from a bounded pool so they never queue
# behind writer flushes on self._lock. See _read_ctx().
#
# The pool is BOUNDED because the previous per-thread
# (threading.local + strong set) scheme pinned one connection per
# (SessionDB x thread) for the life of the process. Starlette
# dispatches sync routes on anyio worker threads, so a SessionDB
# that is never closed accumulated a connection — and two fds, the
# database and its -wal — for every worker thread that ever read,
# until the process hit the 256 soft RLIMIT_NOFILE a service manager
# hands it and every request failed with EMFILE while the process
# stayed alive, so the supervisor's restart-on-exit never fired.
# Same bug class as the closing(...) fix in gateway/readiness.py
# (#69678 / #69567).
self._read_pool: "queue.LifoQueue[sqlite3.Connection]" = queue.LifoQueue(
maxsize=_READ_POOL_MAX
)
# One permit per live read connection, held from before the open in
# _get_read_conn() until after the close in _close_read_conn(). This
# is what bounds PEAK descriptors; _read_pool alone bounds only the
# idle set. See _READ_POOL_MAX. Acquired non-blocking on purpose: a
# reader that cannot get a permit must degrade to the writer lock, not
# queue here — blocking would convert fd exhaustion into a stall, which
# is the same outage with a different stack trace.
#
# Permits are shared per DATABASE PATH, not per instance: the
# descriptors they ration belong to the file, and one process holds
# several SessionDB objects on the same state.db (#98573). See
# _PathReadBudget.
self._read_budget = _read_budget_for(self.db_path)
self._read_budget.register(self)
# Bound to the semaphore itself so every release site
# (_close_read_conn and the _get_read_conn failure paths) is unchanged.
self._read_permits = self._read_budget.permits
# Count of reads that found no permit and fell back to the locked
# writer connection. Not load-bearing; it is the only externally
# visible signal that the ceiling is actually being reached, so a
# too-small _READ_POOL_MAX is diagnosable from a running process
# instead of inferred from latency.
self._read_permit_exhausted = 0
self._read_conns_lock = threading.Lock()
# Set when close() begins. _read_ctx checks this under the lock
# before returning a connection to the pool, so a reader still in
# flight during the drain closes its own connection instead of
# re-populating a pool nobody will drain again.
self._read_conns_closed = False
# "read-only opens are failing against this file" backoff stamp.
# Instance-wide rather than per-thread: with a shared pool the open
# is no longer a per-thread event, and retrying a known-bad open on
# every query is a syscall storm for no benefit. The locked writer
# connection still serves reads while the backoff holds.
# Deliberately a TIMESTAMP, not a sticky bool: the likeliest trigger
# is transient fd pressure (EMFILE) — the very condition this pool
# exists to prevent — and a permanent flag would demote every reader
# on this instance to the writer lock for the life of the process.
# The gateway shares one SessionDB across every agent, so that turns
# a momentary blip into a permanent global convoy. Expires after
# _READ_OPEN_RETRY_SECONDS so the read path self-heals.
self._read_open_failed_at = 0.0
self._wal_active = False
self._write_count = 0
# File identity of the state.db this instance opened. Compared on
# every write (and before FTS fail-open / reopen-after-close) so an
# out-of-band replace cannot limp through in-place surgery.
# Inode catches mv/new-file; application_id catches cp onto the
# same path (same inode, truncate+rewrite).
self._db_file_identity: Optional[tuple] = None
self._db_file_application_id: int = 0
self._db_file_generation_token: str = ""
self._db_replaced = False
# Sticky: set once a write on THIS handle reports bare SQLITE_CORRUPT /
# NOTADB that is not FTS-scoped and not a replaced-file case. Never
# cleared; the recovery boundary is a process restart on a repaired or
# restored file (see StateDbCorruptError).
self._db_corrupt = False
self._db_corrupt_reason = ""
self._db_sidecar_identity: Dict[str, tuple] = {}
self._db_wal_generation_lost = False
# One-shot guard for the usermerge-floor config write on the
# incremental FTS merge cadence (see _merge_fts_incrementally).
self._fts_usermerge_floor_applied = False
self._fts_enabled = False
self._fts_stale = False
self._trigram_available = False
# CJK-bigram index (cjk_unicode61 loadable tokenizer). _fts_cjk_loaded:
# extension present on the writer connection; _fts_cjk_available: the
# messages_fts_cjk table is queryable AND not marked stale. Set during
# _init_schema / _probe_fts_cjk.
self._fts_cjk_loaded = False
self._fts_cjk_available = False
self._fts_unavailable_warned = False
self._conn = None
# Async token accounting (see queue_token_counts). The condition
# guards queue + writer state; it is distinct from self._lock so
# enqueue/flush bookkeeping never contends with SQLite writes.
self._token_queue: deque = deque()
self._token_queue_cond = threading.Condition(threading.Lock())
self._token_writer_thread: Optional[threading.Thread] = None
self._token_writer_stop = False
self._token_writer_busy = False
self._token_atexit_hook: Optional[Callable[[], None]] = None
# Set True when this instance is opened via get_shared_session_db().
# Makes close() a no-op so the registry (not individual callers)
# controls the connection lifecycle (#90837).
self._shared_registry_owned = False
initialization_complete = False
try:
if read_only:
# Read-only attach for cross-profile aggregation: SELECT-only,
# so we skip schema init entirely (no DDL, no FTS probe, no
# column reconcile). Crucially this takes NO write lock, so
# polling another profile's live DB on every sidebar refresh
# never contends with that profile's running backend. The DB
# must already exist + be initialised (callers guard on
# db_path.exists()); a SELECT against an empty file raises and
# the caller degrades per-profile.
open_attempt = 0
while True:
try:
self._conn = _connect_tracked_db(
f"file:{self.db_path}?mode=ro",
tracking_path=self.db_path,
uri=True,
check_same_thread=False,
timeout=1.0,
isolation_level=None,
)
self._conn.row_factory = sqlite3.Row
# FTS capability flags normally come from writable schema
# initialisation. Probe existing virtual tables with
# SELECTs only so read-only search keeps its FTS and
# trigram paths. Close the connection on ANY probe
# failure (e.g. malformed schema raises DatabaseError,
# not the OperationalError the probe handles). The
# constructor's outer finally also covers failures
# before this probe and BaseException paths, so a
# leaked tracked connection cannot block
# _backup_db_file's raw-copy for the rest of the
# process — the writable heal that follows would then
# repair WITHOUT its forensic backup.
try:
apply_database_pragmas(self._conn, db_label="state.db")
cursor = self._conn.cursor()
self._fts_enabled = (
self._fts_table_probe(cursor, "messages_fts")
is True
)
if self._fts_enabled:
self._trigram_available = (
self._fts_table_probe(
cursor,
"messages_fts_trigram",
)
is True
)
except BaseException:
conn, self._conn = self._conn, None
try:
conn.close()
except Exception:
pass
raise
break
except sqlite3.OperationalError as ioerr:
# A WAL checkpoint / reset / frame-flush in flight on
# the writer side can surface SQLITE_IOERR to a
# concurrent mode=ro reader (it cannot perform the
# recovery the read needs — recovery writes the -shm
# index, which mode=ro refuses). The transition closes
# in milliseconds, so retry a bounded number of times
# before classifying the store as failed (#100436).
if not _is_transient_read_only_ioerr(
ioerr, attempt=open_attempt
):
raise
open_attempt += 1
time.sleep(_READ_ONLY_IOERR_RETRY_BACKOFF_S)
self._record_db_file_identity()
initialization_complete = True
return
self.db_path.parent.mkdir(parents=True, exist_ok=True)
# Read-only file/sidecar preflight (port of kilocode#12508):
# repair-or-refuse BEFORE the first connection so users get an
# actionable message instead of an opaque "attempt to write a
# readonly database" from deep inside _init_schema.
if not read_only:
preflight_db_writability(self.db_path, db_label="state.db")
# #68474 / #97568: Serialize startup across zero-byte check, quarantine,
# connect, and schema commit so concurrent openers don't race on an
# absent-path -> connect -> schema-commit window.
needs_startup_guard = (
not read_only
and (
not self.db_path.exists()
or is_zeroed_state_db(self.db_path)
)
)
def _handle_quarantine_if_zeroed(already_locked: bool = False):
if (
self.db_path.exists()
and is_zeroed_state_db(self.db_path)
):
try:
zsize = self.db_path.stat().st_size
except OSError:
zsize = -1
qpath = quarantine_zeroed_state_db(
self.db_path, already_locked=already_locked
)
snaps = self.db_path.parent / "state-snapshots"
msg = (
f"state.db looks ZEROED ({zsize} bytes, no SQLite header). "
f"Preserved at {qpath or '(quarantine failed — file left in place)'}. "
f"Restore from {snaps} via `hermes snapshot list` / "
f"`hermes snapshot restore <id>` if available. "
"Opening a fresh empty database so the agent can start."
)
logger.error(msg)
_set_last_init_error(msg)
# If quarantine failed, do not open the zeroed file (would fail
# opaquely or risk further damage). Raise with the clear message.
if qpath is None and self.db_path.exists() and is_zeroed_state_db(self.db_path):
raise sqlite3.DatabaseError(msg)
def _connect_and_init():
# Refuse before sqlite3.connect (under the startup lock) so we
# cannot mint a replacement WAL while a live writer still
# holds a deleted sidecar inode.
refuse_deleted_wal_generation(self.db_path)
self._conn = _connect_tracked_db(
str(self.db_path),
check_same_thread=False,
# Short timeout — application-level retry with random
# jitter handles contention instead of sitting in
# SQLite's internal busy handler for up to 30s.
timeout=1.0,
# auto-starts transactions on DML, which conflicts with
# our explicit BEGIN IMMEDIATE. None = we manage
# transactions ourselves.
isolation_level=None,
)
self._conn.row_factory = sqlite3.Row
self._wal_active = (
apply_wal_with_fallback(self._conn, db_label="state.db") == "wal"
)
apply_database_pragmas(self._conn, db_label="state.db")
self._conn.execute("PRAGMA foreign_keys=ON")
self._fts_cjk_loaded = load_fts5_cjk_extension(self._conn)
self._init_schema()
def _connect_and_init_with_lock_patience():
# Lock contention during open: _init_schema's DDL/reconcile
# statements run on a 1s-timeout connection with no retry, so
# a sibling process holding the write lock (VACUUM, TRUNCATE
# checkpoint at close, a long FTS pass from an older
# still-running install) used to fail the ENTIRE open —
# callers then disable persistence for the whole run
# ("Failed to initialize SessionDB ... database is locked",
# #74478). The store is healthy; wait it out with the same
# jittered patience the write path uses. Non-lock errors
# (including the malformed class) propagate immediately.
deadline = time.monotonic() + self._WRITE_PATIENCE_S
while True:
try:
_connect_and_init()
return
except sqlite3.OperationalError as exc:
err = str(exc).lower()
if "locked" not in err and "busy" not in err:
raise
try:
if self._conn is not None:
self._conn.close()
except Exception:
pass
now = time.monotonic()
if now >= deadline:
raise
time.sleep(
min(
random.uniform(
self._WRITE_RETRY_SLOW_MIN_S,
self._WRITE_RETRY_SLOW_MAX_S,
),
max(deadline - now, 0.001),
)
)
def _open_with_optional_startup_guard():
if needs_startup_guard:
with quarantine_cross_process_lock(self.db_path) as lock_acquired:
if not lock_acquired:
logger.warning(
"startup quarantine lock for %s not acquired within 5s; proceeding",
self.db_path,
)
_handle_quarantine_if_zeroed(already_locked=lock_acquired)
_connect_and_init_with_lock_patience()
else:
_handle_quarantine_if_zeroed(already_locked=False)
_connect_and_init_with_lock_patience()
try:
_open_with_optional_startup_guard()
except sqlite3.DatabaseError as exc:
# The malformed-schema class (e.g. a duplicate sqlite_master
# row for messages_fts) fails on the very first statement —
# before _init_schema can run — so it can't be caught at the
# FTS-rebuild layer. Recover by repairing sqlite_master in
# place (backup first; canonical sessions/messages preserved),
# then reopen once. This is what lets Desktop/Dashboard
# self-heal instead of silently showing "no sessions".
if not is_malformed_schema_error(exc) or not _claim_repair_attempt(self.db_path):
raise
logger.error(
"state.db schema is malformed (%s) — attempting automatic "
"repair (a backup copy is made first).", exc,
)
try:
if self._conn is not None:
self._conn.close()
except Exception:
pass
report = repair_state_db_schema(self.db_path)
if not report.get("repaired"):
raise
_connect_and_init_with_lock_patience()
# NOTE: the v23 FTS optimization is OPT-IN (`hermes db optimize`),
# never auto-started on open. Legacy installs keep their working
# v22 inline FTS untouched here; only the explicit foreground
# command demotes + rebuilds. This avoids a background worker
# racing session lifecycle and the surprise disk/latency cost on
# an unattended open. (An interrupted optimize resumes when the
# user re-runs the command.)
self._ensure_db_file_generation()
self._record_db_file_identity()
initialization_complete = True
except Exception as exc:
# Capture the cause so /resume and friends can surface WHY the
# session DB is unavailable instead of a bare "Session database
# not available." Callers that catch this exception keep their
# existing ``self._session_db = None`` degradation path.
#
# Note: we deliberately do NOT clear _last_init_error on the
# success path (no else branch). In multi-threaded callers
# (gateway, web_server per-request SessionDB()), a concurrent
# successful open racing past this failure would erase the
# cause that another thread's /resume is about to format.
# Tests that need to reset the state can call
# ``hermes_state._set_last_init_error(None)`` explicitly.
_set_last_init_error(f"{type(exc).__name__}: {exc}")
raise
finally:
if not initialization_complete:
conn, self._conn = self._conn, None
self._close_connection_quietly(conn)
# ── Read-path split ──
def _get_read_conn(self) -> Optional[sqlite3.Connection]:
"""Open a fresh read-only connection, or None when unavailable.
Callers must return the connection to self._read_pool (see
_read_ctx); this opens, it does not track.
Only used under WAL: WAL readers see a consistent snapshot and never
block on (or get blocked by) the writer, so recall/browse queries can
skip self._lock entirely. Under DELETE journal mode (NFS fallback) a
reader can hit SQLITE_BUSY storms during writes, so we keep the
legacy locked single-connection path there.
Fresh read transactions begin per statement (autocommit), so each
query observes everything committed so far — read-your-writes holds
for the flush-then-search patterns in a turn.
"""
if not self._wal_active or self.read_only:
return None
with self._read_conns_lock:
if self._read_conns_closed:
return None
if (
self._read_open_failed_at
and time.monotonic() - self._read_open_failed_at
< _READ_OPEN_RETRY_SECONDS
):
return None
# Take the descriptor permit BEFORE the open, so concurrent openers
# race for permits rather than for file descriptors. Non-blocking:
# losing the race means "use the writer connection", not "wait".
if not self._read_budget.acquire(self):
with self._read_conns_lock:
self._read_permit_exhausted += 1
logger.debug(
"read pool at capacity (%d) for %s; serving this read from the "
"locked writer connection",
_READ_POOL_MAX,
self.db_path,
)
return None
# Bound before the try: the except handlers close it if the open
# half-succeeded, and an unbound name there would raise NameError over
# the top of the real failure.
conn = None
try:
conn = _connect_tracked_db(
f"file:{self.db_path}?mode=ro",
tracking_path=self.db_path,
uri=True,
# Pooled connections are borrowed by whichever thread runs
# the next read, and sqlite3 otherwise refuses cross-thread
# use ("SQLite objects created in a thread can only be used
# in that same thread") — including on close(), which is how
# the old per-thread connections became unclosable and leaked
# their fds. Exclusive ownership is enforced by the pool
# checkout/return, not by sqlite3. Matches the writer opens.
check_same_thread=False,
timeout=5.0,
isolation_level=None,
)
conn.row_factory = sqlite3.Row
apply_database_pragmas(conn, db_label="state.db")
# Load the CJK tokenizer extension on this connection so
# messages_fts_cjk queries work on the read path. The .so
# registers the tokenizer in the connection's in-memory
# registry, not the database file, so mode=ro is fine.
if self._fts_cjk_loaded:
load_fts5_cjk_extension(conn)
except sqlite3.Error:
# A partially-constructed connection — _connect_tracked_db
# succeeded, the CJK extension load did not — must be closed here.
# Dropping it on the floor still open leaves a live descriptor the
# tracking registry still counts: the same leak shape this pool
# exists to fix, one level further down.
self._discard_partial_read_conn(conn)
# Back off from retrying the open on every query; the locked
# writer connection still serves reads until the stamp expires.
with self._read_conns_lock:
self._read_open_failed_at = time.monotonic()
logger.debug("read-only connection open failed for %s", self.db_path, exc_info=True)
self._read_budget.release()
return None
except BaseException:
# Anything else (a non-sqlite3 extension-load failure, MemoryError,
# KeyboardInterrupt landing between open and return) must not
# strand the permit: a stranded permit is not a transient error, it
# permanently shrinks the read path by one slot for the life of the
# process.
self._discard_partial_read_conn(conn)
self._read_budget.release()
raise
return conn
def _evict_one_idle_read_conn(self) -> bool:
"""Close one connection sitting idle in this instance's pool.
Called by _PathReadBudget when a peer SessionDB on the same file wants
a permit this instance is holding but not using. Only the idle set is
reachable from here — a checked-out connection is not in the queue —
so this can never pull a connection out from under a live reader.
Returns whether a connection (and therefore a permit) was released.
"""
try:
conn = self._read_pool.get_nowait()
except queue.Empty:
return False
self._close_read_conn(conn)
return True
def _discard_partial_read_conn(self, conn) -> None:
"""Close a connection that failed between open and hand-off.
Separate from _close_read_conn because that one releases a permit and
this runs on paths that release their own.
"""
if conn is None:
return
try:
conn.close()
except Exception as exc:
logger.warning(
"partially-opened read conn close failed for %s: %s", self.db_path, exc
)
def _close_read_conn(self, conn) -> None:
"""Close a pooled read connection and release its descriptor permit.
This was a bare ``except Exception: pass``, which silently swallowed
the sqlite3.ProgrammingError raised when close() ran on a thread
other than the one that opened the connection — the exact signature
of the fd leak this pool fixes. A close that fails leaks a tracked
fd, so it must not be invisible.
The permit is released even when close() raises: the descriptor is
already lost at that point, and withholding the permit too would turn
one leaked fd into a permanently narrower read path — failing twice for
one fault. The warning is the signal that matters.
Pairs with _get_read_conn(). Calling this on a connection that did not
come from there over-releases the BoundedSemaphore, which raises
ValueError rather than silently widening the ceiling.
"""
try:
conn.close()
except Exception as exc:
logger.warning("read-conn close failed for %s: %s", self.db_path, exc)
finally:
self._read_budget.release()
def _checkout_read_conn(self) -> Optional[sqlite3.Connection]:
"""Borrow a read connection from the pool, opening one on a miss.
The single acquisition seam for the read path: the WAL/read_only gate,
the pool checkout and the open-on-miss all live here, so there is
exactly one place to exercise (and one place for a caller to bypass by
accident). Returns None when the read path is unavailable and the
caller must fall back to the locked writer connection.
A pool hit costs no permit — the connection it hands back is already
holding one. Only the miss path can open, and only _get_read_conn() can
take a permit, so peak live connections is bounded by _READ_POOL_MAX no
matter how many threads miss simultaneously.
"""
if not self._wal_active or self.read_only:
return None
try:
return self._read_pool.get_nowait()
except queue.Empty:
return self._get_read_conn()
@contextmanager
def _read_ctx(self) -> Iterator[sqlite3.Connection]:
"""Yield a connection for read-only statements.
WAL: a read-only connection borrowed from a bounded pool with NO
lock — recall queries never convoy behind writer flushes (the
gateway shares one SessionDB across every agent, so this lock was a
global choke point). The connection is checked out for the duration
of the block, so no two threads ever touch it concurrently.
Non-WAL, read-conn failure, or _READ_POOL_MAX already reached: the
shared writer connection under self._lock, byte-for-byte the legacy
behavior.
That last case is the deliberate degradation. Past the ceiling readers
convoy on the writer lock instead of opening descriptors — measurably
slower under a burst, and the alternative is EMFILE, which takes the
whole process down in a way a restart-on-exit supervisor cannot see.
"""
conn = self._checkout_read_conn()
if conn is not None:
try:
yield conn
finally:
returned = False
with self._read_conns_lock:
if not self._read_conns_closed:
try:
self._read_pool.put_nowait(conn)
returned = True
except queue.Full:
pass
if not returned:
# close() has already drained the pool, so this connection
# is surplus. Close it here — dropping it on the floor is
# what leaked the fd.
#
# queue.Full is now unreachable in practice (permits and
# maxsize are both _READ_POOL_MAX, so there can never be a
# ninth connection to return), but the branch stays: it is
# load-bearing if those two ever drift apart, and a leak is
# the failure mode it prevents.
self._close_read_conn(conn)
return
with self._lock:
if self._conn is None:
# close() ran while a reader was still unwinding (#94736
# class) — reopen instead of yielding None to a .execute.
self._reopen_after_close_locked(context="read")
yield cast(sqlite3.Connection, self._conn)
def _reopen_after_close_locked(self, context: str = "write") -> None:
"""Reopen the writer connection after ``close()`` raced a live caller.
The #94736 failure shape: a teardown owner (cron ``run_job``'s
``finally``, a delegate timeout owner abandoning its worker, agent
``close()``) calls ``SessionDB.close()`` — which sets ``_conn = None``
— while a still-unwinding worker thread has one more transcript flush
to land. The next ``_execute_write`` then died on
``'NoneType' object has no attribute 'execute'``, the conversation
loop force-ended the turn as ``session_persistence_failed``, and the
in-flight tail of the subagent/cron session was silently dropped
(delivered as ``last_status: ok``).
The transcript append is THE critical write — losing it destroys the
turn — so at this shared persistence boundary we self-heal: reopen a
connection to the same database file and let the write land. The
reopen is loud (WARNING names the race) and bounded (only fires when
``_conn`` is ``None``, i.e. after an explicit ``close()`` — the
constructor never leaves a live instance with a ``None`` handle).
``__del__`` delegates to ``close()``, so the reopened connection is
still released at GC/exit time.
Caller must hold ``self._lock``. Raises ``sqlite3.OperationalError``
with an explicit cause when the reopen itself fails, so the surfaced
persistence error names the teardown race instead of the opaque
``NoneType`` attribute error.
"""
if self.read_only:
raise sqlite3.ProgrammingError(
f"SessionDB for {self.db_path} was closed (read-only handle); "
f"cannot serve a {context} after close()"
)
# A reopen resolves the PATH again — if the file at that path is no
# longer the one this instance originally opened (out-of-band
# restore/cp/mv), reconnecting would write into the new generation
# through stale WAL/shm assumptions (#89332). Refuse instead.
if self._db_replaced or self._db_file_was_replaced():
self._halt_db_replaced()
# A quarantined handle must never come back: reopening would hand a
# fresh connection (and its own close-time checkpoint) to a file we
# already know is structurally damaged.
if self._db_corrupt:
raise self._corrupt_error(
f"state.db connection for {self.db_path} is quarantined after "
f"structural corruption; refusing to reopen for a {context} "
"after close(). "
)
if self._db_wal_generation_lost or self._wal_generation_was_lost():
self._halt_deleted_wal_generation()
logger.warning(
"state.db connection for %s was closed while a %s was still in "
"flight — reopening (teardown/worker race, #94736)",
self.db_path,
context,
)
try:
conn = _connect_tracked_db(
str(self.db_path),
check_same_thread=False,
timeout=1.0,
isolation_level=None,
)
except Exception as exc:
raise sqlite3.OperationalError(
f"state.db connection was closed while a {context} was still "
f"in flight (a session-teardown path called close() before "
f"this worker finished — #94736) and the automatic reopen "
f"failed: {exc}"
) from exc
try:
conn.row_factory = sqlite3.Row
self._wal_active = (
apply_wal_with_fallback(conn, db_label="state.db") == "wal"
)
apply_database_pragmas(conn, db_label="state.db")
conn.execute("PRAGMA foreign_keys=ON")
self._fts_cjk_loaded = load_fts5_cjk_extension(conn)
except Exception as exc:
self._close_connection_quietly(conn)
raise sqlite3.OperationalError(
f"state.db reopen after close() succeeded but connection "
f"setup failed: {exc}"
) from exc
# Schema was initialised by this instance's original open; the file
# cannot have lost it, so no _init_schema here (no DDL races with
# sibling processes during teardown).
self._conn = conn
# ── Core write helper ──
@staticmethod
def _is_fts5_unavailable_error(exc: sqlite3.OperationalError) -> bool:
err = str(exc).lower()
if "no such module" in err and "fts5" in err:
return True
# SQLite builds that have FTS5 but lack the optional trigram tokenizer
# raise "no such tokenizer: trigram" instead of "no such module".
# Scope to trigram specifically to avoid masking unrelated tokenizer errors.
if "no such tokenizer: trigram" in err:
return True
# The cjk_unicode61 tokenizer is a loadable extension — a process
# that couldn't load it sees the same capability-error shape.
if "no such tokenizer: cjk_unicode61" in err:
return True
return False
@staticmethod
def _is_trigram_unavailable_error(exc: sqlite3.OperationalError) -> bool:
"""True when only an optional tokenizer is missing (FTS5 itself works).
Covers the built-in trigram tokenizer (needs SQLite >= 3.34) and the
loadable cjk_unicode61 tokenizer — both mean "this one index can't be
served here", never "disable FTS".
"""
err = str(exc).lower()
return (
"no such tokenizer: trigram" in err
or "no such tokenizer: cjk_unicode61" in err
)
@staticmethod
def _db_has_legacy_inline_fts(cursor: sqlite3.Cursor) -> bool:
"""True when messages_fts exists in ANY pre-v23 shape.
v23's messages_fts is external-content over THREE real columns
(content, tool_name, tool_calls). Every pre-v23 shape lacks the
tool_name/tool_calls columns — whether the old inline single-column
form (v11..v22) or the even older external-content single-column form
(v10-era, pre-#16751). We therefore detect "needs optimize" as "the
stored CREATE lacks the tool_name column", which is the precise v23
marker and correctly catches BOTH legacy variants.
Returns False when messages_fts doesn't exist yet (fresh DB mid-init):
the post-migration FTS setup block will create it in the v23 shape.
"""
row = cursor.execute(
"SELECT sql FROM sqlite_master "
"WHERE type = 'table' AND name = 'messages_fts'"
).fetchone()
if row is None:
return False
sql = (row[0] if not isinstance(row, sqlite3.Row) else row["sql"]) or ""
# The v23 table declares tool_name/tool_calls columns. Their absence
# means a legacy shape that doesn't index tool metadata → optimize.
return "tool_name" not in sql
@staticmethod
def _db_has_trigram_tool_calls_projection(cursor: sqlite3.Cursor) -> bool:
"""True when the trigram vtable still includes tool_calls payload."""
row = cursor.execute(
"SELECT sql FROM sqlite_master "
"WHERE type = 'table' AND name = 'messages_fts_trigram'"
).fetchone()
if row is None:
return False
sql = (row[0] if not isinstance(row, sqlite3.Row) else row["sql"]) or ""
return "tool_calls" in sql.lower()
@classmethod
def _db_needs_fts_storage_upgrade(
cls, cursor: sqlite3.Cursor
) -> bool:
"""True when the current FTS storage layout should be treated as stale."""
return (
cls._db_has_legacy_inline_fts(cursor)
or cls._db_has_trigram_tool_calls_projection(cursor)
)
def _warn_trigram_unavailable(self, exc: sqlite3.OperationalError) -> None:
"""Log once that the trigram tokenizer is missing; base FTS5 stays enabled."""
if getattr(self, "_trigram_unavailable_warned", False):
return
self._trigram_unavailable_warned = True
logger.info(
"SQLite trigram tokenizer unavailable for %s "
"(requires SQLite >= 3.34, this build is %s); "
"CJK/substring search will fall back to LIKE: %s",
self.db_path,
sqlite3.sqlite_version,
exc,
)
def _warn_fts5_unavailable(self, exc: sqlite3.OperationalError) -> None:
self._fts_enabled = False
if self._fts_unavailable_warned:
return
self._fts_unavailable_warned = True
logger.warning(
"SQLite FTS5 unavailable for %s; full-text session search "
"disabled. Run `hermes update` to rebuild the venv with a "
"current Python (managed uv guarantees FTS5). "
"(underlying error: %s)",
self.db_path,
exc,
)
def _ensure_fts_cjk_schema(self, cursor) -> None:
"""Create / repair / self-heal the CJK-bigram index surface.
``cursor`` may be a Cursor or a Connection (both expose execute /
executescript). Called only for v23-shape DBs with the base FTS
surface healthy. Sets ``self._fts_cjk_available``. Never raises;
every failure mode degrades to "no cjk index" (trigram/LIKE routing
keeps working).
Cases:
tokenizer loaded, table absent → create. Empty DB: index is
complete by construction (triggers cover everything). Populated
DB: set the cjk backfill markers so the id-gated triggers stay
correct and `optimize-storage` can backfill; the index is NOT
served until the backfill completes.
tokenizer loaded, table present → ensure triggers (recreates any
dropped by a tokenizer-less process), honour the stale
breadcrumb (serve only when absent and no backfill pending).
tokenizer NOT loaded, table present with live triggers → drop the
cjk triggers so message INSERTs don't fail at trigger time,
and leave the stale breadcrumb (#self-heal). The table itself
stays for a later capable open to rebuild.
"""
try:
cjk_present = bool(cursor.execute(
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
"AND name = 'messages_fts_cjk'"
).fetchone())
if not self._fts_cjk_loaded:
if cjk_present:
live = [
r[0] for r in cursor.execute(
"SELECT name FROM sqlite_master WHERE type = 'trigger' "
f"AND name IN ({','.join('?' for _ in _FTS_CJK_TRIGGERS)})",
_FTS_CJK_TRIGGERS,
).fetchall()
]
if live:
# Self-heal: this process cannot tokenize, so every
# message INSERT would die inside the cjk trigger.
# Breadcrumb FIRST (crash between the two statements is
# merely conservative), then drop.
logger.warning(
"messages_fts_cjk triggers present but the "
"cjk_unicode61 tokenizer is unavailable (%s) — "
"dropping the cjk triggers so message writes keep "
"working. CJK search falls back to trigram/LIKE; "
"run `hermes sessions optimize-storage` on a host "
"with the extension to rebuild.",
fts5_cjk_so_path(),
)
cursor.execute(
"INSERT INTO state_meta (key, value) VALUES (?, '1') "
"ON CONFLICT(key) DO UPDATE SET value = '1'",
(FTS_CJK_STALE_KEY,),
)
for trig in live:
cursor.execute(f"DROP TRIGGER IF EXISTS {trig}")
self._fts_cjk_available = False
return
except sqlite3.OperationalError:
# Mirror the tokenizer-loaded except below: the presence check
# and self-heal above run before the loaded/not-loaded branch is
# even known to be safe, so they need the same never-raises
# guarantee the docstring promises for the rest of the method.
logger.warning(
"messages_fts_cjk presence check failed; CJK search stays on "
"trigram/LIKE", exc_info=True,
)
self._fts_cjk_available = False
return
try:
cursor.executescript(FTS_CJK_TABLE_SQL)
if not cjk_present:
# Freshly created. An empty DB's index is complete by
# construction (triggers will cover every future row); a
# populated DB (e.g. a v23 install predating the cjk index)
# gets the dedicated marker pair so the id-gated triggers
# keep NEW rows indexed while old rows await the
# `optimize-storage` backfill. Either way any old stale
# breadcrumb refers to a table that no longer exists.
cursor.execute(
"DELETE FROM state_meta WHERE key = ?",
(FTS_CJK_STALE_KEY,),
)
n_msgs = cursor.execute(
"SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
).fetchone()[0]
if n_msgs > 0:
hw = cursor.execute(
"SELECT COALESCE(MAX(id), 0) FROM messages"
).fetchone()[0]
for k, v in (
("fts_cjk_rebuild_high_water", str(hw)),
("fts_cjk_rebuild_progress", "0"),
):
cursor.execute(
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(k, v),
)
stale = cursor.execute(
"SELECT 1 FROM state_meta WHERE key = ?",
(FTS_CJK_STALE_KEY,),
).fetchone()
if stale:
# A tokenizer-less process dropped the triggers at some
# unknown point — the index has a gap of unknown extent.
# Do NOT reinstall triggers (an external-content 'delete'
# for an unindexed rowid corrupts the index); the next
# `optimize-storage` run rebuilds from scratch.
self._fts_cjk_available = False
return
cursor.executescript(FTS_CJK_TRIGGER_SQL)
backfill_pending = cursor.execute(
"SELECT 1 FROM state_meta "
"WHERE key = 'fts_cjk_rebuild_high_water' LIMIT 1"
).fetchone()
self._fts_cjk_available = not backfill_pending
except sqlite3.OperationalError:
# Includes "no such tokenizer: cjk_unicode61" if the extension
# loaded but registration failed — degrade to trigram/LIKE.
logger.warning(
"messages_fts_cjk ensure failed; CJK search stays on "
"trigram/LIKE", exc_info=True,
)
self._fts_cjk_available = False
@staticmethod
def _drop_fts_triggers(cursor: sqlite3.Cursor) -> None:
for trigger in _FTS_TRIGGERS:
try:
cursor.execute(f"DROP TRIGGER IF EXISTS {trigger}")
except sqlite3.OperationalError:
pass
def _ensure_fts_schema(
self,
cursor: sqlite3.Cursor,
table_name: str,
ddl: str,
) -> bool:
status = self._fts_table_probe(cursor, table_name)
if status is None:
return False
try:
# Run even when the virtual table exists so any dropped or missing
# triggers are recreated after a previous no-FTS5 runtime disabled
# them to keep message writes working.
cursor.executescript(ddl)
return True
except sqlite3.OperationalError as exc:
if not self._is_fts5_unavailable_error(exc):
raise
# Only disable FTS entirely when the whole FTS5 module is missing.
# A missing specific tokenizer (e.g. trigram) means only that
# particular table cannot be created — the base FTS5 table is fine.
if self._is_trigram_unavailable_error(exc):
self._warn_trigram_unavailable(exc)
else:
self._warn_fts5_unavailable(exc)
return False
def _execute_write(
self,
fn: Callable[[sqlite3.Connection], T],
patience_s: Optional[float] = None,
) -> T:
"""Execute a write transaction with BEGIN IMMEDIATE and jitter retry.
*fn* receives the connection and should perform INSERT/UPDATE/DELETE
statements. The caller must NOT call ``commit()`` — that's handled
here after *fn* returns.
BEGIN IMMEDIATE acquires the WAL write lock at transaction start
(not at commit time), so lock contention surfaces immediately.
On ``database is locked``, we release the Python lock, sleep a
random jitter, and retry — breaking the convoy pattern that
SQLite's built-in deterministic backoff creates.
*patience_s* is the total time budget for lock retries (default
``_WRITE_PATIENCE_S``). Transcript-critical writes pass
``_TRANSCRIPT_WRITE_PATIENCE_S`` so a sibling process holding the
lock for a legitimate long operation (VACUUM, TRUNCATE checkpoint,
pre-bounded-merge FTS optimize from an older still-running
install) exhausts routine writers' patience without destroying a
user turn. Jitter starts small (20-150ms) for fast reclaim on
millisecond contention and backs off to 250ms-1s once the lock has
been held longer than ``_WRITE_RETRY_SLOW_AFTER_S``.
Returns whatever *fn* returns.
"""
if patience_s is None:
patience_s = self._WRITE_PATIENCE_S
deadline = time.monotonic() + patience_s
# Set on the first compression-busy collision so the short wait is
# measured from then, not from the start of the write.
compression_deadline: Optional[float] = None
# One retry for SQLITE_IOERR raised by BEGIN IMMEDIATE itself. The
# callback has not run at that point, so there is no durable effect
# to replay and the retry is exactly-once safe (#99502's contract).
# Once the callback starts, an IOERR leaves the write's settlement
# unknown and must propagate — this helper owns non-idempotent
# transcript/counter mutations, not just idempotent UPSERTs.
ioerr_begin_retried = False
# Transient engine-level error observed on contended WAL appends
# (dual gateway/agent writers; FTS5 trigram sync holds the write
# lock). The identical write succeeds standalone, so it is
# retryable like locked/busy. The exception CLASS varies with the
# SQLite build — some surface it as InterfaceError, which lives
# OUTSIDE DatabaseError and escaped the retry net entirely on
# attempt 0 — so the check is message-scoped, not class-scoped.
def _is_no_more_rows(exc: sqlite3.Error) -> bool:
return "no more rows available" in str(exc).lower()
while True:
self._raise_if_db_corrupt()
self._raise_if_db_replaced()
fn_started = False
try:
with self._lock:
if self._conn is None:
# close() ran while this writer was still unwinding
# (#94736) — reopen instead of dying on None.execute.
self._reopen_after_close_locked(context="write")
self._conn.execute("BEGIN IMMEDIATE")
try:
fn_started = True
result = fn(self._conn)
self._conn.commit()
except BaseException:
try:
self._conn.rollback()
except Exception:
pass
raise
# Success — periodic best-effort checkpoint + FTS merge.
self._write_count += 1
if self._write_count % self._CHECKPOINT_EVERY_N_WRITES == 0:
self._try_wal_checkpoint()
if self._write_count % self._FTS_MERGE_EVERY_N_WRITES == 0:
self._try_incremental_merge_fts()
return result
except SessionCompressionInProgressError:
# A live foreign compression lock is transient: the compressor
# publishes in a couple of seconds. Without any wait, a steer
# that lands mid-compression aborts the user's turn as
# session_persistence_failed and sends the operator hunting
# disk space that was never the problem (#75083).
#
# The budget is _COMPRESSION_BUSY_WAIT_S, not the write-lock
# patience: the lease is a correctness boundary, so a writer
# still locked out after a short wait must be refused rather
# than left to land a stale turn once a long-running or wedged
# compression finally lets go.
if compression_deadline is None:
compression_deadline = min(
time.monotonic() + self._COMPRESSION_BUSY_WAIT_S, deadline
)
if self._sleep_before_write_retry(
compression_deadline, self._COMPRESSION_BUSY_WAIT_S
):
continue
raise
except sqlite3.OperationalError as exc:
err_msg = str(exc).lower()
if "locked" in err_msg or "busy" in err_msg:
if self._sleep_before_write_retry(deadline, patience_s):
continue
# Patience exhausted — say what actually happened so the
# surfaced error doesn't read as disk/permission damage.
raise sqlite3.OperationalError(
f"database is locked (another Hermes process held the "
f"state.db write lock for over {patience_s:.0f}s — "
"likely a long maintenance operation such as VACUUM, "
"a large WAL checkpoint, or an older pre-update "
"process; the database itself is healthy)"
) from exc
if _is_no_more_rows(exc) and self._sleep_before_write_retry(deadline, patience_s):
continue
if (
_DISK_IO_ERROR_MARKER in err_msg
and not fn_started
and not ioerr_begin_retried
and self._sleep_before_write_retry(deadline, patience_s)
):
# BEGIN IMMEDIATE itself hit a transient WAL-transition
# IOERR. Nothing has been mutated, so retrying on the SAME
# connection replays nothing. Never close()+reopen to
# "heal" it: close() cancels this process's POSIX locks on
# the file for every sibling connection (howtocorrupt §2.2).
ioerr_begin_retried = True
continue
# Non-lock error, the callback already ran (settlement is
# unknown — do not replay), or patience exhausted.
raise
except sqlite3.DatabaseError as exc:
if _is_no_more_rows(exc) and self._sleep_before_write_retry(deadline, patience_s):
continue
# An out-of-band replace of state.db (restore/cp/mv under a
# live process) surfaces as this same corruption error class.
# In-file repair on a NEW file generation amplifies the
# damage (#89332) — halt writes on this handle instead.
if (
"not a database" in str(exc).lower()
or is_malformed_db_error(exc)
or self._is_fts_write_corruption_error(exc)
):
self._raise_if_db_replaced()
# Corrupt FTS shadow tables make every write raise the
# malformed/corrupt error class through the FTS sync triggers
# while the canonical messages table is intact. Never run a
# full-message FTS5 rebuild from this live persistence path:
# on a multi-gigabyte state.db that can hold the writer lock
# for minutes. Atomically detach the derived indexes instead,
# then retry the canonical write. The existing stale-open and
# explicit repair paths retain rebuild ownership.
if self._enter_fts_fail_open(exc):
continue
# Bare SQLITE_CORRUPT / NOTADB that survived the replaced-file
# check and the FTS-scoped fail-open is structural damage:
# quarantine the handle (see StateDbCorruptError).
if self._is_structural_corruption_error(exc):
self._halt_db_corrupt(exc)
raise
except sqlite3.Error as exc:
# Catch-all for builds that surface 'no more rows available'
# as InterfaceError (a sibling of DatabaseError, not a
# subclass) or another sqlite3.Error class outside the two
# handlers above. Message-scoped: anything else propagates
# untouched.
if _is_no_more_rows(exc) and self._sleep_before_write_retry(deadline, patience_s):
continue
raise
def _ensure_db_file_generation(self) -> None:
"""Mint a once-per-file generation stamp (state_meta + application_id).
First opener wins via INSERT OR IGNORE. application_id is written
only when still 0 so racers converge on the same header value.
PASSIVE checkpoint only — never TRUNCATE (#45383).
"""
if self.read_only or self._conn is None:
return
token = uuid.uuid4().hex
try:
with self._lock:
self._conn.execute(
"INSERT OR IGNORE INTO state_meta (key, value) VALUES (?, ?)",
(_STATE_DB_GENERATION_KEY, token),
)
row = self._conn.execute(
"SELECT value FROM state_meta WHERE key = ?",
(_STATE_DB_GENERATION_KEY,),
).fetchone()
if row and row[0]:
token = str(row[0])
self._db_file_generation_token = token
current = 0
pragma_row = self._conn.execute("PRAGMA application_id").fetchone()
if pragma_row:
current = int(pragma_row[0] or 0)
if current == 0:
app_id = int(token[:8], 16) & 0x7FFFFFFF
if app_id == 0:
app_id = 1
self._conn.execute(f"PRAGMA application_id={app_id}")
current = app_id
self._db_file_application_id = current
try:
self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)")
except sqlite3.Error:
pass
except sqlite3.Error as exc:
logger.debug("state.db generation stamp skipped: %s", exc)
def _record_db_file_identity(self) -> None:
"""Snapshot inode plus the on-disk generation header when present."""
self._db_file_identity = _stat_db_file_identity(self.db_path)
self._db_sidecar_identity = _stat_sqlite_sidecar_identity(self.db_path)
disk_id = _read_sqlite_application_id(self.db_path)
if disk_id:
self._db_file_application_id = disk_id
elif self._conn is not None and not self._db_file_application_id:
try:
with self._read_ctx() as conn:
pragma_row = conn.execute("PRAGMA application_id").fetchone()
if pragma_row and pragma_row[0]:
self._db_file_application_id = int(pragma_row[0])
except sqlite3.Error:
pass
def _db_file_was_replaced(self) -> bool:
"""True when the path no longer names the file this instance opened."""
recorded = self._db_file_identity
if recorded is not None:
current = _stat_db_file_identity(self.db_path)
if current is None or current != recorded:
return True
recorded_app = int(self._db_file_application_id or 0)
if recorded_app:
disk_app = _read_sqlite_application_id(self.db_path)
# Header 0 means the WAL has not been checkpointed yet — not a
# replace. A copied Hermes DB that minted its own id is nonzero.
if disk_app and disk_app != recorded_app:
return True
return False
def _halt_db_replaced(self) -> None:
"""Stop writes and raise; do not run in-file repair on a new generation."""
self._db_replaced = True
logger.error(_STATE_DB_REPLACED_MSG)
raise StateDbReplacedError(_STATE_DB_REPLACED_MSG)
def _wal_generation_was_lost(self) -> bool:
"""True when the WAL/SHM generation this instance opened is gone.
Steady state (a sidecar generation is recorded): pure stat — a
recorded inode that is missing or replaced by a new file at the same
path means the generation split. No /proc walk on healthy writes.
Empty-identity state (fresh DB whose WAL appears only after open, or
identity cleared by a clean ``close()``): fall back to a
``/proc/self/fd`` deleted-fd probe, and adopt the current sidecars as
this handle's generation once the probe comes back clean. The full
``/proc/*/fd`` walk is reserved for
:func:`refuse_deleted_wal_generation` on open, where we must see
*foreign* deleted holders before ``sqlite3.connect`` mints a new WAL.
"""
recorded = self._db_sidecar_identity or {}
base = os.fspath(self.db_path)
if recorded:
for suffix, recorded_ident in recorded.items():
current = _stat_db_file_identity(Path(base + suffix))
if current is None or current != recorded_ident:
return True
return False
if not self._wal_active:
# No WAL on this handle (journal_mode=delete/truncate fallback):
# there is no sidecar generation to lose, and probing every write
# would put a /proc walk on the hot path of exactly the
# delete-mode deployments the field report used as containment.
return False
if sys.platform.startswith("linux"):
watched = _watched_sqlite_sidecar_paths(self.db_path)
fd_dir = f"/proc/{os.getpid()}/fd"
try:
for fd in os.listdir(fd_dir):
try:
target = os.readlink(f"{fd_dir}/{fd}")
except OSError:
continue
if " (deleted)" in target and _canonical_sqlite_path(target) in watched:
return True
except OSError:
return False
# Probe clean (or unavailable on this platform): adopt whatever
# sidecar generation exists now so subsequent writes use the cheap
# stat check.
current_identity = _stat_sqlite_sidecar_identity(self.db_path)
if current_identity:
self._db_sidecar_identity = current_identity
return False
def _halt_deleted_wal_generation(self) -> None:
"""Stop writes; do not mint or keep committing on a split WAL."""
self._db_wal_generation_lost = True
logger.error(_DELETED_WAL_GENERATION_MSG)
raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG)
def _raise_if_db_replaced(self) -> None:
if self._db_replaced:
raise StateDbReplacedError(_STATE_DB_REPLACED_MSG)
if self._db_wal_generation_lost:
raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG)
if self._db_file_was_replaced():
self._halt_db_replaced()
if self._wal_generation_was_lost():
self._halt_deleted_wal_generation()
@classmethod
def _is_structural_corruption_error(cls, exc: BaseException) -> bool:
"""Bare SQLITE_CORRUPT/NOTADB with no FTS provenance.
``_is_fts_write_corruption_error`` is the positive FTS classifier;
everything else in the ``corrupt`` bucket of
``classify_persistence_error`` is damage to a canonical B-tree, the
schema, or the freelist — never repairable from the live write path.
"""
if not isinstance(exc, sqlite3.DatabaseError):
return False
if isinstance(exc, StateDbCorruptError):
return False
if cls._is_fts_write_corruption_error(exc):
return False
return classify_persistence_error(exc) == "corrupt"
def _corrupt_error(self, prefix: str = "") -> "StateDbCorruptError":
"""Build the quarantine error for this handle (message assembled once)."""
return StateDbCorruptError(
f"{prefix}{_STATE_DB_CORRUPT_MSG} (cause: {self._db_corrupt_reason})"
)
def _halt_db_corrupt(self, exc: BaseException) -> None:
"""Quarantine this handle and raise; never run in-file repair here."""
self._db_corrupt = True
self._db_corrupt_reason = str(exc)
self._disable_close_time_checkpoint()
logger.error(
"state.db %s reported structural corruption outside the FTS "
"indexes (%s); quarantining this handle: no further writes, no "
"automatic reopen, no explicit WAL checkpoint at close. Stop the "
"gateway and run `hermes sessions recover --source %s "
"--inspect-only`.",
self.db_path,
exc,
self.db_path,
)
err = self._corrupt_error()
for attr in ("sqlite_errorcode", "sqlite_errorname"):
value = getattr(exc, attr, None)
if value is not None:
setattr(err, attr, value)
raise err from exc
def _disable_close_time_checkpoint(self) -> None:
"""Best-effort: stop SQLite's own last-connection checkpoint on close.
Skipping our explicit ``PRAGMA wal_checkpoint(PASSIVE)`` in
``close()`` is not enough on its own: ``sqlite3.Connection.close()``
still runs SQLite's internal last-connection PASSIVE checkpoint and
unlinks the ``-wal``/``-shm`` sidecars. On the field incident's file
that close-time checkpoint is exactly what wrote 15 pages under the
wrong page numbers. Python 3.12+ exposes the switch as
``Connection.setconfig(SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE)``; on 3.11
neither the constant nor ``setconfig`` exists, so the internal
checkpoint remains (it can only carry pre-quarantine committed
frames — no further writes are accepted on this handle).
"""
flag = getattr(sqlite3, "SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE", None)
if flag is None:
return
conn = self._conn
setconfig = getattr(conn, "setconfig", None)
if conn is None or setconfig is None:
return
try:
setconfig(flag, True)
except Exception:
logger.debug(
"Could not disable SQLite's close-time checkpoint on the "
"quarantined handle for %s",
self.db_path,
exc_info=True,
)
def _raise_if_db_corrupt(self) -> None:
if self._db_corrupt:
raise self._corrupt_error()
def _sleep_before_write_retry(
self, deadline: float, patience_s: float
) -> bool:
"""Sleep one jitter interval if the patience budget still allows it.
Returns True when the caller should retry, False when *deadline* has
passed and the error should propagate. Jitter stays small for the
first ``_WRITE_RETRY_SLOW_AFTER_S`` (fast reclaim on millisecond
contention) and backs off after that, and never overshoots the
deadline by a full slow-jitter.
"""
now = time.monotonic()
if now >= deadline:
return False
elapsed = now - (deadline - patience_s)
if elapsed >= self._WRITE_RETRY_SLOW_AFTER_S:
jitter = random.uniform(
self._WRITE_RETRY_SLOW_MIN_S,
self._WRITE_RETRY_SLOW_MAX_S,
)
else:
jitter = random.uniform(
self._WRITE_RETRY_MIN_S,
self._WRITE_RETRY_MAX_S,
)
time.sleep(min(jitter, max(deadline - now, 0.001)))
return True
@staticmethod
def _is_fts_write_corruption_error(exc: sqlite3.DatabaseError) -> bool:
"""Return true only when SQLite identifies corruption as FTS-scoped.
Newer SQLite builds include ``fts5`` in the error text. Older builds
may emit only ``database disk image is malformed`` while exposing the
extended ``SQLITE_CORRUPT_VTAB`` result code. A bare
``SQLITE_CORRUPT``/malformed-image error is structural and must not
trigger live FTS maintenance: it does not prove that canonical B-trees
are intact.
"""
corrupt_vtab = getattr(sqlite3, "SQLITE_CORRUPT_VTAB", 267)
error_code = getattr(exc, "sqlite_errorcode", None)
if error_code is not None:
return error_code == corrupt_vtab
msg = str(exc).lower()
return msg.startswith("fts5:") and "corrupt structure" in msg
def _foreign_state_db_holders(self) -> List[Tuple[int, str]]:
"""Return foreign processes holding this DB or its WAL sidecars."""
return _foreign_state_db_holders(self.db_path)
def _reap_inactive_orphan_desktop_holders(
self, holders: List[Tuple[int, str]], *, min_age_seconds: float
) -> List[int]:
"""Terminate old PPID-1 Desktop ephemeral backends with no client.
Inspection fails closed: anything whose parent, age, argv, or network
connections cannot be proved safe remains a repair-blocking holder.
"""
if not sys.platform.startswith("linux") or psutil is None:
return []
try:
from hermes_cli.dashboard_procs import _is_ephemeral_port_zero_backend
except Exception:
return []
now = time.time()
candidates = []
for pid in _concrete_state_db_holder_pids(self.db_path, holders):
try:
process = psutil.Process(pid)
statuses = [
conn.status for conn in process.net_connections(kind="inet")
]
if not _is_inactive_orphan_desktop_holder(
ppid=process.ppid(),
age_seconds=now - process.create_time(),
min_age_seconds=min_age_seconds,
ephemeral_backend=_is_ephemeral_port_zero_backend(process.cmdline()),
connection_statuses=statuses,
):
continue
except Exception:
continue
candidates.append(process)
signalled: List[int] = []
for process in candidates:
try:
process.terminate()
signalled.append(process.pid)
except (psutil.Error, OSError):
continue
if not signalled:
return []
try:
_gone, alive = psutil.wait_procs(candidates, timeout=1.5)
except Exception:
alive = []
for process in alive:
try:
process.kill()
except (psutil.Error, OSError):
continue
if alive:
try:
psutil.wait_procs(alive, timeout=1.5)
except Exception:
pass
return signalled
def _enter_fts_fail_open(self, exc: sqlite3.DatabaseError) -> bool:
"""Detach corrupt FTS indexes so canonical writes can continue.
The stale breadcrumb and trigger removal commit atomically. Its
ordering is load-bearing: after triggers are absent, new canonical
rows create an index gap of unknown extent, so another process must
never reinstall the triggers without first rebuilding every row.
"""
if not self._fts_enabled or not self._is_fts_write_corruption_error(exc):
return False
self._raise_if_db_corrupt()
if self._db_replaced or self._db_file_was_replaced():
self._halt_db_replaced()
if self._db_wal_generation_lost or self._wal_generation_was_lost():
self._halt_deleted_wal_generation()
try:
with self._lock:
self._conn.execute("BEGIN IMMEDIATE")
try:
self._conn.execute(
"INSERT INTO state_meta (key, value) VALUES (?, '1') "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(FTS_STALE_KEY,),
)
cjk_triggers_present = self._conn.execute(
"SELECT 1 FROM sqlite_master WHERE type = 'trigger' "
f"AND name IN ({','.join('?' for _ in _FTS_CJK_TRIGGERS)}) "
"LIMIT 1",
_FTS_CJK_TRIGGERS,
).fetchone()
if cjk_triggers_present:
self._conn.execute(
"INSERT INTO state_meta (key, value) VALUES (?, '1') "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(FTS_CJK_STALE_KEY,),
)
self._drop_all_fts_triggers(self._conn.cursor())
self._conn.commit()
except BaseException:
self._conn.rollback()
raise
except sqlite3.Error as detach_exc:
logger.error(
"Could not detach corrupt FTS indexes; canonical write still "
"cannot proceed: %s",
detach_exc,
)
return False
self._fts_stale = True
self._fts_enabled = False
self._trigram_available = False
self._fts_cjk_available = False
logger.error(
"state.db FTS indexes remain corrupt (%s); disabled FTS sync and "
"retrying the canonical write. Search temporarily uses LIKE until "
"a later SessionDB open rebuilds the indexes.",
exc,
)
return True
def _try_wal_checkpoint(self) -> None:
"""Best-effort PASSIVE WAL checkpoint. Never raises.
Flushes committed WAL frames back into the main DB file without
requiring an exclusive lock. PASSIVE is safe for frequent
periodic use because it does not block concurrent writers and
cannot corrupt B-tree pages under I/O pressure.
PASSIVE does not truncate the WAL file — it stays at its
high-water mark. Explicit checkpoints on the shared ``state.db`` no
longer truncate the WAL; it is bounded by ``journal_size_limit`` and
the writer's natural post-checkpoint reset rather than by a TRUNCATE
at every close or maintenance command.
Previous TRUNCATE strategy caused B-tree corruption on large
databases (65K+ pages) due to the exclusive-lock I/O pressure
from checkpointing thousands of frames at once (issue #45383).
"""
if self._db_corrupt:
return # quarantined: never checkpoint over a damaged image
try:
with self._lock:
result = self._conn.execute(
"PRAGMA wal_checkpoint(PASSIVE)"
).fetchone()
if result and result[1] > 0:
logger.debug(
"WAL checkpoint: %d/%d pages checkpointed",
result[2], result[1],
)
except Exception as exc:
logger.warning("WAL checkpoint (PASSIVE) failed: %s", exc)
def __enter__(self) -> "SessionDB":
"""Enter a scope that closes this handle on the way out.
Ownership of a SessionDB should be released explicitly.
Historically an instance with a started token writer pinned ITSELF
(bound-method writer target plus a strong ``atexit`` drain hook), so
``__del__`` never ran for exactly the instances that leaked
descriptors (#88033). The writer now retires after an idle window
and the atexit hook holds only a weak reference, so abandoned
handles are eventually collectible — but "eventually, after the
idle window and a GC cycle" is not a release policy. Call sites
owning a handle are still expected to close it deterministically
(see the ownership comments in ``run_agent.py`` and
``tui_gateway/methods_session.py``).
This makes the correct usage the easy one, so an owning scope can be
exception-safe by construction rather than by remembering a
``try/finally``:
with SessionDB(path) as db:
db.append_message(...)
Purely additive: it changes nothing for callers that already call
``close()`` directly, and ``close()`` stays idempotent, so a scope
that closes early still exits cleanly.
"""
return self
def __exit__(self, exc_type, exc, tb) -> bool:
"""Close the handle, then let any exception propagate.
Returns False (never suppressing), so ``with`` here only manages the
descriptor lifetime and never swallows a caller's error.
"""
self.close()
return False
def close(self):
"""Close the database connection.
Drains queued token deltas first (the background writer needs the
connection). Writable connections then attempt a PASSIVE WAL
checkpoint (NOT TRUNCATE: transient per-cron-run connections close
many times an hour, and a TRUNCATE fires a full WAL reset that
races the gateway's live writer and tears B-tree pages — issue
#45383). Read-only connections never request a checkpoint.
When this instance is shared (opened via ``get_shared_session_db``),
``close()`` RELEASES one refcount instead of tearing down the
connection: the registry owns the lifecycle and only closes on the
final release (#90837). This prevents one caller's close from
tearing down the writer connection that other callers in the same
process are still using — while still letting legacy ``close()``
call sites return their reference instead of leaking it.
"""
if getattr(self, "_shared_registry_owned", False):
from hermes_state_registry import release
release(self)
return
self._stop_token_writer()
hook, self._token_atexit_hook = self._token_atexit_hook, None
if hook is not None:
atexit.unregister(hook)
# Drain the read-only connection pool. Setting the closed flag
# under the lock first means a reader still in flight closes its own
# connection on release instead of re-populating a pool that has
# already been drained.
with self._read_conns_lock:
self._read_conns_closed = True
while True:
try:
conn = self._read_pool.get_nowait()
except queue.Empty:
break
self._close_read_conn(conn)
with self._lock:
if self._conn:
if self._db_corrupt:
# Quarantined handle (see StateDbCorruptError): no explicit
# checkpoint over a damaged page image.
logger.warning(
"Skipping the close-time WAL checkpoint for %s: this "
"handle observed structural corruption (%s). Take a "
"snapshot of state.db, -wal and -shm before restarting, "
"then run `hermes sessions recover --source %s "
"--inspect-only`.",
self.db_path,
self._db_corrupt_reason,
self.db_path,
)
elif not self.read_only:
# PASSIVE, not TRUNCATE. Every cron run_agent opens+closes a
# transient SessionDB, so a TRUNCATE here fires a full WAL
# reset many times/hour, racing the gateway's long-lived
# writer on large WAL databases and tearing hot B-tree
# pages -- the #45383 corruption this class's own periodic
# checkpoint was already made PASSIVE to avoid. TRUNCATE
# belongs only on a sole-opener/quiescent connection.
try:
self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)")
except Exception as exc:
logger.debug(
"WAL checkpoint (PASSIVE) at close failed: %s",
exc,
)
conn, self._conn = self._conn, None
self._close_connection_quietly(conn)
# A clean close of the last connection lets SQLite unlink the
# WAL/SHM sidecars — a legitimate end of this handle's sidecar
# generation, not a split (#94736 late writes must still
# self-heal). Drop the recorded generation so a teardown-race
# reopen re-adopts whatever exists then instead of halting.
self._db_sidecar_identity = {}
def __del__(self) -> None:
"""Safety net: close the connection if the caller forgot.
The async accounting worker retires when idle and its atexit hook
holds only a weak reference, so neither can pin an otherwise orphaned
instance. During interpreter teardown the order of module cleanup is
undefined, so every attribute access remains guarded.
Delegates to ``close()`` so the read pool, token writer, and atexit
hook are all cleaned up — not just the writer connection.
"""
if self.__dict__.get("_conn") is None:
return
try:
self.close()
except Exception:
pass
# ── Chunked FTS rebuild engine (v23 opt-in optimize) ──
#
# `optimize_fts_storage()` (the `hermes sessions optimize-storage`
# command) drops the legacy inline FTS indexes and backfills the new
# external-content ones. A single blocking rebuild measured ~16 minutes
# of held write lock on a real 25 GB DB, so the backfill runs in small
# chunks, each in its own short write transaction:
# - concurrent readers/writers are never starved (WAL stays small,
# each chunk checkpoints via the normal _execute_write cadence);
# - an interrupted run (Ctrl-C, crash) resumes from
# fts_rebuild_progress when the command is re-run;
# - multiple processes sharing the DB don't double-run it — each chunk
# claims work by compare-and-swap on fts_rebuild_progress, so even a
# concurrent second runner just interleaves chunks safely.
#
# THROTTLING (the part that keeps a live gateway sharing the DB
# responsive): a greedy chunk loop re-acquires BEGIN IMMEDIATE nearly
# back-to-back and can starve another process's writer into exhausting
# its lock retries (an early 5000-row/50ms version owned the write lock
# ~85% of the time and visibly froze concurrent CLI sessions on a large
# install). Two layers prevent that:
# 1. Small chunks (500 rows) — a foreground write queues behind a
# chunk for at most ~tens of ms.
# 2. Inter-chunk pause — the loop sleeps max(_FTS_REBUILD_MIN_PAUSE,
# chunk cost x _FTS_REBUILD_DUTY_FACTOR) between chunks, capping
# this process's share of DB bandwidth so concurrent writers always
# find open windows. This works cross-process (unlike any
# same-process activity stamp) because it bounds our own duty
# cycle unconditionally.
_FTS_REBUILD_CHUNK_ROWS = 500
_FTS_REBUILD_DUTY_FACTOR = 4.0 # sleep >= 4x chunk cost (≤20% duty)
_FTS_REBUILD_MIN_PAUSE = 0.2 # seconds — floor between chunks
# Demoted v22 FTS shadow tables awaiting teardown (see the v23 migration:
# DROP of a multi-GB FTS vtable blocks for minutes, so the migration
# demotes the vtable definitions out of sqlite_master and renames the
# orphaned shadow tables — now plain tables — to fts_v22_trash_*; the
# worker empties them in bounded chunks, then drops them cheaply).
_FTS_TRASH_PREFIX = "fts_v22_trash_"
# ── CJK-bigram index backfill (dedicated marker pair) ──
#
# Same chunk engine as the main deferred rebuild, but on the
# ``fts_cjk_rebuild_*`` markers so a cjk-only backfill (the common case:
# an already-optimized v23 DB gaining the cjk index) never gates the
# complete ``messages_fts`` / trigram triggers.
# ── Opt-in v23 FTS storage optimization (`hermes sessions optimize-storage`) ──
#
# This is the ONLY path that migrates an existing legacy (v22 inline) DB
# to the v23 external-content schema. It is deliberately foreground and
# user-invoked, never automatic, because it is disk-heavy and long. It
# runs the throttled/resumable chunk engine above to completion
# synchronously — demote → new schema → chunked backfill → chunked
# teardown — with progress callbacks, a disk preflight in the CLI
# wrapper, a VACUUM at the end, and a defensive schema_version bump.
def _has_fts_trash(self, conn) -> bool:
"""True when demoted v22 shadow tables are still awaiting teardown.
Caller must hold ``self._lock`` (or pass a migration-time cursor)."""
return bool(conn.execute(
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
"AND name LIKE ? ESCAPE '\\' LIMIT 1",
(self._FTS_TRASH_PREFIX.replace("_", "\\_") + "%",),
).fetchone())
# =========================================================================
# Session lifecycle
# =========================================================================
_PROFILE_DIR_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
def _own_profile_name(self) -> Optional[str]:
"""The profile that owns THIS store, derived from ``db_path`` alone.
Every profile-tree ``state.db`` belongs to exactly one profile
(``<root>/state.db`` → ``default``,
``<root>/profiles/<name>/state.db`` → ``<name>``), so the derivation
is a single match, never a guess — the same contract
:meth:`backfill_null_session_profiles` and the web listing's
``row_profile`` stamp rely on. Path-based (not
``get_active_profile_name()``) on purpose: a gateway serving a
NON-launch profile opens that profile's store directly, and the row
must be stamped with the store's owner, not the serving process's
launch profile. Returns ``None`` for stores outside the profile tree
(explicit ``db_path`` in tests, ad-hoc copies) — those rows keep the
legacy NULL rather than a fabricated owner.
"""
try:
from hermes_constants import get_default_hermes_root
root = get_default_hermes_root().resolve()
parent = Path(self.db_path).resolve().parent
if parent == root:
return "default"
if parent.parent == root / "profiles" and self._PROFILE_DIR_RE.match(
parent.name
):
return parent.name
except Exception:
logger.debug("own-profile derivation failed", exc_info=True)
return None
def _insert_session_row(
self,
session_id: str,
source: str,
model: str = None,
model_config: Dict[str, Any] = None,
system_prompt: str = None,
user_id: str = None,
session_key: Optional[str] = None,
chat_id: str = None,
chat_type: str = None,
thread_id: str = None,
parent_session_id: str = None,
cwd: str = None,
profile_name: Optional[str] = None,
git_repo_root: str = None,
origin_json: str = None,
display_name: str = None,
) -> None:
"""Insert a session row, enriching NULL metadata on conflict.
The gateway's ``get_or_create_session`` creates a bare row (source +
user_id) *before* the agent exists; the agent's later
``create_session`` then carries the real ``model`` / ``model_config`` /
``system_prompt``. A plain ``INSERT OR IGNORE`` silently dropped that
enrichment, leaving gateway sessions with NULL model/billing metadata.
The ``ON CONFLICT`` upsert backfills those fields via ``COALESCE`` —
only filling columns that are still NULL, never overwriting values an
earlier writer already set (so a later bare call with source="unknown"
can't clobber a real source/model).
``chat_id``/``thread_id`` record the messaging origin (the chat/room and
thread the session was started in) so that gateway ``/resume`` can prove
a persisted, now-inactive row belongs to the caller's chat/thread before
switching to it (IDOR scoping — without them the ``sessions`` table has
no chat/thread to compare).
When ``parent_session_id`` is set (compression fork, delegate/subagent
spawn, branch continuation) and this row's own ``cwd``/``git_repo_root``/
``git_branch``/``profile_name`` are still NULL after the insert, they are
backfilled from the parent row. Callers of ``create_session`` for a child
session historically didn't propagate these fields themselves (e.g. the
compression-fork path), so a lineage could silently lose its working
directory and drop out of the project sidebar every time it forked
(#64709), or lose its owning profile and be aggregated as "default" every
time it rotated or branched (the cross-profile session-jump bug). This
only fills NULLs — an explicit value on the child is never overwritten.
For compression forks specifically
(parent ended with ``end_reason='compression'``), the gateway origin
columns (``user_id``/``session_key``/``chat_id``/``chat_type``/
``thread_id``/``display_name``/``origin_json``) are inherited too, so a
crash before the gateway re-records the peer can't strand the child
without a recoverable routing mapping (#59527).
When the caller passes no ``profile_name`` at all, the row is stamped
with THIS store's own profile (:meth:`_own_profile_name`) instead of
NULL. Every ``state.db`` belongs to exactly one profile — the same
single-match contract :meth:`backfill_null_session_profiles` relies
on — so the stamp is derivation, not a guess. Rows minted NULL after
that one-shot #94724 backfill ran stayed NULL forever, and
profile-keyed consumers (desktop sidebar scope matching,
``@session:<profile>/<id>`` deep links, the fail-closed owner ladder)
treat NULL as unowned: the session vanishes from the sidebar even
though its transcript is intact (#99222). Stores outside the profile
tree (explicit ``db_path`` in tests, ad-hoc copies) derive nothing
and keep NULL — never guess.
"""
if not (profile_name or "").strip():
profile_name = self._own_profile_name()
def _do(conn):
system_prompt_hash = self._store_system_prompt(conn, system_prompt)
conn.execute(
"""INSERT INTO sessions (
id, source, user_id, session_key, chat_id, chat_type, thread_id,
model, model_config, system_prompt, system_prompt_hash,
parent_session_id, cwd, profile_name, git_repo_root,
origin_json, display_name, started_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
model = COALESCE(sessions.model, excluded.model),
model_config = CASE
WHEN excluded.model_config IS NOT NULL
AND json_type(
sessions.model_config, '$._reset_from'
) IS NOT NULL
AND json_remove(
sessions.model_config, '$._reset_from'
) = '{}'
THEN json_set(
excluded.model_config,
'$._reset_from',
json_extract(
sessions.model_config, '$._reset_from'
)
)
ELSE COALESCE(
sessions.model_config, excluded.model_config
)
END,
system_prompt_hash = COALESCE(
sessions.system_prompt_hash,
excluded.system_prompt_hash
),
system_prompt = CASE
WHEN sessions.system_prompt_hash IS NULL
AND excluded.system_prompt_hash IS NOT NULL
THEN NULL
ELSE sessions.system_prompt
END,
session_key = COALESCE(sessions.session_key, excluded.session_key),
chat_id = COALESCE(sessions.chat_id, excluded.chat_id),
chat_type = COALESCE(sessions.chat_type, excluded.chat_type),
thread_id = COALESCE(sessions.thread_id, excluded.thread_id),
parent_session_id = COALESCE(sessions.parent_session_id, excluded.parent_session_id),
cwd = COALESCE(sessions.cwd, excluded.cwd),
profile_name = COALESCE(sessions.profile_name, excluded.profile_name),
git_repo_root = COALESCE(sessions.git_repo_root, excluded.git_repo_root),
origin_json = COALESCE(sessions.origin_json, excluded.origin_json),
display_name = COALESCE(sessions.display_name, excluded.display_name)""",
(
session_id,
source,
user_id,
session_key,
chat_id,
chat_type,
thread_id,
model,
json.dumps(model_config) if model_config else None,
system_prompt_hash,
parent_session_id,
cwd,
profile_name,
git_repo_root,
origin_json,
display_name,
time.time(),
),
)
if system_prompt_hash is not None:
self._delete_unreferenced_system_prompts(conn)
if parent_session_id:
conn.execute(
f"""UPDATE sessions
SET cwd = COALESCE(sessions.cwd,
(SELECT p.cwd FROM sessions p
WHERE p.id = sessions.parent_session_id)),
git_repo_root = COALESCE(sessions.git_repo_root,
(SELECT p.git_repo_root FROM sessions p
WHERE p.id = sessions.parent_session_id)),
git_branch = COALESCE(sessions.git_branch,
(SELECT p.git_branch FROM sessions p
WHERE p.id = sessions.parent_session_id)),
profile_name = COALESCE(sessions.profile_name,
(SELECT p.profile_name FROM sessions p
WHERE p.id = sessions.parent_session_id
AND ({_SAME_KEY_NAMESPACE_SQL})))
WHERE id = ? AND parent_session_id IS NOT NULL""",
(session_id,),
)
# Belt-and-suspenders for gateway routing metadata (#59527):
# the gateway re-records the peer on the child after rotation
# (d5b4879d4), but a hard crash between child creation and that
# write leaves the child row without origin columns, so
# ``find_latest_gateway_session_for_peer`` can't recover the
# mapping on restart. Inherit them from the parent at creation
# time — but ONLY for compression forks (parent already ended
# with end_reason='compression'). Delegate/subagent children
# are spawned while the parent is still live and must NOT
# inherit routing keys, or peer recovery could repoint gateway
# traffic into a subagent's session.
conn.execute(
"""UPDATE sessions
SET user_id = COALESCE(sessions.user_id,
(SELECT p.user_id FROM sessions p
WHERE p.id = sessions.parent_session_id)),
session_key = COALESCE(sessions.session_key,
(SELECT p.session_key FROM sessions p
WHERE p.id = sessions.parent_session_id)),
chat_id = COALESCE(sessions.chat_id,
(SELECT p.chat_id FROM sessions p
WHERE p.id = sessions.parent_session_id)),
chat_type = COALESCE(sessions.chat_type,
(SELECT p.chat_type FROM sessions p
WHERE p.id = sessions.parent_session_id)),
thread_id = COALESCE(sessions.thread_id,
(SELECT p.thread_id FROM sessions p
WHERE p.id = sessions.parent_session_id)),
display_name = COALESCE(sessions.display_name,
(SELECT p.display_name FROM sessions p
WHERE p.id = sessions.parent_session_id)),
origin_json = COALESCE(sessions.origin_json,
(SELECT p.origin_json FROM sessions p
WHERE p.id = sessions.parent_session_id))
WHERE id = ? AND parent_session_id IS NOT NULL
AND EXISTS (
SELECT 1 FROM sessions p
WHERE p.id = sessions.parent_session_id
AND p.end_reason = 'compression'
)""",
(session_id,),
)
# Session-row creation is transcript-critical: if it fails, the
# first flush of a new session fails and the turn is aborted as
# session_persistence_failed. Ride out long sibling holds.
self._execute_write(_do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S)
def create_session(self, session_id: str, source: str, **kwargs) -> str:
"""Create a new session record. Returns the session_id."""
self._insert_session_row(session_id, source, **kwargs)
return session_id
def record_gateway_session_peer(
self,
session_id: str,
*,
source: str,
user_id: str = None,
session_key: str = None,
chat_id: str = None,
chat_type: str = None,
thread_id: str = None,
display_name: str = None,
origin_json: str = None,
include_compression_ancestors: bool = False,
) -> None:
"""Persist the gateway routing peer for an existing session row.
``display_name`` / ``origin_json`` carry the gateway's presentation
and full origin metadata (#9006) so consumers (mcp_serve, mirror,
channel directory) can read routing data from state.db instead of
sessions.json. They are COALESCE'd only in the sense that ``None``
leaves the existing value untouched.
``include_compression_ancestors`` keeps a logical compression lineage
on one routing peer when an explicit gateway resume moves its tip to a
different lane. Normal per-turn metadata refreshes update only the
supplied row.
Self-healing (#82616): when the target row does not exist yet — the
gateway's ``create_session`` write failed and was deferred, or a
crash landed between routing publication and row creation — this
recorder INSERTs the row with the full identity instead of silently
no-opping. Every per-turn peer refresh is therefore a repair
opportunity: a gateway session row can no longer be first-created by
an identity-less lazy writer (``update_token_counts`` /
``record_auxiliary_usage``) and stay unroutable forever.
"""
if not session_id or not session_key:
return
def _do(conn):
lineage_cte = ""
target_clause = "WHERE id = ?"
query_params = []
if include_compression_ancestors:
lineage_cte = """
WITH RECURSIVE compression_lineage(id) AS (
SELECT ?
UNION
SELECT parent.id
FROM compression_lineage lineage
JOIN sessions child ON child.id = lineage.id
JOIN sessions parent ON parent.id = child.parent_session_id
WHERE parent.end_reason = 'compression'
AND json_extract(
COALESCE(child.model_config, '{}'),
'$._branched_from'
) IS NULL
AND json_extract(
COALESCE(child.model_config, '{}'),
'$._delegate_from'
) IS NULL
AND COALESCE(child.source, '') != 'tool'
)
"""
target_clause = "WHERE id IN (SELECT id FROM compression_lineage)"
query_params.append(session_id)
query_params.extend(
(
session_key,
source,
user_id,
chat_id,
chat_type,
thread_id,
display_name,
origin_json,
)
)
if not include_compression_ancestors:
query_params.append(session_id)
conn.execute(
f"""{lineage_cte}
UPDATE sessions
SET session_key = ?, source = ?, user_id = ?, chat_id = ?,
chat_type = ?, thread_id = ?,
display_name = COALESCE(?, display_name),
origin_json = COALESCE(?, origin_json)
{target_clause}""",
query_params,
)
# Self-heal (#82616): the UPDATE is a silent no-op when the row
# is missing (create_session failed earlier, or a crash landed
# between routing publication and row creation). Insert it with
# the full identity so the session is durably routable — never
# leave first-creation to an identity-less lazy writer.
if not include_compression_ancestors:
cur = conn.execute(
"SELECT 1 FROM sessions WHERE id = ? LIMIT 1", (session_id,)
)
if cur.fetchone() is None:
conn.execute(
"""INSERT INTO sessions (
id, source, user_id, session_key, chat_id,
chat_type, thread_id, display_name, origin_json,
profile_name, started_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
session_key = COALESCE(sessions.session_key, excluded.session_key),
chat_id = COALESCE(sessions.chat_id, excluded.chat_id),
chat_type = COALESCE(sessions.chat_type, excluded.chat_type),
thread_id = COALESCE(sessions.thread_id, excluded.thread_id),
display_name = COALESCE(sessions.display_name, excluded.display_name),
origin_json = COALESCE(sessions.origin_json, excluded.origin_json)""",
(
session_id,
source,
user_id,
session_key,
chat_id,
chat_type,
thread_id,
display_name,
origin_json,
# Same ownership stamp as _insert_session_row: a
# self-healed row is a first creation too, and an
# unowned (NULL) row vanishes from profile-keyed
# consumers (#99222).
self._own_profile_name(),
time.time(),
),
)
self._execute_write(_do)
def set_expiry_finalized(self, session_id: str, finalized: bool = True) -> None:
"""Mark a gateway session's expiry-finalization flag in state.db.
Mirrors ``SessionEntry.expiry_finalized`` (sessions.json) so the flag
survives even if the JSON index is pruned or lost (#9006).
"""
if not session_id:
return
def _do(conn):
conn.execute(
"UPDATE sessions SET expiry_finalized = ? WHERE id = ?",
(1 if finalized else 0, session_id),
)
self._execute_write(_do)
# ── Gateway routing index (replaces sessions.json, #9006 follow-up) ────
def save_gateway_routing_entry(
self, session_key: str, entry_json: str, *, scope: str = ""
) -> None:
"""Upsert one gateway routing entry (session_key -> SessionEntry JSON).
The gateway_routing table is the durable replacement for
sessions.json: one row per routing key, holding the full serialized
``SessionEntry`` so the gateway can rehydrate exactly what it wrote.
``scope`` namespaces the index the way separate sessions.json files
did (one per sessions_dir) — callers pass their sessions_dir path so
two stores with different directories never share routing state.
"""
if not session_key or not entry_json:
return
def _do(conn):
conn.execute(
"""INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(scope, session_key) DO UPDATE SET
entry_json = excluded.entry_json,
updated_at = excluded.updated_at""",
(scope, session_key, entry_json, time.time()),
)
self._execute_write(_do)
def replace_gateway_routing_entries(
self, entries: Dict[str, str], *, scope: str = ""
) -> None:
"""Atomically replace the routing index for *scope* with *entries*.
Mirrors the sessions.json full-rewrite semantics: keys absent from
*entries* are removed (pruned/reset sessions disappear from the
index). Runs as a single write transaction. Other scopes are
untouched.
"""
now = time.time()
def _do(conn):
conn.execute("DELETE FROM gateway_routing WHERE scope = ?", (scope,))
if entries:
conn.executemany(
"INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at) "
"VALUES (?, ?, ?, ?)",
[(scope, k, v, now) for k, v in entries.items() if k and v],
)
self._execute_write(_do)
def load_gateway_routing_entries(self, *, scope: str = "") -> Dict[str, str]:
"""Load routing entries for *scope* as {session_key: entry_json}."""
with self._read_ctx() as conn:
rows = conn.execute(
"SELECT session_key, entry_json FROM gateway_routing WHERE scope = ?",
(scope,),
).fetchall()
return {r["session_key"]: r["entry_json"] for r in rows}
def delete_gateway_routing_entries(
self, session_keys: List[str], *, scope: str = ""
) -> None:
"""Remove routing entries for the given session keys in *scope*."""
if not session_keys:
return
def _do(conn):
conn.executemany(
"DELETE FROM gateway_routing WHERE scope = ? AND session_key = ?",
[(scope, k) for k in session_keys],
)
self._execute_write(_do)
def list_never_active_keyed_sessions(
self, *, older_than_days: float
) -> List[Dict[str, Any]]:
"""Keyed gateway rows that were opened and then never used at all.
Selects rows that are keyed (``session_key IS NOT NULL``), still open
(``ended_at IS NULL``) and carry no evidence of a single turn: no
messages, no tokens, no tool or API calls, no recorded activity, no
title. Such a row is indistinguishable from "never happened".
That is exactly the shape of a leaked test fixture (#82770) — and
also of a chat that was routed but never answered. Both are safe to
drop: there is no transcript to lose, and the gateway mints a fresh
session on the next inbound message either way.
``bulk prune``/``archive`` cannot reach these rows: their shared
selector is pinned to ``ended_at IS NOT NULL`` so that a live session
is never picked, which permanently excludes every never-closed row.
Hence a separate, narrower selector rather than another filter flag.
``pinned`` and ``archived`` rows are excluded — both are explicit
user intent to keep the row around.
"""
cutoff = time.time() - (float(older_than_days) * 86400.0)
with self._read_ctx() as conn:
rows = conn.execute(
"""
SELECT s.id, s.session_key, s.source, s.chat_id,
s.chat_type, s.user_id, s.started_at
FROM sessions s
WHERE s.session_key IS NOT NULL
AND s.ended_at IS NULL
AND s.title IS NULL
AND s.last_activity_at IS NULL
AND COALESCE(s.message_count, 0) = 0
AND COALESCE(s.tool_call_count, 0) = 0
AND COALESCE(s.api_call_count, 0) = 0
AND COALESCE(s.input_tokens, 0) = 0
AND COALESCE(s.output_tokens, 0) = 0
AND COALESCE(s.pinned, 0) = 0
AND COALESCE(s.archived, 0) = 0
AND s.started_at IS NOT NULL
AND s.started_at < ?
AND NOT EXISTS (
SELECT 1 FROM messages m WHERE m.session_id = s.id
)
ORDER BY s.started_at
""",
(cutoff,),
).fetchall()
return [dict(r) for r in rows]
def _delete_routing_entries_for_sessions(self, session_ids: Set[str]) -> int:
"""Drop ``gateway_routing`` rows pointing at any of *session_ids*.
Routing entries are keyed by ``(scope, session_key)`` and record their
target session inside ``entry_json``, so there is no way to reach them
by session id in SQL — the match is done in Python over all scopes.
"""
if not session_ids:
return 0
with self._read_ctx() as conn:
rows = conn.execute(
"SELECT scope, session_key, entry_json FROM gateway_routing"
).fetchall()
doomed: List[Tuple[str, str]] = []
for row in rows:
try:
entry = json.loads(row["entry_json"] or "{}")
except Exception:
continue
if isinstance(entry, dict) and entry.get("session_id") in session_ids:
doomed.append((row["scope"], row["session_key"]))
if not doomed:
return 0
def _do(conn):
conn.executemany(
"DELETE FROM gateway_routing WHERE scope = ? AND session_key = ?",
doomed,
)
self._execute_write(_do)
return len(doomed)
def prune_never_active_keyed_sessions(
self,
*,
older_than_days: float,
sessions_dir: Optional[Path] = None,
) -> Tuple[int, int]:
"""Delete never-active keyed rows and the routing entries naming them.
Returns ``(sessions_deleted, routing_entries_deleted)``.
The routing entries go first: a stale entry that outlived its target
would leave the gateway resuming a session id that no longer exists.
Deleting the pair is what leaving them both would have amounted to
anyway — the target had no transcript to resume.
Deletion goes through :meth:`delete_session` rather than a bulk
``DELETE`` so the delegate cascade, FTS bookkeeping and on-disk
transcript cleanup stay owned by one implementation.
"""
candidates = self.list_never_active_keyed_sessions(
older_than_days=older_than_days
)
if not candidates:
return (0, 0)
ids = {str(row["id"]) for row in candidates}
routing_deleted = self._delete_routing_entries_for_sessions(ids)
deleted = 0
for session_id in ids:
if self.delete_session(session_id, sessions_dir=sessions_dir):
deleted += 1
return (deleted, routing_deleted)
def list_gateway_sessions(
self,
*,
platform: Optional[str] = None,
active_only: bool = True,
) -> List[Dict[str, Any]]:
"""List gateway sessions (rows with a session_key) from state.db.
Returns the newest row per session_key — the same shape consumers got
from sessions.json: one live mapping per routing key. ``platform``
filters on ``source``; ``active_only`` restricts to sessions that
have not ended.
"""
# Full rows carry token/cost totals (MCP listings, /status) — drain
# queued async accounting deltas so consumers see exact counters.
self.flush_token_counts()
query = f"""
SELECT sessions.*,
COALESCE(sp.prompt, sessions.system_prompt)
AS _system_prompt_resolved,
{_sql_session_last_active("sessions")} AS last_active
FROM sessions
LEFT JOIN system_prompts sp
ON sp.hash = sessions.system_prompt_hash
WHERE session_key IS NOT NULL
AND started_at = (
SELECT MAX(s2.started_at) FROM sessions s2
WHERE s2.session_key = sessions.session_key
)
"""
params: list = []
if platform:
query += " AND LOWER(source) = LOWER(?)"
params.append(platform)
if active_only:
query += " AND ended_at IS NULL"
query += " ORDER BY last_active DESC"
with self._read_ctx() as conn:
rows = conn.execute(query, params).fetchall()
return [self._session_row_dict(r) for r in rows]
def find_session_by_origin(
self,
*,
platform: str,
chat_id: str,
thread_id: Optional[str] = None,
user_id: Optional[str] = None,
) -> Optional[str]:
"""Find the most recent live session_id for a platform + chat origin.
Equivalent of gateway/mirror's sessions.json scan: matches on
source + chat_id (+ thread_id when provided). When ``user_id`` is
provided, exact sender matches are preferred; if multiple distinct
users share the chat and none matches, returns None rather than
contaminating another participant's session.
"""
if not platform or chat_id in (None, ""):
return None
query = """
SELECT id, user_id, started_at FROM sessions
WHERE LOWER(source) = LOWER(?)
AND session_key IS NOT NULL
AND chat_id = ?
AND ended_at IS NULL
"""
params: list = [platform, str(chat_id)]
if thread_id is not None:
query += " AND COALESCE(thread_id, '') = ?"
params.append(str(thread_id))
query += " ORDER BY started_at DESC"
with self._read_ctx() as conn:
rows = [dict(r) for r in conn.execute(query, params).fetchall()]
if not rows:
return None
if user_id:
exact = [r for r in rows if str(r.get("user_id") or "") == str(user_id)]
if exact:
return str(exact[0]["id"])
if len(rows) > 1:
return None
elif len(rows) > 1:
distinct_users = {
str(r.get("user_id") or "").strip()
for r in rows
if str(r.get("user_id") or "").strip()
}
if len(distinct_users) > 1:
return None
return str(rows[0]["id"])
def find_latest_gateway_session_for_peer(
self,
*,
source: str,
user_id: Optional[str] = None,
session_key: Optional[str] = None,
chat_id: Optional[str] = None,
chat_type: Optional[str] = None,
thread_id: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
"""Find the latest recoverable gateway session for a routing peer.
``sessions.json`` is the fast routing index, but it can be missing or
pruned after process-level restart bugs. New gateway sessions persist
the deterministic ``session_key`` on the durable session row so the
mapping can be rebuilt exactly. Rows ended only by older gateway
cleanup's ``agent_close`` bug or a mistaken TUI ``ws_orphan_reap``
(dashboard viewer disconnect before #60609) are treated as recoverable;
explicit conversation boundaries such as /new, /resume switches, and
compression splits are not.
Ordering and emptiness (#82616): candidates are ranked by actual
conversation recency (``last_activity_at``, falling back to
``started_at``) — ``started_at`` alone resurrected days-old zombie
rows over the live conversation. Rows with messages are preferred,
but an empty keyed row is still returned rather than ``None``:
returning ``None`` mints a brand-new session id, which is a worse
outcome than resuming an empty-but-correctly-keyed row (and "empty"
may just mean the transcript lives under a compression child).
Reset boundaries fence recovery (#68539): an intentional boundary
such as ``session_reset`` (or any explicit non-recoverable
end_reason) must block fallback to an *older* row for the same
peer. Without the fence, the has-messages ranking above could reach
behind a /new reset and silently restore the exact context the user
reset. Each candidate is therefore rejected when a boundary row for
the peer ended *after* the candidate's last activity — if the
conversation's most recent event is an intentional reset, recovery
returns nothing rather than reaching behind it.
"""
if not session_key:
return None
with self._read_ctx() as conn:
row = conn.execute(
f"""
SELECT s.*,
COALESCE(sp.prompt, s.system_prompt)
AS _system_prompt_resolved,
(COALESCE(s.message_count, 0) > 0 OR EXISTS (
SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1
)) AS _has_messages
FROM sessions s
LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash
WHERE s.session_key = ?
AND s.source = ?
AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL}))
AND NOT EXISTS (
SELECT 1 FROM sessions b
WHERE b.session_key = s.session_key
AND b.source = s.source
AND b.ended_at IS NOT NULL
AND b.end_reason IN ({_RESET_END_REASONS_SQL})
AND b.ended_at
> COALESCE(s.last_activity_at, s.started_at)
)
ORDER BY _has_messages DESC,
COALESCE(s.last_activity_at, s.started_at) DESC
LIMIT 1
""",
(session_key, source),
).fetchone()
if row is not None:
return self._session_row_dict(row)
# Conservative fallback for rows created by current code but with a
# temporarily-missing exact key: still require the complete peer
# tuple so we never cross chats/threads/users.
if chat_id is None or chat_type is None:
return None
# Profile fence (#74285): a Telegram DM's peer tuple is identical
# for every bot (chat_id == user_id, no thread), so a sibling
# profile's row written into this store before the per-profile
# partition (legacy data) would otherwise be adopted here. Every
# profile-tree store has one owner; a row is ours when its
# profile_name is the owner or NULL (legacy rows this store
# minted). Stores outside the tree derive no owner and keep the
# historical unfenced behavior.
owner = self._own_profile_name()
row = conn.execute(
f"""
SELECT s.*,
COALESCE(sp.prompt, s.system_prompt)
AS _system_prompt_resolved,
(COALESCE(s.message_count, 0) > 0 OR EXISTS (
SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1
)) AS _has_messages
FROM sessions s
LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash
WHERE s.source = ?
AND COALESCE(s.user_id, '') = COALESCE(?, '')
AND COALESCE(s.chat_id, '') = COALESCE(?, '')
AND COALESCE(s.chat_type, '') = COALESCE(?, '')
AND COALESCE(s.thread_id, '') = COALESCE(?, '')
AND (? IS NULL OR COALESCE(s.profile_name, ?) = ?)
AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL}))
AND (COALESCE(s.message_count, 0) > 0 OR EXISTS (
SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1
))
AND NOT EXISTS (
SELECT 1 FROM sessions b
WHERE b.source = s.source
AND COALESCE(b.user_id, '') = COALESCE(s.user_id, '')
AND COALESCE(b.chat_id, '') = COALESCE(s.chat_id, '')
AND COALESCE(b.chat_type, '') = COALESCE(s.chat_type, '')
AND COALESCE(b.thread_id, '') = COALESCE(s.thread_id, '')
AND b.ended_at IS NOT NULL
AND b.end_reason IN ({_RESET_END_REASONS_SQL})
AND b.ended_at
> COALESCE(s.last_activity_at, s.started_at)
)
ORDER BY COALESCE(s.last_activity_at, s.started_at) DESC
LIMIT 1
""",
(source, user_id, chat_id, chat_type, thread_id, owner, owner, owner),
).fetchone()
return self._session_row_dict(row) if row else None
# ── Orphaned gateway-session repair (#82616) ──────────────────────────
# A write-path failure (corrupt FTS, crash between routing publication
# and row creation) can leave the live conversation in a session row
# that never received its identity columns. Both queries above require
# those columns, so the row holding the real transcript is invisible to
# recovery: the chat resolves to the last keyed row instead — days older
# — and the conversation time-travels. Hardening the write side cannot
# reach a row that is *already* damaged; these two methods are the
# offline repair path behind ``hermes sessions repair-routing``.
# Widest plausible gap between a keyed predecessor going quiet and its
# unkeyed successor being minted. The reported incident gap was ~60s;
# 15 minutes stays generous without spanning unrelated conversations.
_ORPHAN_ADOPTION_MAX_GAP_S = 900.0
def find_orphaned_gateway_sessions(
self, *, max_gap_s: Optional[float] = None
) -> List[Dict[str, Any]]:
"""Report message-bearing session rows that lost their routing identity.
A row is a candidate orphan when it has messages but no
``session_key``. It is only *adoptable* when exactly one keyed
predecessor can be named as the conversation it continues:
* ``lineage`` — ``parent_session_id`` points at a keyed row of the
same source. That is a recorded fact, so no time window applies.
* ``contiguity`` — exactly one keyed row of the same source (and
compatible ``user_id``) fell quiet within *max_gap_s* of the
orphan's start, and is older than the orphan's own last activity.
Anything ambiguous is reported with ``adoptable=False`` and a reason
rather than guessed at: mis-adopting would splice one person's
conversation into another person's chat. Branch/delegate/tool rows
are excluded outright — they are unkeyed by design, not by damage.
"""
gap = (
self._ORPHAN_ADOPTION_MAX_GAP_S
if max_gap_s is None
else float(max_gap_s)
)
orphan_active = _sql_session_last_active("o")
donor_active = _sql_session_last_active("d")
donor_columns = (
"d.id, d.session_key, d.chat_id, d.chat_type, d.thread_id, "
"d.user_id, d.origin_json, d.display_name, d.end_reason"
)
records: List[Dict[str, Any]] = []
with self._read_ctx() as conn:
orphans = conn.execute(
f"""
SELECT o.id, o.source, o.user_id, o.started_at,
o.parent_session_id,
{orphan_active} AS last_active,
(SELECT COUNT(*) FROM messages m
WHERE m.session_id = o.id) AS message_count
FROM sessions o
WHERE o.session_key IS NULL
AND EXISTS (SELECT 1 FROM messages m
WHERE m.session_id = o.id)
AND COALESCE(o.source, '') != 'tool'
AND json_extract(COALESCE(o.model_config, '{{}}'),
'$._branched_from') IS NULL
AND json_extract(COALESCE(o.model_config, '{{}}'),
'$._delegate_from') IS NULL
ORDER BY o.started_at ASC
"""
).fetchall()
for orphan in orphans:
donor = None
evidence = ""
reason = ""
if orphan["parent_session_id"]:
evidence = "lineage"
donor = conn.execute(
f"""
SELECT {donor_columns}
FROM sessions d
WHERE d.id = ?
AND d.session_key IS NOT NULL
AND COALESCE(d.source, '') = COALESCE(?, '')
""",
(orphan["parent_session_id"], orphan["source"]),
).fetchone()
if donor is None:
reason = (
"parent session carries no gateway identity of "
"this source"
)
else:
evidence = "contiguity"
candidates = conn.execute(
f"""
SELECT {donor_columns}, {donor_active} AS last_active
FROM sessions d
WHERE d.session_key IS NOT NULL
AND d.id != ?
AND COALESCE(d.source, '') = COALESCE(?, '')
AND (COALESCE(d.user_id, '') = ''
OR COALESCE(?, '') = ''
OR d.user_id = ?)
AND {donor_active} BETWEEN ? AND ?
AND {donor_active} < ?
ORDER BY last_active DESC
LIMIT 2
""",
(
orphan["id"],
orphan["source"],
orphan["user_id"],
orphan["user_id"],
(orphan["started_at"] or 0) - gap,
(orphan["started_at"] or 0) + gap,
orphan["last_active"],
),
).fetchall()
if not candidates:
reason = (
f"no keyed predecessor fell quiet within {gap:.0f}s "
"of this session's start"
)
elif len(candidates) > 1:
reason = (
"ambiguous: more than one keyed predecessor "
"matches this window"
)
else:
donor = candidates[0]
records.append(
{
"orphan_id": orphan["id"],
"source": orphan["source"],
"message_count": orphan["message_count"],
"started_at": orphan["started_at"],
"last_active": orphan["last_active"],
"donor_id": donor["id"] if donor else None,
"session_key": donor["session_key"] if donor else None,
"evidence": evidence if donor else "",
"adoptable": donor is not None,
"reason": reason,
}
)
# Two unkeyed successors claiming the same predecessor means at most
# one of them continues that chat, and nothing here says which.
contested = {
r["donor_id"]
for r in records
if r["adoptable"]
and sum(1 for x in records if x["donor_id"] == r["donor_id"]) > 1
}
for record in records:
if record["donor_id"] in contested:
record["adoptable"] = False
record["reason"] = (
"ambiguous: more than one unkeyed session claims this "
"predecessor"
)
return records
def adopt_orphaned_gateway_session(
self, orphan_id: str, donor_id: str
) -> bool:
"""Stamp *orphan_id* with *donor_id*'s routing identity, retire *donor_id*.
Re-verifies the pair inside the write transaction, so a concurrent
gateway that healed either row in the meantime turns this into a
no-op instead of a conflicting write. Existing non-NULL columns on
the orphan are preserved. Returns True when the adoption applied.
"""
if not orphan_id or not donor_id or orphan_id == donor_id:
return False
def _do(conn):
donor = conn.execute(
"SELECT session_key, chat_id, chat_type, thread_id, user_id, "
"origin_json, display_name, source FROM sessions WHERE id = ?",
(donor_id,),
).fetchone()
orphan = conn.execute(
"SELECT session_key, source FROM sessions WHERE id = ?",
(orphan_id,),
).fetchone()
if donor is None or orphan is None:
return False
if not donor["session_key"] or orphan["session_key"]:
return False
if (donor["source"] or "") != (orphan["source"] or ""):
return False
conn.execute(
"""UPDATE sessions
SET session_key = ?,
chat_id = COALESCE(chat_id, ?),
chat_type = COALESCE(chat_type, ?),
thread_id = COALESCE(thread_id, ?),
user_id = COALESCE(user_id, ?),
origin_json = COALESCE(origin_json, ?),
display_name = COALESCE(display_name, ?),
parent_session_id = COALESCE(parent_session_id, ?)
WHERE id = ? AND session_key IS NULL""",
(
donor["session_key"],
donor["chat_id"],
donor["chat_type"],
donor["thread_id"],
donor["user_id"],
donor["origin_json"],
donor["display_name"],
donor_id,
orphan_id,
),
)
# Retire the predecessor under a reason recovery does NOT treat
# as resumable — 'agent_close'/'ws_orphan_reap' would keep it in
# the running, and the newly keyed orphan could lose the chat
# again on the next restart.
conn.execute(
"UPDATE sessions SET ended_at = COALESCE(ended_at, ?), "
"end_reason = 'superseded_by_repair' WHERE id = ?",
(time.time(), donor_id),
)
return True
return self._execute_write(_do)
# Children that carry a ``parent_session_id`` but are NOT compression
# continuations: branches, delegate/subagent runs, and tool sessions.
# A marker only disqualifies a child when it points at the parent being
# queried — compression continuations inherit the rotated agent's
# ``model_config`` verbatim (``publish_compression_child`` callers pass
# ``agent._session_init_model_config``), so a delegate subagent's
# continuation carries ``_delegate_from=<the delegate's own parent>``.
# Matching markers by mere presence misclassified those real
# continuations as delegate children (fail-open for orphan reopen,
# fail-closed for adoption). Bind the parent id for both markers.
_NON_CONTINUATION_CHILD_FILTER_SQL = (
" AND COALESCE(json_extract(COALESCE({alias}model_config, '{{}}'),"
" '$._branched_from'), '') != ?\n"
" AND COALESCE(json_extract(COALESCE({alias}model_config, '{{}}'),"
" '$._delegate_from'), '') != ?\n"
" AND COALESCE({alias}source, '') != 'tool'\n"
)
def find_live_compression_child(
self, parent_session_id: str
) -> Optional[Dict[str, Any]]:
"""Return the unique live direct child of a compression-ended session.
A stale agent may observe that another compression path already rotated
its parent. Recovery is safe only when the durable lineage identifies
exactly one live direct continuation. Multiple children are treated as
ambiguous and fail closed rather than guessing which transcript owns
subsequent messages.
"""
if not parent_session_id:
return None
with self._read_ctx() as conn:
parent = conn.execute(
"SELECT ended_at, end_reason FROM sessions WHERE id = ?",
(parent_session_id,),
).fetchone()
if (
parent is None
or parent["ended_at"] is None
or parent["end_reason"] != "compression"
):
return None
rows = conn.execute(
"""
SELECT s.*,
COALESCE(sp.prompt, s.system_prompt)
AS _system_prompt_resolved
FROM sessions s
LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash
WHERE s.parent_session_id = ?
AND s.ended_at IS NULL
"""
+ self._NON_CONTINUATION_CHILD_FILTER_SQL.format(alias="s.")
+ """
ORDER BY s.started_at ASC
LIMIT 2
""",
(parent_session_id, parent_session_id, parent_session_id),
).fetchall()
return self._session_row_dict(rows[0]) if len(rows) == 1 else None
def reopen_orphaned_compression_session(self, session_id: str) -> bool:
"""Reopen a compression parent only when no continuation was published.
Compression publication is atomic in current builds, but older builds
could leave a closed parent behind after an interrupted handoff. This
recovery is deliberately conservative: an active compression lease or
any canonical child means the lineage is still owned by another path,
so the caller must fail closed instead of reopening the parent.
"""
if not session_id:
return False
def _do(conn):
parent = conn.execute(
"SELECT ended_at, end_reason FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if (
parent is None
or parent["ended_at"] is None
or parent["end_reason"] != "compression"
):
return False
# Treat any direct non-branch/non-delegate/non-tool child as a
# continuation, regardless of its current ended state. Reopening
# in that case could create a second live head for one lineage.
child = conn.execute(
"""
SELECT 1
FROM sessions
WHERE parent_session_id = ?
"""
+ self._NON_CONTINUATION_CHILD_FILTER_SQL.format(alias="")
+ """
LIMIT 1
""",
(session_id, session_id, session_id),
).fetchone()
if child is not None:
return False
# refresh_compression_lock() deliberately lets an owner revive its
# own expired row. Reclaim that row inside this write transaction
# before reopening: refresh-first makes the lease active and aborts
# recovery; recovery-first deletes the holder identity so a later
# refresh cannot resurrect it.
now = time.time()
lock_row = conn.execute(
"SELECT holder, expires_at FROM compression_locks "
"WHERE session_id = ?",
(session_id,),
).fetchone()
if lock_row is not None:
expires_at = lock_row["expires_at"]
if expires_at is None or float(expires_at) >= now:
return False
deleted = conn.execute(
"DELETE FROM compression_locks "
"WHERE session_id = ? AND holder = ? AND expires_at = ?",
(session_id, lock_row["holder"], expires_at),
)
if deleted.rowcount != 1:
return False
updated = conn.execute(
"UPDATE sessions SET ended_at = NULL, end_reason = NULL "
"WHERE id = ? AND ended_at IS NOT NULL "
"AND end_reason = 'compression'",
(session_id,),
)
# rowcount==1 is guaranteed by the parent SELECT at the top of
# this same BEGIN IMMEDIATE transaction. If this is ever edited
# to return False past this point, note that the lease DELETE
# above will still COMMIT (_execute_write commits unless _do
# raises) — raise instead of returning False to roll back.
return updated.rowcount == 1
return bool(self._execute_write(_do))
def publish_compression_child(
self,
*,
parent_session_id: str,
child_session_id: str,
source: str,
messages: List[Dict[str, Any]],
model: str = None,
model_config: Dict[str, Any] = None,
system_prompt: str = None,
cwd: str = None,
profile_name: str = None,
compression_lock_holder: str = None,
require_compression_lease: bool = True,
require_lease_refresh: bool = False,
lease_ttl_seconds: float = 300.0,
watermark: Optional[int] = None,
watermark_ceiling: Optional[int] = None,
) -> None:
"""Atomically close a parent and publish its durable compression child.
The parent closure, child row, and compacted handoff become visible in
one transaction. Readers can therefore observe either the live parent or
a complete child, never an ended parent with a missing/empty child.
Concurrent-append safety (#75316): when *watermark* is provided (the
parent's :meth:`get_active_message_watermark` captured at compression
start), parent rows that arrived during the slow summary call
(``id > watermark``) are cloned into the child AFTER the handoff —
same pure-SQL column clone as :meth:`archive_and_compact`, with the
session id rewritten — so a mid-compression append survives rotation
instead of stranding in the closed parent.
*watermark_ceiling* bounds the clone from above: the rotation path
flushes its OWN un-persisted input transcript to the parent right
before publishing (#47202), and those rows are already represented in
the compacted handoff — cloning them would duplicate the transcript.
The caller captures ``MAX(id)`` immediately BEFORE that flush; only
rows in ``(watermark, watermark_ceiling]`` are foreign concurrent
tail. ``None`` = unbounded (no internal flush happened).
When *require_lease_refresh* is True and *compression_lock_holder* is
set, the lease is refreshed inside the same transaction before the
expiry check. This gives a refresher
that stopped due to transient DB failures one final chance to extend
the lease, preventing wasted compression work. The refresh uses the
same ``conn`` as the publication, so there is no TOCTOU window.
"""
def _do(conn):
if require_lease_refresh and compression_lock_holder:
conn.execute(
"UPDATE compression_locks SET expires_at = ? "
"WHERE session_id = ? AND holder = ?",
(time.time() + lease_ttl_seconds, parent_session_id,
compression_lock_holder),
)
lock_row = conn.execute(
"SELECT holder, expires_at FROM compression_locks WHERE session_id = ?",
(parent_session_id,),
).fetchone()
if require_compression_lease and (
lock_row is None
or not compression_lock_holder
or lock_row["holder"] != compression_lock_holder
or float(lock_row["expires_at"]) <= time.time()
):
raise CompressionSessionBusyError(
f"Compression lease lost before publication: {parent_session_id}"
)
parent = conn.execute(
"""SELECT ended_at, end_reason, cwd, git_branch, git_repo_root,
user_id, session_key, chat_id, chat_type,
thread_id, display_name, origin_json, profile_name
FROM sessions WHERE id = ?""",
(parent_session_id,),
).fetchone()
if parent is None:
raise RuntimeError(f"Compression parent not found: {parent_session_id}")
if parent["ended_at"] is not None:
# A parent stamped ended by AUTOMATIC cleanup (tui_shutdown,
# ws_disconnect, orphan reap, idle/LRU evict) while a live
# agent is publishing its rotation is stale by construction —
# this writer holds the compression lease and is actively
# continuing the conversation the stamp claims is over.
# Left in place it wedges rotation forever: every attempt
# aborts here, nothing clears the stamp, and each attempt's
# pre-publish flush re-grows the parent until the provider
# rejects the request (#88197: 303 unique messages → 2,611
# rows → HTTP 400). Clear it in this same transaction and
# proceed; the closure UPDATE below re-stamps the parent with
# its true boundary (end_reason='compression'). Deliberate
# boundaries (compression, session_reset, explicit close)
# still fail closed — those mean another path owns lineage.
if is_automatic_end_reason(parent["end_reason"]):
conn.execute(
"UPDATE sessions SET ended_at = NULL, end_reason = NULL "
"WHERE id = ?",
(parent_session_id,),
)
else:
raise RuntimeError(
f"Compression parent already ended: {parent_session_id}"
)
if not messages:
raise RuntimeError("Compression child handoff must not be empty")
system_prompt_hash = self._store_system_prompt(conn, system_prompt)
conn.execute(
"""INSERT INTO sessions (
id, source, model, model_config, system_prompt,
system_prompt_hash,
parent_session_id, cwd, git_branch, git_repo_root,
profile_name, user_id, session_key, chat_id, chat_type,
thread_id, display_name, origin_json, started_at
) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
child_session_id,
source,
model,
json.dumps(model_config) if model_config else None,
system_prompt_hash,
parent_session_id,
cwd or parent["cwd"],
parent["git_branch"],
parent["git_repo_root"],
# Same inheritance contract as _insert_session_row's
# compression-fork backfill (#59527 / cross-profile jump
# fix): the child stays on the parent's profile and keeps
# the gateway routing/origin columns so peer recovery
# still works after a crash at the boundary. When neither
# names an owner (legacy NULL parent), stamp this store's
# own profile so the rotated child doesn't extend the
# unowned lineage (#99222).
profile_name
or parent["profile_name"]
or self._own_profile_name(),
parent["user_id"],
parent["session_key"],
parent["chat_id"],
parent["chat_type"],
parent["thread_id"],
parent["display_name"],
parent["origin_json"],
time.time(),
),
)
total_messages, total_tool_calls = self._insert_message_rows(
conn, child_session_id, messages
)
if watermark is not None:
# Clone the parent's concurrent tail (rows landed after the
# watermark, at or below the ceiling — see docstring) into the
# child, after the handoff. Column-exact except id/session_id;
# originals stay in the (closed) parent for lineage recovery.
_ceiling_clause = ""
_params: list = [parent_session_id, int(watermark)]
if watermark_ceiling is not None:
_ceiling_clause = " AND id <= ?"
_params.append(int(watermark_ceiling))
tail_rows = conn.execute(
"SELECT id, tool_calls FROM messages "
"WHERE session_id = ? AND active = 1 AND id > ?"
f"{_ceiling_clause} ORDER BY id",
_params,
).fetchall()
if tail_rows:
tail_ids = [int(r["id"]) for r in tail_rows]
placeholders = ",".join("?" for _ in tail_ids)
clone_cols = [
c for c in self._message_column_names(conn)
if c not in ("id", "session_id", "active", "compacted")
]
col_list = ", ".join(clone_cols)
conn.execute(
f"INSERT INTO messages ({col_list}, session_id, active, compacted) "
f"SELECT {col_list}, ?, 1, 0 FROM messages "
f"WHERE id IN ({placeholders}) ORDER BY id",
[child_session_id, *tail_ids],
)
total_messages += len(tail_ids)
for r in tail_rows:
raw = r["tool_calls"]
if raw:
try:
parsed = json.loads(raw) if isinstance(raw, str) else raw
total_tool_calls += len(parsed) if isinstance(parsed, list) else 0
except (TypeError, ValueError):
pass
conn.execute(
"UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?",
(total_messages, total_tool_calls, child_session_id),
)
updated = conn.execute(
"UPDATE sessions SET ended_at = ?, end_reason = 'compression' "
"WHERE id = ? AND ended_at IS NULL",
(time.time(), parent_session_id),
)
if updated.rowcount != 1:
raise RuntimeError(
f"Compression parent changed during publication: {parent_session_id}"
)
self._execute_write(_do)
def _bump_conversation_generation(self, conn, session_id: str, end_reason: str) -> None:
"""Advance this peer's conversation generation past a boundary.
Called inside the transaction that writes the boundary, so the
generation and the ``end_reason`` that caused it commit together.
Only ``_RESET_END_REASONS`` count: ``compression`` continues one
conversation, and an accidental close is not a replacement. Rows with
no ``session_key`` have no routing peer to advance.
The counter deliberately does NOT read the session rows. An aggregate
over them (COUNT/MAX of boundaries) can return a pair it already
emitted once ``delete_session()`` or bulk pruning removes an ended row,
which would hand a new conversation a retired affinity identity. This
value only ever increments, so a generation is never reused for a peer
even if every row behind it is gone.
"""
if end_reason not in _RESET_END_REASONS:
return
row = conn.execute(
"SELECT source, session_key FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
return
source = str(row["source"] or "").strip()
session_key = str(row["session_key"] or "").strip()
if not source or not session_key:
return
conn.execute(
"""
INSERT INTO conversation_generations (source, session_key, generation)
VALUES (?, ?, 1)
ON CONFLICT(source, session_key) DO UPDATE
SET generation = conversation_generations.generation + 1
""",
(source, session_key),
)
def end_session(self, session_id: str, end_reason: str) -> None:
"""Mark a session as ended.
No-ops when the session is already ended. The first end_reason wins:
compression-split sessions must keep their ``end_reason = 'compression'``
record even if a later stale ``end_session()`` call (e.g. from a
desynced CLI session_id after ``/resume`` or ``/branch``) targets them
with a different reason. Use ``reopen_session()`` first if you
intentionally need to re-end a closed session with a new reason.
"""
def _do(conn):
changed = conn.execute(
"UPDATE sessions SET ended_at = ?, end_reason = ? "
"WHERE id = ? AND ended_at IS NULL",
(time.time(), end_reason, session_id),
).rowcount
# Only a boundary this call actually wrote advances the generation:
# the first end_reason wins, so a no-op must not rotate the peer.
if changed:
self._bump_conversation_generation(conn, session_id, end_reason)
self._execute_write(_do)
def reopen_session(self, session_id: str) -> None:
"""Clear ended_at/end_reason so a session can be resumed.
Before clearing a reset boundary, stabilize markerless legacy reset
children that still depend on the parent's mutable end_reason.
"""
def _do(conn):
placeholders = ",".join("?" for _ in _RESET_END_REASONS)
# WHERE shape shared with _RESET_CHILD_SQL's fallback arm via
# _legacy_reset_child_sql so the stamping and the listing
# predicate cannot drift.
conn.execute(
"UPDATE sessions AS child SET model_config = json_set("
"COALESCE(child.model_config, '{}'), '$._reset_from', "
"child.parent_session_id) "
"WHERE child.parent_session_id = ? "
"AND json_extract(COALESCE(child.model_config, '{}'), "
" '$._reset_from') IS NULL "
f"AND {_legacy_reset_child_sql('child', placeholders)}",
(session_id, *_RESET_END_REASONS),
)
conn.execute(
"UPDATE sessions SET ended_at = NULL, end_reason = NULL WHERE id = ?",
(session_id,),
)
self._execute_write(_do)
def promote_to_session_reset(
self, session_id: str, reason: str = "session_reset"
) -> bool:
"""Durably mark a session as ended by an intentional reset boundary.
Promotes *only* live rows (``ended_at IS NULL``) or rows carrying an
accidental end_reason that the recovery query
(``find_latest_gateway_session_for_peer``) treats as recoverable:
``agent_close`` (older gateway cleanup bug) and ``ws_orphan_reap``
(mistaken TUI reaper). Explicit conversation boundaries such as
``compression``, ``session_reset``, ``session_switch``, etc. are
preserved — the first writer wins for those, and a later expiry
finalization must not silently overwrite them.
Plain ``end_session()`` is NOT sufficient for reset boundaries: it
no-ops on an already-ended row, so a row that agent cleanup already
closed as ``agent_close`` would stay recoverable and stale-route
recovery would resurrect the reset session with its full history
(#61220, #61993, #63539).
Keep this promotion set in sync with the recoverable set in
``find_latest_gateway_session_for_peer`` — any reason recovery would
reopen must be promotable here.
``reason`` lets reset paths keep their auditable specific reasons
(``idle``, ``daily``, ``suspended``, ``resume_pending_expired``).
Returns ``True`` when the row was promoted, ``False`` when skipped
(already has a different explicit end_reason, or row not found).
"""
if not session_id:
return False
now = time.time()
def _do(conn):
cursor = conn.execute(
"UPDATE sessions SET ended_at = ?, end_reason = ? "
"WHERE id = ? AND (ended_at IS NULL "
f"OR end_reason IN ({_RECOVERABLE_END_REASONS_SQL}))",
(now, reason, session_id),
)
# /new and the policy auto-resets promote rather than end_session,
# so the generation has to advance here too — in the same
# transaction as the boundary, and only when one was written.
if cursor.rowcount:
self._bump_conversation_generation(conn, session_id, reason)
return cursor.rowcount
try:
rows = self._execute_write(_do)
return bool(rows)
except Exception:
return False
def update_session_cwd(
self,
session_id: str,
cwd: str,
git_branch: Optional[str] = None,
git_repo_root: Optional[str] = None,
replace_git_meta: bool = False,
) -> Optional[int]:
"""Persist the authoritative cwd and claim a Git metadata generation.
``git_branch`` records the git branch checked out in ``cwd`` at the time
the session started/resumed. The sidebar groups main-checkout sessions
by this so feature-branch work doesn't pile under a single "main" row
(the main checkout's *current* branch is transient and would
misattribute past sessions).
``git_repo_root`` records the git repo this cwd belongs to — the
authoritative project key. Resolving it here, at the lowest level, means
every surface reads the same membership instead of re-probing git in the
GUI over a partial page. Each field is only written when non-empty so a
probe failure never clobbers a previously-captured value.
``replace_git_meta`` inverts that non-empty rule: a deliberate workspace
MOVE (re-homing a session into another project) must overwrite the old
repo identity even when the new cwd resolves to none — keeping the stale
root would leave the session grouped under the project it just left.
Every call increments ``git_metadata_generation`` in the same write
transaction. Async Git probes must publish through
:meth:`publish_session_git_metadata` with the returned generation, so
an older worker cannot overwrite a newer cwd claim even after an
A -> B -> A transition or from another process sharing this database.
Metadata from a different cwd is cleared atomically with the move.
"""
if not session_id or not cwd:
return None
branch = (git_branch or "").strip()
repo_root = (git_repo_root or "").strip()
def _do(conn):
current = conn.execute(
"SELECT cwd FROM sessions WHERE id = ?", (session_id,)
).fetchone()
if current is None:
return None
current_cwd = current["cwd"] if isinstance(current, sqlite3.Row) else current[0]
sets = [
"cwd = ?",
"git_metadata_generation = COALESCE(git_metadata_generation, 0) + 1",
]
params: List[Any] = [cwd]
if current_cwd != cwd or replace_git_meta:
sets.extend(("git_branch = ?", "git_repo_root = ?"))
params.extend((branch or None, repo_root or None))
elif branch:
sets.append("git_branch = ?")
params.append(branch)
if repo_root and current_cwd == cwd and not replace_git_meta:
sets.append("git_repo_root = ?")
params.append(repo_root)
params.append(session_id)
conn.execute(
f"UPDATE sessions SET {', '.join(sets)} WHERE id = ?", params
)
row = conn.execute(
"SELECT git_metadata_generation FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
return None
value = row["git_metadata_generation"] if isinstance(row, sqlite3.Row) else row[0]
return int(value)
return self._execute_write(_do)
def publish_session_git_metadata(
self,
session_id: str,
cwd: str,
generation: int,
git_branch: Optional[str] = None,
git_repo_root: Optional[str] = None,
) -> bool:
"""Publish async Git enrichment only while its cwd claim is current."""
if (
not session_id
or not cwd
or isinstance(generation, bool)
or not isinstance(generation, int)
or generation < 1
):
return False
branch = (git_branch or "").strip()
repo_root = (git_repo_root or "").strip()
if not branch and not repo_root:
return False
sets: List[str] = []
params: List[Any] = []
if branch:
sets.append("git_branch = ?")
params.append(branch)
if repo_root:
sets.append("git_repo_root = ?")
params.append(repo_root)
params.extend((session_id, cwd, generation))
def _do(conn):
cursor = conn.execute(
f"UPDATE sessions SET {', '.join(sets)} "
"WHERE id = ? AND cwd = ? "
"AND git_metadata_generation = ?",
params,
)
return cursor.rowcount == 1
return bool(self._execute_write(_do))
def backfill_repo_roots(self, cwd_to_root: Dict[str, str]) -> None:
"""Persist resolved git repo roots for cwds that don't have one yet.
Backfills history so projects light up for sessions created before the
column existed, without clobbering an already-recorded root. Only
non-empty roots are written (a non-git cwd stays NULL).
"""
pairs = [(root, cwd) for cwd, root in cwd_to_root.items() if root and cwd]
if not pairs:
return
def _do(conn):
for root, cwd in pairs:
conn.execute(
"UPDATE sessions SET git_repo_root = ? "
"WHERE cwd = ? AND COALESCE(git_repo_root, '') = ''",
(root, cwd),
)
self._execute_write(_do)
def record_compression_failure_cooldown(
self,
session_id: str,
cooldown_until: float,
error: Optional[str] = None,
) -> None:
"""Persist the active compression-failure cooldown for a session."""
if not session_id:
return
def _do(conn):
# Merge-max with any longer live deadline so a later shorter
# write cannot reopen the thrash window (#96775). The error
# column always takes the latest diagnostic.
conn.execute(
"UPDATE sessions SET compression_failure_cooldown_until = CASE "
"WHEN compression_failure_cooldown_until IS NOT NULL "
" AND compression_failure_cooldown_until > ? "
"THEN compression_failure_cooldown_until ELSE ? END, "
"compression_failure_error = ? WHERE id = ?",
(cooldown_until, cooldown_until, error, session_id),
)
try:
self._execute_write(_do)
except sqlite3.Error as exc:
logger.warning(
"record_compression_failure_cooldown(%s) failed: %s",
session_id, exc,
)
def get_compression_failure_cooldown(
self,
session_id: str,
) -> Optional[Dict[str, Any]]:
"""Return the active compression-failure cooldown for ``session_id``."""
if not session_id:
return None
now = time.time()
with self._read_ctx() as conn:
row = conn.execute(
"SELECT compression_failure_cooldown_until, compression_failure_error "
"FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
return None
cooldown_until = (
row["compression_failure_cooldown_until"]
if isinstance(row, sqlite3.Row)
else row[0]
)
if cooldown_until is None:
return None
cooldown_until = float(cooldown_until)
if cooldown_until <= now:
return None
error = (
row["compression_failure_error"]
if isinstance(row, sqlite3.Row)
else row[1]
)
return {
"cooldown_until": cooldown_until,
"remaining_seconds": cooldown_until - now,
"error": error,
}
def get_compression_failure_cooldown_row(
self,
session_id: str,
) -> Dict[str, Any]:
"""Return the exact stored cooldown columns without expiry filtering.
Compression cancellation uses this under its session lease so rollback
can preserve an expired row, a partially-null row, or an absent session
exactly instead of converting those states through the active-cooldown
API.
"""
if not session_id:
return {"session_exists": False, "cooldown_until": None, "error": None}
with self._read_ctx() as conn:
row = conn.execute(
"SELECT compression_failure_cooldown_until, compression_failure_error "
"FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
return {"session_exists": False, "cooldown_until": None, "error": None}
cooldown_until = (
row["compression_failure_cooldown_until"]
if isinstance(row, sqlite3.Row)
else row[0]
)
error = (
row["compression_failure_error"]
if isinstance(row, sqlite3.Row)
else row[1]
)
return {
"session_exists": True,
"cooldown_until": (
float(cooldown_until) if cooldown_until is not None else None
),
"error": error,
}
def restore_compression_failure_cooldown_row(
self,
session_id: str,
snapshot: Dict[str, Any],
) -> None:
"""Restore and verify an exact cooldown-row snapshot.
Unlike the ordinary record/clear helpers, this transactional rollback
API deliberately propagates write and verification failures. A caller
must not report cancellation as mutation-free when compensation failed.
"""
expected_exists = bool(snapshot.get("session_exists", False))
if not expected_exists:
actual = self.get_compression_failure_cooldown_row(session_id)
if actual.get("session_exists", False):
raise RuntimeError(
"cannot restore absent compression cooldown row: session now exists"
)
return
deadline = snapshot.get("cooldown_until")
error = snapshot.get("error")
def _do(conn):
cursor = conn.execute(
"UPDATE sessions SET compression_failure_cooldown_until = ?, "
"compression_failure_error = ? WHERE id = ?",
(deadline, error, session_id),
)
if cursor.rowcount != 1:
raise RuntimeError(
f"compression cooldown rollback session missing: {session_id}"
)
self._execute_write(_do)
actual = self.get_compression_failure_cooldown_row(session_id)
expected = {
"session_exists": True,
"cooldown_until": float(deadline) if deadline is not None else None,
"error": error,
}
if actual != expected:
raise RuntimeError(
f"compression cooldown rollback verification failed: "
f"expected={expected!r}, actual={actual!r}"
)
def clear_compression_failure_cooldown(self, session_id: str) -> None:
"""Clear any persisted compression-failure cooldown for a session."""
if not session_id:
return
def _do(conn):
conn.execute(
"UPDATE sessions SET compression_failure_cooldown_until = NULL, "
"compression_failure_error = NULL WHERE id = ?",
(session_id,),
)
try:
self._execute_write(_do)
except sqlite3.Error as exc:
logger.warning(
"clear_compression_failure_cooldown(%s) failed: %s",
session_id, exc,
)
def get_compression_fallback_streak(self, session_id: str) -> int:
"""Return the persisted deterministic-fallback streak."""
if not session_id:
return 0
with self._read_ctx() as conn:
if conn is None:
return 0
row = conn.execute(
"SELECT compression_fallback_streak FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
return 0
value = (
row["compression_fallback_streak"]
if isinstance(row, sqlite3.Row)
else row[0]
)
try:
return max(0, int(value or 0))
except (TypeError, ValueError):
return 0
def set_compression_fallback_streak(self, session_id: str, streak: int) -> None:
"""Persist the deterministic-fallback streak for one session."""
if not session_id:
return
normalized = max(0, int(streak))
def _do(conn):
conn.execute(
"UPDATE sessions SET compression_fallback_streak = ? WHERE id = ?",
(normalized, session_id),
)
self._execute_write(_do)
def increment_hygiene_failure_streak(self, session_key: str) -> int:
"""Atomically increment the session-hygiene failure streak for one chat."""
if not session_key:
return 1
result = []
def _do(conn):
conn.execute(
"""INSERT INTO gateway_hygiene_state (session_key, failure_streak)
VALUES (?, 1)
ON CONFLICT(session_key) DO UPDATE SET
failure_streak = gateway_hygiene_state.failure_streak + 1""",
(session_key,),
)
row = conn.execute(
"SELECT failure_streak FROM gateway_hygiene_state WHERE session_key = ?",
(session_key,),
).fetchone()
result.append(int(row[0]))
self._execute_write(_do)
return result[0]
def reset_hygiene_failure_streak(self, session_key: str) -> None:
"""Clear the persisted session-hygiene failure streak for one chat."""
if not session_key:
return
def _do(conn):
conn.execute(
"DELETE FROM gateway_hygiene_state WHERE session_key = ?",
(session_key,),
)
self._execute_write(_do)
def get_compression_ineffective_count(self, session_id: str) -> int:
"""Return the persisted ineffective-compaction strike count.
Mirrors ``get_compression_fallback_streak``: this is the durable half
of the anti-thrash guard (``_ineffective_compression_count`` on the
built-in compressor), persisted so that a fresh compressor bound to a
resumed session inherits an armed/tripped guard instead of starting
from zero across process restarts (#54923).
"""
if not session_id:
return 0
with self._read_ctx() as conn:
if conn is None:
return 0
row = conn.execute(
"SELECT compression_ineffective_count FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
return 0
value = (
row["compression_ineffective_count"]
if isinstance(row, sqlite3.Row)
else row[0]
)
try:
return max(0, int(value or 0))
except (TypeError, ValueError):
return 0
def set_compression_ineffective_count(self, session_id: str, count: int) -> None:
"""Persist the ineffective-compaction strike count for one session."""
if not session_id:
return
normalized = max(0, int(count))
def _do(conn):
conn.execute(
"UPDATE sessions SET compression_ineffective_count = ? WHERE id = ?",
(normalized, session_id),
)
self._execute_write(_do)
def get_compression_recovery_deadline(self, session_id: str) -> float:
"""Return the persisted anti-thrash recovery deadline (wall-clock epoch).
``0.0`` means "not armed". The deadline is the durable half of the
#14694 recovery clock: the gateway rebuilds the compressor on every
turn / cache eviction, so a process-local deadline restarted the
wait on each rebuild and a tripped session never earned its probe
(#100185).
"""
if not session_id:
return 0.0
with self._read_ctx() as conn:
if conn is None:
return 0.0
row = conn.execute(
"SELECT compression_recovery_deadline FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
return 0.0
value = (
row["compression_recovery_deadline"]
if isinstance(row, sqlite3.Row)
else row[0]
)
try:
return max(0.0, float(value or 0.0))
except (TypeError, ValueError):
return 0.0
def set_compression_recovery_deadline(self, session_id: str, deadline: float) -> None:
"""Persist the anti-thrash recovery deadline; ``0`` / ``None`` disarms it."""
if not session_id:
return
try:
normalized = max(0.0, float(deadline or 0.0))
except (TypeError, ValueError):
normalized = 0.0
stored = normalized if normalized > 0.0 else None
def _do(conn):
conn.execute(
"UPDATE sessions SET compression_recovery_deadline = ? WHERE id = ?",
(stored, session_id),
)
self._execute_write(_do)
# ──────────────────────────────────────────────────────────────────────
# Compression locks
# ──────────────────────────────────────────────────────────────────────
# Atomic per-session locks that prevent two compression paths from
# racing on the same session_id and producing orphan child sessions.
#
# The race: ``conversation_compression.py`` rotates ``agent.session_id``
# as a side effect of a successful compression (end old session, create
# new). That mutation is local to the AIAgent instance — but ``state.db``
# is shared across all instances. Two AIAgents that share the same
# ``session_id`` at the moment they both decide to compress (most
# commonly the parent turn's agent + a background-review fork started
# right after the turn ended) each end the parent and create their own
# NEW session, parented to the same old id. The gateway SessionEntry
# only catches one rotation; the other child silently accumulates
# writes — Damien's "parent → two orphan children" repro shape.
#
# The lock is keyed by ``session_id`` and is held for the duration of
# the compress() call plus the rotation. ``holder`` identifies the
# current owner (pid:tid:nonce) for diagnostics; the lock is recovered
# via ``expires_at`` if the holder process crashed without releasing.
def refresh_compression_lock(
self,
session_id: str,
holder: str,
ttl_seconds: float = 300.0,
) -> bool:
"""Extend the compression lock lease if ``holder`` still owns it.
Ownership is decided by the ``holder`` column alone, deliberately NOT
by ``expires_at``: a live owner whose refresher thread was starved
(GC pause, loaded CI runner, a slow write escaping ``_execute_write``'s
retry budget) past its own TTL must be able to revive its still-unclaimed
row on the next tick. Requiring ``expires_at >= now`` here made such a
stall permanent — every later refresh matched 0 rows, so the owner kept
compressing and rotating with no lease at all, which is exactly the
unprotected window a competing path can fork the session lineage in.
This does not resurrect a lock somebody else already took: SQLite
serialises writes, so a reclaim (DELETE-expired + INSERT-or-IGNORE in
:meth:`try_acquire_compression_lock`) and this UPDATE never interleave.
Reclaim-first replaces ``holder``, so this UPDATE matches nothing and
returns False; refresh-first pushes ``expires_at`` into the future, so
the reclaimer's DELETE-expired matches nothing and its acquire fails.
"""
if not session_id or not holder:
return False
now = time.time()
expires_at = now + ttl_seconds
def _do(conn):
cur = conn.execute(
"UPDATE compression_locks SET expires_at = ? "
"WHERE session_id = ? AND holder = ?",
(expires_at, session_id, holder),
)
return cur.rowcount > 0
try:
return bool(self._execute_write(_do))
except sqlite3.Error as exc:
logger.warning(
"refresh_compression_lock(%s) failed: %s",
session_id, exc,
)
return False
def try_acquire_compression_lock(
self,
session_id: str,
holder: str,
ttl_seconds: float = 300.0,
) -> bool:
"""Try to atomically acquire the compression lock for ``session_id``.
Returns ``True`` on success (caller now owns the lock and must
release via :meth:`release_compression_lock`). Returns ``False``
if another holder already owns a non-expired lock — the caller
MUST NOT proceed with compression in that case (its rotation would
race against the holder's, splitting the session lineage).
Expired locks (``expires_at < now``) are reclaimed transparently.
Structured holders whose local ``pid=`` no longer exists are reclaimed
immediately, so a gateway killed during compression does not stall the
replacement process for the full lease TTL.
Implementation: single-transaction DELETE-expired + INSERT-or-IGNORE,
followed by a SELECT to confirm we got the row. SQLite serialises
writes, so the whole sequence is atomic against other writers.
"""
if not session_id:
return False
now = time.time()
expires_at = now + ttl_seconds
def _do(conn):
reclaimed_holder = None
row = conn.execute(
"SELECT holder, expires_at FROM compression_locks "
"WHERE session_id = ?",
(session_id,),
).fetchone()
if row is not None:
current_holder = (
row["holder"] if isinstance(row, sqlite3.Row) else row[0]
)
current_expires_at = (
row["expires_at"] if isinstance(row, sqlite3.Row) else row[1]
)
if (
current_expires_at < now
or _compression_lock_holder_process_is_dead(current_holder)
):
conn.execute(
"DELETE FROM compression_locks "
"WHERE session_id = ? AND holder = ?",
(session_id, current_holder),
)
reclaimed_holder = current_holder
# Then: try to insert. INSERT OR IGNORE returns no rowcount
# difference — verify ownership via SELECT.
conn.execute(
"INSERT OR IGNORE INTO compression_locks "
"(session_id, holder, acquired_at, expires_at) "
"VALUES (?, ?, ?, ?)",
(session_id, holder, now, expires_at),
)
row = conn.execute(
"SELECT holder FROM compression_locks WHERE session_id = ?",
(session_id,),
).fetchone()
acquired = row is not None and (
row["holder"] if isinstance(row, sqlite3.Row) else row[0]
) == holder
return acquired, reclaimed_holder
try:
acquired, reclaimed_holder = self._execute_write(_do)
if reclaimed_holder:
logger.warning(
"Reclaimed stale compression lock for session=%s "
"(holder=%s)",
session_id,
reclaimed_holder,
)
return bool(acquired)
except sqlite3.Error as exc:
logger.warning(
"try_acquire_compression_lock(%s) failed: %s",
session_id, exc,
)
# Fail open: returning False makes the caller skip compression,
# which is the safe behaviour when the lock subsystem is broken.
return False
def release_compression_lock(self, session_id: str, holder: str) -> None:
"""Release the compression lock for ``session_id`` iff we own it.
Idempotent: no-op when the lock has already expired and been
reclaimed by a different holder, or when no lock exists. The
``holder`` check prevents a late-returning compressor from
clobbering a fresh lock held by someone else.
"""
if not session_id:
return
def _do(conn):
conn.execute(
"DELETE FROM compression_locks "
"WHERE session_id = ? AND holder = ?",
(session_id, holder),
)
try:
self._execute_write(_do)
except sqlite3.Error as exc:
logger.warning(
"release_compression_lock(%s) failed: %s",
session_id, exc,
)
def _session_turn_lease_key_on_conn(self, conn, session_id: str) -> str:
"""Walk compression parents on ``conn`` to the conversation lease key.
Must run on the same connection as the lease INSERT/UPDATE/DELETE.
A prior ``get_session`` failure must not compute a child id that the
later write then persists: refresh would walk to the parent and
fail-close. Markers bind to ``parent_session_id`` (same contract as
``_NON_CONTINUATION_CHILD_FILTER_SQL``). Lock errors propagate so
``_execute_write`` / ``acquire_session_turn_lease`` can retry.
"""
if not session_id:
return session_id
def _row(sid: str):
row = conn.execute(
"SELECT id, parent_session_id, source, model_config, end_reason "
"FROM sessions WHERE id = ?",
(sid,),
).fetchone()
return dict(row) if row else None
current = _row(session_id)
seen = {session_id}
while current:
parent_id = current.get("parent_session_id")
if (
not parent_id
or parent_id in seen
or self._is_explicit_fork_child_row(current)
):
break
parent = _row(parent_id)
if not parent or parent.get("end_reason") != "compression":
break
seen.add(parent_id)
current = parent
return str(current.get("id") or session_id) if current else session_id
def _session_turn_lease_key(self, session_id: str) -> str:
"""Return the stable serialization key for every compression segment.
Acquire/refresh/release resolve this inside their write transaction.
This helper is for tests and diagnostics; it does not swallow lock
errors (a swallowed walk plus a later successful write was the
fail-open that replayed the post-rotation refresh miss).
"""
if not session_id:
return session_id
with self._read_ctx() as conn:
return self._session_turn_lease_key_on_conn(conn, session_id)
def try_acquire_session_turn_lease(
self,
session_id: str,
holder: str,
*,
ttl_seconds: float = 300.0,
patience_s: Optional[float] = None,
) -> bool:
"""Atomically acquire the cross-process turn lease for a conversation.
Compression rotates a session into child segments, so the durable key
is the lineage root rather than the current segment id. The walk and
INSERT share one write transaction. Expired leases and leases whose
structured local holder PID is known dead are reclaimed in that same
transaction.
"""
if not session_id or not holder:
return False
now = time.time()
expires_at = now + max(0.1, float(ttl_seconds))
def _do(conn):
conversation_id = self._session_turn_lease_key_on_conn(conn, session_id)
row = conn.execute(
"SELECT holder, expires_at FROM session_turn_leases "
"WHERE conversation_id = ?",
(conversation_id,),
).fetchone()
if row is not None:
current_holder = row["holder"]
if (
float(row["expires_at"]) <= now
or _compression_lock_holder_process_is_dead(current_holder)
):
conn.execute(
"DELETE FROM session_turn_leases "
"WHERE conversation_id = ? AND holder = ?",
(conversation_id, current_holder),
)
conn.execute(
"INSERT OR IGNORE INTO session_turn_leases "
"(conversation_id, holder, acquired_at, expires_at) "
"VALUES (?, ?, ?, ?)",
(conversation_id, holder, now, expires_at),
)
owner = conn.execute(
"SELECT holder FROM session_turn_leases WHERE conversation_id = ?",
(conversation_id,),
).fetchone()
return owner is not None and owner["holder"] == holder
return bool(self._execute_write(_do, patience_s=patience_s))
def acquire_session_turn_lease(
self,
session_id: str,
holder: str,
*,
ttl_seconds: float = 300.0,
wait_seconds: float = 1800.0,
poll_interval_seconds: float = 1.0,
on_wait=None,
wait_notice_interval_seconds: float = 15.0,
should_abort=None,
acquire_patience_s: float = 0.5,
) -> bool:
"""Wait for a cross-process turn lease without holding a SQLite lock.
``on_wait(elapsed_seconds)`` is best-effort: invoked when the first
attempt fails (elapsed ~0) and again about every
``wait_notice_interval_seconds`` while still waiting, so UIs can show
that another process holds the conversation.
When ``should_abort()`` returns True (for example the agent received
``/stop`` while waiting), acquisition stops immediately and returns
False without consuming the full ``wait_seconds`` budget.
"""
deadline = time.monotonic() + max(0.0, float(wait_seconds))
wait_started = None
last_notice_at = None
notice_every = max(0.0, float(wait_notice_interval_seconds))
while True:
if should_abort is not None:
try:
if should_abort():
return False
except Exception:
logger.debug(
"session turn lease should_abort callback failed",
exc_info=True,
)
try:
if self.try_acquire_session_turn_lease(
session_id,
holder,
ttl_seconds=ttl_seconds,
patience_s=acquire_patience_s,
):
return True
except sqlite3.Error as exc:
# Long holder transactions (compression publish, large
# flushes) can exhaust a single write-patience budget.
# Keep polling until wait_seconds or should_abort.
if classify_persistence_error(exc) != "locked":
raise
now = time.monotonic()
remaining = deadline - now
if remaining <= 0:
return False
if wait_started is None:
wait_started = now
if on_wait is not None and (
last_notice_at is None
or notice_every == 0.0
or (now - last_notice_at) >= notice_every
):
try:
on_wait(max(0.0, now - wait_started))
except Exception:
logger.debug(
"session turn lease on_wait callback failed",
exc_info=True,
)
last_notice_at = now
time.sleep(min(max(0.01, float(poll_interval_seconds)), remaining))
def refresh_session_turn_lease(
self,
session_id: str,
holder: str,
*,
ttl_seconds: float = 300.0,
) -> bool:
"""Extend a turn lease only while ``holder`` still owns it."""
if not session_id or not holder:
return False
expires_at = time.time() + max(0.1, float(ttl_seconds))
def _do(conn):
conversation_id = self._session_turn_lease_key_on_conn(conn, session_id)
cursor = conn.execute(
"UPDATE session_turn_leases SET expires_at = ? "
"WHERE conversation_id = ? AND holder = ?",
(expires_at, conversation_id, holder),
)
return cursor.rowcount > 0
return bool(self._execute_write(_do))
def release_session_turn_lease(self, session_id: str, holder: str) -> None:
"""Release a turn lease iff ``holder`` still owns it; idempotent."""
if not session_id or not holder:
return
def _do(conn):
conversation_id = self._session_turn_lease_key_on_conn(conn, session_id)
conn.execute(
"DELETE FROM session_turn_leases "
"WHERE conversation_id = ? AND holder = ?",
(conversation_id, holder),
)
self._execute_write(_do)
def get_compression_lock_holder(self, session_id: str) -> Optional[str]:
"""Return the current (non-expired) holder for ``session_id``, or None.
Diagnostic helper — not used by the locking protocol itself.
"""
if not session_id:
return None
now = time.time()
with self._read_ctx() as conn:
row = conn.execute(
"SELECT holder FROM compression_locks "
"WHERE session_id = ? AND expires_at >= ?",
(session_id, now),
).fetchone()
if row is None:
return None
return row["holder"] if isinstance(row, sqlite3.Row) else row[0]
def touch_session_activity(
self,
session_id: str,
ts: Optional[float] = None,
*,
description: Optional[str] = None,
provenance: Optional[ActivityProvenance] = None,
) -> None:
"""Stamp durable mid-turn session activity (observation-only).
Called (rate-limited) from ``AIAgent._touch_activity`` so gateway/CLI
surfaces and stall consumers observe API/tool/compaction activity
even when no new message row has been written yet (#72016 / #72039).
Never moves ``last_activity_at`` backwards. When the timestamp
advances, bounded ``last_activity_description`` /
``last_activity_provenance`` are written with it. No-ops when
``session_id`` is empty or the row does not exist.
"""
if not session_id:
return
from agent.session_activity import (
bound_activity_description,
normalize_activity_provenance,
)
when = float(ts if ts is not None else time.time())
desc = bound_activity_description(description)
prov = normalize_activity_provenance(provenance).value
def _do(conn):
conn.execute(
"UPDATE sessions SET "
"last_activity_at = ?, "
"last_activity_description = ?, "
"last_activity_provenance = ? "
"WHERE id = ? AND (last_activity_at IS NULL OR last_activity_at < ?)",
(when, desc, prov, session_id, when),
)
# Observation-only write: never let it ride the full routine
# write-patience budget (#76354 review S1). Under contention a
# heartbeat that waits ~20s would delay the response-critical path
# it is merely observing; give up after a sub-second budget instead
# (the next due window retries naturally).
self._execute_write(_do, patience_s=self._ACTIVITY_WRITE_PATIENCE_S)
def clear_session_activity_labels(self, session_id: str) -> None:
"""Clear mid-turn activity labels after a turn ends.
Keeps ``last_activity_at`` intact so idle / watchdog clocks stay
continuous. Description and provenance are observation labels for
*what was happening at* that timestamp during an active turn; once
the turn is idle they must not keep advertising "compressing" /
"executing tool" (#72039).
Response-critical-path contract (#76354 review S1): runs in the
turn's ``finally``; a no-op clear (labels already empty) skips the
write transaction entirely, and a real clear uses the same short
sub-second busy budget as :meth:`touch_session_activity` instead of
the full routine write patience.
"""
if not session_id:
return
from agent.session_activity import ActivityProvenance
# No-op fast path: skip the transaction when there is nothing to
# clear. Read-only, no write lock.
try:
with self._read_ctx() as conn:
row = conn.execute(
"SELECT last_activity_description, last_activity_provenance "
"FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
except sqlite3.Error:
row = None
if row is not None:
desc = row[0] if not isinstance(row, sqlite3.Row) else row["last_activity_description"]
prov = row[1] if not isinstance(row, sqlite3.Row) else row["last_activity_provenance"]
if not desc and (
not prov or prov == ActivityProvenance.UNKNOWN.value
):
return
def _do(conn):
conn.execute(
"UPDATE sessions SET "
"last_activity_description = ?, "
"last_activity_provenance = ? "
"WHERE id = ?",
("", ActivityProvenance.UNKNOWN.value, session_id),
)
self._execute_write(_do, patience_s=self._ACTIVITY_WRITE_PATIENCE_S)
def get_session_activity(self, session_id: str) -> Optional[Dict[str, Any]]:
"""Return the durable activity snapshot for *session_id*, or None."""
if not session_id:
return None
row = self.get_session(session_id)
if not row:
return None
from agent.session_activity import build_activity_snapshot
return build_activity_snapshot(
last_activity_at=row.get("last_activity_at"),
last_activity_description=row.get("last_activity_description"),
last_activity_provenance=row.get("last_activity_provenance"),
)
def update_session_meta(
self,
session_id: str,
model_config_json: str,
model: Optional[str] = None,
) -> None:
"""Update model_config and optionally model for an existing session.
Uses COALESCE so that passing model=None leaves the stored model
column unchanged. Routes through _execute_write for the standard
BEGIN IMMEDIATE + jitter-retry + lock guarantee.
"""
# Barrier against queued token deltas — see update_session_model.
self.flush_token_counts()
def _do(conn):
conn.execute(
"UPDATE sessions SET model_config = ?, model = COALESCE(?, model) WHERE id = ?",
(model_config_json, model, session_id),
)
self._execute_write(_do)
def update_system_prompt(
self, session_id: str, system_prompt: Optional[str]
) -> None:
"""Store the full assembled system prompt snapshot."""
def _do(conn):
system_prompt_hash = self._store_system_prompt(conn, system_prompt)
conn.execute(
"UPDATE sessions "
"SET system_prompt_hash = ?, system_prompt = NULL WHERE id = ?",
(system_prompt_hash, session_id),
)
self._delete_unreferenced_system_prompts(conn)
self._execute_write(_do)
def update_session_tool_names(
self, session_id: str, tool_names: Optional[List[str]]
) -> None:
"""Persist the session's resolved ``tools[]`` name order (JSON array).
Read back by ``tools.mcp_tool.restore_agent_tool_prefix`` when a fresh
``AIAgent`` is rebuilt for an existing session (gateway agent-cache
eviction) so a flipped ``check_fn`` verdict can't fork the cached tool
prefix. ``None`` clears the pin.
"""
payload = json.dumps(list(tool_names)) if tool_names is not None else None
def _do(conn):
conn.execute(
"UPDATE sessions SET tool_names = ? WHERE id = ?",
(payload, session_id),
)
self._execute_write(_do)
def update_session_model(
self, session_id: str, model: str, provider: Optional[str] = None
) -> None:
"""Update the model for a session after a mid-session switch.
Unlike ``update_token_counts`` which uses ``COALESCE(model, ?)``
(only filling in NULL), this unconditionally sets the model column
so that the dashboard reflects the user's latest /model choice.
Also nulls ``system_prompt`` so stale ``Model:`` / ``Provider:``
footer metadata is rebuilt on the next turn. A successful /model
switch explicitly replaces any confirmed Browser runtime lock while
preserving unrelated lineage markers in ``model_config``.
When *provider* is given, it is merged into ``model_config``
alongside the model (``$.model`` / ``$.provider``) so a later
resume recombines the persisted model with the provider that
actually serves it instead of the config.yaml primary provider
(#79536). Callers without provider knowledge leave any stored
provider untouched.
"""
# This write bypasses the token queue, so deltas enqueued before the
# switch must land first: a still-queued first delta carries the
# pre-switch route, and applying it after this UPDATE would trip the
# first_accounted_route overwrite in update_token_counts (row sees
# api_call_count == 0 + a route mismatch) and resurrect the old
# model/provider. Flushing here restores the pre-queue ordering.
self.flush_token_counts()
def _do(conn):
# Use the shared merge discipline so lineage markers like
# _branched_from / _delegate_from survive. browser_model_lock
# is deleted via a None patch value (same semantics as the
# old json_remove).
patch: Dict[str, Any] = {"browser_model_lock": None}
if model:
patch["model"] = model
if provider:
patch["provider"] = provider
merged = self._merge_model_config_json(conn, session_id, patch)
if merged is _MODEL_CONFIG_ROW_MISSING:
return
conn.execute(
"UPDATE sessions SET "
"model = ?, model_config = ?, "
"system_prompt = NULL, system_prompt_hash = NULL "
"WHERE id = ?",
(model, merged, session_id),
)
self._delete_unreferenced_system_prompts(conn)
self._execute_write(_do)
def _merge_model_config_json(
self,
conn,
session_id: str,
patch: Dict[str, Any],
*,
on_missing: str = "skip",
):
"""SELECT + tolerant-parse + merge ``patch`` into a session's model_config.
Shared by every model_config writer (``update_session_runtime_lock``,
``set_session_yolo``, ``archive_and_compact``,
``patch_session_model_config``) so the merge discipline that keeps
lineage markers like ``_branched_from`` / ``_delegate_from`` alive
lives in exactly one place. A ``None`` patch value deletes that key.
Must run inside an open write transaction (callers own the UPDATE).
Returns the serialized merged JSON — ``None`` when the merged dict is
empty (matching ``create_session``'s NULL convention) — or the
``_MODEL_CONFIG_ROW_MISSING`` sentinel when the row doesn't exist and
``on_missing == "skip"``; ``on_missing == "raise"`` raises ValueError.
"""
row = conn.execute(
"SELECT model_config FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
if on_missing == "raise":
raise ValueError(f"Session not found: {session_id}")
return _MODEL_CONFIG_ROW_MISSING
raw = row["model_config"] if isinstance(row, sqlite3.Row) else row[0]
config: Dict[str, Any] = {}
if isinstance(raw, str) and raw.strip():
try:
parsed = json.loads(raw)
if isinstance(parsed, dict):
config = parsed
except (json.JSONDecodeError, TypeError):
config = {}
elif isinstance(raw, dict):
config = dict(raw)
for key, value in patch.items():
if value is None:
config.pop(key, None)
else:
config[key] = value
return json.dumps(config) if config else None
def patch_session_model_config(
self, session_id: str, patch: Dict[str, Any]
) -> None:
"""Merge ``patch`` into a session's model_config JSON atomically.
A ``None`` patch value removes that key. No-op when the session row
doesn't exist or the patch is empty. This is the standalone setter for
callers that need to update model_config *without* rewriting the
transcript (the transcript-coupled path is ``archive_and_compact``'s
``model_config_patch``, which shares the same merge helper).
"""
if not session_id or not patch:
return
def _do(conn):
merged = self._merge_model_config_json(conn, session_id, patch)
if merged is _MODEL_CONFIG_ROW_MISSING:
return
conn.execute(
"UPDATE sessions SET model_config = ? WHERE id = ?",
(merged, session_id),
)
self._execute_write(_do)
def get_session_model_config_value(
self, session_id: str, key: str, default: Any = None
) -> Any:
"""Read one key out of a session's model_config JSON (tolerant parse)."""
session = self.get_session(session_id) or {}
raw = session.get("model_config")
config: Dict[str, Any] = {}
if isinstance(raw, str) and raw.strip():
try:
parsed = json.loads(raw)
if isinstance(parsed, dict):
config = parsed
except (json.JSONDecodeError, TypeError):
config = {}
elif isinstance(raw, dict):
config = raw
return config.get(key, default)
def update_session_runtime_lock(
self,
session_id: str,
*,
model: Optional[str] = None,
provider: Optional[str] = None,
model_options: Optional[Dict[str, Any]] = None,
route_source: Optional[str] = None,
confirmed: bool = False,
) -> None:
"""Persist a Browser / API client runtime lock without clobbering lineage markers.
Merges ``browser_model_lock`` into the existing ``model_config`` JSON so
``_branched_from`` / ``_delegate_from`` survive. Nulls ``system_prompt``
so cached ``Model:`` / ``Provider:`` footers cannot lie after a switch.
"""
lock = {
"provider": provider or "",
"model": model or "",
"model_options": model_options or {},
"route_source": route_source or "",
"confirmed": bool(confirmed),
"updated_at": time.time(),
}
def _do(conn):
merged = self._merge_model_config_json(
conn, session_id, {"browser_model_lock": lock}
)
if merged is _MODEL_CONFIG_ROW_MISSING:
return
conn.execute(
"""UPDATE sessions SET
model_config = ?,
model = COALESCE(?, model),
system_prompt = NULL,
system_prompt_hash = NULL
WHERE id = ?""",
(merged, model, session_id),
)
self._delete_unreferenced_system_prompts(conn)
self._execute_write(_do)
def set_session_yolo(self, session_id: str, enabled: bool) -> None:
"""Persist the per-session YOLO bypass flag into ``model_config``.
Merges ``yolo_mode`` into the existing ``model_config`` JSON (same
merge discipline as ``update_session_runtime_lock`` so lineage
markers like ``_branched_from`` / ``_delegate_from`` survive). The
CLI resume paths read this flag back so a ``/yolo ON`` toggle — or a
``--yolo`` launch — survives ``hermes --resume`` into a fresh
process. No-op when the session row doesn't exist yet; the
creation-time ``model_config`` carries the flag for ``--yolo``
launches.
"""
if not session_id:
return
def _do(conn):
merged = self._merge_model_config_json(
conn, session_id, {"yolo_mode": bool(enabled)}
)
if merged is _MODEL_CONFIG_ROW_MISSING:
return
conn.execute(
"UPDATE sessions SET model_config = ? WHERE id = ?",
(merged, session_id),
)
self._execute_write(_do)
@staticmethod
def session_yolo_enabled(session_meta: Optional[Dict[str, Any]]) -> bool:
"""Read the persisted YOLO flag off a session row dict.
Accepts the dict returned by ``get_session`` (``model_config`` is a
JSON string) or an already-parsed dict. Returns False on any parse
failure — resume must never enable the bypass by accident.
"""
raw = (session_meta or {}).get("model_config")
if isinstance(raw, str):
try:
raw = json.loads(raw)
except Exception:
return False
if not isinstance(raw, dict):
return False
return bool(raw.get("yolo_mode"))
@staticmethod
def session_gateway_runtime(session_meta: Optional[Dict[str, Any]]) -> Dict[str, Any]:
"""Read the persisted runtime route off a session row dict.
Accepts the dict returned by ``get_session`` (``model_config`` is a
JSON string) or an already-parsed dict. Prefers the nested
``gateway_runtime`` key (written by the gateway's
``_sync_session_model_from_agent`` and the CLI ``/model`` persist),
falling back to the top-level ``provider``/``base_url``/``api_mode``
keys the TUI gateway's ``_runtime_model_config`` writes. As a last
resort, falls back to the ``billing_provider`` column (written on
every session's first accounted API call) so sessions that never ran
``/model`` still restore the provider that actually served them.
Returns an empty dict on any parse failure — resume falls back to
ambient config resolution.
"""
raw = (session_meta or {}).get("model_config")
if isinstance(raw, str):
try:
raw = json.loads(raw)
except Exception:
raw = {}
if not isinstance(raw, dict):
raw = {}
runtime = raw.get("gateway_runtime")
if isinstance(runtime, dict) and runtime.get("provider"):
# Filter None values: the persist path writes or-None to trigger
# deletion in the top-level merge, but gateway_runtime is replaced
# as a whole dict (not deep-merged), so None values survive here.
return {k: v for k, v in runtime.items() if v is not None}
top_level = {
key: raw.get(key)
for key in ("provider", "base_url", "api_mode")
if raw.get(key)
}
if top_level:
return top_level
# Last resort: billing_provider column. Written via COALESCE on every
# session's first accounted API call — the only durable record for
# sessions that never ran /model. Mirrors the TUI gateway's
# _stored_session_runtime_overrides fallback. Bare billing buckets
# ("auto"/"custom") are not routable identities — filter them out so
# resume falls back to the ambient config default instead.
billing_provider = str(
(session_meta or {}).get("billing_provider") or ""
).strip()
if (
billing_provider
and billing_provider.lower() not in _BARE_BILLING_PROVIDERS
):
return {"provider": billing_provider}
return {k: v for k, v in (runtime or {}).items() if v is not None} if isinstance(runtime, dict) else {}
def update_session_billing_route(
self,
session_id: str,
*,
provider: str,
base_url: str,
billing_mode: Optional[str] = None,
) -> None:
"""Unconditionally update the billing provider/base_url for a session.
Unlike ``update_token_counts`` which uses ``COALESCE(billing_provider, ?)``
(only filling in NULL), this unconditionally sets the billing fields so
that the dashboard reflects the user's latest /model switch.
Also nulls ``system_prompt`` so the cached snapshot (which embeds a
stale ``Model:`` / ``Provider:`` header) is rebuilt — matching the
behavior of ``update_session_model`` (see #48173, #48248).
"""
# Barrier against queued token deltas — see update_session_model.
self.flush_token_counts()
def _do(conn):
conn.execute(
"""UPDATE sessions SET
billing_provider = ?,
billing_base_url = ?,
billing_mode = COALESCE(?, billing_mode),
system_prompt = NULL,
system_prompt_hash = NULL
WHERE id = ?""",
(provider, base_url, billing_mode, session_id),
)
self._delete_unreferenced_system_prompts(conn)
self._execute_write(_do)
# ── Async token accounting ──
# update_token_counts() runs a sessions UPDATE (plus a per-model usage
# upsert) inside BEGIN IMMEDIATE; against a cold multi-GB state.db one
# call can stall the turn thread for tens to hundreds of ms, and the
# tool loop pays it after EVERY API call (measured p50 3.3ms / p95 70ms
# per call in production). queue_token_counts() reduces the critical
# path to a deque append: a dedicated single-writer thread applies
# deltas in enqueue order, coalescing consecutive same-route deltas
# into one UPDATE when a backlog forms. Readers that need exact
# mid-turn totals (get_session and friends) call flush_token_counts()
# first — a plain attribute check when nothing is queued.
# Delta fields summed when coalescing. Route fields must be equal for
# two deltas to merge: model/billing_* feed COALESCE backfill and the
# per-model usage attribution key, and cost_status/cost_source are
# last-non-None-wins — equality makes the merged UPDATE byte-for-byte
# equivalent to applying the deltas sequentially.
_TOKEN_DELTA_SUM_FIELDS = (
"input_tokens", "output_tokens", "cache_read_tokens",
"cache_write_tokens", "reasoning_tokens", "api_call_count",
)
_TOKEN_DELTA_COST_FIELDS = ("estimated_cost_usd", "actual_cost_usd")
_TOKEN_DELTA_ROUTE_FIELDS = (
"model", "cost_status", "cost_source", "pricing_version",
"billing_provider", "billing_base_url", "billing_mode",
)
def queue_token_counts(self, session_id: str, **kwargs) -> None:
"""Enqueue a token/cost delta for the background writer.
Accepts the same keyword arguments as :meth:`update_token_counts`
and applies them asynchronously with identical semantics. Cheap
(append + notify) — safe to call on the turn thread after every
API call. After close() has stopped the writer, falls back to the
synchronous path and may raise like :meth:`update_token_counts`.
"""
with self._token_queue_cond:
thread = self._token_writer_thread
writer_stopped = self._token_writer_stop and (
thread is None or not thread.is_alive()
)
if not writer_stopped:
self._token_queue.append((session_id, kwargs))
if thread is None or not thread.is_alive():
# Daemon so process exit never hangs on accounting; the
# atexit hook drains anything still queued at interpreter
# shutdown (registered once per instance, on first use).
# ``not is_alive()`` (rather than ``is None`` only)
# respawns the writer if it ever died from an unexpected
# escape — otherwise a dead thread object would block
# respawn forever and deltas would pile up on the deque
# until a reader's flush drained them synchronously.
thread = threading.Thread(
target=self._token_writer_loop,
name="session-db-token-writer",
daemon=True,
)
self._token_writer_thread = thread
thread.start()
if self._token_atexit_hook is None:
self_ref = weakref.ref(self)
def _drain_at_exit() -> None:
db = self_ref()
if db is not None:
db._drain_token_queue_at_exit()
self._token_atexit_hook = _drain_at_exit
atexit.register(_drain_at_exit)
self._token_queue_cond.notify_all()
if writer_stopped:
# Writer permanently stopped (close() ran; a stop-flagged but
# still-live writer keeps accepting — its loop drains before
# exiting). Enqueueing now would drop the delta silently: no
# writer will run and close() already unregistered the atexit
# hook. Apply inline instead so a closed-connection failure
# raises at the call site, exactly like the old synchronous
# update_token_counts path these call sites still guard for.
self.update_token_counts(session_id, **kwargs)
def flush_token_counts(self, timeout: float = 5.0) -> bool:
"""Block until every queued token delta has been applied.
Returns True when the queue is fully drained, False on timeout
(callers then read totals that are stale by the still-queued
deltas — no worse than reading before the flush existed).
Never raises: apply failures are logged by the writer.
"""
# Fast path — nothing queued, nothing in flight.
if not self._token_queue and not self._token_writer_busy:
return True
batch = None
with self._token_queue_cond:
deadline = time.monotonic() + timeout
while self._token_queue or self._token_writer_busy:
# A live writer is authoritative even when stop-flagged
# (close() in progress): its loop drains the queue before
# exiting, and draining here instead would race its
# in-flight batch — newer deltas committing before older
# ones breaks the last-non-None-wins / first-accounted-
# route / COALESCE-backfill fields. Only when the writer is
# dead (or never started for these deltas) does the caller
# take the leftovers. Re-checked each wakeup: the writer
# can exit mid-wait with deltas enqueued after its final
# empty-queue check. busy is claimed while draining (same
# protocol as the writer) so a concurrent flush cannot
# report drained — or pop a newer delta — while this batch
# is still unapplied; a claimed busy therefore also means
# "wait", never "drain alongside".
thread = self._token_writer_thread
if (
(thread is None or not thread.is_alive())
and not self._token_writer_busy
):
self._token_writer_busy = True
batch = list(self._token_queue)
self._token_queue.clear()
break
remaining = deadline - time.monotonic()
if remaining <= 0:
return False
self._token_queue_cond.wait(remaining)
if batch:
try:
self._apply_token_batch(batch)
finally:
with self._token_queue_cond:
self._token_writer_busy = False
self._token_queue_cond.notify_all()
return True
def _token_writer_loop(self) -> None:
while True:
with self._token_queue_cond:
idle_deadline = time.monotonic() + self._TOKEN_WRITER_IDLE_SECONDS
while not self._token_queue and not self._token_writer_stop:
remaining = idle_deadline - time.monotonic()
if remaining <= 0:
# Publish retirement under the same lock used by
# queue_token_counts() to decide whether to spawn. An
# enqueue cannot strand a delta behind an exiting worker.
self._token_writer_thread = None
return
self._token_queue_cond.wait(remaining)
if not self._token_queue:
self._token_writer_thread = None
return # stop requested and fully drained
# busy is set BEFORE the queue is cleared: the lock-free
# fast path in flush_token_counts() reads queue-then-busy,
# so this order guarantees it can never observe an empty
# queue while the popped batch is still unapplied.
self._token_writer_busy = True
batch = list(self._token_queue)
self._token_queue.clear()
try:
self._apply_token_batch(batch)
finally:
with self._token_queue_cond:
self._token_writer_busy = False
self._token_queue_cond.notify_all()
def _apply_token_batch(self, batch: List[Tuple[str, Dict[str, Any]]]) -> None:
"""Apply queued deltas in order, coalescing where safe. Never raises."""
try:
coalesced = self._coalesce_token_deltas(batch)
except Exception as exc:
# Coalescing must never kill the writer thread (a dead writer
# can't be observed by callers). Fall back to applying the raw
# batch delta-by-delta — the merge is an optimization only.
logger.warning(
"async token accounting: coalesce failed, applying raw "
"batch: %s", exc,
)
coalesced = batch
for session_id, kwargs in coalesced:
try:
self.update_token_counts(session_id, **kwargs)
except Exception as exc:
# Same contract as the old inline call sites: accounting
# loss is logged, never raised into a turn.
logger.warning(
"async token accounting: apply failed (session=%s): %s",
session_id, exc,
)
def _coalesce_token_deltas(
self, batch: List[Tuple[str, Dict[str, Any]]]
) -> List[Tuple[str, Dict[str, Any]]]:
"""Merge consecutive incremental deltas with an identical route.
Only adjacent deltas merge, so ordering across sessions and across
a mid-session /model switch is preserved exactly. absolute=True
deltas (cumulative overwrites) never merge.
"""
groups: List[Tuple[Optional[tuple], str, Dict[str, Any]]] = []
for session_id, kwargs in batch:
key = None
if not kwargs.get("absolute"):
key = (session_id,) + tuple(
kwargs.get(f) for f in self._TOKEN_DELTA_ROUTE_FIELDS
)
if groups and key is not None and groups[-1][0] == key:
merged = groups[-1][2]
for f in self._TOKEN_DELTA_SUM_FIELDS:
merged[f] = merged.get(f, 0) + kwargs.get(f, 0)
for f in self._TOKEN_DELTA_COST_FIELDS:
value = kwargs.get(f)
if value is not None:
# None-preserving sum: an all-None run must stay
# None so COALESCE keeps the stored value untouched.
merged[f] = (merged.get(f) or 0.0) + value
else:
groups.append((key, session_id, dict(kwargs)))
return [(sid, kw) for _, sid, kw in groups]
def _stop_token_writer(self, join_timeout: float = 10.0) -> None:
"""Stop the writer thread and drain remaining deltas. Never raises."""
with self._token_queue_cond:
self._token_writer_stop = True
self._token_queue_cond.notify_all()
thread = self._token_writer_thread
if thread is not None and thread.is_alive():
thread.join(timeout=join_timeout)
if thread.is_alive():
# Writer stuck mid-apply (pathological lock contention).
# Leave any queued deltas unapplied rather than racing the
# stuck apply and misordering/double-counting.
logger.warning(
"async token accounting: writer did not stop within %.0fs; "
"%d queued delta(s) not persisted",
join_timeout, len(self._token_queue),
)
return
# Writer exited (or never started) — apply leftovers synchronously.
# Claim busy like the writer/flush drains do, so a concurrent
# flush_token_counts cannot fast-path True while this batch is
# still being applied; conversely, wait out a flush caller-drain
# that already claimed busy — close() nulls the connection right
# after this returns, and must not yank it mid-batch.
with self._token_queue_cond:
deadline = time.monotonic() + join_timeout
while self._token_writer_busy:
remaining = deadline - time.monotonic()
if remaining <= 0:
logger.warning(
"async token accounting: concurrent drain did not "
"finish within %.0fs; %d queued delta(s) not persisted",
join_timeout, len(self._token_queue),
)
return
self._token_queue_cond.wait(remaining)
# busy is claimed BEFORE the queue is cleared — same ordering
# as the writer loop and the flush caller-drain. The lock-free
# fast path in flush_token_counts() reads queue-then-busy
# without the cond, so clearing first would let a concurrent
# flush observe "empty and idle" and return True while this
# popped batch is still unapplied.
batch = list(self._token_queue)
if batch:
self._token_writer_busy = True
self._token_queue.clear()
if batch:
try:
self._apply_token_batch(batch)
finally:
with self._token_queue_cond:
self._token_writer_busy = False
self._token_queue_cond.notify_all()
def _drain_token_queue_at_exit(self) -> None:
try:
self._stop_token_writer()
except Exception:
pass # Best effort — never fatal at interpreter shutdown.
def update_token_counts(
self,
session_id: str,
input_tokens: int = 0,
output_tokens: int = 0,
model: str = None,
cache_read_tokens: int = 0,
cache_write_tokens: int = 0,
reasoning_tokens: int = 0,
estimated_cost_usd: Optional[float] = None,
actual_cost_usd: Optional[float] = None,
cost_status: Optional[str] = None,
cost_source: Optional[str] = None,
pricing_version: Optional[str] = None,
billing_provider: Optional[str] = None,
billing_base_url: Optional[str] = None,
billing_mode: Optional[str] = None,
api_call_count: int = 0,
absolute: bool = False,
) -> None:
"""Update token counters and backfill model if not already set.
When *absolute* is False (default), values are **incremented** — use
this for per-API-call deltas (CLI path).
When *absolute* is True, values are **set directly** — use this when
the caller already holds cumulative totals (gateway path, where the
cached agent accumulates across messages).
"""
# Ensure the session row exists so the UPDATE doesn't silently affect
# 0 rows. Under concurrent load (cron + kanban + delegate_task) the
# initial create_session() may have failed due to SQLite locking.
# INSERT OR IGNORE is cheap and idempotent.
self._insert_session_row(session_id, "unknown", model=model)
if absolute:
sql = """UPDATE sessions SET
input_tokens = ?,
output_tokens = ?,
cache_read_tokens = ?,
cache_write_tokens = ?,
reasoning_tokens = ?,
estimated_cost_usd = COALESCE(?, 0),
actual_cost_usd = CASE
WHEN ? IS NULL THEN actual_cost_usd
ELSE ?
END,
cost_status = COALESCE(?, cost_status),
cost_source = COALESCE(?, cost_source),
pricing_version = COALESCE(?, pricing_version),
billing_provider = COALESCE(billing_provider, ?),
billing_base_url = COALESCE(billing_base_url, ?),
billing_mode = COALESCE(billing_mode, ?),
model = COALESCE(model, ?),
api_call_count = ?
WHERE id = ?"""
else:
sql = """UPDATE sessions SET
input_tokens = input_tokens + ?,
output_tokens = output_tokens + ?,
cache_read_tokens = cache_read_tokens + ?,
cache_write_tokens = cache_write_tokens + ?,
reasoning_tokens = reasoning_tokens + ?,
estimated_cost_usd = COALESCE(estimated_cost_usd, 0) + COALESCE(?, 0),
actual_cost_usd = CASE
WHEN ? IS NULL THEN actual_cost_usd
ELSE COALESCE(actual_cost_usd, 0) + ?
END,
cost_status = COALESCE(?, cost_status),
cost_source = COALESCE(?, cost_source),
pricing_version = COALESCE(?, pricing_version),
billing_provider = COALESCE(billing_provider, ?),
billing_base_url = COALESCE(billing_base_url, ?),
billing_mode = COALESCE(billing_mode, ?),
model = COALESCE(model, ?),
api_call_count = COALESCE(api_call_count, 0) + ?
WHERE id = ?"""
has_accounted_usage = bool(
input_tokens or output_tokens or cache_read_tokens
or cache_write_tokens or reasoning_tokens or api_call_count
or estimated_cost_usd or actual_cost_usd
)
params = (
input_tokens,
output_tokens,
cache_read_tokens,
cache_write_tokens,
reasoning_tokens,
estimated_cost_usd,
actual_cost_usd,
actual_cost_usd,
cost_status,
cost_source,
pricing_version,
billing_provider if has_accounted_usage else None,
billing_base_url if has_accounted_usage else None,
billing_mode if has_accounted_usage else None,
model if has_accounted_usage else None,
api_call_count,
session_id,
)
# Per-model usage attribution. ``update_token_counts`` is the single
# chokepoint every per-API-call delta flows through (CLI, gateway, cron,
# delegated runs — see conversation_loop / codex_runtime), and each call
# carries the model/provider *active at the time of that call*. The
# ``sessions`` row only keeps one (model, billing_provider) pair, so a
# mid-session ``/model`` switch otherwise attributes every token to the
# initial model (issue #51607). Recording the per-call delta into
# session_model_usage keyed by the live model preserves an accurate
# per-model breakdown regardless of how many times the user switches.
#
# Only the incremental path records here. Absolute cumulative updates
# cannot be split back into routes; Insights reconciles any positive
# residual against the aggregate session row instead.
record_model_usage = (not absolute) and (
input_tokens or output_tokens or cache_read_tokens
or cache_write_tokens or reasoning_tokens or api_call_count
or estimated_cost_usd
)
def _do(conn):
row = conn.execute(
"SELECT model, billing_provider, api_call_count FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
existing_model = row["model"] if row is not None else None
existing_provider = row["billing_provider"] if row is not None else None
existing_api_calls = int((row["api_call_count"] if row is not None else 0) or 0)
# Session creation records the requested primary route before any API
# call. If it fails and fallback succeeds, the first accounted usage
# event is the first authoritative route. After that, preserve the
# legacy row: one row cannot represent mixed-provider usage.
first_accounted_route = (
existing_api_calls == 0
and has_accounted_usage
and bool(model)
and bool(billing_provider)
and (existing_model != model or existing_provider != billing_provider)
)
if first_accounted_route:
conn.execute(
"""UPDATE sessions
SET model = ?, billing_provider = ?,
billing_base_url = ?, billing_mode = ?
WHERE id = ?""",
(model, billing_provider, billing_base_url, billing_mode, session_id),
)
conn.execute(sql, params)
if record_model_usage:
self._record_model_usage(
conn,
session_id,
model=model,
billing_provider=billing_provider,
billing_base_url=billing_base_url,
billing_mode=billing_mode,
input_tokens=input_tokens,
output_tokens=output_tokens,
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
reasoning_tokens=reasoning_tokens,
estimated_cost_usd=estimated_cost_usd,
actual_cost_usd=actual_cost_usd,
cost_status=cost_status,
cost_source=cost_source,
api_call_count=api_call_count,
)
self._execute_write(_do)
def _record_model_usage(
self,
conn,
session_id: str,
*,
model: Optional[str],
billing_provider: Optional[str],
billing_base_url: Optional[str],
billing_mode: Optional[str],
input_tokens: int,
output_tokens: int,
cache_read_tokens: int,
cache_write_tokens: int,
reasoning_tokens: int,
estimated_cost_usd: Optional[float],
actual_cost_usd: Optional[float],
cost_status: Optional[str],
cost_source: Optional[str],
api_call_count: int,
task: str = "",
) -> None:
"""Accumulate a per-API-call usage delta into session_model_usage.
Runs inside the caller's write transaction (after the ``sessions``
UPDATE) so the per-model rows stay consistent with the summary row.
When the caller omits the model/provider (some paths only pass token
deltas), fall back to the values already recorded on the session row —
the same COALESCE-from-session behaviour the summary update uses.
``task`` distinguishes what kind of work consumed the tokens:
``''`` (empty) is the main agent loop; auxiliary calls record their
task name (``vision``, ``compression``, ``title_generation``, ...)
via :meth:`record_auxiliary_usage` (issue #23270).
"""
row = conn.execute(
"SELECT model, billing_provider, billing_base_url, billing_mode "
"FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
sess_model = row["model"] if row is not None else None
sess_provider = row["billing_provider"] if row is not None else None
sess_base_url = row["billing_base_url"] if row is not None else None
sess_billing_mode = row["billing_mode"] if row is not None else None
# Aux-task rows (task != '') must NOT inherit the session's main-loop
# route: an aux call may use a completely different provider/model
# (vision on gemini while the main loop runs anthropic). Missing info
# stays 'unknown'/empty rather than borrowing a misleading route.
if task:
eff_model = model or "unknown"
eff_provider = billing_provider or ""
eff_base_url = billing_base_url or ""
eff_billing_mode = billing_mode or ""
else:
eff_model = model or sess_model or "unknown"
eff_provider = billing_provider or sess_provider or ""
eff_base_url = billing_base_url or sess_base_url or ""
eff_billing_mode = billing_mode or sess_billing_mode or ""
now = time.time()
conn.execute(
"""INSERT INTO session_model_usage (
session_id, model, billing_provider, billing_base_url, billing_mode,
task, api_call_count, input_tokens, output_tokens,
cache_read_tokens, cache_write_tokens, reasoning_tokens,
estimated_cost_usd, actual_cost_usd, cost_status, cost_source,
first_seen, last_seen
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(session_id, model, billing_provider, billing_base_url, billing_mode, task)
DO UPDATE SET
api_call_count = api_call_count + excluded.api_call_count,
input_tokens = input_tokens + excluded.input_tokens,
output_tokens = output_tokens + excluded.output_tokens,
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
reasoning_tokens = reasoning_tokens + excluded.reasoning_tokens,
estimated_cost_usd = estimated_cost_usd + excluded.estimated_cost_usd,
actual_cost_usd = actual_cost_usd + excluded.actual_cost_usd,
cost_status = COALESCE(excluded.cost_status, cost_status),
cost_source = COALESCE(excluded.cost_source, cost_source),
last_seen = excluded.last_seen""",
(
session_id,
eff_model,
eff_provider,
eff_base_url,
eff_billing_mode,
task or "",
api_call_count or 0,
input_tokens or 0,
output_tokens or 0,
cache_read_tokens or 0,
cache_write_tokens or 0,
reasoning_tokens or 0,
float(estimated_cost_usd or 0.0),
float(actual_cost_usd or 0.0),
cost_status,
cost_source,
now,
now,
),
)
def ensure_session(
self,
session_id: str,
source: str = "unknown",
model: str = None,
**kwargs,
) -> str:
"""Ensure a session row exists (INSERT OR IGNORE). Accepts optional kwargs."""
self._insert_session_row(session_id, source, model=model, **kwargs)
return session_id
def record_auxiliary_usage(
self,
session_id: str,
task: str,
*,
model: Optional[str] = None,
billing_provider: Optional[str] = None,
billing_base_url: Optional[str] = None,
input_tokens: int = 0,
output_tokens: int = 0,
cache_read_tokens: int = 0,
cache_write_tokens: int = 0,
reasoning_tokens: int = 0,
estimated_cost_usd: Optional[float] = None,
api_call_count: int = 1,
) -> None:
"""Record an auxiliary LLM call's usage against *session_id* (issue #23270).
Auxiliary calls (vision, compression, title_generation, web_extract,
session_search, ...) historically discarded their usage, leaving the
dashboard's per-model analytics blind to aux model spend. This writes
a per-(model, provider, task) delta into ``session_model_usage`` —
the same table the main loop's ``update_token_counts`` feeds — WITHOUT
touching the ``sessions`` summary row. That separation is deliberate:
the gateway overwrites session counters with absolute main-loop totals,
so folding aux tokens into the summary row would either be clobbered
or double-counted. Insights/analytics read the union of both.
``api_call_count`` defaults to 1 (one aux LLM call). Background-review
forks record an aggregate of N fork API calls in one write with
``task='background_review'`` (issue #87250).
Best-effort by contract: callers must never fail an aux call because
accounting failed.
"""
if not session_id or not task:
return
# FK on session_model_usage.session_id → sessions.id: ensure the row
# exists (same INSERT OR IGNORE guard update_token_counts uses — the
# initial create_session() can fail under concurrent SQLite locking).
self._insert_session_row(session_id, "unknown")
def _do(conn):
self._record_model_usage(
conn,
session_id,
model=model,
billing_provider=billing_provider,
billing_base_url=billing_base_url,
billing_mode=None,
input_tokens=input_tokens or 0,
output_tokens=output_tokens or 0,
cache_read_tokens=cache_read_tokens or 0,
cache_write_tokens=cache_write_tokens or 0,
reasoning_tokens=reasoning_tokens or 0,
estimated_cost_usd=estimated_cost_usd,
actual_cost_usd=None,
cost_status=None,
cost_source=None,
api_call_count=(
1 if api_call_count is None else int(api_call_count)
),
task=task,
)
self._execute_write(_do)
def prune_empty_ghost_sessions(self, sessions_dir: "Optional[Path]" = None) -> int:
"""Remove empty TUI ghost sessions (no messages, no title, >24hr old)."""
cutoff = time.time() - 86400 # Only sessions older than 24 hours
def _do(conn):
rows = conn.execute("""
SELECT id FROM sessions
WHERE source = 'tui'
AND title IS NULL
AND ended_at IS NOT NULL
AND started_at < ?
AND NOT EXISTS (
SELECT 1 FROM messages WHERE messages.session_id = sessions.id
)
""", (cutoff,)).fetchall()
ids = [r[0] if isinstance(r, (tuple, list)) else r["id"] for r in rows]
if ids:
placeholders = ",".join("?" * len(ids))
conn.execute(
f"DELETE FROM sessions WHERE id IN ({placeholders})", ids
)
self._delete_unreferenced_system_prompts(conn)
return ids
removed_ids = self._execute_write(_do) or []
# Clean up any on-disk session files (belt-and-suspenders)
if sessions_dir and removed_ids:
for sid in removed_ids:
self._remove_session_files(sessions_dir, sid)
return len(removed_ids)
def finalize_orphaned_compression_sessions(self) -> int:
"""Mark orphaned compression continuation sessions as ended.
Targets child sessions that were never finalized: parent is ended
with reason='compression', child has messages but no end_reason/ended_at
and api_call_count=0. Non-destructive: preserves all messages and sets
end_reason='orphaned_compression'. Fix for #20001.
"""
cutoff = time.time() - 604800 # 7 days
def _do(conn):
now = time.time()
result = conn.execute(
"""
UPDATE sessions
SET ended_at = ?,
end_reason = 'orphaned_compression'
WHERE api_call_count = 0
AND end_reason IS NULL
AND ended_at IS NULL
AND started_at < ?
AND parent_session_id IS NOT NULL
AND EXISTS (
SELECT 1 FROM sessions p
WHERE p.id = sessions.parent_session_id
AND p.end_reason = 'compression'
AND p.ended_at IS NOT NULL
)
AND EXISTS (
SELECT 1 FROM messages m
WHERE m.session_id = sessions.id
)
""",
(now, cutoff),
)
return result.rowcount
return self._execute_write(_do) or 0
def sweep_orphaned_sessions(
self,
*,
max_idle_seconds: float,
sources: Tuple[str, ...] = ("tui", "desktop", "subagent"),
exclude_ids: Tuple[str, ...] = (),
exclude_pinned: bool = False,
heartbeat_staleness_seconds: Optional[float] = None,
heartbeat_ownership_grace_seconds: Optional[float] = None,
respect_gateway_heartbeats: bool = True,
) -> List[str]:
"""Close session rows orphaned by a dead gateway process (#65194, #94895).
The TUI/desktop gateway reaps disconnected websocket sessions with an
in-process ``threading.Timer`` grace timer; a gateway restart destroys
the timer and leaves the row ``ended_at IS NULL`` forever. This is the
startup-time complement: it closes rows for the given ``sources`` whose
``started_at`` and canonical last-activity time are both older than
``max_idle_seconds``, with a distinct
``end_reason='startup_orphan_reap'`` for traceability.
Canonical activity is the newest of ``last_activity_at`` (the in-turn
heartbeat) and the newest durable message timestamp, falling back to
``started_at``. The separate ``started_at`` predicate protects freshly
created compression/branch children whose copied activity is old.
Only pass sources whose lifecycle the caller owns (never messaging-gateway
platforms like ``telegram`` — ending those triggers the #60609 routing
loop). ``exclude_ids`` spares rows this process still holds in memory
(a ``session.resume`` that landed during the startup grace window).
``exclude_pinned`` is intended for broad automatic sweeps; pinned rows
remain explicitly recoverable. Non-destructive: messages are preserved
and the row remains resumable. First-reason-wins is preserved via
``ended_at IS NULL``.
Cross-backend liveness (#94895): when one ``state.db`` is shared by N
serve / gateway processes, each backend refreshes a row in
``gateway_heartbeats``. With ``respect_gateway_heartbeats`` enabled, a
row is only reaped when activity staleness holds AND no live backend
(heartbeat refreshed within ``heartbeat_staleness_seconds``, default
``2 * max_idle_seconds``) could plausibly own it. Disable that gate only
for sources whose lifecycle is explicitly owned by state.db itself.
Ownership inference: a live backend B ``owns`` a session S if
``B.started_at <= S.started_at + heartbeat_ownership_grace_seconds``
(default ``heartbeat_staleness_seconds``). The grace window covers a
migrating backend whose existing sessions predate its first heartbeat,
but is bounded so a fresh PID-reuse respawn cannot protect rows forever.
With no fresh heartbeat the predicate falls back to the legacy sweep.
The SELECT, live-lease validation, and UPDATE run in one
``BEGIN IMMEDIATE`` transaction. Active turn leases or compression
locks spare the row; expired/reclaimed guards are removed so their
former owner is fenced. Returns the swept session ids.
"""
srcs = tuple(s for s in sources if s)
if max_idle_seconds <= 0 or not srcs:
return []
hb_staleness = (
heartbeat_staleness_seconds
if heartbeat_staleness_seconds and heartbeat_staleness_seconds > 0
else max_idle_seconds * 2
)
hb_grace = (
heartbeat_ownership_grace_seconds
if heartbeat_ownership_grace_seconds is not None
and heartbeat_ownership_grace_seconds >= 0
else hb_staleness
)
now = time.time()
cutoff = now - max_idle_seconds
hb_cutoff = now - hb_staleness
placeholders = ",".join("?" for _ in srcs)
staleness = (
f"started_at < ? AND {_sql_session_last_active('sessions')} < ?"
)
pin_scope = " AND COALESCE(pinned, 0) = 0" if exclude_pinned else ""
heartbeat_params: Tuple[float, ...] = ()
orphan_predicate = staleness
if respect_gateway_heartbeats:
orphan_predicate += (
" AND NOT EXISTS ("
"SELECT 1 FROM gateway_heartbeats h"
" WHERE h.last_heartbeat >= ?"
" AND h.started_at <= sessions.started_at + ?"
")"
)
heartbeat_params = (hb_cutoff, hb_grace)
def _do(conn):
rows = conn.execute(
f"SELECT id FROM sessions WHERE ended_at IS NULL"
f" AND source IN ({placeholders}){pin_scope}"
f" AND {orphan_predicate}",
(*srcs, cutoff, cutoff, *heartbeat_params),
).fetchall()
excluded = {str(x) for x in exclude_ids if x}
victims = []
for row in rows:
sid = str(row["id"])
if sid in excluded:
continue
try:
self._check_transcript_write_guards(
conn,
sid,
compression_lock_holder=None,
turn_lease_holder=None,
reject_active_turn_lease=True,
reject_active_compression_lock=True,
)
except (
SessionCompressionInProgressError,
SessionTurnLeaseLostError,
):
continue
victims.append(sid)
if not victims:
return []
closed_at = time.time()
marks = ",".join("?" for _ in victims)
# Re-apply every scope/liveness predicate under the write lock.
conn.execute(
f"UPDATE sessions SET ended_at = ?, end_reason = 'startup_orphan_reap'"
f" WHERE id IN ({marks}) AND ended_at IS NULL"
f" AND source IN ({placeholders}){pin_scope}"
f" AND {orphan_predicate}",
(
closed_at,
*victims,
*srcs,
cutoff,
cutoff,
*heartbeat_params,
),
)
return victims
return self._execute_write(_do) or []
# ── Cross-backend heartbeat API (#94895) ───────────────────────────
# Each serve / tui_gateway process registers a heartbeat row at startup
# and refreshes ``last_heartbeat`` periodically. The startup orphan
# sweep reads these rows to avoid reaping sessions owned by another
# still-live backend that just happens to be idle. Backends remove
# their own row on graceful shutdown; a row that survives a crash is
# reclaimed by the staleness sweep once ``last_heartbeat`` ages out.
def register_backend_heartbeat(
self,
*,
backend_id: str,
pid: int,
started_at: float,
last_heartbeat: Optional[float] = None,
profile: str = "",
host: str = "",
) -> None:
"""Upsert this backend's liveness row (#94895).
``backend_id`` MUST be stable for the lifetime of the process
(e.g. ``f"{profile}@{host}:{pid}"``) so a respawn cannot accidentally
inherit the dead predecessor's heartbeat and protect stale rows.
``started_at`` records when THIS process started (not the wall clock
at first refresh) so a long-lived backend whose previous run died
cannot be confused with a freshly-spawned sibling.
"""
if not backend_id:
return
ts = time.time() if last_heartbeat is None else float(last_heartbeat)
def _do(conn):
conn.execute(
"INSERT INTO gateway_heartbeats"
" (backend_id, pid, started_at, last_heartbeat, profile, host)"
" VALUES (?, ?, ?, ?, ?, ?)"
" ON CONFLICT(backend_id) DO UPDATE SET"
" pid = excluded.pid,"
" started_at = excluded.started_at,"
" last_heartbeat = excluded.last_heartbeat,"
" profile = excluded.profile,"
" host = excluded.host",
(str(backend_id), int(pid), float(started_at), ts,
str(profile), str(host)),
)
self._execute_write(_do)
def clear_backend_heartbeat(self, backend_id: str) -> bool:
"""Remove this backend's heartbeat row (#94895).
Called from ``atexit`` so a graceful shutdown doesn't leave a stale
row behind. A crashed backend's row is reclaimed later by
``prune_stale_heartbeats``. Returns True if a row was removed.
"""
if not backend_id:
return False
def _do(conn):
cur = conn.execute(
"DELETE FROM gateway_heartbeats WHERE backend_id = ?",
(str(backend_id),),
)
return cur.rowcount > 0
return bool(self._execute_write(_do))
def prune_stale_heartbeats(self, *, max_age_seconds: float) -> List[str]:
"""Drop heartbeat rows whose ``last_heartbeat`` is older than the
staleness window. Returns the removed backend ids. Safe to call
from any process; only stale rows are touched.
"""
if max_age_seconds <= 0:
return []
cutoff = time.time() - max_age_seconds
def _do(conn):
cur = conn.execute(
"DELETE FROM gateway_heartbeats WHERE last_heartbeat < ?"
" RETURNING backend_id",
(cutoff,),
)
return [str(r[0]) for r in cur.fetchall()]
return list(self._execute_write(_do) or [])
def list_backend_heartbeats(self) -> List[Dict[str, Any]]:
"""Snapshot of every registered backend's heartbeat (for diagnostics
and tests). The fields mirror ``gateway_heartbeats`` exactly.
"""
with self._read_ctx() as conn:
rows = conn.execute(
"SELECT backend_id, pid, started_at, last_heartbeat,"
" profile, host FROM gateway_heartbeats"
" ORDER BY last_heartbeat DESC"
).fetchall()
out: List[Dict[str, Any]] = []
for r in rows:
if isinstance(r, sqlite3.Row):
out.append({k: r[k] for k in r.keys()})
else:
out.append({
"backend_id": r[0], "pid": r[1], "started_at": r[2],
"last_heartbeat": r[3], "profile": r[4], "host": r[5],
})
return out
def get_session(self, session_id: str) -> Optional[Dict[str, Any]]:
"""Get a session by ID."""
# Cost/usage readers (/status, /usage, gateway endpoints) reach the
# row through here; drain queued token deltas so they see exact
# totals. No-op attribute check when nothing is queued.
self.flush_token_counts()
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT s.*, "
"COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved "
"FROM sessions s "
"LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash "
"WHERE s.id = ?",
(session_id,),
)
row = cursor.fetchone()
return self._session_row_dict(row) if row else None
def get_dominant_session_model_route(
self, session_id: str
) -> Optional[Dict[str, Any]]:
"""Return the main-loop model route that served most API calls.
``sessions`` is a legacy aggregate row and can hold model/provider fields
written by different route changes. ``session_model_usage`` keeps the
coherent per-call tuple, so persisted status and billing reads should use
its dominant main-loop route when one is available.
"""
self.flush_token_counts()
with self._read_ctx() as conn:
row = conn.execute(
"""SELECT model, billing_provider, billing_base_url, billing_mode,
api_call_count
FROM session_model_usage
WHERE session_id = ?
AND task = ''
AND model <> 'unknown'
AND billing_provider <> ''
ORDER BY api_call_count DESC,
(input_tokens + output_tokens + cache_read_tokens +
cache_write_tokens + reasoning_tokens) DESC,
last_seen DESC
LIMIT 1""",
(session_id,),
).fetchone()
return dict(row) if row else None
def resolve_session_id(self, session_id_or_prefix: str) -> Optional[str]:
"""Resolve an exact or uniquely prefixed session ID to the full ID.
Returns the exact ID when it exists. Otherwise treats the input as a
prefix and returns the single matching session ID if the prefix is
unambiguous. Returns None for no matches or ambiguous prefixes.
"""
exact = self.get_session(session_id_or_prefix)
if exact:
return exact["id"]
escaped = _escape_like(session_id_or_prefix)
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT id FROM sessions WHERE id LIKE ? ESCAPE '\\' ORDER BY started_at DESC LIMIT 2",
(f"{escaped}%",),
)
matches = [row["id"] for row in cursor.fetchall()]
if len(matches) == 1:
return matches[0]
return None
# Maximum length for session titles
MAX_TITLE_LENGTH = 100
# Title provenance, lowest to highest authority. An auto-titling write may
# only replace a title of strictly lower authority, so the instant
# ``derived`` title upgrades to the model's ``llm`` title exactly once and
# nothing the agent generates can ever clobber a name the user typed.
TITLE_SOURCE_DERIVED = "derived"
TITLE_SOURCE_LLM = "llm"
TITLE_SOURCE_USER = "user"
_TITLE_SOURCE_RANK = {
TITLE_SOURCE_DERIVED: 0,
TITLE_SOURCE_LLM: 1,
TITLE_SOURCE_USER: 2,
}
# Bot Mode's forever-chat registry: the session titled exactly this, on a
# bot's profile, IS the bot's canonical chat — resolved by exact-title
# lookup on every open (no session-id pointer exists). The title is the
# identity, which is why _set_session_title refuses renames of a hidden
# row holding it (#92473).
CANONICAL_BOT_CHAT_TITLE = "Bot Chat"
@classmethod
def _title_rank(cls, source: Optional[str]) -> int:
"""Rank a stored title_source. NULL means a pre-provenance row.
Rows written before this column existed carry NULL. They were almost
always set by the old auto-titler, but a manual ``/title`` from that
era is indistinguishable — so treat NULL as ``user`` and refuse to
overwrite it. Auto-titling only ever fills genuinely empty titles on
legacy rows, which is the conservative direction.
"""
if source is None:
return cls._TITLE_SOURCE_RANK[cls.TITLE_SOURCE_USER]
return cls._TITLE_SOURCE_RANK.get(str(source), 0)
@staticmethod
def sanitize_title(title: Optional[str]) -> Optional[str]:
"""Validate and sanitize a session title.
- Strips leading/trailing whitespace
- Removes ASCII control characters (0x00-0x1F, 0x7F) and problematic
Unicode control chars (zero-width, RTL/LTR overrides, etc.)
- Collapses internal whitespace runs to single spaces
- Normalizes empty/whitespace-only strings to None
- Enforces MAX_TITLE_LENGTH
Returns the cleaned title string or None.
Raises ValueError if the title exceeds MAX_TITLE_LENGTH after cleaning.
"""
if not title:
return None
# Lone surrogates cannot be bound by sqlite3 (UnicodeEncodeError at
# UTF-8 encode time) — scrub them like every other write path here.
title = _sanitize_surrogates(title)
# Remove ASCII control characters (0x00-0x1F, 0x7F) but keep
# whitespace chars (\t=0x09, \n=0x0A, \r=0x0D) so they can be
# normalized to spaces by the whitespace collapsing step below
cleaned = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', title)
# Remove problematic Unicode control characters:
# - Zero-width chars (U+200B-U+200F, U+FEFF)
# - Directional overrides (U+202A-U+202E, U+2066-U+2069)
# - Object replacement (U+FFFC), interlinear annotation (U+FFF9-U+FFFB)
cleaned = re.sub(
r'[\u200b-\u200f\u2028-\u202e\u2060-\u2069\ufeff\ufffc\ufff9-\ufffb]',
'', cleaned,
)
# Collapse internal whitespace runs and strip
cleaned = re.sub(r'\s+', ' ', cleaned).strip()
if not cleaned:
return None
if len(cleaned) > SessionDB.MAX_TITLE_LENGTH:
raise ValueError(
f"Title too long ({len(cleaned)} chars, max {SessionDB.MAX_TITLE_LENGTH})"
)
return cleaned
def _is_compression_ancestor(
self, conn, *, ancestor_id: str, descendant_id: str
) -> bool:
"""Return True if *ancestor_id* is a compression predecessor of
*descendant_id* (walking parent links up the continuation chain).
The continuation edge is the canonical one shared with
:func:`_ephemeral_child_sql` / :meth:`set_session_archived`
(``_COMPRESSION_CHILD_SQL``): a parent → child edge counts only when the
parent ended with ``end_reason = 'compression'`` and the child started
at or after the parent's ``ended_at``, which distinguishes continuations
from delegate subagents / branch children that also carry a
``parent_session_id``. Expressed as a single recursive CTE rather than a
per-hop Python walk so the edge definition lives in exactly one place.
"""
if not ancestor_id or not descendant_id or ancestor_id == descendant_id:
return False
# Walk parent links up from the descendant, following only compression
# continuation edges, and check whether ancestor_id is reached.
edge = _COMPRESSION_CHILD_SQL.format(a="child")
row = conn.execute(
f"""
WITH RECURSIVE ancestors(id) AS (
SELECT ?
UNION
SELECT parent.id
FROM ancestors a
JOIN sessions child ON child.id = a.id
JOIN sessions parent ON parent.id = child.parent_session_id
WHERE {edge}
)
SELECT 1 FROM ancestors WHERE id = ? AND id != ? LIMIT 1
""",
(descendant_id, ancestor_id, descendant_id),
).fetchone()
return row is not None
def _set_session_title(
self,
session_id: str,
title: str,
*,
source: str,
) -> bool:
"""Write a title, enforcing provenance precedence.
``source`` is one of ``TITLE_SOURCE_{DERIVED,LLM,USER}``. A ``user``
write always lands — an explicit rename is authoritative. An automatic
write (``derived``/``llm``) lands only when the row is untitled or the
stored title has strictly lower authority, so the instant ``derived``
title upgrades to ``llm`` exactly once and neither can ever overwrite a
name the user typed. Re-running the titler on an already-``llm`` row is
a no-op, which is what stops a session renaming itself. The one thing
no writer may do is move a hidden canonical Bot Chat off its title.
The read and the write are one compare-and-swap inside a single
transaction, so a manual ``/title`` racing an in-flight generation
cannot be clobbered by the late arrival.
"""
title = self.sanitize_title(title)
is_user = source == self.TITLE_SOURCE_USER
new_rank = self._title_rank(source) if not is_user else None
def _do(conn):
current = conn.execute(
"SELECT title, title_source, hidden FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if current is None:
return 0
# The canonical Bot Chat's NAME is its identity: Bot Mode resolves
# the forever-chat by exact-title lookup on every open, so renaming
# the row orphans the entire conversation — the next click mints an
# empty replacement and UNIQUE(title) then blocks ever renaming
# back (#92473). Refuse the rename at the single write path every
# surface funnels through (gateway session.title, /title, CLI
# rename, REST). Hidden is the discriminator: canonical chats are
# born hidden; an ordinary visible session a user happens to call
# "Bot Chat" stays freely renameable. Provenance-blind: an
# automatic llm write outranks a derived title, so the auto-titler
# would otherwise rename the row too (#99517) — it no-ops instead.
if (
(current["title"] or "") == self.CANONICAL_BOT_CHAT_TITLE
and bool(current["hidden"])
and title != self.CANONICAL_BOT_CHAT_TITLE
):
if is_user:
raise ValueError(
"This is the bot's canonical Bot Chat — its name is its "
"identity, and renaming it would orphan the conversation. "
"To start fresh, create a new bot instead."
)
return 0
if not is_user and current["title"] is not None:
if self._title_rank(current["title_source"]) >= new_rank:
return 0
if title:
# Check uniqueness (allow the same session to keep its own title)
cursor = conn.execute(
"SELECT id FROM sessions WHERE title = ? AND id != ?",
(title, session_id),
)
conflict = cursor.fetchone()
if conflict:
conflict_id = conflict["id"]
# A compression continuation is the live, projected-forward
# head of its conversation; its compressed predecessors are
# ended and hidden from the session list (list_sessions_rich
# projects roots → tip). When the title that "conflicts" is
# held by such a hidden ancestor, the user has no way to free
# it — renaming the visible tip back to the base name would
# dead-end with "already in use by <session they can't see>".
# Treat this as a transfer: move the title off the ancestor
# onto the continuation. Uniqueness is preserved (still only
# one session carries the exact title) and the parent-link
# lineage is untouched.
if self._is_compression_ancestor(
conn, ancestor_id=conflict_id, descendant_id=session_id
):
conn.execute(
"UPDATE sessions SET title = NULL WHERE id = ?",
(conflict_id,),
)
else:
raise ValueError(
f"Title '{title}' is already in use by session {conflict_id}"
)
# Compare-and-swap on the exact values we just read (``IS`` is
# NULL-safe in SQLite), so a concurrent write between the SELECT
# and here loses instead of being silently overwritten.
cursor = conn.execute(
"UPDATE sessions SET title = ?, title_source = ? "
"WHERE id = ? AND title IS ? AND title_source IS ?",
(
title,
source if title else None,
session_id,
current["title"],
current["title_source"],
),
)
return cursor.rowcount
rowcount = self._execute_write(_do)
return rowcount > 0
def set_session_title(self, session_id: str, title: str) -> bool:
"""Set or update a session's title on the user's behalf.
Returns True if session was found and title was set.
Raises ValueError if title is already in use by another session,
or if the title fails validation (too long, invalid characters).
Empty/whitespace-only strings are normalized to None (clearing the title).
This records ``user`` provenance, so auto-titling will never replace
the result. Automatic callers must use :meth:`set_auto_title`.
"""
return self._set_session_title(
session_id, title, source=self.TITLE_SOURCE_USER
)
def set_auto_title(self, session_id: str, title: str, *, source: str) -> bool:
"""Set an automatically generated title, honoring provenance precedence.
Returns True when the title was written, False when a higher-authority
title already holds the row (nothing is modified in that case).
"""
if source not in (self.TITLE_SOURCE_DERIVED, self.TITLE_SOURCE_LLM):
raise ValueError(f"invalid automatic title source: {source!r}")
return self._set_session_title(session_id, title, source=source)
def set_auto_title_if_empty(self, session_id: str, title: str) -> bool:
"""Back-compat shim: set an LLM title only if nothing better exists.
Retained because older callers (and third-party plugins) reference it
by name. New code should call :meth:`set_auto_title` with an explicit
source.
"""
return self.set_auto_title(
session_id, title, source=self.TITLE_SOURCE_LLM
)
def get_session_title(self, session_id: str) -> Optional[str]:
"""Get the title for a session, or None."""
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT title FROM sessions WHERE id = ?", (session_id,)
)
row = cursor.fetchone()
return row["title"] if row else None
def get_session_title_source(self, session_id: str) -> Optional[str]:
"""Get the provenance of a session's title, or None when untitled."""
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT title, title_source FROM sessions WHERE id = ?",
(session_id,),
)
row = cursor.fetchone()
if not row or row["title"] is None:
return None
return row["title_source"]
def set_session_title_source(self, session_id: str, source: str) -> bool:
"""Overwrite a title's provenance without touching the title text.
Used when a title is carried across a session boundary (compression
rotation) and the copy must keep the original's authority rather than
the authority of whichever setter performed the copy.
"""
if source not in self._TITLE_SOURCE_RANK:
raise ValueError(f"invalid title source: {source!r}")
def _do(conn):
cursor = conn.execute(
"UPDATE sessions SET title_source = ? "
"WHERE id = ? AND title IS NOT NULL",
(source, session_id),
)
return cursor.rowcount
return self._execute_write(_do) > 0
def backfill_null_session_profiles(self, profile_name: str) -> int:
"""One-shot owner backfill for legacy pre-ownership session rows.
Sessions created before the durable-ownership work (#95407 lineage)
carry ``profile_name = NULL``. On single-backend installs that was
harmless, but once a Desktop registers a second connection the
fail-closed owner ladder (which is correct for new sessions) can no
longer route those rows anywhere — every pre-campaign session becomes
unresumable after upgrade (#94724, field report).
This store belongs to exactly one profile — the profile whose
``state.db`` this is — so stamping its own name onto rows that never
recorded one is a single-match backfill, not a guess. Rules mirror the
``create_session`` COALESCE contract:
* only ``NULL``/empty ``profile_name`` rows are touched — a non-NULL
owner is NEVER overwritten;
* idempotent and one-shot-per-row: a second run matches zero rows.
Returns the number of rows stamped (0 when nothing was legacy).
"""
stamp = (profile_name or "").strip()
if not stamp:
return 0
def _do(conn):
cursor = conn.execute(
"""UPDATE sessions
SET profile_name = ?
WHERE profile_name IS NULL OR TRIM(profile_name) = ''""",
(stamp,),
)
rowcount = cursor.rowcount
if rowcount is None or rowcount < 0:
rowcount = conn.execute("SELECT changes()").fetchone()[0]
return rowcount
return int(self._execute_write(_do) or 0)
def set_session_archived(self, session_id: str, archived: bool) -> bool:
"""Archive or unarchive a session.
Archived sessions are hidden from the default session list but keep all
their messages — this is a soft hide, not a delete. For compression
chains, archive the whole logical conversation. Desktop lists compression
roots projected forward to their latest continuation; updating only the
displayed tip lets the still-unarchived root resurrect it on refresh.
Returns True when at least one row was updated.
"""
def _do(conn):
cursor = conn.execute(
"""
WITH RECURSIVE
ancestors(id) AS (
SELECT ?
UNION
SELECT parent.id
FROM ancestors a
JOIN sessions child ON child.id = a.id
JOIN sessions parent ON parent.id = child.parent_session_id
WHERE parent.end_reason = 'compression'
),
descendants(id) AS (
SELECT ?
UNION
SELECT child.id
FROM descendants d
JOIN sessions parent ON parent.id = d.id
JOIN sessions child ON child.parent_session_id = parent.id
WHERE parent.end_reason = 'compression'
),
lineage(id) AS (
SELECT id FROM ancestors
UNION
SELECT id FROM descendants
)
UPDATE sessions
SET archived = ?
WHERE id IN (SELECT id FROM lineage)
""",
(session_id, session_id, 1 if archived else 0),
)
rowcount = cursor.rowcount
if rowcount is None or rowcount < 0:
rowcount = conn.execute("SELECT changes()").fetchone()[0]
return rowcount
rowcount = self._execute_write(_do)
return rowcount > 0
# Accidental end reasons that recovery treats as resumable. Single source
# of truth: hermes_state_common._RECOVERABLE_END_REASONS, interpolated
# into find_latest_gateway_session_for_peer / promote_to_session_reset
# SQL — literals cannot drift (docs/session-lifecycle.md "recoverable
# accidental reasons").
RECOVERABLE_END_REASONS = _RECOVERABLE_END_REASONS
def unarchive_recoverable_session(self, session_id: str) -> bool:
"""Un-archive a session that was archived by a recoverable accident.
Registry-style lookups (Bot Mode's canonical "Bot Chat") use this to
resurrect a row the ws-orphan reaper (``ws_orphan_reap``) or older
agent cleanup (``agent_close``) archived: those ends are accidents,
not user intent, so the identity-scoped canonical chat must survive
them (#92687). Sessions archived with no end_reason or an explicit
boundary reason (user archived deliberately, ``session_reset``, …)
are left untouched — returns ``False`` for those, ``True`` only when
the row was archived for a recoverable reason and is now un-archived
(whole compression lineage, via :meth:`set_session_archived`).
"""
if not session_id:
return False
try:
row = self.get_session(session_id)
except Exception:
return False
if not row or not row.get("archived"):
return False
# A compressed lineage's registry row keeps end_reason='compression';
# the accidental stamp lives on the live TIP. Judge recoverability at
# the tip (== the row itself when uncompressed).
tip = row
try:
tip_id = self.get_compression_tip(session_id) or session_id
if tip_id != session_id:
tip = self.get_session(tip_id) or row
except Exception:
tip_id = session_id
if (tip.get("end_reason") or "") not in self.RECOVERABLE_END_REASONS:
return False
if not self.set_session_archived(session_id, False):
return False
# Clear the accidental end stamp: the session is live again, and a
# surviving ws_orphan_reap/agent_close reason would make a LATER
# deliberate archive (which never writes end_reason) auto-resurrect
# on the next lookup — permanently overriding user intent.
def _clear_end(conn):
conn.execute(
"UPDATE sessions SET ended_at = NULL, end_reason = NULL WHERE id = ?",
(tip["id"],),
)
return 1
self._execute_write(_clear_end)
return True
def set_session_pinned(self, session_id: str, pinned: bool) -> bool:
"""Pin or unpin a session (and its whole compression lineage).
``pinned`` is a durable "keep" flag: pinned sessions are exempt from
the ``sessions.auto_archive`` stale sweep (see
:meth:`archive_stale_sessions`). Desktop is the current writer — its
sidebar pins mirror here so a backend/other-surface sweep honours
them. Like :meth:`set_session_archived` the whole compression chain is
flipped as a unit, so pinning the surfaced tip protects the root (and
vice-versa) no matter which id the caller holds. Returns True when at
least one row changed.
"""
def _do(conn):
cursor = conn.execute(
"""
WITH RECURSIVE
ancestors(id) AS (
SELECT ?
UNION
SELECT parent.id
FROM ancestors a
JOIN sessions child ON child.id = a.id
JOIN sessions parent ON parent.id = child.parent_session_id
WHERE parent.end_reason = 'compression'
),
descendants(id) AS (
SELECT ?
UNION
SELECT child.id
FROM descendants d
JOIN sessions parent ON parent.id = d.id
JOIN sessions child ON child.parent_session_id = parent.id
WHERE parent.end_reason = 'compression'
),
lineage(id) AS (
SELECT id FROM ancestors
UNION
SELECT id FROM descendants
)
UPDATE sessions
SET pinned = ?
WHERE id IN (SELECT id FROM lineage)
""",
(session_id, session_id, 1 if pinned else 0),
)
rowcount = cursor.rowcount
if rowcount is None or rowcount < 0:
rowcount = conn.execute("SELECT changes()").fetchone()[0]
return rowcount
rowcount = self._execute_write(_do)
return rowcount > 0
def set_session_hidden(self, session_id: str, hidden: bool) -> bool:
"""Hide or unhide a session (and its whole compression lineage).
``hidden`` is a generic "don't show in the global Sessions sidebar"
flag: a hidden session is dropped from the default
:meth:`list_sessions_rich` listing (which omits ``include_hidden``) but
stays fully resumable by the surface that owns it — useful for plugins
that manage their own sessions (e.g. kanban) and don't want them
cluttering the shared recents list. Like :meth:`set_session_archived`
/ :meth:`set_session_pinned` the whole compression chain is flipped as
a unit, so hiding the surfaced tip hides the root (and vice-versa) no
matter which id the caller holds. Returns True when at least one row
changed.
"""
def _do(conn):
cursor = conn.execute(
"""
WITH RECURSIVE
ancestors(id) AS (
SELECT ?
UNION
SELECT parent.id
FROM ancestors a
JOIN sessions child ON child.id = a.id
JOIN sessions parent ON parent.id = child.parent_session_id
WHERE parent.end_reason = 'compression'
),
descendants(id) AS (
SELECT ?
UNION
SELECT child.id
FROM descendants d
JOIN sessions parent ON parent.id = d.id
JOIN sessions child ON child.parent_session_id = parent.id
WHERE parent.end_reason = 'compression'
),
lineage(id) AS (
SELECT id FROM ancestors
UNION
SELECT id FROM descendants
)
UPDATE sessions
SET hidden = ?
WHERE id IN (SELECT id FROM lineage)
""",
(session_id, session_id, 1 if hidden else 0),
)
rowcount = cursor.rowcount
if rowcount is None or rowcount < 0:
rowcount = conn.execute("SELECT changes()").fetchone()[0]
return rowcount
rowcount = self._execute_write(_do)
return rowcount > 0
def set_session_read(self, session_id: str, read: bool = True) -> bool:
"""Mark a session read or unread (and its whole compression lineage).
Read state is a watermark, not a flag: ``last_read_at`` records when
the conversation was last read, and it counts as unread when activity
postdates that watermark (the derived ``unread`` key on
:meth:`list_sessions_rich` rows). New messages therefore flip a read
conversation back to unread without any write on the message path.
Three states:
* NULL — never tracked (every pre-feature row): treated as read, so
shipping the column doesn't badge a user's entire history at once.
* 0 — explicitly marked unread: any activity postdates it.
* timestamp — read up to that moment.
Like :meth:`set_session_archived` / :meth:`set_session_pinned`, the
whole compression chain is stamped as a unit, so reading the surfaced
tip clears the root (and vice-versa) no matter which id the caller
holds. Returns True when at least one row changed.
"""
def _do(conn):
cursor = conn.execute(
"""
WITH RECURSIVE
ancestors(id) AS (
SELECT ?
UNION
SELECT parent.id
FROM ancestors a
JOIN sessions child ON child.id = a.id
JOIN sessions parent ON parent.id = child.parent_session_id
WHERE parent.end_reason = 'compression'
),
descendants(id) AS (
SELECT ?
UNION
SELECT child.id
FROM descendants d
JOIN sessions parent ON parent.id = d.id
JOIN sessions child ON child.parent_session_id = parent.id
WHERE parent.end_reason = 'compression'
),
lineage(id) AS (
SELECT id FROM ancestors
UNION
SELECT id FROM descendants
)
UPDATE sessions
SET last_read_at = ?
WHERE id IN (SELECT id FROM lineage)
""",
(session_id, session_id, time.time() if read else 0.0),
)
rowcount = cursor.rowcount
if rowcount is None or rowcount < 0:
rowcount = conn.execute("SELECT changes()").fetchone()[0]
return rowcount
rowcount = self._execute_write(_do)
return rowcount > 0
@staticmethod
def session_unread(session_row: Dict[str, Any]) -> bool:
"""Derive unread from a session row's watermark and activity.
Shared by ``list_sessions_rich`` and any future surface that holds a
row (or projected row) with ``last_read_at`` and ``last_active``.
NULL watermark = never tracked = read.
"""
last_read = session_row.get("last_read_at")
if last_read is None:
return False
last_active = session_row.get("last_active") or session_row.get("started_at")
return float(last_active or 0) > float(last_read)
def get_session_by_title(self, title: str) -> Optional[Dict[str, Any]]:
"""Look up a session by exact title. Returns session dict or None."""
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT s.*, "
"COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved "
"FROM sessions s "
"LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash "
"WHERE s.title = ?",
(title,),
)
row = cursor.fetchone()
return self._session_row_dict(row) if row else None
def resolve_session_by_title(self, title: str) -> Optional[str]:
"""Resolve a title to a session ID, preferring the latest in a lineage.
If the exact title exists, returns that session's ID.
If not, searches for "title #N" variants and returns the latest one.
If the exact title exists AND numbered variants exist, returns the
latest numbered variant (the most recent continuation).
"""
# First try exact match
exact = self.get_session_by_title(title)
# Also search for numbered variants: "title #2", "title #3", etc.
# Escape SQL LIKE wildcards (%, _) in the title to prevent false matches
escaped = _escape_like(title)
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT id, title, started_at FROM sessions "
"WHERE title LIKE ? ESCAPE '\\' ORDER BY started_at DESC",
(f"{escaped} #%",),
)
numbered = cursor.fetchall()
if numbered:
# Return the most recent numbered variant
return numbered[0]["id"]
elif exact:
return exact["id"]
return None
def get_next_title_in_lineage(self, base_title: str) -> str:
"""Generate the next title in a lineage (e.g., "my session" → "my session #2").
Strips any existing " #N" suffix to find the base name, then finds
the highest existing number and increments.
"""
# Strip existing #N suffix to find the true base
match = re.match(r'^(.*?) #(\d+)$', base_title)
if match:
base = match.group(1)
else:
base = base_title
# Find all existing numbered variants
# Escape SQL LIKE wildcards (%, _) in the base to prevent false matches
escaped = _escape_like(base)
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT title FROM sessions WHERE title = ? OR title LIKE ? ESCAPE '\\'",
(base, f"{escaped} #%"),
)
existing = [row["title"] for row in cursor.fetchall()]
if not existing:
return base # No conflict, use the base name as-is
# Find the highest number
max_num = 1 # The unnumbered original counts as #1
for t in existing:
m = re.match(r'^.* #(\d+)$', t)
if m:
max_num = max(max_num, int(m.group(1)))
return f"{base} #{max_num + 1}"
def get_compression_chain(self, session_id: str) -> List[str]:
"""Walk the compression-continuation chain forward and return every id.
Root-first order, ending at the tip; ``[session_id]`` when no
continuation exists. ``get_compression_tip`` is this walk's last
element — kept as the single implementation so the two can never
disagree about what the chain is.
A compression continuation is a child of a session whose
``end_reason = 'compression'``. Older builds tried to distinguish
continuations from branches/subagents by requiring
``child.started_at >= parent.ended_at``. That ordering is too brittle:
gateway + compression races can insert the real continuation row before
the parent row's ``ended_at`` is written, while a stale websocket later
creates/reuses a sibling that *does* satisfy the timestamp test. The
visible symptom is brutal: desktop resume follows the stale sibling and
the user's latest messages look "lost" even though they are persisted in
the real continuation chain.
Instead, only follow children of compression-ended parents, exclude
explicit branch/delegate/tool children, and prefer children that are
themselves continuing the compression chain (``end_reason='compression'``)
or still live over stale closed siblings such as ``ws_orphan_reap``.
Returns the latest continuation tip, or the input id when no
continuation exists.
"""
current = session_id
chain = [current] if current else []
seen = {current} if current else set()
# Bound the walk defensively — compression chains this deep are
# pathological and shouldn't happen in practice. 100 = plenty.
for _ in range(100):
with self._read_ctx() as conn:
cursor = conn.execute(
f"""
SELECT child.id
FROM sessions parent
JOIN sessions child ON child.parent_session_id = parent.id
WHERE parent.id = ?
AND parent.end_reason = 'compression'
AND json_extract(COALESCE(child.model_config, '{{}}'), '$._branched_from') IS NULL
AND json_extract(COALESCE(child.model_config, '{{}}'), '$._delegate_from') IS NULL
AND COALESCE(child.source, '') != 'tool'
ORDER BY
CASE
WHEN child.end_reason = 'compression' THEN 0
WHEN child.ended_at IS NULL THEN 1
ELSE 2
END,
{_sql_session_last_active("child")} DESC,
child.started_at DESC,
child.id DESC
LIMIT 1
""",
(current,),
)
row = cursor.fetchone()
if row is None:
return chain
child_id = row["id"]
if not child_id or child_id in seen:
return chain
seen.add(child_id)
current = child_id
chain.append(child_id)
return chain
def get_compression_tip(self, session_id: str) -> Optional[str]:
"""The live tip of a compression-continuation chain (see
``get_compression_chain`` for the walk's semantics). Returns the input
id when no continuation exists."""
chain = self.get_compression_chain(session_id)
return chain[-1] if chain else session_id
# Columns excluded from compact_rows projections: only the payload-heavy
# blob no list consumer renders. Everything else — including gateway
# routing fields and desktop sidebar fields like git_branch — stays, and
# the projection is derived from SCHEMA_SQL so columns added later via
# declarative reconciliation are included automatically instead of
# silently dropping out of list rows.
_SESSION_COMPACT_EXCLUDED = frozenset(
{"system_prompt", "system_prompt_hash", "git_metadata_generation"}
)
_session_compact_cols_sql: Optional[str] = None
def usage_totals(self, *, min_message_count: int = 1, include_archived: bool = False) -> Dict[str, float]:
"""Tokens and spend across this store, as one aggregate.
The sidebar shows a profile's totals beside a page of its sessions, so
summing the rows it happens to have loaded would report a fraction of
the truth and shrink as paging changed. SQLite adds the columns up over
every row instead, at the cost of one scan.
Spend is the billed figure when the provider returned one and the
estimate otherwise — the same precedence a single row renders.
"""
where = ["parent_session_id IS NULL", "message_count >= ?"]
params: List[Any] = [min_message_count]
if not include_archived:
where.append("COALESCE(archived, 0) = 0")
with self._read_ctx() as conn:
row = conn.execute(
f"""
SELECT COALESCE(SUM(COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0)), 0),
COALESCE(SUM(COALESCE(actual_cost_usd, estimated_cost_usd, 0)), 0)
FROM sessions
WHERE {' AND '.join(where)}
""",
params,
).fetchone()
return {"tokens": int(row[0] or 0), "cost_usd": float(row[1] or 0.0)}
def list_recent_sessions_bounded(
self,
*,
limit: int = 20,
exclude_sources: List[str] = None,
timeout_seconds: float = 3.0,
candidate_limit: int = None,
lineage_limit: int = None,
) -> List[Dict[str, Any]]:
"""List recent user conversations without an unbounded message scan.
This is the latency-bounded browse path used by ``session_search()``.
It deliberately separates cheap candidate selection from expensive
hydration:
1. preselect a small set of session ids from the indexed durable
activity timestamp (falling back to ``started_at``);
2. resolve only those candidates across compression ancestry/chains;
3. calculate message-derived activity and previews only for that
bounded set.
Compression ancestry and descendant traversal use ``UNION`` so a
corrupt cycle cannot revisit the same session for one logical root,
plus a total-row ceiling so a deep or highly branching lineage cannot
defeat the candidate bound. If that ceiling is reached before a
candidate resolves to a terminal root/tip, that incomplete lineage is
omitted from the browse result rather than expanded without bound.
The query has a cooperative SQLite VM progress deadline. Expensive
statements that remain active beyond ``timeout_seconds`` are
interrupted at the next progress callback and this method raises
``TimeoutError`` instead of holding a gateway callback indefinitely.
Cheap statements may finish between callbacks; the deadline is a
fail-safe for sustained work, not a real-time scheduler guarantee.
This method intentionally supports only the filters needed by the
agent-tool browse shape. Rich dashboard/search callers keep using
:meth:`list_sessions_rich`.
"""
limit = max(1, int(limit))
timeout_seconds = max(0.0, float(timeout_seconds))
if candidate_limit is None:
candidate_limit = max(128, limit * 8)
candidate_limit = max(limit, min(int(candidate_limit), 2048))
if lineage_limit is None:
lineage_limit = min(8192, candidate_limit * 8)
lineage_limit = max(candidate_limit, min(int(lineage_limit), 8192))
candidate_clauses = [
"s.archived = 0",
"s.hidden = 0",
f"{_delegate_from_json('s.model_config')} IS NULL",
]
candidate_params: List[Any] = []
if exclude_sources:
placeholders = ",".join("?" for _ in exclude_sources)
candidate_clauses.append(f"s.source NOT IN ({placeholders})")
candidate_params.extend(exclude_sources)
candidate_where = " AND ".join(candidate_clauses)
# A compression continuation is an implementation edge, unlike /new
# reset and /branch children which are independent user-visible
# conversations. The same predicate is used in both directions so a
# candidate tip maps to its logical root and the root maps back to the
# freshest live tip.
compression_parent_edge = f"""
parent.end_reason = 'compression'
AND child.parent_session_id = parent.id
AND json_extract(
COALESCE(child.model_config, '{{}}'), '$._branched_from'
) IS NULL
AND {_delegate_from_json('child.model_config')} IS NULL
AND COALESCE(child.source, '') != 'tool'
"""
query = f"""
WITH RECURSIVE
recent_candidates(id) AS (
SELECT s.id
FROM sessions s
WHERE {candidate_where}
ORDER BY COALESCE(s.last_activity_at, s.started_at) DESC,
s.started_at DESC, s.id DESC
LIMIT ?
),
ancestors(candidate_id, cur_id) AS (
SELECT id, id FROM recent_candidates
UNION
SELECT a.candidate_id, parent.id
FROM ancestors a
JOIN sessions child ON child.id = a.cur_id
JOIN sessions parent ON {compression_parent_edge}
LIMIT ?
),
candidate_roots(root_id) AS (
SELECT DISTINCT a.cur_id
FROM ancestors a
JOIN sessions child ON child.id = a.cur_id
WHERE NOT EXISTS (
SELECT 1
FROM sessions parent
WHERE {compression_parent_edge}
)
),
chain(root_id, cur_id) AS (
SELECT root_id, root_id FROM candidate_roots
UNION
SELECT c.root_id, child.id
FROM chain c
JOIN sessions parent ON parent.id = c.cur_id
JOIN sessions child ON {compression_parent_edge}
LIMIT ?
),
chain_rows AS (
SELECT
c.root_id,
c.cur_id,
{_sql_session_last_active_by_id('c.cur_id')} AS activity,
CASE WHEN EXISTS (
SELECT 1
FROM sessions parent
JOIN sessions child ON {compression_parent_edge}
WHERE parent.id = c.cur_id
) THEN 0 ELSE 1 END AS is_tip
FROM chain c
),
ranked_tips AS (
SELECT root_id, cur_id, activity,
ROW_NUMBER() OVER (
PARTITION BY root_id
ORDER BY activity DESC, cur_id DESC
) AS rank_in_root
FROM chain_rows
WHERE is_tip = 1
)
SELECT
tip.id,
tip.source,
tip.model,
tip.title,
s.started_at AS started_at,
tip.ended_at,
tip.end_reason,
tip.message_count,
tip.tool_call_count,
rt.activity AS last_active,
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = tip.id
AND m.role = 'user'
AND m.content IS NOT NULL
AND {_PREVIEW_ELIGIBLE_SQL}
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
CASE WHEN s.id != tip.id THEN s.id ELSE NULL END
AS _lineage_root_id
FROM ranked_tips rt
JOIN sessions s ON s.id = rt.root_id
JOIN sessions tip ON tip.id = rt.cur_id
WHERE rt.rank_in_root = 1
AND s.archived = 0
AND s.hidden = 0
AND {_LISTABLE_CHILD_SQL}
AND {_delegate_from_json('s.model_config')} IS NULL
ORDER BY rt.activity DESC, s.started_at DESC, tip.id DESC
LIMIT ?
"""
params = candidate_params + [
candidate_limit,
lineage_limit,
lineage_limit,
limit,
]
deadline = time.monotonic() + timeout_seconds
interrupted_by_deadline = False
def _deadline_progress_handler() -> int:
nonlocal interrupted_by_deadline
if time.monotonic() >= deadline:
interrupted_by_deadline = True
return 1
return 0
try:
with self._read_ctx() as conn:
conn.set_progress_handler(_deadline_progress_handler, 1000)
try:
rows = conn.execute(query, params).fetchall()
finally:
conn.set_progress_handler(None, 0)
except sqlite3.OperationalError as exc:
if interrupted_by_deadline and "interrupt" in str(exc).lower():
raise TimeoutError(
f"recent-session browse exceeded {timeout_seconds:g}s deadline"
) from exc
raise
sessions = []
for row in rows:
session = self._session_row_dict(row)
session["preview"] = _shape_preview(session.pop("_preview_raw", ""))
session["unread"] = self.session_unread(session)
sessions.append(session)
return sessions
def list_sessions_rich(
self,
source: str = None,
sources: List[str] = None,
exclude_sources: List[str] = None,
cwd_prefix: str = None,
limit: int = 20,
offset: int = 0,
include_children: bool = False,
min_message_count: int = 0,
project_compression_tips: bool = True,
order_by_last_active: bool = False,
include_archived: bool = False,
archived_only: bool = False,
id_query: str = None,
search_query: str = None,
compact_rows: bool = False,
include_pinned: bool = False,
session_key: str = None,
include_hidden: bool = False,
) -> List[Dict[str, Any]]:
"""List sessions with preview (first user message) and last active timestamp.
Returns dicts with keys: id, source, model, title, started_at, ended_at,
message_count, preview (first 60 chars of first user message),
last_active (freshest of last_activity_at heartbeat and latest
message timestamp, else started_at).
Uses a single query with correlated subqueries instead of N+2 queries.
By default, child sessions that represent implementation details
(subagent runs, compression continuations) are excluded. User-visible
branch and reset children remain listable. Pass ``include_children=True``
to include every child.
With ``project_compression_tips=True`` (default), sessions that are
roots of compression chains are projected forward to their latest
continuation — one logical conversation = one list entry, showing the
live continuation's id/message_count/title/last_active. This prevents
compressed continuations from being invisible to users while keeping
delegate subagents and branches hidden. Pass ``False`` to return the
raw root rows (useful for admin/debug UIs).
Pass ``order_by_last_active=True`` to sort by most-recent activity
instead of original conversation start time. For compression chains,
the "most-recent activity" is taken from the live tip (not the root),
so an old conversation that was compressed and continued recently
surfaces in the correct slot. Ordering is computed at SQL level via
a recursive CTE that walks compression-continuation edges, so LIMIT
and OFFSET still apply efficiently.
``search_query`` matches case-insensitive substrings against each
surfaced row's title and id (and, like ``id_query``, every title/id in
its forward compression chain). A punctuation-stripped variant is also
matched so e.g. ``an94`` finds ``AN-94``. Only honored in the
``order_by_last_active`` path.
Pass ``compact_rows=True`` for dashboard and picker callers that only
need lightweight metadata. This omits the ``system_prompt`` blob from
the SELECT so SQLite never copies it out of the B-tree page — a
significant I/O saving on large databases where the blob routinely
runs to tens of kilobytes per row.
Pass ``include_pinned=True`` to back-fill any conversation carrying the
durable ``pinned`` flag that the LIMIT/OFFSET window left out. A pin is
a "this must always be reachable" statement, so a pinned conversation
aging past the requested page is a bug, not a paging outcome — the
desktop sidebar would render an empty Pinned section. Back-filled rows
obey the same filters (source, archived, min_message_count) as the
page: an archived or filtered-out conversation stays out.
Pass ``session_key`` to restrict results to one stable gateway
conversation scope (DM, group, channel, or thread, including the
configured per-user isolation policy).
"""
# Rows carry token/cost totals — drain queued deltas first so
# listings (sidebar, /resume, dashboards) show exact counters.
self.flush_token_counts()
where_clauses = []
params = []
if not include_children:
# Show roots and user-visible branch/reset sessions, while still
# hiding sub-agent runs and compression continuations. All four
# carry parent_session_id, so the shared predicate classifies the
# edge from stable markers plus legacy-compatible parent metadata.
#
# Branch sessions are identified two ways, OR'd for robustness:
# 1. A stable ``_branched_from`` marker in model_config, written
# by /branch at creation time. This survives the parent being
# reopened and re-ended with a different end_reason (e.g.
# tui_shutdown overwriting 'branched'), which otherwise hides
# the branch — see issue #20856.
# 2. The legacy heuristic (parent ended with 'branched' before the
# child started), covering branch sessions created before the
# marker existed.
where_clauses.append(_LISTABLE_CHILD_SQL)
where_clauses.append(f"{_delegate_from_json('s.model_config')} IS NULL")
include_sources = [source] if source else list(sources or [])
if include_sources:
placeholders = ",".join("?" for _ in include_sources)
where_clauses.append(f"s.source IN ({placeholders})")
params.extend(include_sources)
if session_key:
where_clauses.append("s.session_key = ?")
params.append(session_key)
if exclude_sources:
placeholders = ",".join("?" for _ in exclude_sources)
where_clauses.append(f"s.source NOT IN ({placeholders})")
params.extend(exclude_sources)
if cwd_prefix:
clause, clause_params = _cwd_prefix_clause(cwd_prefix)
where_clauses.append(clause)
params.extend(clause_params)
if min_message_count > 0:
where_clauses.append("s.message_count >= ?")
params.append(min_message_count)
if archived_only:
where_clauses.append("s.archived = 1")
elif not include_archived:
where_clauses.append("s.archived = 0")
if not include_hidden:
where_clauses.append("s.hidden = 0")
where_sql = f"WHERE {' AND '.join(where_clauses)}" if where_clauses else ""
# Snapshot the filter params before the query builders below extend
# them with LIMIT/OFFSET — the pinned back-fill reuses the same WHERE.
base_where_params = list(params)
prompt_select = (
"" if compact_rows
else ", COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved"
)
prompt_join = (
"" if compact_rows
else "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash"
)
# Optional session-id filter, pushed into SQL so callers (Desktop
# session-id search) don't have to fetch every row and filter in
# Python. ``id_query`` is matched as a case-insensitive substring
# against each surfaced row's id AND every id in its forward
# compression chain — so searching a compression *root* id or a *tip*
# id both resolve to the same projected conversation. Only used in the
# order_by_last_active path (which builds the chain CTE); other callers
# pass id_query=None.
id_needle = (id_query or "").strip().lower()
search_needle = (search_query or "").strip().lower()
if order_by_last_active:
# Compute effective_last_active by walking each surfaced session's
# compression-continuation chain forward in SQL and taking the MAX
# timestamp across the chain. This lets us ORDER BY + LIMIT at SQL
# level instead of fetching every row and sorting in Python, while
# still surfacing old compression roots whose live tip is fresh.
#
# The CTE seeds from rows the outer WHERE admits (roots +
# user-visible branch/reset children), then recursively joins through
# compression-continuation edges. Do NOT require
# child.started_at >= parent.ended_at here: real desktop/gateway
# races can insert the continuation row before the parent's
# ended_at is written, while stale websocket siblings may satisfy
# the timestamp test and hijack resume/list projection.
outer_where = where_sql
id_params: List[Any] = []
filter_clauses: List[str] = []
def _like_pattern(needle: str) -> str:
return f"%{_escape_like(needle)}%"
if id_needle:
# Admit a surfaced row if its own id or any id in its forward
# compression chain matches the needle. LIKE with a leading
# wildcard can't use an index, but the chain membership and
# the small result set keep this bounded — far cheaper than
# fetching every session and scanning in Python.
filter_clauses.append(
"EXISTS (SELECT 1 FROM chain cq"
" WHERE cq.root_id = s.id"
" AND LOWER(cq.cur_id) LIKE ? ESCAPE '\\')"
)
id_params.append(_like_pattern(id_needle))
if search_needle:
# Same chain-membership trick as id_query, but matching either
# the title or the id of any session in the chain. The compact
# (punctuation-stripped) variant lets `an94` match `AN-94`.
compact_needle = re.sub(r"[\W_]+", "", search_needle)
compact_sql = (
"REPLACE(REPLACE(REPLACE(REPLACE(LOWER(COALESCE({0}, '')),"
" '-', ''), '_', ''), '.', ''), ' ', '')"
)
search_clause = (
"EXISTS (SELECT 1 FROM chain cq"
" JOIN sessions cs ON cs.id = cq.cur_id"
" WHERE cq.root_id = s.id"
" AND (LOWER(COALESCE(cs.title, '')) LIKE ? ESCAPE '\\'"
" OR LOWER(cq.cur_id) LIKE ? ESCAPE '\\'"
)
id_params.extend([_like_pattern(search_needle)] * 2)
if compact_needle:
search_clause += (
f" OR {compact_sql.format('cs.title')} LIKE ? ESCAPE '\\'"
)
id_params.append(_like_pattern(compact_needle))
filter_clauses.append(search_clause + "))")
if filter_clauses:
combined = " AND ".join(filter_clauses)
outer_where = (
f"{where_sql} AND {combined}" if where_sql else f"WHERE {combined}"
)
_sel = self._compact_session_cols() if compact_rows else "s.*"
query = f"""
WITH RECURSIVE chain(root_id, cur_id) AS (
SELECT s.id, s.id FROM sessions s {where_sql}
UNION ALL
SELECT c.root_id, child.id
FROM chain c
JOIN sessions parent ON parent.id = c.cur_id
JOIN sessions child ON child.parent_session_id = c.cur_id
WHERE parent.end_reason = 'compression'
AND json_extract(COALESCE(child.model_config, '{{}}'), '$._branched_from') IS NULL
AND json_extract(COALESCE(child.model_config, '{{}}'), '$._delegate_from') IS NULL
AND COALESCE(child.source, '') != 'tool'
),
chain_max AS (
SELECT
root_id,
MAX({_sql_session_last_active_by_id("cur_id")}) AS effective_last_active
FROM chain
GROUP BY root_id
)
SELECT {_sel}{prompt_select},
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
AND {_PREVIEW_ELIGIBLE_SQL}
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
{_sql_session_last_active("s")} AS last_active,
COALESCE(cm.effective_last_active, s.started_at) AS _effective_last_active
FROM sessions s
LEFT JOIN chain_max cm ON cm.root_id = s.id
{prompt_join}
{outer_where}
ORDER BY _effective_last_active DESC, s.started_at DESC, s.id DESC
LIMIT ? OFFSET ?
"""
# WHERE params apply twice (CTE seed + outer select); the id filter
# only applies to the outer select.
params = params + params + id_params + [limit, offset]
else:
_sel = self._compact_session_cols() if compact_rows else "s.*"
query = f"""
SELECT {_sel}{prompt_select},
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
AND {_PREVIEW_ELIGIBLE_SQL}
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
{_sql_session_last_active("s")} AS last_active
FROM sessions s
{prompt_join}
{where_sql}
ORDER BY s.started_at DESC
LIMIT ? OFFSET ?
"""
params.extend([limit, offset])
with self._read_ctx() as conn:
cursor = conn.execute(query, params)
rows = cursor.fetchall()
sessions = []
for row in rows:
s = self._session_row_dict(row)
s["preview"] = _shape_preview(s.pop("_preview_raw", ""))
# Drop the internal ordering column so callers see a clean dict.
s.pop("_effective_last_active", None)
sessions.append(s)
# Back-fill pinned conversations the page missed. A pin outlives
# recency, so this runs BEFORE compression projection below — a
# back-filled root then projects to its live tip exactly like a row
# that had made the page on its own. One extra query, bounded by the
# number of pins (a handful), never N+1 per pin.
if include_pinned:
seen_ids = {s["id"] for s in sessions}
pinned_where = (
f"{where_sql} AND s.pinned = 1" if where_sql else "WHERE s.pinned = 1"
)
_sel = self._compact_session_cols() if compact_rows else "s.*"
pinned_query = f"""
SELECT {_sel}{prompt_select},
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
AND {_PREVIEW_ELIGIBLE_SQL}
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
COALESCE(
(SELECT MAX(m2.timestamp) FROM messages m2 WHERE m2.session_id = s.id),
s.started_at
) AS last_active
FROM sessions s
{prompt_join}
{pinned_where}
ORDER BY s.started_at DESC
"""
with self._read_ctx() as conn:
pinned_cursor = conn.execute(pinned_query, base_where_params)
pinned_rows = pinned_cursor.fetchall()
for row in pinned_rows:
s = self._session_row_dict(row)
if s["id"] in seen_ids:
continue
s["preview"] = _shape_preview(s.pop("_preview_raw", ""))
seen_ids.add(s["id"])
sessions.append(s)
# Project compression roots forward to their tips. Each row whose
# end_reason is 'compression' has a continuation child; replace the
# surfaced fields (id, message_count, title, last_active, ended_at,
# end_reason, preview) with the tip's values so the list entry acts
# as the live conversation. Keep the root's started_at to preserve
# chronological ordering by original conversation start.
if project_compression_tips and not include_children:
# get_compression_tip() walks each root's chain individually (it's
# a per-session graph walk, not batchable in one query), but the
# tip *row* fetch afterward was previously one _get_session_rich_row()
# call per compression root. Batch that half instead: resolve
# every tip id first, then fetch all tip rows in a single query.
tip_ids_by_root: Dict[str, str] = {}
chain_by_root: Dict[str, List[str]] = {}
for s in sessions:
if s.get("end_reason") != "compression":
continue
chain = self.get_compression_chain(s["id"])
tip_id = chain[-1] if chain else s["id"]
if tip_id != s["id"]:
tip_ids_by_root[s["id"]] = tip_id
chain_by_root[s["id"]] = chain
tip_rows = (
self._get_session_rich_rows_batch(
set(tip_ids_by_root.values()), compact_rows=compact_rows
)
if tip_ids_by_root
else {}
)
projected = []
for s in sessions:
tip_id = tip_ids_by_root.get(s["id"])
tip_row = tip_rows.get(tip_id) if tip_id else None
if not tip_row:
projected.append(s)
continue
# Preserve the root's started_at for stable sort order, but
# surface the tip's identity and activity data.
merged = dict(s)
for key in (
"id", "ended_at", "end_reason", "message_count",
"tool_call_count", "title", "last_active", "preview",
"model", "system_prompt", "cwd", "git_branch", "git_repo_root",
):
if key in tip_row:
merged[key] = tip_row[key]
merged["_lineage_root_id"] = s["id"]
# Every id on the chain, intermediates included. Root and tip
# alone are not enough client-side: a persisted tile or route
# can hold a MIDDLE segment's id (it was the tip when opened,
# then rotated again), and with only the root/tip pair such a
# surface can no longer prove it names this conversation —
# which is how one chat ends up open twice after compaction.
merged["_lineage_ids"] = chain_by_root.get(s["id"]) or None
projected.append(merged)
sessions = projected
# Derive read state per surfaced conversation. ``last_read_at`` is
# lineage-stamped by set_session_read, so a projected row's root
# watermark and its tip's are the same value — comparing it against
# the tip's last_active is correct either way.
for s in sessions:
s["unread"] = self.session_unread(s)
return sessions
def session_lifecycle_statuses(
self, session_ids: List[str]
) -> Dict[str, str]:
"""Classify each session's lifecycle state from its LAST message row.
Returns ``{session_id: status}`` where status is one of:
- ``'complete'`` — last message is a normal assistant reply
- ``'interrupted'`` — last message is a user turn, a pending assistant
tool call (no tool result followed), or a tool result the assistant
never responded to
- ``'error'`` — last message carries an error finish_reason
- ``'empty'`` — session has no messages
Cost-bounded by design: one query that resolves each listed session's
newest message id via ``MAX(id)`` (an index seek on
``idx_messages_session_id``) and joins back for that single row's
role/tool_calls/finish_reason. Never scans transcripts, so it stays
cheap on large databases regardless of total message volume.
"""
ids = [sid for sid in (session_ids or []) if sid]
if not ids:
return {}
statuses: Dict[str, str] = {sid: "empty" for sid in ids}
placeholders = ",".join("?" for _ in ids)
query = f"""
SELECT m.session_id, m.role,
m.tool_calls IS NOT NULL AS has_tool_calls,
m.finish_reason
FROM messages m
JOIN (
SELECT session_id, MAX(id) AS max_id
FROM messages
WHERE session_id IN ({placeholders})
GROUP BY session_id
) latest ON m.id = latest.max_id
"""
with self._read_ctx() as conn:
rows = conn.execute(query, ids).fetchall()
for row in rows:
statuses[row["session_id"]] = classify_session_status(
role=row["role"],
has_tool_calls=bool(row["has_tool_calls"]),
finish_reason=row["finish_reason"],
)
return statuses
# =========================================================================
# Message storage
# =========================================================================
# Sentinel prefix used to distinguish JSON-encoded structured content
# (multimodal messages: lists of parts like text + image_url) from plain
# string content. The NUL byte is not legal in normal text, so this
# cannot collide with real user content.
_CONTENT_JSON_PREFIX = "\x00json:"
@classmethod
def _encode_content(cls, content: Any) -> Any:
"""Serialize structured (list/dict) message content for sqlite.
sqlite3 can only bind ``str``, ``bytes``, ``int``, ``float``, and ``None``
to query parameters. Multimodal messages have ``content`` as a list of
parts (``[{"type": "text", ...}, {"type": "image_url", ...}]``), which
raises ``ProgrammingError: Error binding parameter N: type 'list' is
not supported`` when bound directly.
Returns the value unchanged when it's already a safe scalar, or a
sentinel-prefixed JSON string for lists/dicts. Paired with
:meth:`_decode_content` on read.
"""
if isinstance(content, str):
# Lone UTF-16 surrogates reach here inside tool results scraped
# from the web/social platforms (the same input that crashed the
# guardrail hasher). The proactive sanitizer upstream only cleans
# the *api_messages* copy, and the recovery sanitizer only runs
# after the API call itself raises — which it no longer does — so
# the canonical history keeps them and this write is where they
# land. Left raw, sqlite3 raises UnicodeEncodeError, the flush is
# abandoned, and the session silently stops persisting for the
# rest of its life. Scrub so persistence never fails.
return _sanitize_surrogates(content)
if content is None or isinstance(content, (bytes, int, float)):
return content
try:
# json.dumps defaults to ensure_ascii=True, which escapes any
# surrogate as \udXXX — already safe to bind.
return cls._CONTENT_JSON_PREFIX + json.dumps(content)
except (TypeError, ValueError):
# Last-resort fallback: stringify so persistence never fails.
return _sanitize_surrogates(str(content))
@classmethod
def _decode_content(cls, content: Any) -> Any:
"""Reverse :meth:`_encode_content`; returns scalars unchanged."""
if isinstance(content, str) and content.startswith(cls._CONTENT_JSON_PREFIX):
try:
return json.loads(content[len(cls._CONTENT_JSON_PREFIX):])
except (json.JSONDecodeError, TypeError):
logger.warning(
"Failed to decode JSON-encoded message content; "
"returning raw string"
)
return content
return content
@staticmethod
def _encode_display_metadata(display_metadata: Any) -> Optional[str]:
"""Serialize ``display_metadata`` for its TEXT column without double-encoding.
Import/replace paths can hand us an already-serialized JSON string (the
same hazard ``tool_calls`` guards against above). ``json.dumps`` on that
string would store a quoted JSON string, and the single ``json.loads``
on read then yields a ``str`` instead of a dict.
"""
if not display_metadata:
return None
if isinstance(display_metadata, str):
try:
parsed = json.loads(display_metadata)
except (json.JSONDecodeError, TypeError):
logger.warning("Ignoring non-JSON display metadata on write")
return None
if not isinstance(parsed, dict):
logger.warning("Ignoring non-object display metadata on write")
return None
return json.dumps(parsed)
if isinstance(display_metadata, dict):
return json.dumps(display_metadata)
logger.warning(
"Ignoring unexpected display metadata type on write: %s",
type(display_metadata).__name__,
)
return None
def _check_transcript_write_guards(
self,
conn,
session_id: str,
compression_lock_holder: Optional[str],
turn_lease_holder: Optional[str] = None,
turn_lease_ttl_seconds: float = 300.0,
reject_active_turn_lease: bool = False,
reject_active_compression_lock: bool = False,
allow_closed_compression_parent: bool = False,
) -> None:
"""Transcript-write admission checks, run INSIDE the write txn.
Shared by :meth:`append_message` and :meth:`append_messages_batch` so
the two writers can never diverge on these correctness invariants
(this guard has already needed targeted fixes — see the #74478 patience
note below). User-initiated transcript mutations may opt in to rejecting
an active unowned turn lease in that same transaction.
"""
# NOTE (#75316 redesign): appends do NOT check compression_locks.
# The lock's job is to stop two COMPRESSIONS colliding, not to fence
# ordinary transcript writes. Concurrent appends during a compression
# are safe by construction: archive_and_compact() commits against a
# watermark captured at compression start and clones every row that
# arrived after it back into the live transcript, in the same write
# transaction. Blocking appends here was the root cause of a whole
# symptom family — turns dying as session_persistence_failed while a
# slow provider summary held the lease (#74568, #77386), including
# stale locks from dead PIDs blocking writes for the full TTL.
# Destructive user mutations are different: a compressor that already
# captured its watermark can otherwise publish the pre-rewind snapshot
# after the mutation and resurrect the removed turn. Keep that narrow
# fence opt-in so ordinary appends retain the watermark behavior.
if reject_active_compression_lock:
active_lock = conn.execute(
"SELECT holder, expires_at FROM compression_locks "
"WHERE session_id = ?",
(session_id,),
).fetchone()
if active_lock is not None:
current_holder = active_lock["holder"]
if (
float(active_lock["expires_at"]) <= time.time()
or _compression_lock_holder_process_is_dead(current_holder)
):
conn.execute(
"DELETE FROM compression_locks "
"WHERE session_id = ? AND holder = ?",
(session_id, current_holder),
)
elif current_holder != compression_lock_holder:
raise SessionCompressionInProgressError(
f"Session {session_id!r} is being compressed by another writer"
)
if turn_lease_holder or reject_active_turn_lease:
conversation_id = self._session_turn_lease_key_on_conn(conn, session_id)
lease = conn.execute(
"SELECT holder, expires_at FROM session_turn_leases "
"WHERE conversation_id = ?",
(conversation_id,),
).fetchone()
now = time.time()
if turn_lease_holder:
if lease is None or lease["holder"] != turn_lease_holder:
raise SessionTurnLeaseLostError(
f"Session turn lease lost; refusing transcript write "
f"for {session_id!r}"
)
if float(lease["expires_at"]) <= now:
# Expiry makes the row reclaimable; it does not prove that a
# takeover occurred. BEGIN IMMEDIATE serializes this renewal
# with acquisition, so a still-matching owner can recover from
# a starved refresher without weakening the foreign-holder fence.
conn.execute(
"UPDATE session_turn_leases SET expires_at = ? "
"WHERE conversation_id = ? AND holder = ?",
(
now + max(0.1, float(turn_lease_ttl_seconds)),
conversation_id,
turn_lease_holder,
),
)
elif lease is not None:
current_holder = lease["holder"]
if (
float(lease["expires_at"]) <= now
or _compression_lock_holder_process_is_dead(current_holder)
):
# Match acquisition semantics: an expired or provably dead
# owner is reclaimable. Deleting it inside this BEGIN IMMEDIATE
# transaction also fences a stale late flush after the mutation.
conn.execute(
"DELETE FROM session_turn_leases "
"WHERE conversation_id = ? AND holder = ?",
(conversation_id, current_holder),
)
else:
raise SessionTurnLeaseLostError(
f"Session has an active turn lease; refusing transcript "
f"mutation for {session_id!r}"
)
session = conn.execute(
"SELECT ended_at, end_reason FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if (
session is not None
and session["ended_at"] is not None
and session["end_reason"] == "compression"
and not allow_closed_compression_parent
):
raise CompressionSessionClosedError(session_id)
@staticmethod
def _decode_display_metadata(raw: Any) -> Optional[Dict[str, Any]]:
"""Decode a ``display_metadata`` column into the dict every reader expects.
Every message read path must go through this. Returning the raw TEXT
instead reaches the desktop as a string, where ``'task_count' in meta``
throws and fails the whole resume. Rows written before the encode guard
landed are double-encoded, so unwrap a second layer when we find one.
"""
if raw is None:
return None
try:
meta = json.loads(raw) if isinstance(raw, str) else raw
if isinstance(meta, str):
meta = json.loads(meta)
except (json.JSONDecodeError, TypeError):
logger.warning("Ignoring invalid display metadata on message row")
return None
if not isinstance(meta, dict):
logger.warning("Ignoring non-object display metadata on message row")
return None
return meta
@staticmethod
def _reasoning_json_text(value: Any) -> Optional[str]:
"""Serialize a structured reasoning field for its TEXT column.
``reasoning_details`` / ``codex_reasoning_items`` / ``codex_message_items``
arrive as list/dict structures from the live runtime, but callers that
round-trip stored rows — ``get_messages`` straight into
``replace_messages``, e.g. the POST /api/sessions/{id}/fork handler —
hand back the raw TEXT these columns already hold, because
``get_messages`` only deserializes ``content`` and ``tool_calls``.
Re-dumping that TEXT double-encodes it, and the forked session's next
``get_messages_as_conversation`` json.loads then yields the inner
string instead of the original list, so every reasoning-replay consumer
(all of which check ``isinstance(..., list)``) silently drops it.
Strings are therefore stored as-is; structures are dumped.
"""
if not value:
return None
if isinstance(value, str):
return value
return json.dumps(value)
def append_message(
self,
session_id: str,
role: str,
content: str = None,
tool_name: str = None,
tool_calls: Any = None,
tool_call_id: str = None,
token_count: int = None,
finish_reason: str = None,
reasoning: str = None,
reasoning_content: str = None,
reasoning_details: Any = None,
codex_reasoning_items: Any = None,
codex_message_items: Any = None,
platform_message_id: str = None,
observed: bool = False,
effect_disposition: Optional[str] = None,
_compressed_summary: bool = False,
timestamp: Any = None,
api_content: Optional[str] = None,
display_kind: Optional[str] = None,
display_metadata: Optional[Dict[str, Any]] = None,
compression_lock_holder: Optional[str] = None,
turn_lease_holder: Optional[str] = None,
turn_lease_ttl_seconds: float = 300.0,
) -> int:
"""
Append a message to a session. Returns the message row ID.
Also increments the session's message_count (and tool_call_count
if role is 'tool' or tool_calls is present).
``platform_message_id`` is the external messaging platform's own
message ID (e.g. Telegram update_id, Yuanbao msg_id). It is
independent of the SQLite autoincrement primary key and is used by
platform-specific flows like yuanbao's recall guard to redact a
message by its platform-side identifier.
``api_content`` is the exact content string sent to the API for this
message when it differs from ``content`` (ephemeral memory/plugin
injections, persist overrides). It is a byte-fidelity sidecar for
prompt-cache-stable replay — stored as sent, except lone surrogates
(which sqlite3 cannot bind and which the conversation loop scrubs
from every outgoing payload anyway, so the scrubbed form IS the
wire bytes).
"""
# Display metadata is presentation-only and never changes the model
# context role/content replayed to providers.
display_metadata_json = self._encode_display_metadata(display_metadata)
# Serialize structured fields to JSON before entering the write txn
reasoning_details_json = self._reasoning_json_text(reasoning_details)
codex_items_json = self._reasoning_json_text(codex_reasoning_items)
codex_message_items_json = self._reasoning_json_text(codex_message_items)
# tool_calls may arrive as a Python list (from the live agent) or
# as a JSON string (from import/export). Parse first to avoid
# double-encoding.
if isinstance(tool_calls, str):
try:
tool_calls = json.loads(tool_calls)
except (json.JSONDecodeError, TypeError):
tool_calls = []
tool_calls_json = json.dumps(tool_calls) if tool_calls else None
# Multimodal content (list of parts) must be JSON-encoded: sqlite3
# cannot bind list/dict parameters directly.
stored_content = self._encode_content(content)
message_timestamp = time.time()
if timestamp is not None:
try:
if hasattr(timestamp, "timestamp"):
message_timestamp = float(timestamp.timestamp())
else:
message_timestamp = float(timestamp)
except (TypeError, ValueError):
logger.debug("Ignoring invalid explicit message timestamp: %r", timestamp)
# Pre-compute tool call count
num_tool_calls = 0
if tool_calls is not None:
num_tool_calls = len(tool_calls) if isinstance(tool_calls, list) else 1
def _do(conn):
self._check_transcript_write_guards(
conn,
session_id,
compression_lock_holder,
turn_lease_holder=turn_lease_holder,
turn_lease_ttl_seconds=turn_lease_ttl_seconds,
)
cursor = conn.execute(
"""INSERT INTO messages (session_id, role, content, tool_call_id,
tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason,
reasoning, reasoning_content, reasoning_details, codex_reasoning_items,
codex_message_items, platform_message_id, observed, _compressed_summary, active, api_content, display_kind, display_metadata)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
session_id,
role,
stored_content,
tool_call_id,
tool_calls_json,
_scrub_surrogates(tool_name),
effect_disposition,
message_timestamp,
token_count,
finish_reason,
_scrub_surrogates(reasoning),
_scrub_surrogates(reasoning_content),
reasoning_details_json,
codex_items_json,
codex_message_items_json,
platform_message_id,
1 if observed else 0,
1 if _compressed_summary else 0,
1,
_scrub_surrogates(api_content) if isinstance(api_content, str) else None,
_scrub_surrogates(display_kind) if isinstance(display_kind, str) else None,
display_metadata_json,
),
)
msg_id = cursor.lastrowid
# Update counters
if num_tool_calls > 0:
conn.execute(
"""UPDATE sessions SET message_count = message_count + 1,
tool_call_count = tool_call_count + ? WHERE id = ?""",
(num_tool_calls, session_id),
)
else:
conn.execute(
"UPDATE sessions SET message_count = message_count + 1 WHERE id = ?",
(session_id,),
)
return msg_id
# Transcript append is THE critical write: its failure aborts the
# user's turn (session_persistence_failed). Use the long patience so
# a sibling process legitimately holding the write lock for seconds
# (VACUUM, TRUNCATE checkpoint at close, an older pre-bounded-merge
# process's FTS optimize) can't destroy a healthy turn (#74478).
return self._execute_write(
_do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S
)
def append_messages_batch(
self,
session_id: str,
messages: List[Dict[str, Any]],
compression_lock_holder: Optional[str] = None,
turn_lease_holder: Optional[str] = None,
chunk_rows: Optional[int] = None,
turn_lease_ttl_seconds: float = 300.0,
) -> int:
"""Append multiple messages atomically in ONE write transaction.
``messages`` is a list of dicts in the same shape
:meth:`_insert_message_rows` already consumes for replace/compact/
import (role, content, tool_name, tool_calls, tool_call_id,
finish_reason, reasoning*, codex_*, timestamp, api_content,
display_kind, display_metadata, ...). Reusing that helper keeps ONE
row-serialization path for every multi-row writer.
A turn-boundary flush writes the whole turn (user + assistant + tool
rows, typically 3-8 messages) as one BEGIN IMMEDIATE / commit pair
instead of one transaction (and, off WAL, one fsync) per row.
Atomicity contract: all rows land or none do (the caller re-flushes
unstamped messages on the next attempt). The same admission guards
as :meth:`append_message` run once for the batch — same session,
same instant.
``chunk_rows`` bounds the transaction size for LARGE copies (branch
seeds can be thousands of rows; measured: 10k rows ≈ 2.4s inside one
BEGIN IMMEDIATE because the FTS triggers run per row, which would
monopolize the write lock and starve concurrent writers). When set,
the batch commits in chunks of at most that many rows — same
recovery semantics as the old per-row loops (a mid-copy failure
leaves a partial seed), just with bounded lock holds. A turn flush
never needs it. Returns the inserted row count.
"""
if not messages:
return 0
if chunk_rows is not None and len(messages) > chunk_rows:
inserted_total = 0
for start in range(0, len(messages), chunk_rows):
inserted_total += self.append_messages_batch(
session_id,
messages[start:start + chunk_rows],
compression_lock_holder=compression_lock_holder,
turn_lease_holder=turn_lease_holder,
turn_lease_ttl_seconds=turn_lease_ttl_seconds,
)
return inserted_total
def _do(conn):
self._check_transcript_write_guards(
conn,
session_id,
compression_lock_holder,
turn_lease_holder=turn_lease_holder,
turn_lease_ttl_seconds=turn_lease_ttl_seconds,
)
from agent.transcript_repair import resolve_and_repair_transcript_batch
inserted_rows = resolve_and_repair_transcript_batch(
conn,
session_id,
messages,
encode_content_fn=self._encode_content,
decode_content_fn=self._decode_content,
)
inserted = 0
tool_calls_total = 0
if inserted_rows:
inserted, tool_calls_total = self._insert_message_rows(
conn, session_id, inserted_rows
)
# One aggregated counter update for the newly inserted rows.
if tool_calls_total > 0:
conn.execute(
"""UPDATE sessions SET message_count = message_count + ?,
tool_call_count = tool_call_count + ? WHERE id = ?""",
(inserted, tool_calls_total, session_id),
)
elif inserted > 0:
conn.execute(
"UPDATE sessions SET message_count = message_count + ? WHERE id = ?",
(inserted, session_id),
)
return inserted
# Same criticality as append_message: this IS the turn's transcript.
return self._execute_write(
_do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S
)
def set_latest_matching_message_display_kind(
self, session_id: str, *, role: str, content: str, display_kind: str,
display_metadata: Optional[Dict[str, Any]] = None,
) -> bool:
"""Stamp presentation metadata on this turn's freshly persisted row.
The model still receives ``role`` and ``content`` unchanged. Gateway and
CLI synthetic inputs call this immediately after their serial turn has
flushed, preserving producer provenance without classifying by content
during transcript rendering.
"""
if not session_id or not content or not display_kind:
return False
def _do(conn):
row = conn.execute(
"SELECT id FROM messages WHERE session_id = ? AND role = ? "
"AND content = ? AND active = 1 ORDER BY id DESC LIMIT 1",
(session_id, role, self._encode_content(content)),
).fetchone()
if row is None:
return False
conn.execute(
"UPDATE messages SET display_kind = ?, display_metadata = ? WHERE id = ?",
(
_scrub_surrogates(display_kind),
self._encode_display_metadata(display_metadata),
row[0],
),
)
return True
return bool(self._execute_write(_do))
#: Key under which message reactions live inside ``display_metadata``.
#: Reactions share the existing per-message JSON column rather than a side
#: table so they survive rewind/compaction row rewrites with the row itself.
REACTIONS_METADATA_KEY = "reactions"
def set_message_reaction(
self,
session_id: str,
message_row_id: int,
emoji: Optional[str],
*,
author: str = "user",
) -> Optional[List[Dict[str, Any]]]:
"""Set (or with ``emoji=None`` clear) *author*'s reaction on one message.
iOS Tapback semantics: one reaction per author per message. Re-sending
the same emoji clears it, a different emoji replaces it. Returns the
message's full reaction list after the write, or ``None`` when the row
doesn't exist or isn't part of *session_id*.
"""
if not session_id or message_row_id is None:
return None
def _do(conn):
row = conn.execute(
"SELECT display_metadata FROM messages WHERE id = ? AND session_id = ?",
(message_row_id, session_id),
).fetchone()
if row is None:
return None
meta = self._decode_display_metadata(row[0]) or {}
existing = meta.get(self.REACTIONS_METADATA_KEY)
reactions = [
r
for r in (existing if isinstance(existing, list) else [])
if isinstance(r, dict) and r.get("author") != author
]
previous = next(
(
r
for r in (existing if isinstance(existing, list) else [])
if isinstance(r, dict) and r.get("author") == author
),
None,
)
# Tapping the live reaction again retracts it.
toggling_off = (
emoji is not None and previous is not None and previous.get("emoji") == emoji
)
if emoji and not toggling_off:
reactions.append(
{"emoji": _scrub_surrogates(emoji), "author": author, "at": time.time()}
)
if reactions:
meta[self.REACTIONS_METADATA_KEY] = reactions
else:
meta.pop(self.REACTIONS_METADATA_KEY, None)
conn.execute(
"UPDATE messages SET display_metadata = ? WHERE id = ?",
(self._encode_display_metadata(meta) if meta else None, message_row_id),
)
return reactions
return self._execute_write(_do)
def get_message_reactions(
self, session_id: str, message_row_id: int
) -> List[Dict[str, Any]]:
"""Return the reaction list persisted on one message row (never ``None``)."""
if not session_id or message_row_id is None:
return []
with self._read_ctx() as conn:
row = conn.execute(
"SELECT display_metadata FROM messages WHERE id = ? AND session_id = ?",
(message_row_id, session_id),
).fetchone()
if row is None:
return []
meta = self._decode_display_metadata(row[0]) or {}
reactions = meta.get(self.REACTIONS_METADATA_KEY)
return [r for r in reactions if isinstance(r, dict)] if isinstance(reactions, list) else []
def take_unseen_reactions(
self, session_id: str, *, author: str = "user"
) -> List[Dict[str, Any]]:
"""Return *author*'s not-yet-surfaced reactions and mark them seen.
Powers the cache-safe model-context path: reactions are announced on the
NEXT user turn (never by rewriting the message that was reacted to), and
the ``seen`` stamp guarantees each one is announced exactly once.
"""
if not session_id:
return []
def _do(conn):
rows = conn.execute(
"SELECT id, role, content, display_metadata FROM messages "
"WHERE session_id = ? AND active = 1 AND display_metadata IS NOT NULL "
"ORDER BY id",
(session_id,),
).fetchall()
pending = []
for row in rows:
meta = self._decode_display_metadata(row["display_metadata"])
if not meta:
continue
reactions = meta.get(self.REACTIONS_METADATA_KEY)
if not isinstance(reactions, list):
continue
changed = False
for reaction in reactions:
if (
not isinstance(reaction, dict)
or reaction.get("author") != author
or reaction.get("seen")
):
continue
reaction["seen"] = True
changed = True
content = self._decode_content(row["content"])
pending.append(
{
"row_id": row["id"],
"role": row["role"],
"emoji": reaction.get("emoji") or "",
"text": content if isinstance(content, str) else "",
}
)
if changed:
conn.execute(
"UPDATE messages SET display_metadata = ? WHERE id = ?",
(self._encode_display_metadata(meta), row["id"]),
)
return pending
return self._execute_write(_do) or []
def latest_message_row_id(
self, session_id: str, *, role: str = "user", offset: int = 0, require_text: bool = True
) -> Optional[int]:
"""Row id of the most recent active message with *role*, or ``None``.
Two callers, same need — "the message I mean, without an id": the agent
defaulting to the turn that triggered it, and the desktop reacting to a
live message that hasn't round-tripped through a resume yet.
``offset`` steps to earlier turns (1 = the one before the latest) so a
reaction can land retroactively — "two messages ago" is how the caller
thinks about it.
``require_text`` (default) skips rows with no plain-text content —
tool-call-only assistant turns and attachment stubs don't render as
bubbles, so "the latest message" as a HUMAN means it must never
resolve to one (a reaction landing on an invisible row looks dropped,
and its annotation quotes an empty string).
"""
if not session_id or role not in {"user", "assistant"} or offset < 0:
return None
text_filter = (
"AND content IS NOT NULL AND TRIM(content) != '' " if require_text else ""
)
with self._read_ctx() as conn:
row = conn.execute(
"SELECT id FROM messages WHERE session_id = ? AND role = ? "
f"AND active = 1 {text_filter}ORDER BY id DESC LIMIT 1 OFFSET ?",
(session_id, role, int(offset)),
).fetchone()
return row[0] if row else None
def latest_user_message_row_id(self, session_id: str) -> Optional[int]:
"""Row id of the most recent active user message, or ``None``.
The agent's default reaction target: "the message that triggered me",
so the model never has to thread row ids through a tool call (mirrors
the photon adapter's ``_record_last_inbound``).
"""
return self.latest_message_row_id(session_id, role="user")
def get_message_role(self, session_id: str, row_id: int) -> Optional[str]:
"""Role of the active message at *row_id* in *session_id*, or ``None``.
Lets a reaction event carry the target's role so a renderer can match
a live message that doesn't know its durable row id yet.
"""
if not session_id:
return None
with self._read_ctx() as conn:
row = conn.execute(
"SELECT role FROM messages WHERE id = ? AND session_id = ? AND active = 1",
(int(row_id), session_id),
).fetchone()
return row[0] if row else None
def _insert_message_rows(self, conn, session_id: str, messages: List[Dict[str, Any]]) -> tuple[int, int]:
"""Insert *messages* as fresh active rows for *session_id*.
Shared by :meth:`replace_messages` (delete-then-insert) and
:meth:`archive_and_compact` (soft-archive-then-insert). Runs inside the
caller's write transaction (takes the live ``conn``). Returns
``(inserted_count, tool_call_count)``. Does NOT touch sessions.* counters
— the caller owns that, since the two flows reconcile counts differently.
"""
now_ts = time.time()
inserted = 0
tool_calls_total = 0
for msg in messages:
role = msg.get("role", "unknown")
tool_calls = msg.get("tool_calls")
message_timestamp = now_ts
if msg.get("timestamp") is not None:
try:
ts_value = msg.get("timestamp")
if hasattr(ts_value, "timestamp"):
message_timestamp = float(ts_value.timestamp())
else:
message_timestamp = float(ts_value)
except (TypeError, ValueError):
logger.debug("Ignoring invalid explicit message timestamp: %r", msg.get("timestamp"))
reasoning_details = msg.get("reasoning_details") if role == "assistant" else None
codex_reasoning_items = (
msg.get("codex_reasoning_items") if role == "assistant" else None
)
codex_message_items = (
msg.get("codex_message_items") if role == "assistant" else None
)
reasoning_details_json = self._reasoning_json_text(reasoning_details)
codex_items_json = self._reasoning_json_text(codex_reasoning_items)
codex_message_items_json = self._reasoning_json_text(codex_message_items)
# tool_calls may arrive as a Python list (from the live agent)
# or as a JSON string (from import_sessions / export_session,
# which store it as TEXT). json.dumps on an already-serialized
# string double-encodes it, so parse first.
if isinstance(tool_calls, str):
try:
tool_calls = json.loads(tool_calls)
except (json.JSONDecodeError, TypeError):
tool_calls = []
tool_calls_json = json.dumps(tool_calls) if tool_calls else None
# Accept either `platform_message_id` (new explicit name) or
# `message_id` (yuanbao's existing convention on message dicts).
platform_msg_id = (
msg.get("platform_message_id") or msg.get("message_id")
)
api_content = msg.get("api_content")
cur = conn.execute(
"""INSERT INTO messages (session_id, role, content, tool_call_id,
tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason,
reasoning, reasoning_content, reasoning_details, codex_reasoning_items,
codex_message_items, platform_message_id, observed, _compressed_summary, active, api_content, display_kind, display_metadata)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
session_id,
role,
self._encode_content(msg.get("content")),
msg.get("tool_call_id"),
tool_calls_json,
_scrub_surrogates(msg.get("tool_name")),
msg.get("effect_disposition"),
message_timestamp,
msg.get("token_count"),
msg.get("finish_reason"),
_scrub_surrogates(msg.get("reasoning")) if role == "assistant" else None,
_scrub_surrogates(msg.get("reasoning_content")) if role == "assistant" else None,
reasoning_details_json,
codex_items_json,
codex_message_items_json,
platform_msg_id,
1 if msg.get("observed") else 0,
1 if msg.get("_compressed_summary") else 0,
1,
_scrub_surrogates(api_content) if isinstance(api_content, str) else None,
_scrub_surrogates(msg.get("display_kind")) if isinstance(msg.get("display_kind"), str) else None,
self._encode_display_metadata(msg.get("display_metadata")),
),
)
if isinstance(msg, dict) and cur.lastrowid is not None:
msg["_row_id"] = cur.lastrowid
inserted += 1
if tool_calls is not None:
tool_calls_total += (
len(tool_calls) if isinstance(tool_calls, list) else 1
)
now_ts = max(now_ts + 1e-6, message_timestamp + 1e-6)
return inserted, tool_calls_total
def replace_messages(
self,
session_id: str,
messages: List[Dict[str, Any]],
active_only: bool = False,
archive_dropped: bool = False,
reject_active_turn_lease: bool = False,
) -> None:
"""Atomically replace the stored messages for a session.
Used by transcript-rewrite flows such as /retry, /undo, and /compress.
The delete + reinsert sequence must commit as one transaction so a
mid-rewrite failure does not leave SQLite with a partial transcript.
DESTRUCTIVE by default: every row for the session is DELETEd (and drops
out of the FTS index). For compaction that must preserve the
pre-compaction transcript under the same id, use
:meth:`archive_and_compact` instead.
Pass ``active_only=True`` to replace ONLY the live (``active = 1``) rows,
leaving soft-archived rows (``active = 0`` — e.g. the ``compacted = 1``
turns that :meth:`archive_and_compact` keeps on disk for #38763
durability, or rewind/undo rows) untouched. Callers that share a session
id with an agent already running in-place compaction must use this so a
full-history rewrite doesn't wipe the rows the agent deliberately
archived. ``message_count``/``tool_call_count`` then track the live set,
matching :meth:`archive_and_compact`.
Pass ``archive_dropped=True`` to SOFT-archive the live rows instead of
DELETEing them: the replaced turns stay on disk with ``active = 0``,
``compacted = 0`` — the same "the user took it back" marking
:meth:`rewind_to_message` applies — and stay readable via
:meth:`get_messages` with ``include_inactive=True``. This is the mode a
rewind/edit/regenerate must use: those flows overwrite a transcript the
user may not have meant to drop, and a plain DELETE also evicts the rows
from the FTS index, leaving nothing to recover from (#82756). It implies
active-only handling — already-archived rows are never touched — so
``active_only`` is redundant with it. The rewritten set is inserted as
fresh active rows exactly as in the destructive path, so the live view
is identical either way; only the durability of the dropped turns
differs.
Pass ``reject_active_turn_lease=True`` for user-initiated rewrites that
do not already own the cross-process turn lease. The lease check and
transcript mutation then share one write transaction, so a second
process cannot archive or replace a turn that is still being produced.
"""
active_clause = " AND active = 1" if active_only else ""
def _do(conn):
if reject_active_turn_lease:
self._check_transcript_write_guards(
conn,
session_id,
None,
reject_active_turn_lease=True,
reject_active_compression_lock=True,
)
else:
session = conn.execute(
"SELECT ended_at, end_reason FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if (
session is not None
and session["ended_at"] is not None
and session["end_reason"] == "compression"
):
raise CompressionSessionClosedError(session_id)
if archive_dropped:
# Content-preserving UPDATE: the rows keep their FTS entries
# (the messages_fts triggers fire on INSERT / DELETE / UPDATE
# of content columns, not on `active`), so the replaced turns
# stay readable via get_messages(include_inactive=True) and
# searchable with include_inactive=True after the rewrite.
conn.execute(
"UPDATE messages SET active = 0 "
"WHERE session_id = ? AND active = 1",
(session_id,),
)
else:
conn.execute(
f"DELETE FROM messages WHERE session_id = ?{active_clause}",
(session_id,),
)
conn.execute(
"UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?",
(session_id,),
)
total_messages, total_tool_calls = self._insert_message_rows(
conn, session_id, messages
)
conn.execute(
"UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?",
(total_messages, total_tool_calls, session_id),
)
self._execute_write(_do)
def has_archived_messages(self, session_id: str) -> bool:
"""Return True if the session has any soft-archived (``active = 0``) rows.
Cheap existence probe — does not load rows. NOTE: production rewrite
paths no longer branch on this (they pass ``active_only=True``
unconditionally — a probe can fail open or race a concurrent
``archive_and_compact``, #80216); kept for tests and diagnostics.
"""
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT 1 FROM messages WHERE session_id = ? AND active = 0 LIMIT 1",
(session_id,),
)
return cursor.fetchone() is not None
def get_active_message_watermark(self, session_id: str) -> int:
"""MAX(id) of the session's active rows — the compression watermark.
Captured at compression START (before the slow provider summary call).
Every active row with ``id > watermark`` at commit time arrived
concurrently and must survive the compaction verbatim. Returns 0 for
an empty/unknown session.
"""
if not session_id:
return 0
with self._read_ctx() as conn:
row = conn.execute(
"SELECT COALESCE(MAX(id), 0) FROM messages "
"WHERE session_id = ? AND active = 1",
(session_id,),
).fetchone()
return int(row[0]) if row else 0
def archive_and_compact(
self,
session_id: str,
compacted_messages: List[Dict[str, Any]],
model_config_patch: Optional[Dict[str, Any]] = None,
watermark: Optional[int] = None,
lock_holder: Optional[str] = None,
tail_count: int = 0,
) -> int:
"""Non-destructive in-place compaction for a single durable session id.
Soft-archives the active messages (``active = 0``) and inserts
*compacted_messages* as fresh active rows — atomically, in one write
transaction. The conversation keeps ONE session id for life (#38763)
WITHOUT destroying history:
- The live-context load (:meth:`get_messages_as_conversation`,
:meth:`get_messages`) filters ``active = 1`` by default, so the model
reloads ONLY the compacted set.
- The archived pre-compaction turns stay on disk (active=0) and stay
DISCOVERABLE: they are marked compacted=1, and search_messages()
includes compacted=1 rows by default — so session_search still finds
them, unlike rewind/undo rows (active=0, compacted=0) which stay
hidden. They remain in the FTS index (the messages_fts* triggers
index on INSERT / drop on DELETE and don't key on active/compacted;
flipping to active=0 is a content-preserving UPDATE) and are
recoverable via get_messages(..., include_inactive=True).
Concurrent-append safety (#75316): when *watermark* is provided (the
value of :meth:`get_active_message_watermark` captured at compression
START), rows that arrived during the slow provider summary call
(``id > watermark``) are NOT summarized away. They are re-sequenced
after the compacted set by a pure-SQL column clone (every column
except ``id`` — content, api_content, platform_message_id, token
counts, reasoning sidecars all survive byte-exact, and the FTS
triggers index the clones naturally), and the originals are archived.
NOTE: re-sequencing assigns the tail rows fresh ids; consumers that
reference durable row ids re-resolve by content (see 3e8ab0610).
``watermark=None`` preserves the historical archive-everything
behavior.
Commit-fence safety: when *lock_holder* is provided, the commit
verifies INSIDE the transaction that the compression lock is still
held by that holder and unexpired — a compression whose lease was
reclaimed (crash cleanup, TTL expiry, competing writer) fails the
commit instead of clobbering the winner's transcript.
*tail_count* (default 0) names how many of the LAST rows of
*compacted_messages* are the verbatim carried-forward tail the
compressor protected rather than summarized (#86366). Those rows'
ORIGINALS — which this call archives as a side effect of the blanket
soft-archive — are superseded byte-identical duplicates, not
"summarized away" content, so they are stamped rewind-style
(``active=0, compacted=0``, hidden from search_messages) instead of
``compacted=1``. Without this the tail originals satisfy the recall
filter alongside their live clones and session_search returns every
carried-forward message once per compaction. Callers that cannot know
their tail shape keep the historical archive-everything behavior.
``message_count`` is set to the ACTIVE count after commit, matching
what the live load returns. ``model_config_patch`` is merged into the
session's JSON config in the same transaction; a ``None`` value
removes that key. Returns the new active count.
"""
def _do(conn):
if lock_holder is not None:
lock_row = conn.execute(
"SELECT holder, expires_at FROM compression_locks "
"WHERE session_id = ?",
(session_id,),
).fetchone()
if (
lock_row is None
or lock_row["holder"] != lock_holder
or float(lock_row["expires_at"]) <= time.time()
):
raise SessionCompressionInProgressError(
f"Compression lease for {session_id!r} lost before "
"commit; refusing to publish a stale compaction"
)
patched_model_config = None
if model_config_patch is not None:
# on_missing="raise": a prune/compaction must not commit
# against a vanished session row (the compressor's caller
# converts the raised error into a safe keep-the-original
# no-op), unlike the flag setters which tolerate missing rows.
patched_model_config = self._merge_model_config_json(
conn, session_id, model_config_patch, on_missing="raise"
)
# Concurrent tail: active rows that arrived after the watermark.
# Snapshot their ids and tool_calls now — the clone below needs a
# stable id list, and the tool-call count keeps sessions.* honest.
tail_ids: list[int] = []
tail_tool_calls = 0
if watermark is not None:
for row in conn.execute(
"SELECT id, tool_calls FROM messages "
"WHERE session_id = ? AND active = 1 AND id > ? "
"ORDER BY id",
(session_id, int(watermark)),
).fetchall():
tail_ids.append(int(row["id"]))
raw = row["tool_calls"]
if raw:
try:
parsed = json.loads(raw) if isinstance(raw, str) else raw
tail_tool_calls += len(parsed) if isinstance(parsed, list) else 0
except (TypeError, ValueError):
pass
# Soft-archive the live turns: active=0 hides them from the live
# context load, compacted=1 marks them as "summarized away" (vs
# rewind/undo's active=0+compacted=0, which means "user took it
# back"). search_messages includes compacted=1 rows by default so
# the pre-compaction transcript stays discoverable; live-context
# loads (active=1 only) still exclude them. Tail originals whose
# verbatim clones ride inside *compacted_messages* (tail_count)
# are superseded duplicates instead (#86366): they get the
# rewind-style flags so they stop matching the recall filter.
# Rewind-target ids: the originals of the carried-forward tail
# rows (tail_count), captured BEFORE any flag flips. Named apart
# from the watermark `tail_ids` below on purpose — the two are
# different sets (#86366): rewind targets sit AT/BELOW the
# watermark (the compressor only saw rows up to it), while
# `tail_ids` are concurrent appends ABOVE it. Without the bound,
# a concurrent append would steal a LIMIT slot and leave a real
# carried-forward original stamped compacted=1.
rewind_tail_ids: Optional[list[int]] = None
if tail_count > 0:
if watermark is not None:
tail_rows = conn.execute(
"SELECT id FROM messages "
"WHERE session_id = ? AND active = 1 AND id <= ? "
"ORDER BY id DESC LIMIT ?",
(session_id, int(watermark), int(tail_count)),
).fetchall()
else:
tail_rows = conn.execute(
"SELECT id FROM messages "
"WHERE session_id = ? AND active = 1 ORDER BY id DESC LIMIT ?",
(session_id, int(tail_count)),
).fetchall()
rewind_tail_ids = [int(row["id"]) for row in tail_rows]
# The watermark clone below re-inserts `tail_ids` rows byte-exact
# as live rows — their originals are the SAME superseded-duplicate
# class as the carried-forward tail (#86366), so they take the
# rewind flags too instead of double-matching the recall filter.
rewind_ids = [*(rewind_tail_ids or []), *tail_ids]
if rewind_ids:
placeholders = ",".join("?" for _ in rewind_ids)
conn.execute(
"UPDATE messages SET active = 0, compacted = 0 "
f"WHERE session_id = ? AND id IN ({placeholders})",
[session_id, *rewind_ids],
)
conn.execute(
"UPDATE messages SET active = 0, compacted = 1 "
"WHERE session_id = ? AND active = 1 "
f"AND id NOT IN ({placeholders})",
[session_id, *rewind_ids],
)
else:
conn.execute(
"UPDATE messages SET active = 0, compacted = 1 "
"WHERE session_id = ? AND active = 1",
(session_id,),
)
inserted, tool_calls_total = self._insert_message_rows(
conn, session_id, compacted_messages
)
if tail_ids:
# Re-sequence the concurrent tail after the compacted set via
# a pure-SQL column clone: no decode/re-encode round trip, no
# field drift — new id, active=1, compacted=0, all else exact.
placeholders = ",".join("?" for _ in tail_ids)
clone_cols = [
c for c in self._message_column_names(conn)
if c not in ("id", "active", "compacted")
]
col_list = ", ".join(clone_cols)
conn.execute(
f"INSERT INTO messages ({col_list}, active, compacted) "
f"SELECT {col_list}, 1, 0 FROM messages "
f"WHERE id IN ({placeholders}) ORDER BY id",
tail_ids,
)
inserted += len(tail_ids)
tool_calls_total += tail_tool_calls
# message_count / tool_call_count reflect the LIVE (active) set —
# the archived rows are still on disk but not part of the live count.
if model_config_patch is None:
conn.execute(
"UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?",
(inserted, tool_calls_total, session_id),
)
else:
conn.execute(
"UPDATE sessions SET message_count = ?, tool_call_count = ?, "
"model_config = ? WHERE id = ?",
(inserted, tool_calls_total, patched_model_config, session_id),
)
return inserted
return self._execute_write(_do)
def _message_column_names(self, conn) -> List[str]:
"""Column names of the messages table, cached per-connection era."""
cached = getattr(self, "_message_columns_cache", None)
if cached:
return cached
cols = [r[1] for r in conn.execute("PRAGMA table_info(messages)").fetchall()]
self._message_columns_cache = cols
return cols
def set_latest_user_api_content(
self, session_id: str, content: Any, api_content: str
) -> int:
"""Backfill the ``api_content`` sidecar onto the newest ACTIVE user row.
In-place preflight compaction (:meth:`archive_and_compact`) inserts the
current turn's user row BEFORE the turn prologue composes the
prefetch/plugin sidecar, and the subsequent crash persist identity-skips
every compacted dict — without this backfill the stamped sidecar would
never land in the DB and any reload would replay clean content,
re-introducing the prompt-cache divergence the sidecar exists to close.
The ``content`` match is a defensive guard: if the newest active user
row is not the message the caller stamped (racing rewrite, unexpected
tail shape), nothing is written. Returns the number of rows updated
(0 or 1).
"""
encoded = self._encode_content(content)
def _do(conn):
cursor = conn.execute(
"UPDATE messages SET api_content = ? WHERE id = ("
"SELECT id FROM messages "
"WHERE session_id = ? AND role = 'user' AND active = 1 "
"ORDER BY id DESC LIMIT 1"
") AND content IS ?",
(_scrub_surrogates(api_content), session_id, encoded),
)
return cursor.rowcount
return self._execute_write(_do)
def _dedupe_display_generations(self, rows):
"""Collapse compaction generations so each message appears once.
Compaction epochs copy the protected tail into each new generation, so
one logical message can exist as several rows (identical
role/content/timestamp) with different ``active`` flags and ids. A
display read must surface each exactly once: prefer the live row, then
the newest generation.
This is the ONE definition shared by every display projection —
:meth:`get_messages` (REST), :meth:`get_resume_conversations` and
:meth:`get_ancestor_display_prefix` (gateway resume), and
:meth:`get_messages_as_conversation` (warm-session payload) — so the
surfaces cannot disagree about the same transcript. *rows* must already
be ordered by ``id``; the returned list keeps that order.
"""
seen: Dict[Tuple[Any, ...], Any] = {}
for row in rows:
dedupe_content = row["content"]
if row["role"] == "user":
from agent.context_compressor import split_user_originated_turn
candidate = {
"role": "user",
"content": self._decode_content(row["content"]),
"display_kind": row["display_kind"],
"display_metadata": self._decode_display_metadata(
row["display_metadata"]
),
}
handoff, live_view = split_user_originated_turn(candidate)
if handoff is not None and live_view is not None:
dedupe_content = self._encode_content(live_view.get("content"))
# Tool fields participate in the dedupe key: compaction copies them
# verbatim, so identical tool messages across generations still
# collapse, while distinct tool calls that happen to share
# role/content/timestamp are never merged.
key = (
row["role"],
dedupe_content,
row["timestamp"],
row["tool_call_id"],
row["tool_calls"],
row["tool_name"],
)
cur = seen.get(key)
if cur is None or (row["active"], row["id"]) > (cur["active"], cur["id"]):
seen[key] = row
return sorted(seen.values(), key=lambda r: r["id"])
def get_messages(
self,
session_id: str,
include_inactive: bool = False,
include_compacted: bool = False,
limit: Optional[int] = None,
offset: int = 0,
latest: bool = False,
after_id: Optional[int] = None,
) -> List[Dict[str, Any]]:
"""Load messages for a session in insertion order.
By default only active messages are returned. Pass
``include_inactive=True`` to load soft-deleted rows (e.g. for
audit / debug views of rewound history). See
:meth:`rewind_to_message` for the soft-delete mechanic.
Pass ``include_compacted=True`` to additionally load rows preserved
by in-place context compaction (``active=0, compacted=1``). Those are
durable display history, not soft-deleted rows — a user-visible
transcript read must not drop them, or earlier turns silently become
unreachable once the UI exhausts its active-only window. Soft-deleted
Undo/Rewind rows (``active=0, compacted=0``) stay excluded; use
``include_inactive`` for those.
Ordered by AUTOINCREMENT id (true insertion order) rather than
timestamp — see c03acca50 for the WSL2 clock-regression rationale.
When ``limit`` is provided, returns at most ``limit`` messages
starting from ``offset`` (0-based, in insertion order). Enables
pagination for the API endpoint to avoid loading entire transcripts.
With ``latest=True``, the offset is measured back from the newest
message and the selected page is still returned in chronological
order. ``offset`` alone (without ``limit``) also pages — SQLite
requires a LIMIT clause for OFFSET, so it's emitted as ``LIMIT -1``
(unbounded).
``after_id`` enables keyset pagination (``id > after_id``): O(1)
page seeks on huge transcripts where OFFSET degrades to O(n) per
page. Ascending order only (incompatible with ``latest``/``offset``).
"""
if after_id is not None and (latest or offset):
raise ValueError("after_id is incompatible with latest/offset paging")
if after_id is not None and include_compacted:
raise ValueError("after_id is incompatible with include_compacted (deduped display reads use offset paging)")
if include_inactive:
# Audit / debug reads: every row, including soft-deleted.
active_clause = ""
elif include_compacted:
# Display history: active rows plus rows preserved by in-place
# compaction (active=0, compacted=1), but never soft-deleted
# Undo/Rewind rows (active=0, compacted=0).
active_clause = " AND (active = 1 OR compacted = 1)"
else:
active_clause = " AND active = 1"
keyset_clause = " AND id > ?" if after_id is not None else ""
sql = (
"SELECT * FROM messages WHERE session_id = ?"
f"{active_clause}{keyset_clause} ORDER BY id {'DESC' if latest else 'ASC'}"
)
params: list = [session_id]
if after_id is not None:
params.append(after_id)
if include_compacted:
# Read the full display set (a session's rows are bounded; the
# UI-level 500-row cap lives in the endpoint, not here), dedupe
# generations, then apply paging.
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT * FROM messages WHERE session_id = ?" + active_clause
+ " ORDER BY id ASC",
[session_id],
)
all_rows = cursor.fetchall()
rows = self._dedupe_display_generations(all_rows)
if latest:
rows = rows[::-1]
rows = rows[offset:]
if limit is not None:
rows = rows[:limit]
if latest:
rows = rows[::-1]
else:
if limit is not None or offset:
# SQLite's OFFSET requires LIMIT; -1 means "no limit".
sql += " LIMIT ? OFFSET ?"
params.extend([-1 if limit is None else limit, offset])
with self._read_ctx() as conn:
cursor = conn.execute(sql, params)
rows = cursor.fetchall()
if latest:
rows.reverse()
result = []
for row in rows:
msg = dict(row)
if msg.pop("_compressed_summary", 0):
msg["_compressed_summary"] = True
if "content" in msg:
msg["content"] = self._decode_content(msg["content"])
if msg.get("tool_calls"):
try:
msg["tool_calls"] = json.loads(msg["tool_calls"])
except (json.JSONDecodeError, TypeError):
logger.warning("Failed to deserialize tool_calls in get_messages, falling back to []")
msg["tool_calls"] = []
if msg.get("display_metadata") is not None:
msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"])
result.append(msg)
return result
def find_pr_url_messages(self, session_ids: List[str]) -> List[Dict[str, Any]]:
"""Tool results in these sessions that mention a GitHub PR url.
A candidate scan, deliberately loose: it hands back every tool result
containing ``/pull/`` and leaves the caller to decide which ones make a
claim (see the desktop's PR recovery, which only accepts an output that
is a bare PR url — the signature of ``gh pr create``). Ordered
oldest-first per session so the caller can take the last match.
"""
found: List[Dict[str, Any]] = []
ids = [s for s in session_ids if s]
for start in range(0, len(ids), 900): # SQLite's bound-variable ceiling.
chunk = ids[start : start + 900]
placeholders = ",".join("?" * len(chunk))
with self._read_ctx() as conn:
rows = conn.execute(
f"""SELECT session_id, content FROM messages
WHERE session_id IN ({placeholders})
AND role = 'tool' AND content LIKE '%/pull/%'
ORDER BY id ASC""",
chunk,
).fetchall()
found.extend({"session_id": row[0], "content": row[1]} for row in rows)
return found
def get_messages_around(
self,
session_id: str,
around_message_id: int,
window: int = 5,
) -> Dict[str, Any]:
"""Load a window of messages anchored on a specific message id.
Returns a dict with:
- ``window``: up to ``window`` messages before the anchor, the anchor
itself, and up to ``window`` messages after, ordered by id ascending.
- ``messages_before``: count of messages strictly before the anchor
still in the session (== window unless we hit the start).
- ``messages_after``: count of messages strictly after the anchor
still in the session (== window unless we hit the end).
Used by ``session_search`` for both the discovery shape (anchored on the
FTS5 match) and the scroll shape (anchored on any message id). The
``messages_before`` / ``messages_after`` counts let the caller detect
session boundaries: when either is less than ``window``, the agent has
reached one end of the session.
Returns an empty window when ``around_message_id`` is not a real id in
``session_id`` — callers decide how to surface that.
"""
if window < 0:
window = 0
with self._read_ctx() as conn:
# Confirm the anchor exists in this session.
anchor_exists = conn.execute(
"SELECT 1 FROM messages WHERE id = ? AND session_id = ? LIMIT 1",
(around_message_id, session_id),
).fetchone()
if not anchor_exists:
return {"window": [], "messages_before": 0, "messages_after": 0}
# Two queries: anchor + before (DESC, take window+1), and after
# (ASC, take window). Final order is id ASC.
before_rows = conn.execute(
"SELECT * FROM messages "
"WHERE session_id = ? AND id <= ? "
"ORDER BY id DESC LIMIT ?",
(session_id, around_message_id, window + 1),
).fetchall()
after_rows = conn.execute(
"SELECT * FROM messages "
"WHERE session_id = ? AND id > ? "
"ORDER BY id ASC LIMIT ?",
(session_id, around_message_id, window),
).fetchall()
# before_rows is DESC; reverse so it's ASC, then concatenate after_rows.
rows = list(reversed(before_rows)) + list(after_rows)
result = []
for row in rows:
msg = dict(row)
if "content" in msg:
msg["content"] = self._decode_content(msg["content"])
if msg.get("tool_calls"):
try:
msg["tool_calls"] = json.loads(msg["tool_calls"])
except (json.JSONDecodeError, TypeError):
logger.warning(
"Failed to deserialize tool_calls in get_messages_around, falling back to []"
)
msg["tool_calls"] = []
if msg.get("display_metadata") is not None:
msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"])
result.append(msg)
# before_rows includes the anchor itself; subtract 1 for the count of
# messages strictly before the anchor in the returned slice.
messages_before = max(0, len(before_rows) - 1)
messages_after = len(after_rows)
return {
"window": result,
"messages_before": messages_before,
"messages_after": messages_after,
}
def resolve_resume_session_id(self, session_id: str) -> str:
"""Redirect a resume target to the descendant session that holds the messages.
Context compression ends the current session and forks a new child session
(linked via ``parent_session_id``). The flush cursor is reset, so the
child is where new messages actually land — the parent ends up with
``message_count = 0`` rows unless messages had already been flushed to
it before compression. See #15000.
This helper walks ``parent_session_id`` forward from ``session_id`` and
returns the descendant in the chain that has the **most recent** messages.
Unlike the original logic, it does NOT short-circuit when the starting
session already has messages — a descendant that was created by
compression may hold the continuation content and should be preferred
by the WebUI and gateway for ``--resume`` and session loading.
If no descendant (including the starting session) has any messages,
the original ``session_id`` is returned unchanged.
The chain is always walked via the child whose ``started_at`` is
latest; that matches the single-chain shape that compression creates.
A depth cap (32) guards against accidental loops in malformed data.
"""
if not session_id:
return session_id
# Follow the compression-continuation chain forward to the live tip
# FIRST. Auto-compression ends the current session and forks a
# continuation child, but a long-lived parent keeps its own flushed
# message rows — so the empty-head walk below never redirects it, and
# resuming the parent id reloads the pre-compression transcript while
# the turns generated *after* compression (and their responses) sit in
# the continuation. ``get_compression_tip`` is lineage-aware: it only
# follows children whose parent ended with ``end_reason='compression'``
# (created after the parent was ended), so delegation / branch children
# never hijack the resume. This is the fix for the desktop "I came back
# and the reply isn't there" report on large sessions.
try:
tip = self.get_compression_tip(session_id)
except Exception:
tip = session_id
if tip and tip != session_id:
session_id = tip
with self._read_ctx() as conn:
current = session_id
seen = {current}
best = None # tracks the last (deepest) node with messages
for _ in range(32):
# Check if the current node has messages.
try:
row = conn.execute(
"SELECT 1 FROM messages WHERE session_id = ? LIMIT 1",
(current,),
).fetchone()
except Exception:
return session_id
if row is not None:
best = current
# Walk to the most-recently-started child — but skip explicit
# branch (`_branched_from`), delegate/subagent (`_delegate_from`),
# reset-continuation (`_reset_from` or the legacy same-key
# heuristic — a post-reset conversation must never be reached
# by resuming the parent the user reset away), and tool
# children. They also carry a ``parent_session_id`` yet
# are NOT compression continuations; following them would hijack
# the resume target to an unrelated session (e.g. a subagent
# run). This mirrors the child-exclusion in ``get_compression_tip``.
try:
child_row = conn.execute(
"SELECT id FROM sessions AS child "
"WHERE child.parent_session_id = ? "
" AND json_extract(COALESCE(child.model_config, '{}'), '$._branched_from') IS NULL "
" AND json_extract(COALESCE(child.model_config, '{}'), '$._delegate_from') IS NULL "
" AND json_extract(COALESCE(child.model_config, '{}'), '$._reset_from') IS NULL "
f" AND NOT {_legacy_reset_child_sql('child', _RESET_END_REASONS_SQL)} "
" AND COALESCE(child.source, '') != 'tool' "
"ORDER BY child.started_at DESC, child.id DESC LIMIT 1",
(current,),
).fetchone()
except Exception:
return session_id
if child_row is None:
break
child_id = child_row["id"] if hasattr(child_row, "keys") else child_row[0]
if not child_id or child_id in seen:
break
seen.add(child_id)
current = child_id
return best if best is not None else session_id
def get_messages_as_conversation(
self,
session_id: str,
include_ancestors: bool = False,
include_inactive: bool = False,
repair_alternation: bool = False,
include_row_ids: bool = False,
include_compacted: bool = False,
) -> List[Dict[str, Any]]:
"""
Load messages in the OpenAI conversation format (role + content dicts).
Used by the gateway to restore conversation history.
By default only active messages are returned. Pass
``include_inactive=True`` to load soft-deleted (rewound) rows
as well. See :meth:`rewind_to_message`.
``include_compacted=True`` additionally loads rows preserved by
in-place compaction (``active=0, compacted=1``), deduped by
:meth:`_dedupe_display_generations`. DISPLAY reads want this; the
model-fed restore must NOT pass it, or a resumed session regrows the
very history compaction just summarized away.
``repair_alternation=True`` runs ``repair_message_sequence`` over the
loaded list before returning it. Callers that restore a session for
LIVE REPLAY should pass it: a durable alternation violation (e.g. a
``user;user`` pair left by a turn that persisted no assistant row)
otherwise re-triggers the pre-request defensive repair on every
single request for the rest of the session's life — the repair
mutates only the per-request list, never the stored transcript.
Inspection/export consumers keep the default and see the transcript
verbatim.
"""
session_ids = [session_id]
if include_ancestors and not self._is_explicit_branch_session(session_id):
session_ids = self._session_lineage_root_to_tip(session_id)
if include_inactive:
active_clause = ""
elif include_compacted:
active_clause = " AND (active = 1 OR compacted = 1)"
else:
active_clause = " AND active = 1"
with self._read_ctx() as conn:
placeholders = ",".join("?" for _ in session_ids)
rows = conn.execute(
f"SELECT {self._CONVERSATION_ROW_COLUMNS} "
f"FROM messages WHERE session_id IN ({placeholders})"
# Order by AUTOINCREMENT id (true insertion order), NOT timestamp:
# append_message stamps rows with time.time(), which is not
# monotonic (WSL2, NTP steps, VM/laptop sleep resume). A later
# row can carry an earlier timestamp than its predecessor, and
# ORDER BY timestamp would then sort an assistant tool_calls row
# after its tool response, breaking tool-call/response adjacency
# and triggering an HTTP 400 on replay. This matches get_messages
# — see c03acca50 for the original fix.
f"{active_clause} ORDER BY id",
tuple(session_ids),
).fetchall()
if include_compacted:
rows = self._dedupe_display_generations(rows)
return self._rows_to_conversation(
rows,
session_id=session_id,
include_ancestors=include_ancestors,
repair_alternation=repair_alternation,
include_row_ids=include_row_ids,
)
# Columns every conversation projection decodes. Shared by
# get_messages_as_conversation and get_resume_conversations so a single
# SELECT can feed both the model-fed and display views. ``active`` rides
# along so a display read can split the compaction-archived rows from the
# live set (and feed _dedupe_display_generations) without a second query.
_CONVERSATION_ROW_COLUMNS = (
"id, role, content, tool_call_id, tool_calls, tool_name, effect_disposition, "
"finish_reason, reasoning, reasoning_content, reasoning_details, "
"codex_reasoning_items, codex_message_items, platform_message_id, observed, "
"_compressed_summary, timestamp, active, "
"api_content, display_kind, display_metadata"
)
def _rows_to_conversation(
self,
rows,
*,
session_id: str,
include_ancestors: bool,
repair_alternation: bool,
include_row_ids: bool = False,
include_summary_markers: bool = False,
) -> List[Dict[str, Any]]:
"""Decode fetched message rows into the OpenAI conversation format.
Extracted from get_messages_as_conversation so get_resume_conversations
can build the model-fed and display views from one SELECT. ``rows`` must
already be ordered by ``id`` (insertion order) and filtered to the
desired session set / active state by the caller.
"""
messages = []
# Watermark rotation column-clones concurrent tail rows into the child
# after the new summary, so the copies need not be adjacent. Index the
# exact durable clone identity while decoding instead of rescanning the
# whole accumulated lineage for every user row.
exact_user_clones: Dict[Tuple[Any, str], Dict[str, Any]] = {}
for row in rows:
content = self._decode_content(row["content"])
if row["role"] in {"user", "assistant"} and isinstance(content, str):
content = sanitize_context(content).strip()
msg = {"role": row["role"], "content": content}
# Born durable (#92231): this dict is materialized FROM a durable
# row, so stamp the persistence marker at the source instead of
# relying on every restore caller to thread the loaded list back
# through a flush as ``conversation_history=`` — any
# identity-losing handoff (compression's durable-snapshot
# adoption, incremental persists with no history arg) would
# otherwise re-append the ENTIRE transcript on flush.
# Underscore-prefixed like ``_row_id``: every transport strips it
# before the wire, and compression's assembly copies deliberately
# strip it so rotated child handoffs still flush (see
# _fresh_compaction_message_copy).
msg[_DB_PERSISTED_MARKER_KEY] = True
# Durable per-message identity for surfaces that need to address a
# specific row later (desktop reactions). OPT-IN: only the gateway
# asks for it — every other consumer (ACP restore, export,
# inspection) gets the transcript in its historical shape.
# Underscore-prefixed so every transport's convert_messages()
# strips it before the wire.
if include_row_ids and row["id"] is not None:
msg["_row_id"] = row["id"]
# api_content is the byte-fidelity sidecar: the exact string sent
# to the API when it differed from the clean content. Returned
# VERBATIM — no sanitize_context, no strip — because the replay
# path substitutes it for content to keep the provider prompt
# cache prefix byte-stable across turns. Cleaning it here would
# re-introduce the divergence it exists to remove.
if row["api_content"]:
msg["api_content"] = row["api_content"]
if row["display_kind"]:
msg["display_kind"] = row["display_kind"]
if row["display_metadata"]:
decoded = self._decode_display_metadata(row["display_metadata"])
if decoded is not None:
msg["display_metadata"] = decoded
if include_summary_markers and row["_compressed_summary"]:
msg["_compressed_summary"] = True
if row["timestamp"]:
msg["timestamp"] = row["timestamp"]
if row["tool_call_id"]:
msg["tool_call_id"] = row["tool_call_id"]
if row["tool_name"]:
msg["tool_name"] = row["tool_name"]
if row["effect_disposition"]:
msg["effect_disposition"] = row["effect_disposition"]
if row["tool_calls"]:
try:
msg["tool_calls"] = json.loads(row["tool_calls"])
except (json.JSONDecodeError, TypeError):
logger.warning("Failed to deserialize tool_calls in conversation replay, falling back to []")
msg["tool_calls"] = []
# Surface the platform-side message id (e.g. yuanbao msg_id,
# telegram update_id) so platform-specific flows like recall
# can match by external identifier instead of having to fall
# back to content-match heuristics. Exposed as ``message_id``
# for backward compatibility with the JSONL transcript shape.
if row["platform_message_id"]:
msg["message_id"] = row["platform_message_id"]
if row["observed"]:
msg["observed"] = True
# Restore reasoning fields on assistant messages so providers
# that replay reasoning (OpenRouter, OpenAI, Nous) receive
# coherent multi-turn reasoning context.
if row["role"] == "assistant":
if row["finish_reason"]:
msg["finish_reason"] = row["finish_reason"]
if row["reasoning"]:
msg["reasoning"] = row["reasoning"]
if row["reasoning_content"] is not None:
msg["reasoning_content"] = row["reasoning_content"]
if row["reasoning_details"]:
try:
msg["reasoning_details"] = json.loads(row["reasoning_details"])
except (json.JSONDecodeError, TypeError):
logger.warning("Failed to deserialize reasoning_details, falling back to None")
msg["reasoning_details"] = None
if row["codex_reasoning_items"]:
try:
msg["codex_reasoning_items"] = json.loads(row["codex_reasoning_items"])
except (json.JSONDecodeError, TypeError):
logger.warning("Failed to deserialize codex_reasoning_items, falling back to None")
msg["codex_reasoning_items"] = None
if row["codex_message_items"]:
try:
msg["codex_message_items"] = json.loads(row["codex_message_items"])
except (json.JSONDecodeError, TypeError):
logger.warning("Failed to deserialize codex_message_items, falling back to None")
msg["codex_message_items"] = None
if include_ancestors:
canonical_content, _is_composite = (
self._canonical_replayed_user_content(msg)
)
exact_clone_key = self._exact_replayed_user_clone_key(
msg.get("timestamp"), canonical_content
)
previous_exact = (
exact_user_clones.get(exact_clone_key)
if exact_clone_key is not None
else None
)
duplicate = None
if previous_exact is not None:
previous_index = next(
(
index
for index, candidate in enumerate(messages)
if candidate is previous_exact
),
None,
)
if previous_index is not None:
duplicate = (previous_index, True)
if duplicate is None:
duplicate = self._find_duplicate_replayed_user_message(
messages, msg
)
if duplicate is not None:
duplicate_index, prefer_current = duplicate
if prefer_current:
# A rotated compression child can carry the same live
# ask as the parent row plus the only surviving summary
# scaffold. Keep the child carrier (and its durable row
# id), not the simpler ancestor copy.
messages.pop(duplicate_index)
else:
continue
messages.append(msg)
if include_ancestors and exact_clone_key is not None:
exact_user_clones[exact_clone_key] = msg
# DEFENSE-IN-DEPTH against background-review session pollution: a forked
# skill/memory review that (in older builds, before the _persist_disabled
# fix) shared the parent's session_id wrote its harness turn into this
# real session. The harness is a user/system message instructing the
# agent to "Review the conversation above and update the skill library /
# save to memory" under a hard tool restriction; re-loading it as live
# history makes the agent adopt the curator role and refuse the user's
# actual task. Strip any such harness message AND the curator-mode
# assistant reply immediately following it, so a polluted session
# resumes clean even if stray rows exist.
messages = _strip_background_review_harness(messages)
# DEFENSE-IN-DEPTH against #78148: before that fix, a bare tool-call
# marker (e.g. "[memory]") could get cached as a fallback and
# persisted as if it were the model's real answer. Sessions written
# before the fix can still carry those rows — clear the stray
# content on load so replaying history doesn't re-teach the model
# to keep emitting the marker. No-op for unaffected sessions.
messages = _strip_stale_tool_call_markers(messages)
if repair_alternation and messages:
# Lazy import: hermes_state already depends on agent.* (see
# sanitize_context above), but keep this optional path from
# widening the import surface at module load.
from agent.agent_runtime_helpers import repair_message_sequence
repaired = repair_message_sequence(None, messages)
if repaired:
logger.info(
"Repaired %d message-alternation violation(s) while "
"restoring session %s — durable transcript kept them, "
"see repair_message_sequence",
repaired,
session_id,
)
return messages
def get_resume_conversations(
self, session_id: str
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
"""Return ``(model_history, display_history)`` for a session resume in ONE SELECT.
``session.resume`` needs two projections of the same lineage:
- ``model_history`` — the tip session's active rows, alternation-repaired
(the live-replay working conversation). Equivalent to
``get_messages_as_conversation(session_id, repair_alternation=True)``.
- ``display_history`` — the full compression lineage (ancestors → tip),
verbatim, with replayed-user dedup. Explicit ``/branch`` sessions are
excluded from this lineage because their own rows already contain the
copied transcript; including the live parent's rows would let messages
written to the original after the fork leak into the branch.
The display projection also includes rows preserved by IN-PLACE
compaction (``active=0, compacted=1``), deduped by
:meth:`_dedupe_display_generations`. Without them a compacted
conversation resumes showing only its summary plus the carried-forward
tail — the user's own turns read as deleted even though every row is
still on disk, and the REST transcript read (which has always included
them) disagreed with this one about the same session (#92080).
The display fetch already reads a superset of the model fetch (the tip
rows are part of the lineage), so serving both from one lineage SELECT
halves the resume's DB work versus two separate calls, with byte-identical
output (see test_get_resume_conversations_matches_separate_reads).
"""
session_ids = self._resume_lineage_ids(session_id)
with self._read_ctx() as conn:
placeholders = ",".join("?" for _ in session_ids)
rows = conn.execute(
f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} "
f"FROM messages WHERE session_id IN ({placeholders}) "
# Compaction-archived rows (active=0, compacted=1) are display
# history; Undo/Rewind rows (active=0, compacted=0) are not.
"AND (active = 1 OR compacted = 1) "
# ORDER BY id (insertion order) — see get_messages_as_conversation
# for why timestamp ordering is unsafe.
"ORDER BY id",
tuple(session_ids),
).fetchall()
# Tip rows are exactly the model-fed set (get_messages_as_conversation
# with session_ids=[session_id]); filtering the lineage fetch preserves
# their relative id order. The model projection stays active-only — it
# is the compressed working context and must not regrow the history
# compaction just summarized away.
tip_rows = [r for r in rows if r["session_id"] == session_id and r["active"]]
model_history = self._rows_to_conversation(
tip_rows,
session_id=session_id,
include_ancestors=False,
repair_alternation=True,
include_row_ids=True,
# Pre-compress checkpointing: the resumed model history must keep
# the summary marker so checkpoint providers can exclude derivative
# summaries after a process restart (marker survives restart).
include_summary_markers=True,
)
display_history = self._rows_to_conversation(
self._dedupe_display_generations(rows),
session_id=session_id,
include_ancestors=True,
repair_alternation=False,
include_row_ids=True,
)
return model_history, display_history
def _resume_lineage_ids(self, session_id: str) -> List[str]:
"""Session ids a full (display) resume materializes for *session_id*.
Compression continuations need their ended ancestors' rows for the
display transcript; an explicit ``/branch`` copy already owns its
transcript, so its lineage is itself alone. This is the ONE definition
shared by the resume readers (``get_resume_conversations``,
``get_ancestor_display_prefix``) and the resume guard
(``assert_resume_safe`` / ``get_resume_message_count``) — the guard must
count exactly the rows a resume would load, never a superset.
"""
if self._is_explicit_branch_session(session_id):
return [session_id]
return self._session_lineage_root_to_tip(session_id)
def get_resume_message_count(
self, session_id: str, *, tip_only: bool = False
) -> int:
"""Count the rows that a resume would materialize.
``tip_only=True`` counts the tip segment's ACTIVE rows — the set a
model-history restore loads (``get_messages_as_conversation`` without
ancestors, or the deferred Desktop resume that pages the display
transcript over REST and never materializes the ancestor prefix in
memory).
Otherwise this counts the full-lineage DISPLAY set — active rows plus
the compaction-archived rows ``get_resume_conversations`` now loads
for the transcript. Counting only active rows here would let a
heavily-compacted conversation pass a limit sized for a handful of
live rows and then materialize tens of thousands.
"""
session_ids = [session_id] if tip_only else self._resume_lineage_ids(session_id)
active_clause = "active = 1" if tip_only else "(active = 1 OR compacted = 1)"
placeholders = ",".join("?" for _ in session_ids)
with self._read_ctx() as conn:
row = conn.execute(
f"SELECT COUNT(*) FROM messages "
f"WHERE session_id IN ({placeholders}) AND {active_clause}",
tuple(session_ids),
).fetchone()
return int(row[0] if row else 0)
def assert_resume_safe(
self,
session_id: str,
max_messages: Optional[int] = None,
*,
tip_only: bool = False,
) -> int:
"""Return resume row count or reject a transcript too large to load.
``max_messages=None`` resolves the limit from config
(``sessions.max_resume_messages``); 0 disables the guard and returns
the (bounded) count without raising.
``tip_only=True`` bounds only the tip segment's ACTIVE rows, for
callers that never materialize the ancestor lineage or the
compaction archive in memory (tip-only model restore, deferred
Desktop resume whose display history is REST-paginated). A
heavily-compressed conversation — 85 compaction segments and ~29k
lineage rows behind a ~700-row tip — is exactly the shape compression
is supposed to produce; counting its whole lineage against a limit
sized for in-memory materialization rejected the healthiest sessions
(Desktop Bot Chat stuck on "Waking up…" with code 4130) while the
process would only ever have held the tip.
The full (non-``tip_only``) bound counts the DISPLAY set — active plus
compaction-archived rows — because that is what
``get_resume_conversations`` materializes for the transcript.
"""
if max_messages is None:
max_messages = resolved_max_resume_messages()
if max_messages < 0:
raise ValueError("max_messages must be non-negative")
if max_messages == 0:
# Guard disabled by config — skip counting entirely. Every live
# caller invokes this for its raise side effect and ignores the
# return value, and an unbounded lineage COUNT here would do the
# exact pathological work the disable exists to avoid.
return 0
session_ids = [session_id] if tip_only else self._resume_lineage_ids(session_id)
active_clause = "active = 1" if tip_only else "(active = 1 OR compacted = 1)"
placeholders = ",".join("?" for _ in session_ids)
with self._read_ctx() as conn:
row = conn.execute(
"SELECT COUNT(*) FROM ("
f"SELECT 1 FROM messages WHERE session_id IN ({placeholders}) "
f"AND {active_clause} LIMIT ?"
")",
(*session_ids, max_messages + 1),
).fetchone()
message_count = int(row[0] if row else 0)
if message_count > max_messages:
raise SessionResumeTooLargeError(
message_count,
max_messages,
scope="in its tip segment" if tip_only else "across its lineage",
)
return message_count
def assert_export_safe(
self,
session_id: str,
max_messages: Optional[int] = None,
) -> int:
"""Return active row count or reject an unsafe in-memory export.
Exporting one session does not include compression ancestors, so this
guard deliberately counts only the requested segment. The limited
subquery stops as soon as it proves the transcript exceeds the bound.
``max_messages=None`` resolves the limit from config
(``sessions.max_export_messages``); 0 disables the guard and returns
the active row count without raising.
"""
if max_messages is None:
max_messages = resolved_max_export_messages()
if max_messages < 0:
raise ValueError("max_messages must be non-negative")
if max_messages == 0:
# Guard disabled by config — skip the COUNT; live callers use
# this for its raise side effect only (and skip calling it
# entirely when the limit is 0).
return 0
with self._read_ctx() as conn:
row = conn.execute(
"SELECT COUNT(*) FROM ("
"SELECT 1 FROM messages WHERE session_id = ? AND active = 1 LIMIT ?"
")",
(session_id, max_messages + 1),
).fetchone()
message_count = int(row[0] if row else 0)
if message_count > max_messages:
raise SessionExportTooLargeError(session_id, message_count, max_messages)
return message_count
def get_ancestor_display_prefix(self, session_id: str) -> List[Dict[str, Any]]:
"""Return the ancestor-only display messages for a session lineage.
These are messages from parent/grandparent sessions (compression
ancestors) that appear in the display transcript but NOT in the
tip session's model-fed history. Used by ``session.resume`` to
build the ``display_history_prefix`` that ``_live_session_payload``
prepends to the live model history.
Previously the prefix was calculated as
``display_history[:len(display) - len(raw)]``, but that overcounts
when ``repair_message_sequence`` removes messages from the MIDDLE
of the tip history (e.g. verification candidates collapsed by the
consecutive-assistant merge) — the length difference includes both
ancestor messages AND repair-removed tip messages, but the slice
only captures the first N display messages (which are tip messages
when there are no ancestors), causing duplication. This method
returns ONLY the genuine ancestor messages, identified by
``session_id != tip_session_id``. (#65919)
"""
session_ids = self._resume_lineage_ids(session_id)
if len(session_ids) <= 1:
return []
with self._read_ctx() as conn:
placeholders = ",".join("?" for _ in session_ids)
rows = conn.execute(
f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} "
f"FROM messages WHERE session_id IN ({placeholders}) "
# Display read: compaction-archived rows included, Undo/Rewind
# rows excluded (see get_resume_conversations).
"AND (active = 1 OR compacted = 1) "
"ORDER BY id",
tuple(session_ids),
).fetchall()
rows = self._dedupe_display_generations(rows)
ancestor_ids = {
int(row["id"])
for row in rows
if row["session_id"] != session_id and row["id"] is not None
}
if not ancestor_ids:
return []
lineage = self._rows_to_conversation(
rows,
session_id=session_id,
include_ancestors=True,
repair_alternation=False,
include_row_ids=True,
)
prefix: List[Dict[str, Any]] = []
for message in lineage:
if message.get("_row_id") not in ancestor_ids:
continue
projected = message.copy()
projected.pop("_row_id", None)
prefix.append(projected)
return prefix
def _is_explicit_branch_session(self, session_id: str) -> bool:
"""Return whether *session_id* is a copied user-facing branch.
Branches and compression continuations both use ``parent_session_id``,
but they have different history semantics: a branch owns a copied
transcript, while a compression continuation needs its ended parent's
archived rows for display. The durable ``_branched_from`` marker is the
existing discriminator written by all branch creation paths.
"""
if not session_id:
return False
with self._read_ctx() as conn:
row = conn.execute(
"SELECT model_config FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
return False
raw_config = row["model_config"] if hasattr(row, "keys") else row[0]
if not raw_config:
return False
try:
config = json.loads(raw_config) if isinstance(raw_config, str) else raw_config
except (json.JSONDecodeError, TypeError):
return False
return isinstance(config, dict) and bool(config.get("_branched_from"))
def get_conversation_root(self, session_id: str) -> str:
"""Return the ROOT id of *session_id*'s lineage chain.
The root is the stable "conversation id": context compression
rotates ``session_id`` to a new segment linked via
``parent_session_id``, and delegate subagents hang off their
parent the same way. Walking to the root gives every segment of
one user-facing conversation (and its delegation tree) a single
identifier — used for Nous Portal ``conversation=`` usage tagging.
Returns *session_id* unchanged when it has no recorded parent.
"""
chain = self._session_lineage_root_to_tip(session_id)
return (chain[0] if chain and chain[0] else session_id)
def _session_lineage_root_to_tip(self, session_id: str) -> List[str]:
if not session_id:
return [session_id]
chain = []
current = session_id
seen = set()
with self._read_ctx() as conn:
for _ in range(100):
if not current or current in seen:
break
seen.add(current)
chain.append(current)
row = conn.execute(
"SELECT parent_session_id FROM sessions WHERE id = ?",
(current,),
).fetchone()
if row is None:
break
current = row["parent_session_id"] if hasattr(row, "keys") else row[0]
return list(reversed(chain)) or [session_id]
@staticmethod
def _canonical_replayed_user_content(
msg: Dict[str, Any],
) -> Tuple[Any, bool]:
"""Return canonical live content and whether *msg* is composite."""
if msg.get("role") != "user":
return None, False
from agent.context_compressor import split_user_originated_turn
handoff, live_view = split_user_originated_turn(msg)
is_composite = handoff is not None and live_view is not None
return (
live_view.get("content")
if is_composite and live_view is not None
else msg.get("content"),
is_composite,
)
@staticmethod
def _exact_replayed_user_clone_key(
timestamp: Any, content: Any
) -> Optional[Tuple[Any, str]]:
"""Return a hashable key for a column-exact rotation clone."""
if timestamp is None or content in (None, "", []):
return None
try:
encoded = json.dumps(
content,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
except (TypeError, ValueError):
return None
return timestamp, encoded
@staticmethod
def _find_duplicate_replayed_user_message(
messages: List[Dict[str, Any]], msg: Dict[str, Any]
) -> Optional[Tuple[int, bool]]:
"""Return an adjacent replay duplicate and whether *msg* must win.
Compression rotation may persist the current ask once in the parent
and again inside a composite child carrier. Compare the canonical live
payload for that carrier, while retaining the historical exact-string
dedupe for ordinary replayed users. The child carrier wins because it
owns both the current durable row identity and the retained scaffold.
"""
if msg.get("role") != "user":
return None
content, prefer_current = SessionDB._canonical_replayed_user_content(msg)
if content in (None, "", []):
return None
for index in range(len(messages) - 1, -1, -1):
prev = messages[index]
if prev.get("role") == "user":
prev_content, prev_is_composite = (
SessionDB._canonical_replayed_user_content(prev)
)
if prev_content == content and (
prefer_current
or prev_is_composite
or isinstance(content, str)
):
return index, prefer_current
if prev.get("role") == "assistant" and (prev.get("content") or prev.get("tool_calls")):
return None
return None
@staticmethod
def _is_duplicate_replayed_user_message(
messages: List[Dict[str, Any]], msg: Dict[str, Any]
) -> bool:
return SessionDB._find_duplicate_replayed_user_message(messages, msg) is not None
# =========================================================================
# Rewind (soft-delete) — see /rewind slash command + issue #21910
# =========================================================================
def get_active_message_ids(self, session_id: str) -> List[int]:
"""Return the ordered physical ids pinned by rewind CAS checks.
Conversation projections intentionally omit legacy background-review
harness rows. Destructive rewinds must nevertheless pin every active
physical row so the caller snapshot matches the transaction-local
comparison in :meth:`rewind_to_message`.
"""
with self._read_ctx() as conn:
rows = conn.execute(
"SELECT id FROM messages "
"WHERE session_id = ? AND active = 1 ORDER BY id",
(session_id,),
).fetchall()
return [int(row[0]) for row in rows]
@staticmethod
def _active_transcript_counts(conn, session_id: str) -> tuple[int, int]:
"""Return active message/tool-call counts inside the caller's txn."""
rows = conn.execute(
"SELECT tool_calls FROM messages "
"WHERE session_id = ? AND active = 1",
(session_id,),
).fetchall()
tool_call_count = 0
for row in rows:
raw = row[0]
if not raw:
continue
try:
decoded = json.loads(raw) if isinstance(raw, str) else raw
except (json.JSONDecodeError, TypeError):
continue
if isinstance(decoded, list):
tool_call_count += len(decoded)
elif decoded:
tool_call_count += 1
return len(rows), tool_call_count
def rewind_to_message(
self,
session_id: str,
target_message_id: int,
*,
preserve_compaction_handoff: bool = False,
expected_active_ids: Optional[List[int]] = None,
expected_target_content: Any = None,
) -> Dict[str, Any]:
"""Soft-delete all messages with id >= ``target_message_id`` in *session_id*.
The target message itself becomes inactive as well so the caller
can pre-fill it as the next user prompt without it appearing
twice in the replayed transcript. Rewound rows are kept on
disk with ``active=0`` for audit / forensic inspection — use
:meth:`get_messages` with ``include_inactive=True`` to see them.
Returns a dict::
{
"rewound_count": int, # number of rows newly flipped to active=0
"target_message": dict, # full row dict of the target
"new_head_id": int|None # id of the last still-active row, or None
}
Raises ``ValueError`` if the target message does not exist in
*session_id* or if its role is not ``"user"``. With
``preserve_compaction_handoff=True``, a composite summary carrier is
split inside the same write transaction: its original row is archived
and its canonical hidden handoff scaffold is inserted as the new head.
That opt-in result also contains ``replacement_message_id``.
``expected_active_ids`` optionally pins the ordered active row set.
``expected_target_content`` additionally pins the selected canonical
live-user payload. Both checks run inside the write transaction before
any row or counter mutation. Presentation-only metadata changes (for
example Desktop reactions) deliberately do not invalidate a rewind.
A live cross-process turn lease always refuses the rewind; expired or
provably dead holders are reclaimed inside the mutation transaction.
Always increments ``sessions.rewind_count`` — even when the
target is already inactive — so the counter accurately reflects
the number of rewind operations performed against the session.
Idempotent on the ``active`` flag: re-rewinding past the same
target is a no-op on row state but still bumps the counter.
"""
def _do(conn):
# Rewind changes the active transcript and must honor the same
# compression/closed-parent and cross-process turn guards as
# append writers.
self._check_transcript_write_guards(
conn,
session_id,
None,
reject_active_turn_lease=True,
reject_active_compression_lock=True,
)
if expected_active_ids is not None:
active_rows = conn.execute(
"SELECT id FROM messages "
"WHERE session_id = ? AND active = 1 ORDER BY id",
(session_id,),
).fetchall()
active_ids = [int(active_row[0]) for active_row in active_rows]
if active_ids != expected_active_ids:
raise RuntimeError(
"active transcript changed before the rewind could be persisted"
)
row = conn.execute(
"SELECT * FROM messages WHERE id = ? AND session_id = ?",
(target_message_id, session_id),
).fetchone()
if row is None:
raise ValueError(
f"message {target_message_id} not found in session {session_id}"
)
target_row = dict(row)
if target_row.get("role") != "user":
raise ValueError(
f"rewind target must be a 'user' message (got role="
f"{target_row.get('role')!r}, id={target_message_id})"
)
replacement_message_id: Optional[int] = None
replacement: Optional[Dict[str, Any]] = None
if preserve_compaction_handoff or expected_target_content is not None:
if not target_row.get("active"):
raise ValueError("rewind target is not active")
from agent.context_compressor import split_user_originated_turn
split_target = target_row.copy()
split_target["content"] = self._decode_content(
split_target.get("content")
)
split_target["display_metadata"] = self._decode_display_metadata(
split_target.get("display_metadata")
)
handoff, live_view = split_user_originated_turn(split_target)
if live_view is None:
raise ValueError("rewind target is not a user-originated turn")
live_content = live_view.get("content")
if isinstance(live_content, str):
live_content = sanitize_context(live_content).strip()
if (
expected_target_content is not None
and live_content != expected_target_content
):
raise RuntimeError(
"rewind target changed before it could be persisted"
)
if preserve_compaction_handoff and handoff is None:
raise ValueError(
"preserve_compaction_handoff requires an active composite carrier"
)
replacement = handoff if preserve_compaction_handoff else None
cursor = conn.execute(
"SELECT id FROM messages "
"WHERE session_id = ? AND id >= ? AND active = 1",
(session_id, target_message_id),
)
ids = [r[0] for r in cursor.fetchall()]
if ids:
placeholders = ",".join("?" for _ in ids)
conn.execute(
f"UPDATE messages SET active = 0 WHERE id IN ({placeholders})",
ids,
)
if replacement is not None:
self._insert_message_rows(conn, session_id, [replacement])
inserted = conn.execute("SELECT last_insert_rowid()").fetchone()
replacement_message_id = int(inserted[0])
conn.execute(
"UPDATE sessions SET rewind_count = COALESCE(rewind_count, 0) + 1 "
"WHERE id = ?",
(session_id,),
)
message_count, tool_call_count = self._active_transcript_counts(
conn, session_id
)
conn.execute(
"UPDATE sessions SET message_count = ?, tool_call_count = ? "
"WHERE id = ?",
(message_count, tool_call_count, session_id),
)
head_row = conn.execute(
"SELECT MAX(id) FROM messages WHERE session_id = ? AND active = 1",
(session_id,),
).fetchone()
new_head_id = (
head_row[0] if head_row and head_row[0] is not None else None
)
return target_row, ids, new_head_id, replacement_message_id
target_row, rewound, new_head_id, replacement_message_id = (
self._execute_write(_do)
)
# Decode content for callers (prefill the prompt buffer) without a
# second fallible database operation after the transaction commits.
target_row["content"] = self._decode_content(target_row.get("content"))
result = {
"rewound_count": len(rewound),
"target_message": target_row,
"new_head_id": new_head_id,
}
if preserve_compaction_handoff:
result["replacement_message_id"] = replacement_message_id
return result
def restore_rewound(self, session_id: str, since_message_id: int) -> int:
"""Mark inactive messages with id >= *since_message_id* active again.
Returns the number of rows flipped back to ``active=1``.
Intended for undo-of-rewind and test cleanup; not wired to a
slash command in v1.
"""
def _do(conn):
cursor = conn.execute(
"SELECT id FROM messages "
"WHERE session_id = ? AND id >= ? AND active = 0",
(session_id, since_message_id),
)
ids = [r[0] for r in cursor.fetchall()]
if ids:
placeholders = ",".join("?" for _ in ids)
conn.execute(
f"UPDATE messages SET active = 1 WHERE id IN ({placeholders})",
ids,
)
return len(ids)
return self._execute_write(_do)
# =========================================================================
# Search
# =========================================================================
def search_sessions(
self,
source: str = None,
limit: int = 20,
offset: int = 0,
workspace_key: str = None,
) -> List[Dict[str, Any]]:
"""List sessions, optionally filtered by source.
Returns rows enriched with a computed ``last_active`` column
(freshest of ``last_activity_at`` and latest message timestamp,
else ``started_at``), ordered by most-recently-used first.
Pass ``workspace_key`` to scope rows to one workspace - matching
:func:`workspace_key` semantics (git repo root, else cwd). Used by
``hermes -c``/``--resume`` so the "last" session is the last one in
the *current* workspace, not the global MRU.
"""
select_with_last_active = (
"SELECT s.*, "
"COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved, "
f"{_sql_session_last_active('s')} AS last_active "
"FROM sessions s "
"LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash "
)
where_clauses = []
params: list = []
if source:
where_clauses.append("s.source = ?")
params.append(source)
if workspace_key:
ws_clause, ws_params = _workspace_key_clause(workspace_key)
where_clauses.append(ws_clause)
params.extend(ws_params)
where_sql = f" WHERE {' AND '.join(where_clauses)}" if where_clauses else ""
params.extend([limit, offset])
with self._read_ctx() as conn:
cursor = conn.execute(
f"{select_with_last_active}"
f"{where_sql} "
"ORDER BY last_active DESC, s.started_at DESC, s.id DESC LIMIT ? OFFSET ?",
params,
)
return [self._session_row_dict(row) for row in cursor.fetchall()]
# =========================================================================
# Utility
# =========================================================================
def session_count(
self,
source: str = None,
sources: List[str] = None,
cwd_prefix: str = None,
min_message_count: int = 0,
include_archived: bool = False,
archived_only: bool = False,
exclude_children: bool = False,
exclude_sources: List[str] = None,
) -> int:
"""Count sessions, optionally filtered by source.
Pass ``exclude_children=True`` to count only the conversations that
``list_sessions_rich`` surfaces (root + branch/reset sessions), hiding
sub-agent runs and compression continuations. Use it whenever the count
is paired with a ``list_sessions_rich`` page (e.g. sidebar "load more"
totals) so the total matches the number of listable rows — otherwise the
raw row count is inflated by children and "load more" never settles.
Pass ``exclude_sources`` to drop whole source classes from the count
(e.g. ``["cron"]`` so the recents "load more" total matches a
cron-excluded ``list_sessions_rich`` page and doesn't keep "load more"
stuck on for buried scheduler sessions).
"""
where_clauses = []
params = []
if exclude_children:
# Mirror list_sessions_rich's child-exclusion clause exactly so the
# count lines up with the rows: roots plus user-visible branch/reset
# children.
where_clauses.append(_LISTABLE_CHILD_SQL)
where_clauses.append(f"{_delegate_from_json('s.model_config')} IS NULL")
include_sources = [source] if source else list(sources or [])
if include_sources:
placeholders = ",".join("?" for _ in include_sources)
where_clauses.append(f"s.source IN ({placeholders})")
params.extend(include_sources)
if exclude_sources:
placeholders = ",".join("?" for _ in exclude_sources)
where_clauses.append(f"s.source NOT IN ({placeholders})")
params.extend(exclude_sources)
if cwd_prefix:
clause, clause_params = _cwd_prefix_clause(cwd_prefix)
where_clauses.append(clause)
params.extend(clause_params)
if min_message_count > 0:
where_clauses.append("s.message_count >= ?")
params.append(min_message_count)
if archived_only:
where_clauses.append("s.archived = 1")
elif not include_archived:
where_clauses.append("s.archived = 0")
where_sql = f" WHERE {' AND '.join(where_clauses)}" if where_clauses else ""
with self._read_ctx() as conn:
cursor = conn.execute(f"SELECT COUNT(*) FROM sessions s{where_sql}", params)
return cursor.fetchone()[0]
def session_count_ge(self, n: int = 1) -> bool:
"""Check if at least N sessions exist (archived included).
Short-circuits via LIMIT — much cheaper than ``session_count()``,
which pays a full index scan for its default ``archived = 0``
filter (measured 543us vs 4us on a 20k-session DB). Archived
sessions count: every caller so far asks "has this install ever
had sessions", and an archived session is still a created one.
Use this instead of ``session_count() >= n`` when the exact count
is irrelevant.
"""
with self._read_ctx() as conn:
cursor = conn.execute("SELECT 1 FROM sessions LIMIT ?", (n,))
rows = cursor.fetchall()
return len(rows) >= n
def session_count_by_source(
self,
*,
include_archived: bool = False,
archived_only: bool = False,
exclude_children: bool = False,
) -> Dict[str, int]:
"""Return a ``{source: count}`` dict via a single ``GROUP BY`` query.
Replaces the O(N) ``list_sessions_rich`` histogram loop with an
aggregate query. When ``exclude_children`` is False the query uses
``idx_sessions_source``; when True, the child-exclusion predicates
require a full table scan (same as ``session_count`` and
``list_sessions_rich``).
``exclude_children=True`` mirrors ``list_sessions_rich`` visibility
(roots + branch/reset sessions, excluding sub-agent runs, delegates,
and compression continuations) so the source counts match what the
Sessions page actually lists.
"""
where_clauses = []
params: list = []
if exclude_children:
where_clauses.append(_LISTABLE_CHILD_SQL)
where_clauses.append(f"{_delegate_from_json('s.model_config')} IS NULL")
if archived_only:
where_clauses.append("s.archived = 1")
elif not include_archived:
where_clauses.append("s.archived = 0")
where_sql = f" WHERE {' AND '.join(where_clauses)}" if where_clauses else ""
with self._read_ctx() as conn:
if self._conn is None:
raise RuntimeError("SessionDB connection is closed")
rows = conn.execute(
"SELECT COALESCE(NULLIF(s.source, ''), 'cli') AS source, COUNT(*) AS count "
f"FROM sessions s{where_sql} "
"GROUP BY COALESCE(NULLIF(s.source, ''), 'cli') "
"ORDER BY count DESC",
params,
).fetchall()
return {str(row["source"]): int(row["count"] or 0) for row in rows}
def message_count(self, session_id: str = None) -> int:
"""Count messages, optionally for a specific session."""
with self._read_ctx() as conn:
if session_id:
cursor = conn.execute(
"SELECT COUNT(*) FROM messages WHERE session_id = ?", (session_id,)
)
else:
cursor = conn.execute("SELECT COUNT(*) FROM messages")
return cursor.fetchone()[0]
def has_platform_message_id(
self, session_id: str, platform_message_id: str
) -> bool:
"""Check if a message with the given platform_message_id exists.
Uses the idx_messages_platform_msg_id partial index for efficient
lookup. Used by the gateway's transient-failure dedupe guard (#47237)
to skip re-persisting a user message that was already saved on a
prior retry of the same inbound platform message.
"""
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT 1 FROM messages "
"WHERE session_id = ? AND platform_message_id = ? LIMIT 1",
(session_id, platform_message_id),
)
return cursor.fetchone() is not None
# =========================================================================
# Export and cleanup
# =========================================================================
def _is_explicit_fork_child_row(self, session: Dict[str, Any]) -> bool:
"""True when ``session`` is a branch, delegate, or tool child of its parent.
Markers only count as a fork when they point at ``parent_session_id``.
Compression copies ``model_config`` onto the continuation
(``publish_compression_child`` callers pass
``agent._session_init_model_config``), so a delegate's continuation
carries ``_delegate_from=<the delegate's own parent>``. Presence-only
matching would treat that real continuation as a fork — the same
misclassification ``_NON_CONTINUATION_CHILD_FILTER_SQL`` already
avoids by binding both markers to the queried parent.
"""
if session.get("source") == "tool":
return True
raw = session.get("model_config")
if not raw:
return False
try:
cfg = json.loads(raw) if isinstance(raw, str) else raw
except (TypeError, json.JSONDecodeError):
return False
if not isinstance(cfg, dict):
return False
parent_id = session.get("parent_session_id")
branched = cfg.get("_branched_from")
delegated = cfg.get("_delegate_from")
if parent_id:
return branched == parent_id or delegated == parent_id
return branched is not None or delegated is not None
def is_explicit_fork_child(self, session_id: str) -> bool:
"""True when ``session_id`` is a /branch, delegate, or tool child row.
Read-only public view of :meth:`_is_explicit_fork_child_row` for
callers that must respect the fork boundary without re-implementing
its marker rules (``agent/prompt_cache_scope.py`` keeps a declared
conversation key from crossing it). A missing row is not a fork.
"""
session = self.get_session(session_id)
return bool(session and self._is_explicit_fork_child_row(session))
def declared_scope_identity(self, session_id: str) -> Tuple[bool, str]:
"""Fork verdict and recorded ``source`` for *session_id*, in ONE read.
``agent/prompt_cache_scope.py`` needs both to resolve a host-declared
conversation scope, and both live on the same ``sessions`` row; asking
for them separately read that row twice per resolution (@teknium1 on
#98811). The marker rules stay here, beside
:meth:`is_explicit_fork_child`, instead of being re-implemented by the
caller.
A missing row is not a fork and has no source, which is the right
answer before ``_ensure_db_session`` persists it. This raises whatever
:meth:`get_session` raises: the caller fails closed on a DB error, and
merging the two reads cannot weaken that, because the fork check was
already the first of the two.
"""
session = self.get_session(session_id)
if not session:
return False, ""
return (
self._is_explicit_fork_child_row(session),
str(session.get("source") or "").strip(),
)
def latest_conversation_boundary(
self, session_key: str, source: str
) -> Optional[int]:
"""How many conversation boundaries this routing peer has crossed.
A boundary is a row this peer ended at an intentional conversation
break — the ``_RESET_END_REASONS`` set (``/new``, ``/switch``, idle,
daily, suspended, resume_pending_expired). That is the same fence
:meth:`find_latest_gateway_session_for_peer` refuses to reach behind,
so the two agree on where one conversation stops and the next begins
and cannot drift.
The peer is ``(session_key, source)``, the SAME identity tuple recovery
uses — never the key alone. ``X-Hermes-Session-Key`` accepts any
authenticated caller-supplied string, so an API conversation may
legally carry the same key as a Telegram row in one database; keying
on the string alone would let a ``/new`` on that unrelated row rotate
this conversation's affinity identity while recovery correctly refuses
to cross the same line.
Returns the count, or ``None`` when this peer has never been reset.
The value comes from ``conversation_generations``, which
:meth:`_bump_conversation_generation` advances inside the transaction
that writes each boundary — NOT from an aggregate over the session
rows. An aggregate cannot prove non-reuse: ``delete_session()``
orphans children and deletes the row, and bulk prune selects ended
rows, so ``COUNT``/``MAX`` over boundaries can return a pair it already
emitted and hand a new conversation a retired affinity identity. It is
also wall-clock-free, so a backwards NTP correction cannot reorder it.
Databases upgraded mid-conversation start at no generation and take
their first one from the next boundary written; a conversation that
reset before the upgrade shares its predecessor's scope once, which
costs a warm prompt-cache bucket and never crosses an identity.
These rows are never garbage-collected, by design: dropping one resets
the peer to "no generation", so its next boundary writes ``1`` again
and re-issues a scope a retired conversation already used — the ABA
this counter exists to prevent. See the schema comment in
``hermes_state_common.py``.
"""
if not session_key or not source:
return None
with self._read_ctx() as conn:
row = conn.execute(
"SELECT generation FROM conversation_generations "
"WHERE source = ? AND session_key = ?",
(source, session_key),
).fetchone()
if row is None or row["generation"] is None:
return None
generation = int(row["generation"])
return generation if generation > 0 else None
def _is_compression_child_row(self, child: Dict[str, Any]) -> bool:
parent_id = child.get("parent_session_id")
if not parent_id or self._is_explicit_fork_child_row(child):
return False
parent = self.get_session(parent_id)
return bool(parent and parent.get("end_reason") == "compression")
def get_compression_lineage(self, session_id: str) -> List[str]:
"""Return compression ancestors through tip in chronological order."""
session = self.get_session(session_id)
if not session or self._is_explicit_fork_child_row(session):
return [session_id] if session else []
root = session
ancestors = {root["id"]}
while self._is_compression_child_row(root):
parent = self.get_session(root["parent_session_id"])
if not parent or parent["id"] in ancestors:
break
root = parent
ancestors.add(root["id"])
lineage = [root["id"]]
seen = {root["id"]}
current = root
while current.get("end_reason") == "compression":
with self._read_ctx() as conn:
rows = conn.execute(
"""
SELECT * FROM sessions
WHERE parent_session_id = ?
ORDER BY started_at ASC
""",
(current["id"],),
).fetchall()
next_child = None
for row in rows:
candidate = dict(row)
if self._is_compression_child_row(candidate):
next_child = candidate
break
if not next_child or next_child["id"] in seen:
break
lineage.append(next_child["id"])
seen.add(next_child["id"])
current = next_child
if current["id"] == session_id:
# Continue to include later compression tips only when the
# requested session itself was compacted.
continue
return lineage if session_id in lineage else [session_id]
def clear_messages(self, session_id: str) -> None:
"""Delete all messages for a session and reset its counters."""
def _do(conn):
conn.execute(
"DELETE FROM messages WHERE session_id = ?", (session_id,)
)
conn.execute(
"UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?",
(session_id,),
)
self._execute_write(_do)
@staticmethod
def _remove_session_files(sessions_dir: Optional[Path], session_id: str) -> None:
"""Remove on-disk transcript files for a session.
Cleans up ``{session_id}.json``, ``{session_id}.jsonl``, and any
``request_dump_{session_id}_*.json`` files left by the gateway.
Silently skips files that don't exist and swallows OSError so a
filesystem hiccup never blocks a DB operation.
"""
if sessions_dir is None:
return
for suffix in (".json", ".jsonl"):
p = sessions_dir / f"{session_id}{suffix}"
try:
p.unlink(missing_ok=True)
except OSError:
pass
# request_dump files use session_id as a prefix component
try:
for p in sessions_dir.glob(f"request_dump_{session_id}_*.json"):
try:
p.unlink(missing_ok=True)
except OSError:
pass
except OSError:
pass
def get_session_delete_targets(self, session_id: str) -> List[str]:
"""Return every session row that :meth:`delete_session` would remove.
The requested session is first, followed by its recursively discovered
delegate/subagent children. Branch and compression children are not
included because deletion preserves them by orphaning their parent
reference.
"""
with self._read_ctx() as conn:
exists = conn.execute(
"SELECT 1 FROM sessions WHERE id = ? LIMIT 1", (session_id,)
).fetchone()
if not exists:
return []
# Use the borrowed read connection, never self._conn: handing the
# shared writer connection to a helper here executes on it without
# self._lock — the same unsynchronized-read class as #99349/#90734.
delegate_ids = _collect_delegate_child_ids(conn, [session_id])
return [session_id, *sorted(delegate_ids)]
def delete_session(
self,
session_id: str,
sessions_dir: Optional[Path] = None,
expected_delete_ids: Optional[List[str]] = None,
) -> bool:
"""Delete a session and all its messages.
Delegate subagent children (``model_config._delegate_from``) are
cascade-deleted with the parent so they never resurface in session
pickers as orphaned rows. Branch / compression children are orphaned
(``parent_session_id → NULL``) so they remain accessible independently.
When *sessions_dir* is provided, also removes on-disk transcript
files (``.json`` / ``.jsonl`` / ``request_dump_*``) for every deleted
session. When *expected_delete_ids* is provided, deletion proceeds only
if the parent plus delegate cascade still matches that exact set. This
lets export-before-delete callers fail closed if a new delegate appears
after they materialize their archive. The delegate tree is re-walked
inside the write transaction on purpose (TOCTOU guard); the cost is
accepted for correctness. Returns True if the session was found and
deleted.
"""
removed_delegate_ids: List[str] = []
expected_ids = (
set(expected_delete_ids) if expected_delete_ids is not None else None
)
def _do(conn):
cursor = conn.execute(
"SELECT 1 FROM sessions WHERE id = ? LIMIT 1", (session_id,)
)
if cursor.fetchone() is None:
return False
if expected_ids is not None:
actual_ids = {
session_id,
*_collect_delegate_child_ids(conn, [session_id]),
}
if actual_ids != expected_ids:
return False
removed_delegate_ids.extend(_delete_delegate_children(conn, [session_id]))
# Orphan remaining child sessions (branches, etc.) so FK is satisfied.
conn.execute(
"UPDATE sessions SET parent_session_id = NULL "
"WHERE parent_session_id = ?",
(session_id,),
)
conn.execute("DELETE FROM messages WHERE session_id = ?", (session_id,))
conn.execute("DELETE FROM sessions WHERE id = ?", (session_id,))
self._delete_unreferenced_system_prompts(conn)
return True
deleted = self._execute_write(_do)
if deleted:
for delegate_id in removed_delegate_ids:
self._remove_session_files(sessions_dir, delegate_id)
self._remove_session_files(sessions_dir, session_id)
return bool(deleted)
def delete_session_if_empty(
self,
session_id: str,
sessions_dir: Optional[Path] = None,
) -> bool:
"""Delete *session_id* only when it never gained resumable content.
A session is considered empty when it has no messages and no
user-assigned title. Used by CLI exit / session-rotation paths so
immediately-started-and-quit sessions don't pile up in ``/resume``
and ``hermes sessions list`` output. (Pattern ported from
google-gemini/gemini-cli#27770.)
The emptiness check and delete run in one transaction, so a message
flushed concurrently by another writer can't be lost. Sessions with
children (delegate subagent runs) are preserved — a parent that
spawned work is not "empty" even if its own transcript never
flushed. Returns True if the session was deleted.
"""
def _do(conn):
cursor = conn.execute(
"""
DELETE FROM sessions
WHERE id = ?
AND title IS NULL
AND NOT EXISTS (
SELECT 1 FROM messages WHERE messages.session_id = sessions.id
)
AND NOT EXISTS (
SELECT 1 FROM sessions child
WHERE child.parent_session_id = sessions.id
)
""",
(session_id,),
)
if cursor.rowcount > 0:
self._delete_unreferenced_system_prompts(conn)
return cursor.rowcount > 0
deleted = self._execute_write(_do)
if deleted:
self._remove_session_files(sessions_dir, session_id)
return bool(deleted)
def delete_sessions(
self,
session_ids: List[str],
sessions_dir: Optional[Path] = None,
) -> int:
"""Delete every session in *session_ids* in a single transaction.
Backs the dashboard's bulk-select-then-delete flow on the
sessions page (``POST /api/sessions/bulk-delete``). Mirrors the
single-session :meth:`delete_session` contract per row:
* Unknown IDs are silently skipped (no 404) — selection state
in the UI can race against another tab's delete, and we'd
rather succeed-on-the-rest than fail-the-whole-batch.
* Delegate subagent children (``model_config._delegate_from``) are
cascade-deleted with their parent; branch children are orphaned
(``parent_session_id → NULL``) so they stay accessible.
* Messages and the session row both go in one
``_execute_write`` call so a partial failure can't leave the
DB in a "messages gone but session row still there" state.
* On-disk transcript / ``request_dump_*`` files are cleaned up
outside the DB transaction when *sessions_dir* is provided,
matching :meth:`prune_sessions` and
:meth:`delete_empty_sessions`.
Returns the count of sessions that actually existed and were
deleted (may be less than ``len(session_ids)`` if some IDs were
already gone).
"""
if not session_ids:
return 0
# Dedup + drop any non-string entries up-front. Avoids
# double-counting in the WHERE-IN list and protects against
# callers that pass a list with stray ``None`` values.
unique_ids = list({sid for sid in session_ids if isinstance(sid, str) and sid})
if not unique_ids:
return 0
removed_ids: list[str] = []
removed_delegate_ids: list[str] = []
def _do(conn):
placeholders = ",".join("?" * len(unique_ids))
# First, filter to IDs that actually exist — we want to
# return the real deleted count, not the input length.
cursor = conn.execute(
f"SELECT id FROM sessions WHERE id IN ({placeholders})",
unique_ids,
)
existing = [row["id"] for row in cursor.fetchall()]
if not existing:
return 0
existing_placeholders = ",".join("?" * len(existing))
removed_delegate_ids.extend(_delete_delegate_children(conn, existing))
# Orphan remaining children whose parent is in the kill list so the
# FK constraint stays satisfied. Pin children whose parent
# is itself in the kill list rather than NULL-ing parents
# of survivors — the IN list on ``parent_session_id`` does
# exactly this.
conn.execute(
f"UPDATE sessions SET parent_session_id = NULL "
f"WHERE parent_session_id IN ({existing_placeholders})",
existing,
)
conn.execute(
f"DELETE FROM messages WHERE session_id IN ({existing_placeholders})",
existing,
)
conn.execute(
f"DELETE FROM sessions WHERE id IN ({existing_placeholders})",
existing,
)
self._delete_unreferenced_system_prompts(conn)
removed_ids.extend(existing)
return len(existing)
count = self._execute_write(_do)
for sid in removed_delegate_ids:
self._remove_session_files(sessions_dir, sid)
for sid in removed_ids:
self._remove_session_files(sessions_dir, sid)
return count
#: Shared selector for :meth:`count_empty_sessions` and
#: :meth:`delete_empty_sessions` so the badge and the sweep agree.
#:
#: ``message_count`` tracks live (``active = 1``) rows only; rewind
#: (:meth:`replace_messages` w/ ``archive_dropped``) and in-place
#: compaction (:meth:`archive_and_compact`) reset it to 0 while keeping
#: dropped turns on disk as ``active = 0`` — the only recoverable copy
#: (#70516 / #80763 / #82756). The ``NOT EXISTS`` probe is the authority;
#: ``message_count = 0`` stays as a cheap prefilter. Same shape as every
#: other emptiness guard in this module. (#95868)
_EMPTY_SESSION_WHERE = (
"message_count = 0 "
"AND ended_at IS NOT NULL "
"AND archived = 0 "
"AND NOT EXISTS ("
"SELECT 1 FROM messages WHERE messages.session_id = sessions.id"
")"
)
def count_empty_sessions(self) -> int:
"""Return the count of empty, non-active, non-archived sessions.
"Empty" = the session holds no message rows at all AND has ended
(``ended_at IS NOT NULL``) AND is not archived. The ``ended_at``
guard matches the safety contract used by :meth:`prune_sessions`:
only ended sessions are candidates for bulk deletion, so a freshly
spawned session whose first message hasn't landed yet — or one
held open by the live agent — is never sniped out from under
the runtime.
Emptiness is decided by :data:`_EMPTY_SESSION_WHERE` — see that
constant for why the ``NOT EXISTS`` probe is needed instead of
trusting ``message_count`` alone.
Backs the ``GET /api/sessions/empty/count`` endpoint that lets the
web dashboard hide its "Delete empty" button when there's nothing
to clean up, and pre-populate the confirm dialog with the actual
count.
"""
with self._read_ctx() as conn:
cursor = conn.execute(
f"SELECT COUNT(*) FROM sessions WHERE {self._EMPTY_SESSION_WHERE}"
)
return cursor.fetchone()[0]
def delete_empty_sessions(
self,
sessions_dir: Optional[Path] = None,
) -> int:
"""Delete every empty, ended, non-archived session.
Mirrors :meth:`prune_sessions`' transactional shape:
* Selects candidate IDs first (:data:`_EMPTY_SESSION_WHERE`) so we
never touch a live session, one the user deliberately archived,
or one whose transcript survives as soft-archived rows.
* Orphans any child whose parent is in the kill list — children
of an empty parent are kept and re-parented to ``NULL`` rather
than cascade-deleted, matching ``delete_session`` /
``prune_sessions`` semantics so branch/subagent transcripts
survive an inadvertent parent cleanup.
* Deletes the rows in a single ``_execute_write`` callback so
the operation is atomic — a partial failure (e.g. SIGKILL
mid-loop) doesn't leave the DB in a "messages-deleted but
session-row-still-there" half-state.
* Cleans up on-disk transcript files (``.json`` / ``.jsonl`` /
``request_dump_*``) outside the DB transaction when
``sessions_dir`` is provided. Empty sessions don't typically
have transcript files, but the gateway can leave a stub
``request_dump_*`` if it crashed before the first reply —
so we still sweep, matching ``prune_sessions``.
Returns the number of sessions deleted.
"""
removed_ids: list[str] = []
def _do(conn):
cursor = conn.execute(
f"SELECT id FROM sessions WHERE {self._EMPTY_SESSION_WHERE}"
)
session_ids = {row["id"] for row in cursor.fetchall()}
if not session_ids:
return 0
placeholders = ",".join("?" * len(session_ids))
conn.execute(
f"UPDATE sessions SET parent_session_id = NULL "
f"WHERE parent_session_id IN ({placeholders})",
list(session_ids),
)
for sid in session_ids:
# DELETE FROM messages is paranoia — the selector's
# ``NOT EXISTS`` probe already proved these sessions own no
# message rows — but a row inserted between the SELECT and
# this statement would otherwise be left dangling, so we
# still leave a clean FK state.
conn.execute(
"DELETE FROM messages WHERE session_id = ?", (sid,)
)
conn.execute("DELETE FROM sessions WHERE id = ?", (sid,))
removed_ids.append(sid)
self._delete_unreferenced_system_prompts(conn)
return len(session_ids)
count = self._execute_write(_do)
for sid in removed_ids:
self._remove_session_files(sessions_dir, sid)
return count
@staticmethod
def _prune_filter_where(
*,
last_active_before: Optional[float] = None,
last_active_after: Optional[float] = None,
started_before: Optional[float] = None,
started_after: Optional[float] = None,
source: Optional[str] = None,
title_like: Optional[str] = None,
end_reason: Optional[str] = None,
cwd_prefix: Optional[str] = None,
min_messages: Optional[int] = None,
max_messages: Optional[int] = None,
archived: Optional[bool] = None,
model_like: Optional[str] = None,
provider: Optional[str] = None,
user_id: Optional[str] = None,
chat_id: Optional[str] = None,
chat_type: Optional[str] = None,
branch_like: Optional[str] = None,
min_tokens: Optional[int] = None,
max_tokens: Optional[int] = None,
min_cost: Optional[float] = None,
max_cost: Optional[float] = None,
min_tool_calls: Optional[int] = None,
max_tool_calls: Optional[int] = None,
include_pinned: bool = False,
) -> Tuple[str, list]:
"""Build the shared WHERE clause for bulk prune/archive selection.
All filters AND together. Only ended sessions are ever candidates
(``ended_at IS NOT NULL``) so a live session is never selected.
``archived`` is a tri-state: ``None`` = both, ``True`` = only
archived rows, ``False`` = only unarchived rows.
String matching conventions: ``model_like`` / ``branch_like`` /
``title_like`` are case-insensitive substring matches (model slugs
and branch names vary in prefix format); ``provider`` / ``user_id``
/ ``chat_id`` / ``chat_type`` / ``source`` / ``end_reason`` are
exact (case-insensitive for provider). Token bounds apply to
``input_tokens + output_tokens``; cost bounds apply to
``COALESCE(actual_cost_usd, estimated_cost_usd)``.
The clause references the ``s`` table alias — callers must select
``FROM sessions s``.
"""
clauses = ["s.ended_at IS NOT NULL"]
params: list = []
if last_active_before is not None:
clauses.append(
"""COALESCE(
(SELECT MAX(m.timestamp) FROM messages m
WHERE m.session_id = s.id),
s.started_at
) < ?"""
)
params.append(last_active_before)
# An automatic orphan sweep closes a stale open row so the user can
# still recover it. Age those rows from the sweep, not from their old
# activity, or the next prune pass can delete them immediately.
clauses.append(
"(COALESCE(s.end_reason, '') != 'startup_orphan_reap' "
"OR s.ended_at < ?)"
)
params.append(last_active_before)
if last_active_after is not None:
clauses.append(
"""COALESCE(
(SELECT MAX(m.timestamp) FROM messages m
WHERE m.session_id = s.id),
s.started_at
) >= ?"""
)
params.append(last_active_after)
if started_before is not None:
clauses.append("s.started_at < ?")
params.append(started_before)
if started_after is not None:
clauses.append("s.started_at >= ?")
params.append(started_after)
if source:
clauses.append("s.source = ?")
params.append(source)
if title_like:
clauses.append("LOWER(COALESCE(s.title, '')) LIKE ? ESCAPE '\\'")
params.append(f"%{_escape_like(title_like.lower())}%")
if end_reason:
clauses.append("s.end_reason = ?")
params.append(end_reason)
if cwd_prefix:
clause, clause_params = _cwd_prefix_clause(cwd_prefix)
clauses.append(clause)
params.extend(clause_params)
if min_messages is not None:
clauses.append("s.message_count >= ?")
params.append(min_messages)
if max_messages is not None:
clauses.append("s.message_count <= ?")
params.append(max_messages)
if model_like:
clauses.append("LOWER(COALESCE(s.model, '')) LIKE ? ESCAPE '\\'")
params.append(f"%{_escape_like(model_like.lower())}%")
if provider:
clauses.append("LOWER(COALESCE(s.billing_provider, '')) = ?")
params.append(provider.lower())
if user_id:
clauses.append("s.user_id = ?")
params.append(user_id)
if chat_id:
clauses.append("s.chat_id = ?")
params.append(chat_id)
if chat_type:
clauses.append("s.chat_type = ?")
params.append(chat_type)
if branch_like:
clauses.append("LOWER(COALESCE(s.git_branch, '')) LIKE ? ESCAPE '\\'")
params.append(f"%{_escape_like(branch_like.lower())}%")
if min_tokens is not None:
clauses.append(
"(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) >= ?"
)
params.append(min_tokens)
if max_tokens is not None:
clauses.append(
"(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) <= ?"
)
params.append(max_tokens)
if min_cost is not None:
clauses.append(
"COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) >= ?"
)
params.append(min_cost)
if max_cost is not None:
clauses.append(
"COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) <= ?"
)
params.append(max_cost)
if min_tool_calls is not None:
clauses.append("COALESCE(s.tool_call_count, 0) >= ?")
params.append(min_tool_calls)
if max_tool_calls is not None:
clauses.append("COALESCE(s.tool_call_count, 0) <= ?")
params.append(max_tool_calls)
if archived is True:
clauses.append("s.archived = 1")
elif archived is False:
clauses.append("s.archived = 0")
# Pinned sessions are a durable "keep" flag (exempt from the stale
# auto-archive sweep). Bulk prune/delete/archive must honor that too:
# exclude pinned rows unless the caller explicitly opts in. Without
# this, `sessions prune`/`delete`/`archive` with a filter silently
# destroyed pinned conversations (round-3 QA SES-01, data loss).
if not include_pinned:
clauses.append("COALESCE(s.pinned, 0) = 0")
return " AND ".join(clauses), params
@staticmethod
def _apply_prune_age_filter(
older_than_days: Optional[float], filters: Dict[str, Any]
) -> None:
"""Translate the legacy age window into the shared activity filter."""
if (
filters.get("last_active_before") is None
and filters.get("started_before") is None
and older_than_days is not None
):
filters["last_active_before"] = time.time() - (
older_than_days * 86400
)
def list_prune_candidates(
self,
older_than_days: Optional[float] = None,
source: str = None,
**filters,
) -> List[Dict[str, Any]]:
"""Return the sessions a matching :meth:`prune_sessions` /
:meth:`archive_sessions` call would touch, without modifying anything.
Backs ``--dry-run`` and pre-confirmation counts. Accepts the same
keyword filters as :meth:`_prune_filter_where` (unknown names raise
``TypeError`` there). Rows are ordered oldest-first and carry
``id, source, title, model, started_at, last_active, ended_at,
message_count, archived``. ``older_than_days`` is an inactivity
threshold: it uses the latest message timestamp, falling back to
``started_at`` for sessions without messages.
"""
self._apply_prune_age_filter(older_than_days, filters)
where, params = self._prune_filter_where(source=source, **filters)
with self._read_ctx() as conn:
cursor = conn.execute(
f"""SELECT s.id, s.source, s.title, s.model, s.started_at,
COALESCE(
(SELECT MAX(m.timestamp) FROM messages m
WHERE m.session_id = s.id),
s.started_at
) AS last_active,
s.ended_at, s.message_count, s.archived
FROM sessions s WHERE {where}
ORDER BY last_active ASC, s.started_at ASC""",
params,
)
return [dict(row) for row in cursor.fetchall()]
def count_prune_matches(
self,
older_than_days: Optional[float] = None,
source: str = None,
**filters,
) -> int:
"""Count sessions a matching prune/archive would touch.
Same filter surface as :meth:`list_prune_candidates` (including the
``include_pinned`` tri-state), but returns only a count. Used by the
CLI to report how many pinned sessions are being spared.
"""
self._apply_prune_age_filter(older_than_days, filters)
where, params = self._prune_filter_where(source=source, **filters)
with self._read_ctx() as conn:
cursor = conn.execute(
f"SELECT COUNT(*) FROM sessions s WHERE {where}", params
)
return int(cursor.fetchone()[0])
def count_open_prune_matches(
self,
older_than_days: Optional[float] = None,
source: str = None,
**filters,
) -> int:
"""Count open sessions excluded from a matching bulk prune.
This applies every normal prune filter, but inverts only the
``ended_at`` safety guard. It is visibility-only: callers can explain
why an otherwise matching session was skipped without making live
sessions eligible for destructive pruning.
"""
self._apply_prune_age_filter(older_than_days, filters)
where, params = self._prune_filter_where(source=source, **filters)
ended_guard = "s.ended_at IS NOT NULL"
if not where.startswith(ended_guard):
raise RuntimeError("prune filter lost its ended-session safety guard")
open_where = f"s.ended_at IS NULL{where[len(ended_guard):]}"
with self._read_ctx() as conn:
cursor = conn.execute(
f"SELECT COUNT(*) FROM sessions s WHERE {open_where}", params
)
return int(cursor.fetchone()[0])
def archive_sessions(
self,
older_than_days: Optional[float] = None,
source: str = None,
**filters,
) -> int:
"""Bulk-archive (soft-hide) every session matching the filters.
Same filter surface as :meth:`prune_sessions`, but instead of deleting
rows it flips ``archived = 1`` via :meth:`set_session_archived` so
each match's compression lineage is archived as a unit (an unarchived
compression root would otherwise resurrect the conversation in
Desktop's projected list). Nothing is deleted; messages and transcript
files are untouched. Returns the number of sessions matched.
``archived`` defaults to ``False`` here (only select rows not yet
archived) so repeat runs are idempotent no-ops.
"""
filters.setdefault("archived", False)
rows = self.list_prune_candidates(
older_than_days=older_than_days, source=source, **filters
)
for row in rows:
self.set_session_archived(row["id"], True)
return len(rows)
def archive_stale_sessions(
self, idle_days: float, *, exclude_pinned: bool = True
) -> int:
"""Archive every session untouched for at least ``idle_days`` days.
"Touched" is the freshest of ``last_activity_at`` and the latest
message timestamp (else ``started_at``) — i.e. real recency, not
creation time — so a session
created long ago but active yesterday is spared, while an old
abandoned one (even a still-open one) is swept. Unlike
:meth:`archive_sessions`, this method can also archive unended
sessions.
Guards:
* ``pinned = 0`` when ``exclude_pinned`` (the Desktop "keep" flag).
* ``archived = 0`` so repeat runs are idempotent no-ops.
* only lineage *tips* / standalone rows are candidates
(``end_reason <> 'compression'``); a stale tip archives its whole
chain via :meth:`set_session_archived`, so we never resurrect an
active conversation by matching an old compressed-away root whose
live continuation is recent.
Returns the number of sessions archived. Never raises for an empty or
non-positive ``idle_days`` — it simply archives nothing.
"""
if idle_days is None or idle_days < 0:
return 0
cutoff = time.time() - float(idle_days) * 86400.0
pin_clause = "AND s.pinned = 0" if exclude_pinned else ""
with self._read_ctx() as conn:
rows = conn.execute(
f"""
SELECT s.id FROM sessions s
WHERE s.archived = 0
AND COALESCE(s.end_reason, '') <> 'compression'
{pin_clause}
AND {_sql_session_last_active("s")} < ?
ORDER BY s.started_at ASC
""",
(cutoff,),
).fetchall()
ids = [(r["id"] if isinstance(r, sqlite3.Row) else r[0]) for r in rows]
for sid in ids:
self.set_session_archived(sid, True)
return len(ids)
def prune_sessions(
self,
older_than_days: Optional[float] = 90,
source: str = None,
sessions_dir: Optional[Path] = None,
exclude_active_write_guards: bool = False,
**filters,
) -> int:
"""Delete sessions matching the filters. Returns count deleted.
By default, delete ended sessions inactive for
``older_than_days`` days, optionally restricted to ``source``.
Activity is the latest message timestamp, falling back to
``started_at`` for sessions without messages. Additional keyword
filters AND together — the full set is defined by
:meth:`_prune_filter_where`:
* ``last_active_before`` / ``last_active_after`` — epoch bounds on
the latest message timestamp (falling back to ``started_at``).
* ``started_before`` / ``started_after`` — epoch bounds on
``started_at``. An explicit ``started_before`` overrides the
default ``older_than_days`` inactivity cutoff; pass
``older_than_days=None`` for no implicit upper age bound.
* ``title_like`` / ``model_like`` / ``branch_like`` —
case-insensitive substring matches.
* ``end_reason`` / ``provider`` / ``user_id`` / ``chat_id`` /
``chat_type`` — exact matches (provider case-insensitive, against
``billing_provider``).
* ``cwd_prefix`` — session cwd equals or is under this path.
* ``min_messages`` / ``max_messages`` — bounds on message_count.
* ``min_tokens`` / ``max_tokens`` — bounds on input+output tokens.
* ``min_cost`` / ``max_cost`` — bounds on USD cost
(actual, falling back to estimated).
* ``min_tool_calls`` / ``max_tool_calls`` — bounds on tool_call_count.
* ``archived`` — tri-state: None = both (default), True = only
archived, False = only unarchived.
Only prunes ended sessions (not active ones). Child sessions outside
the prune window are orphaned (parent_session_id set to NULL) rather
than cascade-deleted. When *sessions_dir* is provided, also removes
on-disk transcript files (``.json`` / ``.jsonl`` /
``request_dump_*``) for every pruned session, outside the DB
transaction.
``exclude_active_write_guards`` is for destructive automatic
maintenance: rows protected by a live turn lease or compression lock
are skipped, while expired or provably dead holders are reclaimed and
fenced in the same write transaction.
"""
self._apply_prune_age_filter(older_than_days, filters)
where, where_params = self._prune_filter_where(source=source, **filters)
removed_ids: list[str] = []
def _do(conn):
cursor = conn.execute(
f"SELECT s.id FROM sessions s WHERE {where}", where_params
)
session_ids = {row["id"] for row in cursor.fetchall()}
if exclude_active_write_guards:
protected = set()
for sid in session_ids:
try:
self._check_transcript_write_guards(
conn,
sid,
compression_lock_holder=None,
turn_lease_holder=None,
reject_active_turn_lease=True,
reject_active_compression_lock=True,
allow_closed_compression_parent=True,
)
except (
SessionCompressionInProgressError,
SessionTurnLeaseLostError,
):
protected.add(sid)
session_ids.difference_update(protected)
if not session_ids:
return 0
# Orphan any sessions whose parent is about to be deleted
placeholders = ",".join("?" * len(session_ids))
conn.execute(
f"UPDATE sessions SET parent_session_id = NULL "
f"WHERE parent_session_id IN ({placeholders})",
list(session_ids),
)
for sid in session_ids:
conn.execute("DELETE FROM messages WHERE session_id = ?", (sid,))
conn.execute("DELETE FROM sessions WHERE id = ?", (sid,))
removed_ids.append(sid)
self._delete_unreferenced_system_prompts(conn)
return len(session_ids)
count = self._execute_write(_do)
# Clean up on-disk files outside the DB transaction
for sid in removed_ids:
self._remove_session_files(sessions_dir, sid)
return count
def purge_stale_tool_call_markers(
self, *, dry_run: bool = False, backup: bool = True
) -> Dict[str, Any]:
"""Permanently clear bare tool-call marker content (e.g. "[memory]")
left in the ``messages`` table by sessions persisted before the
#78148 fix in ``agent.conversation_loop``.
``_strip_stale_tool_call_markers`` already repairs this in memory on
every session load (see ``_rows_to_conversation``), so running this
is optional — but for long-lived sessions the same rows get
re-scanned and re-repaired on every resume, which is wasted work
and keeps the contaminated bytes sitting in the DB (and in any
downstream cache/backup snapshot of it) indefinitely. This rewrites
the affected rows once, in place.
Only the ``content`` column is touched — ``role``, ``tool_calls``,
and every other column on the row are left exactly as they are, so
provider tool_call/tool_result pairing is unaffected.
Unlike the in-memory repair, this UPDATE is permanent and can't be
undone from within the DB. Since ``backup`` defaults to True, a
timestamped full snapshot is taken via ``VACUUM INTO`` (safe against
a live connection, unlike the raw-copy ``_backup_db_file`` used for
malformed-schema repair) before any row is touched — mirroring
``repair_state_db_schema``'s backup-by-default convention for
destructive state.db operations. No snapshot is taken when there is
nothing to change.
With ``dry_run=True``, reports the affected row count/ids without
writing or backing up (read-only, no write lock taken).
Returns ``{"dry_run": bool, "rows_affected": int, "row_ids": [...],
"backup_path": str|None}``.
"""
def _find_affected(conn) -> List[int]:
cursor = conn.execute(
"SELECT id, content FROM messages "
"WHERE role = 'assistant' AND tool_calls IS NOT NULL AND tool_calls != ''"
)
affected: List[int] = []
for row in cursor.fetchall():
content = row["content"]
if isinstance(content, str) and _STALE_TOOL_CALL_MARKER_RE.fullmatch(content.strip()):
affected.append(row["id"])
return affected
with self._read_ctx() as conn:
affected_ids = _find_affected(conn)
if dry_run:
return {
"dry_run": True,
"rows_affected": len(affected_ids),
"row_ids": affected_ids,
"backup_path": None,
}
if not affected_ids:
return {
"dry_run": False,
"rows_affected": 0,
"row_ids": [],
"backup_path": None,
}
backup_path: Optional[str] = None
if backup:
import datetime
stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
dest = self.db_path.with_name(
f"{self.db_path.name}.pre-clean-markers-backup-{stamp}"
)
with self._lock:
self._conn.execute("VACUUM INTO ?", (str(dest),))
backup_path = str(dest)
logger.info("Backed up state.db to %s before clean-markers write", backup_path)
def _do(conn):
ids = _find_affected(conn)
if ids:
placeholders = ",".join("?" * len(ids))
conn.execute(
f"UPDATE messages SET content = '' WHERE id IN ({placeholders})",
ids,
)
return ids
affected_ids = self._execute_write(_do)
if affected_ids:
logger.info(
"Permanently cleared %d stale tool-call marker row(s) in state.db (#78148)",
len(affected_ids),
)
return {
"dry_run": False,
"rows_affected": len(affected_ids),
"row_ids": affected_ids,
"backup_path": backup_path,
}
# ── Meta key/value (for scheduler bookkeeping) ──
def get_meta(self, key: str) -> Optional[str]:
"""Read a value from the state_meta key/value store."""
# Kept on self._lock (not _read_ctx) because callers like
# fts_rebuild_step read progress before entering a write
# transaction, and the read-only WAL connection sees only
# committed data — a pending write transaction's uncommitted
# meta writes would be invisible. This is a cheap point lookup,
# not the convoy bottleneck the read-path split targets.
with self._lock:
row = self._conn.execute(
"SELECT value FROM state_meta WHERE key = ?", (key,)
).fetchone()
if row is None:
return None
return row["value"] if isinstance(row, sqlite3.Row) else row[0]
def set_meta(
self, key: str, value: str, *, cursor: Optional[sqlite3.Cursor] = None
) -> None:
"""Write a value to the state_meta key/value store.
When ``cursor`` is provided the write is issued on that cursor
inline (used during ``_init_schema``, which already holds an open
transaction — routing through ``_execute_write`` there would nest
BEGIN IMMEDIATE and deadlock). Otherwise a normal write transaction
is used.
"""
if cursor is not None:
cursor.execute(
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(key, value),
)
return
def _do(conn):
conn.execute(
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(key, value),
)
self._execute_write(_do)
def retag_kanban_worker_sessions(self, workspaces_root: str) -> int:
"""Retag legacy kanban worker rows from ``cli`` to ``kanban``.
Workers used to spawn without ``HERMES_SESSION_SOURCE``, so their runs
landed as untitled ``cli`` rows and the sidebar rendered one per attempt
labeled with the worker's own prompt. New workers tag themselves; this
reclaims the rows already on disk so they drop out of the session lists
too. Identified by cwd under the board's workspaces root — a path only
the dispatcher ever runs a session in.
Gated per workspaces root (``state_meta``) so each board reclaims its
own rows exactly once. Returns the number of rows retagged.
"""
prefix = str(workspaces_root).rstrip("/\\")
if not prefix:
return 0
gate = f"kanban_worker_source_retagged:{prefix}"
if self.get_meta(gate) == "1":
return 0
def _do(conn):
cursor = conn.execute(
"UPDATE sessions SET source = 'kanban' "
"WHERE source = 'cli' AND (cwd = ? OR cwd LIKE ? ESCAPE '\\')",
(prefix, _escape_like(prefix) + "/%"),
)
# Read rowcount before set_meta reuses this cursor for its INSERT,
# which would otherwise overwrite it with the meta write's count.
retagged = cursor.rowcount or 0
self.set_meta(gate, "1", cursor=cursor)
return retagged
return self._execute_write(_do)
def list_meta_prefix(self, prefix: str) -> List[Tuple[str, str]]:
"""Return ``[(key, value), ...]`` for state_meta keys with ``prefix``.
Used by feature stores that persist one row per session under a
namespaced key (e.g. ``loop:<session_id>``) and need to enumerate
them across sessions (the gateway's idle /loop wakeup watcher).
``prefix`` is matched literally — LIKE wildcards in it are escaped.
"""
if not prefix:
return []
escaped = prefix.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
with self._read_ctx() as conn:
rows = conn.execute(
"SELECT key, value FROM state_meta WHERE key LIKE ? ESCAPE '\\'",
(escaped + "%",),
).fetchall()
return [(row[0], row[1]) for row in rows]
def apply_telegram_topic_migration(self) -> None:
"""Create Telegram DM topic-mode tables on explicit /topic opt-in.
This migration is deliberately not part of automatic SessionDB startup
reconciliation. Operators must be able to upgrade Hermes, keep the old
Telegram bot behavior running, and only mutate topic-mode state when the
user executes /topic to opt into the feature.
Schema versions:
v1 — initial shape (no ON DELETE CASCADE on session_id FK)
v2 — session_id FK gets ON DELETE CASCADE so session pruning
automatically clears bindings.
v3 — ``profile_name`` dimension on both tables so multiplexed
gateways (shared ``state.db``) isolate topic mode/bindings
per Hermes profile (issue #76423).
"""
# (table, column list, DDL body). ``profile_name`` leads the primary
# key so multiplexed profiles sharing one state.db never collide on a
# private chat_id (which is the user id, identical across bots).
tables = (
(
"telegram_dm_topic_mode",
"profile_name, chat_id, user_id, enabled, activated_at, updated_at, "
"has_topics_enabled, allows_users_to_create_topics, "
"capability_checked_at, intro_message_id, pinned_message_id",
"""
profile_name TEXT NOT NULL DEFAULT 'default',
chat_id TEXT NOT NULL,
user_id TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 1,
activated_at REAL NOT NULL,
updated_at REAL NOT NULL,
has_topics_enabled INTEGER,
allows_users_to_create_topics INTEGER,
capability_checked_at REAL,
intro_message_id TEXT,
pinned_message_id TEXT,
PRIMARY KEY (profile_name, chat_id)
""",
),
(
"telegram_dm_topic_bindings",
"profile_name, chat_id, thread_id, user_id, session_key, "
"session_id, managed_mode, linked_at, updated_at",
"""
profile_name TEXT NOT NULL DEFAULT 'default',
chat_id TEXT NOT NULL,
thread_id TEXT NOT NULL,
user_id TEXT NOT NULL,
session_key TEXT NOT NULL,
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
managed_mode TEXT NOT NULL DEFAULT 'auto',
linked_at REAL NOT NULL,
updated_at REAL NOT NULL,
PRIMARY KEY (profile_name, chat_id, thread_id)
""",
),
)
def _do(conn):
for table, columns, ddl in tables:
# Fresh installs get the v3 shape immediately.
conn.execute(f"CREATE TABLE IF NOT EXISTS {table} ({ddl})")
have = {row[1] for row in conn.execute(f"PRAGMA table_info('{table}')")}
if "profile_name" in have:
continue
# Pre-profile shape (v1 or v2) → v3. SQLite can't ALTER a
# primary key (or a foreign key), so rebuild; this also
# supplies the v2 ON DELETE CASCADE for v1 bindings tables.
# Legacy rows land in the "default" namespace only — never
# replicated across profiles.
legacy_columns = columns.replace("profile_name, ", "", 1)
conn.executescript(
f"""
CREATE TABLE {table}_new ({ddl});
INSERT INTO {table}_new ({columns})
SELECT 'default', {legacy_columns} FROM {table};
DROP TABLE {table};
ALTER TABLE {table}_new RENAME TO {table};
"""
)
# Indexes after any rebuild so they always target the v3 shape
# (a legacy table lacking profile_name can't take the user index).
conn.executescript(
"""
CREATE UNIQUE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_session
ON telegram_dm_topic_bindings(session_id);
CREATE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_user
ON telegram_dm_topic_bindings(profile_name, user_id, chat_id);
"""
)
conn.execute(
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
("telegram_dm_topic_schema_version", "3"),
)
self._execute_write(_do)
def enable_telegram_topic_mode(
self,
*,
chat_id: str,
user_id: str,
profile_name: str = "default",
has_topics_enabled: Optional[bool] = None,
allows_users_to_create_topics: Optional[bool] = None,
) -> None:
"""Enable Telegram DM topic mode for one private chat/user.
This method intentionally owns the explicit topic migration. Ordinary
SessionDB startup must not create these side tables.
``profile_name`` namespaces rows under a shared multiplex ``state.db``
(issue #76423). Callers handling a multiplexed event must pass the
routed profile from ``source.profile``, not the process-global active
profile.
"""
self.apply_telegram_topic_migration()
now = time.time()
profile_name = _normalize_telegram_topic_profile_name(profile_name)
def _to_int(value: Optional[bool]) -> Optional[int]:
if value is None:
return None
return 1 if value else 0
def _do(conn):
conn.execute(
"""
INSERT INTO telegram_dm_topic_mode (
profile_name, chat_id, user_id, enabled, activated_at, updated_at,
has_topics_enabled, allows_users_to_create_topics,
capability_checked_at
) VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?)
ON CONFLICT(profile_name, chat_id) DO UPDATE SET
user_id = excluded.user_id,
enabled = 1,
updated_at = excluded.updated_at,
has_topics_enabled = excluded.has_topics_enabled,
allows_users_to_create_topics = excluded.allows_users_to_create_topics,
capability_checked_at = excluded.capability_checked_at
""",
(
profile_name,
str(chat_id),
str(user_id),
now,
now,
_to_int(has_topics_enabled),
_to_int(allows_users_to_create_topics),
now,
),
)
self._execute_write(_do)
def disable_telegram_topic_mode(
self,
*,
chat_id: str,
profile_name: str = "default",
clear_bindings: bool = True,
) -> None:
"""Disable Telegram DM topic mode for one private chat.
When ``clear_bindings`` is True (default) the (chat_id, thread_id)
bindings for this chat are also cleared so re-enabling later
starts from a clean slate. Set to False if the operator wants to
preserve bindings for a later re-enable.
Never creates the topic-mode tables from scratch; if they don't
exist there is nothing to disable and the call is a no-op.
"""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
def _do(conn):
try:
conn.execute(
"UPDATE telegram_dm_topic_mode SET enabled = 0, updated_at = ? "
"WHERE profile_name = ? AND chat_id = ?",
(time.time(), profile_name, str(chat_id)),
)
if clear_bindings:
conn.execute(
"DELETE FROM telegram_dm_topic_bindings "
"WHERE profile_name = ? AND chat_id = ?",
(profile_name, str(chat_id)),
)
except sqlite3.OperationalError:
# Tables don't exist yet — nothing to disable.
return
self._execute_write(_do)
def is_telegram_topic_mode_enabled(
self,
*,
chat_id: str,
user_id: str,
profile_name: str = "default",
) -> bool:
"""Return whether Telegram DM topic mode is enabled for this chat/user."""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
with self._read_ctx() as conn:
try:
row = conn.execute(
"""
SELECT enabled FROM telegram_dm_topic_mode
WHERE profile_name = ? AND chat_id = ? AND user_id = ?
""",
(profile_name, str(chat_id), str(user_id)),
).fetchone()
except sqlite3.OperationalError:
return False
if row is None:
return False
enabled = row["enabled"] if isinstance(row, sqlite3.Row) else row[0]
return bool(enabled)
def get_telegram_topic_binding(
self,
*,
chat_id: str,
thread_id: str,
profile_name: str = "default",
) -> Optional[Dict[str, Any]]:
"""Return the session binding for a Telegram DM topic, if present."""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
with self._read_ctx() as conn:
try:
row = conn.execute(
"""
SELECT * FROM telegram_dm_topic_bindings
WHERE profile_name = ? AND chat_id = ? AND thread_id = ?
""",
(profile_name, str(chat_id), str(thread_id)),
).fetchone()
except sqlite3.OperationalError:
return None
return dict(row) if row else None
def list_telegram_topic_bindings_for_chat(
self,
*,
chat_id: str,
profile_name: str = "default",
) -> List[Dict[str, Any]]:
"""All Telegram DM topic bindings for one chat, newest first.
Read-only; returns [] if the bindings table doesn't exist yet
(does not trigger the topic-mode migration).
"""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
with self._read_ctx() as conn:
try:
rows = conn.execute(
"SELECT * FROM telegram_dm_topic_bindings "
"WHERE profile_name = ? AND chat_id = ? "
"ORDER BY updated_at DESC",
(profile_name, str(chat_id)),
).fetchall()
except sqlite3.OperationalError:
return []
return [dict(row) for row in rows]
def get_telegram_topic_binding_by_session(
self,
*,
session_id: str,
) -> Optional[Dict[str, Any]]:
"""Return the Telegram DM topic binding for a given session_id, if present.
Uses the UNIQUE INDEX on telegram_dm_topic_bindings(session_id) for an
efficient reverse lookup. Returns None when the session has no binding or
the table does not exist yet.
"""
with self._read_ctx() as conn:
try:
row = conn.execute(
"""
SELECT * FROM telegram_dm_topic_bindings
WHERE session_id = ?
""",
(str(session_id),),
).fetchone()
except sqlite3.OperationalError:
return None
return dict(row) if row else None
def delete_telegram_topic_binding(
self,
*,
chat_id: str,
thread_id: str,
profile_name: str = "default",
) -> int:
"""Remove the binding row for a single (chat, thread) pair.
Called when the Telegram Bot API confirms a topic was deleted
externally (``Thread not found`` after the same-thread retry
already failed). Without this prune, the stale row keeps
living in ``telegram_dm_topic_bindings`` and the
recovery logic in ``gateway.run._recover_telegram_topic_thread_id``
cheerfully redirects future inbound messages to the deleted
topic, causing tool progress, approvals, and replies to land
in the wrong place. Issue #31501.
When this prune removes the chat's *last* remaining binding,
the chat's row in ``telegram_dm_topic_mode`` is also flipped to
``enabled = 0`` in the same transaction. Otherwise the chat
would be left in topic mode with zero lanes — and
``gateway.run._recover_telegram_topic_thread_id`` keeps treating
the chat as topic-enabled, lobby messages keep hunting for a
binding that no longer exists, and a user who disabled topics in
the Telegram client (rather than via ``/topic off``) stays stuck
until the next send happens to fail. Clearing the flag makes
recovery fully stand down once the dead topics are gone.
Returns the number of binding rows deleted (0 when the binding
was already absent or the topic-mode tables haven't been
migrated yet — both are silent no-ops; we never raise from
a cleanup hot path).
"""
chat_id = str(chat_id)
thread_id = str(thread_id)
profile_name = _normalize_telegram_topic_profile_name(profile_name)
deleted = {"count": 0}
def _do(conn):
try:
cursor = conn.execute(
"""
DELETE FROM telegram_dm_topic_bindings
WHERE profile_name = ? AND chat_id = ? AND thread_id = ?
""",
(profile_name, chat_id, thread_id),
)
deleted["count"] = cursor.rowcount or 0
except sqlite3.OperationalError:
# Tables don't exist yet — nothing to prune.
deleted["count"] = 0
return
if not deleted["count"]:
return
# If that was the chat's last binding, disable topic mode for
# the chat so recovery stops steering lobby messages at a now
# empty lane set. Same transaction → no read-after-prune race.
try:
remaining = conn.execute(
"""
SELECT 1 FROM telegram_dm_topic_bindings
WHERE profile_name = ? AND chat_id = ? LIMIT 1
""",
(profile_name, chat_id),
).fetchone()
if remaining is None:
conn.execute(
"UPDATE telegram_dm_topic_mode "
"SET enabled = 0, updated_at = ? "
"WHERE profile_name = ? AND chat_id = ?",
(time.time(), profile_name, chat_id),
)
except sqlite3.OperationalError:
# telegram_dm_topic_mode absent — binding prune still stands.
pass
self._execute_write(_do)
return deleted["count"]
def bind_telegram_topic(
self,
*,
chat_id: str,
thread_id: str,
user_id: str,
session_key: str,
session_id: str,
managed_mode: str = "auto",
profile_name: str = "default",
) -> None:
"""Bind one Telegram DM topic thread to one Hermes session.
A Hermes session may only be linked to one Telegram topic in MVP.
Rebinding the same topic to the same session is idempotent; trying to
link the same session to a different topic raises ValueError.
"""
self.apply_telegram_topic_migration()
now = time.time()
chat_id = str(chat_id)
thread_id = str(thread_id)
user_id = str(user_id)
session_key = str(session_key)
session_id = str(session_id)
profile_name = _normalize_telegram_topic_profile_name(profile_name)
def _do(conn):
existing_session = conn.execute(
"""
SELECT profile_name, chat_id, thread_id
FROM telegram_dm_topic_bindings
WHERE session_id = ?
""",
(session_id,),
).fetchone()
if existing_session is not None:
if isinstance(existing_session, sqlite3.Row):
linked_profile = existing_session["profile_name"]
linked_chat = existing_session["chat_id"]
linked_thread = existing_session["thread_id"]
else:
linked_profile, linked_chat, linked_thread = existing_session
if (
str(linked_profile) != profile_name
or str(linked_chat) != chat_id
or str(linked_thread) != thread_id
):
raise ValueError("session is already linked to another Telegram topic")
conn.execute(
"""
INSERT INTO telegram_dm_topic_bindings (
profile_name, chat_id, thread_id, user_id, session_key, session_id,
managed_mode, linked_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(profile_name, chat_id, thread_id) DO UPDATE SET
user_id = excluded.user_id,
session_key = excluded.session_key,
session_id = excluded.session_id,
managed_mode = excluded.managed_mode,
updated_at = excluded.updated_at
""",
(
profile_name,
chat_id,
thread_id,
user_id,
session_key,
session_id,
managed_mode,
now,
now,
),
)
self._execute_write(_do)
def is_telegram_session_linked_to_topic(self, *, session_id: str) -> bool:
"""Return True if a Hermes session is already bound to any Telegram DM topic.
Read-only: does NOT trigger the telegram-topic migration. If the
topic-mode tables have not been created yet (i.e. nobody has run
``/topic`` in this profile), the session is by definition unbound
and we return False.
"""
with self._read_ctx() as conn:
try:
row = conn.execute(
"""
SELECT 1 FROM telegram_dm_topic_bindings
WHERE session_id = ?
LIMIT 1
""",
(str(session_id),),
).fetchone()
except sqlite3.OperationalError:
return False
return row is not None
def list_unlinked_telegram_sessions_for_user(
self,
*,
chat_id: str,
user_id: str,
profile_name: str = "default",
limit: int = 10,
) -> List[Dict[str, Any]]:
"""List previous Telegram sessions for this user that are not bound to a topic.
Read-only: does NOT trigger the telegram-topic migration. If the
topic-mode tables are absent, fall back to a simpler query that
just returns this user's Telegram sessions — there can't be any
bindings yet.
Scoped by ``profile_name`` so multiplexed profiles do not surface
each other's unlinked sessions (issue #76423).
"""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
# sessions.profile_name is NULL/empty for legacy rows → treat as default.
profile_clause = "AND COALESCE(NULLIF(TRIM(s.profile_name), ''), 'default') = ?"
with self._read_ctx() as conn:
try:
rows = conn.execute(
f"""
SELECT s.*,
COALESCE(sp.prompt, s.system_prompt)
AS _system_prompt_resolved,
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
AND {_PREVIEW_ELIGIBLE_SQL}
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
{_sql_session_last_active("s")} AS last_active
FROM sessions s
LEFT JOIN system_prompts sp
ON sp.hash = s.system_prompt_hash
WHERE s.source = 'telegram'
AND s.user_id = ?
{profile_clause}
AND NOT EXISTS (
SELECT 1 FROM telegram_dm_topic_bindings b
WHERE b.session_id = s.id
)
ORDER BY last_active DESC, s.started_at DESC
LIMIT ?
""",
(str(user_id), profile_name, int(limit)),
).fetchall()
except sqlite3.OperationalError:
# telegram_dm_topic_bindings doesn't exist yet — no bindings
# means every telegram session for this user is "unlinked".
rows = conn.execute(
f"""
SELECT s.*,
COALESCE(sp.prompt, s.system_prompt)
AS _system_prompt_resolved,
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
AND {_PREVIEW_ELIGIBLE_SQL}
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
{_sql_session_last_active("s")} AS last_active
FROM sessions s
LEFT JOIN system_prompts sp
ON sp.hash = s.system_prompt_hash
WHERE s.source = 'telegram'
AND s.user_id = ?
ORDER BY last_active DESC, s.started_at DESC
LIMIT ?
""",
(str(user_id), int(limit)),
).fetchall()
sessions: List[Dict[str, Any]] = []
for row in rows:
session = self._session_row_dict(row)
session["preview"] = _shape_preview(session.pop("_preview_raw", ""))
sessions.append(session)
return sessions
# ── Space reclamation ──
# FTS5 virtual tables whose b-tree segments we merge on optimize. The
# trigram table is created lazily / may be disabled, and the cjk-bigram
# table only exists (and is only queryable) when the loadable tokenizer
# is present — so we probe each before touching it (see optimize_fts).
_FTS_TABLES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
def logical_size_bytes(self) -> Optional[int]:
"""Database size in bytes as SQLite itself accounts for it.
``page_count * page_size`` — the size the main DB file will have once
the WAL is checkpointed back into it.
Prefer this over ``os.path.getsize(db_path)`` when reporting the effect
of a VACUUM. In WAL mode a VACUUM's rewrite lands in the ``-wal`` file,
and the checkpoint that folds it back is refused while any other
connection (a live gateway) holds a read-mark. Until that happens the
main file on disk still carries its pre-VACUUM size and keeps growing,
so a stat()-based before/after delta understates the win and can go
negative — the "reclaimed -3820.1 MB" report on a database that had
actually shrunk 60%.
Returns None if the pragmas cannot be read.
"""
try:
with self._read_ctx() as conn:
if self._conn is None:
return None
page_count = conn.execute("PRAGMA page_count").fetchone()[0]
page_size = conn.execute("PRAGMA page_size").fetchone()[0]
return int(page_count) * int(page_size)
except Exception as exc:
logger.debug("Could not read logical DB size: %s", exc)
return None
def _freelist_ratio(self) -> Optional[float]:
"""Fraction of database pages that are on the freelist (reclaimable).
``PRAGMA freelist_count / PRAGMA page_count`` read over the existing
connection (never a byte-level probe of the live file — see
``sqlite_safe_read``). This is what VACUUM would actually give back;
it is the gate :meth:`maybe_auto_prune_and_vacuum` uses to decide
whether a full rewrite pays off (#54189).
Returns None if the pragmas cannot be read (callers treat that as
"unknown" and fall back to the time throttle alone).
"""
try:
with self._read_ctx() as conn:
if self._conn is None:
return None
page_count = int(conn.execute("PRAGMA page_count").fetchone()[0])
freelist = int(conn.execute("PRAGMA freelist_count").fetchone()[0])
if page_count <= 0:
return 0.0
return freelist / page_count
except Exception as exc:
logger.debug("Could not read freelist ratio: %s", exc)
return None
def vacuum(self) -> int:
"""Run VACUUM to reclaim disk space after large deletes.
SQLite does not shrink the database file when rows are deleted —
freed pages just get reused on the next insert. After a prune that
removed hundreds of sessions, the file stays bloated unless we
explicitly VACUUM.
VACUUM rewrites the entire DB, so it's expensive (seconds per
100MB) and cannot run inside a transaction. It also acquires an
exclusive lock, so callers must ensure no other writers are
active. Safe to call at startup before the gateway/CLI starts
serving traffic.
FTS5 segments are merged first via :meth:`optimize_fts` so the
subsequent VACUUM reclaims the pages freed by the merge. This is a
layout-only optimization — search results are unchanged.
Returns the number of FTS indexes that were optimized (0 if the
merge step failed or no FTS tables exist).
"""
# Merge FTS5 segments before VACUUM so the freed pages are returned
# to the OS in the same pass. optimize_fts() manages its own lock.
optimized = 0
try:
optimized = self.optimize_fts()
except Exception as exc:
logger.warning("FTS optimize before VACUUM failed: %s", exc)
# VACUUM cannot be executed inside a transaction.
with self._lock:
# Best-effort WAL checkpoint first, then VACUUM. PASSIVE, not
# TRUNCATE: a manual `hermes sessions vacuum` runs in a transient
# CLI process, and a TRUNCATE reset here would race a live gateway
# writer and tear B-tree pages (#45383). VACUUM folds the WAL back
# itself; journal_size_limit bounds the file.
try:
self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)")
except Exception as exc:
logger.debug("WAL checkpoint (PASSIVE) before VACUUM failed: %s", exc)
self._conn.execute("VACUUM")
# ...and again afterwards. VACUUM rewrites every page THROUGH the
# WAL, so the pre-VACUUM checkpoint above does nothing for the
# slack VACUUM itself creates: on a 3.0 GB database it left a
# 3.07 GB state.db-wal behind, so `sessions optimize` reported
# "reclaimed -11.2 MB" while actually consuming 3 GB of disk and
# filling the host to 100%. Truncating here is what makes the
# command a net win instead of a net loss on large databases.
try:
self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
except Exception as exc:
logger.debug("WAL checkpoint (TRUNCATE) after VACUUM failed: %s", exc)
# TRUNCATE may replace the WAL inode; adopt the post-VACUUM
# sidecars so the write-path generation guard does not halt a
# healthy exclusive maintenance connection.
self._record_db_file_identity()
return optimized
def maybe_auto_prune_and_vacuum(
self,
retention_days: int = 90,
min_interval_hours: int = 24,
vacuum: bool = True,
sessions_dir: Optional[Path] = None,
min_vacuum_interval_days: int = 30,
min_vacuum_freelist_ratio: float = AUTO_VACUUM_MIN_FREELIST_RATIO,
) -> Dict[str, Any]:
"""Idempotent auto-maintenance: prune inactive sessions + optional VACUUM.
Records the last run timestamp in state_meta so subsequent calls
within ``min_interval_hours`` no-op. VACUUM has its own, typically
longer, throttle controlled by ``min_vacuum_interval_days`` so routine
pruning does not repeatedly rewrite the database, and is additionally
gated on the reclaimable fraction of the file: it only runs when
``PRAGMA freelist_count / PRAGMA page_count`` exceeds
``min_vacuum_freelist_ratio`` (default
:data:`AUTO_VACUUM_MIN_FREELIST_RATIO`, 25%), so pruning a few small
sessions on a dense multi-GB database never triggers a full rewrite
(#54189). Designed to be called once at startup from long-lived
entrypoints (CLI, gateway, cron scheduler).
When *sessions_dir* is provided, on-disk transcript files
(``.json`` / ``.jsonl`` / ``request_dump_*``) for pruned sessions
are removed as part of the same sweep (issue #3015).
Stale-open reconciliation (#54189): several state-owned producers
(cron, kanban workers, subagents, one-shot CLI runs) never set
``ended_at`` when their process dies, and ``prune_sessions`` only
deletes ended rows — so retention was a no-op exactly where growth
concentrates. After pruning, this pass closes open rows from
:attr:`_AUTO_PRUNE_STALE_OPEN_SOURCES` whose activity is older than
``retention_days`` (``end_reason='startup_orphan_reap'``). Closed rows
stay resumable and are aged from their close, so they get one more
full retention window before a later pass deletes them. Messaging
and UI sources are never touched here.
Never raises. On any failure, logs a warning and returns a dict
with ``"error"`` set.
Returns a dict with keys:
- ``"skipped"`` (bool) — true if within min_interval_hours of last run
- ``"pruned"`` (int) — number of sessions deleted
- ``"closed"`` (int) — stale open state-owned sessions marked ended
- ``"vacuumed"`` (bool) — true if VACUUM ran
- ``"freelist_ratio"`` (float|None) — reclaimable fraction measured
when a VACUUM was considered (absent when it was not)
- ``"error"`` (str, optional) — present only on failure
"""
result: Dict[str, Any] = {
"skipped": False,
"pruned": 0,
"closed": 0,
"vacuumed": False,
}
maintenance_lock = _try_acquire_auto_maintenance_lock(self.db_path)
if maintenance_lock is None:
result["skipped"] = True
return result
try:
# Skip if another process/call did maintenance recently.
last_raw = self.get_meta("last_auto_prune")
now = time.time()
if last_raw:
try:
last_ts = float(last_raw)
if now - last_ts < min_interval_hours * 3600:
result["skipped"] = True
return result
except (TypeError, ValueError):
pass # corrupt meta; treat as no prior run
# Delete only sessions that were already explicitly closed. A
# startup orphan discovered by this pass is closed *after* pruning,
# preserving a full retention window in which it can be resumed.
pruned = self.prune_sessions(
older_than_days=retention_days,
sessions_dir=sessions_dir,
exclude_active_write_guards=True,
)
result["pruned"] = pruned
# Reap stale state-owned rows only. Runtime-owned messaging sources
# are intentionally outside this automatic destructive scope.
closed = self.sweep_orphaned_sessions(
max_idle_seconds=float(retention_days) * 86400.0,
sources=self._AUTO_PRUNE_STALE_OPEN_SOURCES,
exclude_pinned=True,
# These sources are owned by state.db lifecycles, not by the
# dashboard/TUI gateway heartbeats used by startup recovery.
respect_gateway_heartbeats=False,
)
result["closed"] = len(closed)
# Only VACUUM if we actually freed rows, no more often than once
# every min_vacuum_interval_days, AND only when the rewrite pays
# off: the reclaimable fraction of the file (freelist_count /
# page_count) must exceed AUTO_VACUUM_MIN_FREELIST_RATIO (#54189).
# A large prune (e.g. the first one to cross retention_days on a
# DB with tens of thousands of rows) can free enough pages that
# pruned > 0 fires on every subsequent startup even though a
# VACUUM already ran recently; and pruning one tiny session on a
# dense multi-GB DB would otherwise rewrite the whole file to
# reclaim a few MB. VACUUM on this DB's size (FTS5 shadow tables)
# is not cheap -- it holds an exclusive lock for the full rewrite.
# The time throttle says "not too often"; the ratio gate says
# "only when it pays off". Both must pass.
last_vacuum_raw = self.get_meta("last_vacuum")
vacuum_due = True
if last_vacuum_raw:
try:
vacuum_due = (now - float(last_vacuum_raw)) >= min_vacuum_interval_days * 86400
except (TypeError, ValueError):
vacuum_due = True
if vacuum and pruned > 0 and vacuum_due:
ratio = self._freelist_ratio()
result["freelist_ratio"] = ratio
if ratio is None or ratio > min_vacuum_freelist_ratio:
try:
self.vacuum()
result["vacuumed"] = True
self.set_meta("last_vacuum", str(now))
except Exception as exc:
logger.warning("state.db VACUUM failed: %s", exc)
else:
logger.debug(
"state.db auto-maintenance: skipping VACUUM, only "
"%.1f%% of pages reclaimable (threshold %.0f%%)",
ratio * 100.0,
min_vacuum_freelist_ratio * 100.0,
)
# Record the attempt even if pruned == 0, so we don't retry
# every startup within the min_interval_hours window.
self.set_meta("last_auto_prune", str(now))
if closed or pruned > 0:
logger.info(
"state.db auto-maintenance: closed %d stale open session(s), "
"pruned %d session(s) inactive for %d days%s",
len(closed),
pruned,
retention_days,
" + VACUUM" if result["vacuumed"] else "",
)
except Exception as exc:
# Maintenance must never block startup. Log and return error marker.
logger.warning("state.db auto-maintenance failed: %s", exc)
result["error"] = str(exc)
finally:
_release_auto_maintenance_lock(maintenance_lock)
return result
def maybe_auto_archive(
self,
idle_days: float = 3,
min_interval_hours: int = 24,
exclude_pinned: bool = True,
) -> Dict[str, Any]:
"""Idempotent auto-archive: soft-hide sessions idle for ``idle_days``.
Sibling of :meth:`maybe_auto_prune_and_vacuum` but non-destructive —
it archives (hides) rather than deletes, and ages on last activity
(see :meth:`archive_stale_sessions`) rather than creation. Records the
last run in ``state_meta['last_auto_archive']`` so calls within
``min_interval_hours`` no-op; safe to call opportunistically (startup
hooks, or when the Desktop backend lists sessions).
Never raises. Returns a dict with:
- ``"skipped"`` (bool) — within min_interval_hours of last run
- ``"archived"`` (int) — sessions archived this run
- ``"error"`` (str, optional) — present only on failure
"""
result: Dict[str, Any] = {"skipped": False, "archived": 0}
try:
last_raw = self.get_meta("last_auto_archive")
now = time.time()
if last_raw:
try:
if now - float(last_raw) < min_interval_hours * 3600:
result["skipped"] = True
return result
except (TypeError, ValueError):
pass # corrupt meta; treat as no prior run
archived = self.archive_stale_sessions(
idle_days, exclude_pinned=exclude_pinned
)
result["archived"] = archived
# Record even a zero-archive run so we don't re-sweep every call
# within the interval window.
self.set_meta("last_auto_archive", str(now))
if archived > 0:
logger.info(
"state.db auto-archive: archived %d session(s) idle >= %s days",
archived,
idle_days,
)
except Exception as exc:
logger.warning("state.db auto-archive failed: %s", exc)
result["error"] = str(exc)
return result
# ── Handoff (cross-platform session transfer) ──────────────────────────
#
# State machine:
# None — no handoff in flight
# "pending" — CLI requested handoff, gateway hasn't picked it up yet
# "running" — gateway is processing (session switch + synthetic turn)
# "completed"— gateway successfully delivered the synthetic turn
# "failed" — gateway hit an error; reason in handoff_error
#
# The CLI writes "pending" then poll-waits for terminal state. The gateway
# watcher transitions pending→running→{completed,failed}.
def request_handoff(self, session_id: str, platform: str) -> bool:
"""Mark a session as pending handoff to the given platform.
Returns True if the row was found and not already in flight; False if
the session is already in a non-terminal handoff state.
"""
def _do(conn):
cur = conn.execute(
"UPDATE sessions "
"SET handoff_state = 'pending', "
" handoff_platform = ?, "
" handoff_error = NULL "
"WHERE id = ? AND (handoff_state IS NULL "
" OR handoff_state IN ('completed', 'failed'))",
(platform, session_id),
)
return cur.rowcount > 0
return self._execute_write(_do)
def get_handoff_state(self, session_id: str) -> Optional[Dict[str, Any]]:
"""Read the current handoff state for a session.
Returns ``{"state", "platform", "error"}`` or None if the session has
no handoff record.
"""
try:
with self._read_ctx() as conn:
row = conn.execute(
"SELECT handoff_state, handoff_platform, handoff_error "
"FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if not row:
return None
return {
"state": row["handoff_state"],
"platform": row["handoff_platform"],
"error": row["handoff_error"],
}
except Exception:
return None
def list_pending_handoffs(self) -> List[Dict[str, Any]]:
"""Return all sessions in handoff_state='pending', oldest first.
Used by the gateway's handoff watcher.
"""
try:
with self._read_ctx() as conn:
rows = conn.execute(
"SELECT s.*, "
"COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved "
"FROM sessions s "
"LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash "
"WHERE s.handoff_state = 'pending' "
"ORDER BY s.started_at ASC"
).fetchall()
return [self._session_row_dict(r) for r in rows]
except Exception:
return []
def claim_handoff(self, session_id: str) -> bool:
"""Atomically transition pending → running. Returns True if claimed."""
def _do(conn):
cur = conn.execute(
"UPDATE sessions SET handoff_state = 'running' "
"WHERE id = ? AND handoff_state = 'pending'",
(session_id,),
)
return cur.rowcount > 0
return self._execute_write(_do)
def complete_handoff(self, session_id: str) -> None:
"""Mark a handoff as completed."""
def _do(conn):
conn.execute(
"UPDATE sessions SET handoff_state = 'completed', "
"handoff_error = NULL WHERE id = ?",
(session_id,),
)
self._execute_write(_do)
def fail_handoff(
self,
session_id: str,
error: str,
*,
only_states: Optional[Tuple[str, ...]] = None,
) -> bool:
"""Mark a handoff as failed and record the reason.
``only_states`` makes the write a compare-and-swap: the row is only
failed when its current ``handoff_state`` is in the given tuple.
Waiters that give up (CLI 60s poll, Desktop bounded poll) MUST pass
``only_states=("pending",)`` — once the gateway watcher has claimed
the row (``running``) it owns the terminal state, and a waiter-side
unconditional fail races the dispatch: the gateway later overwrites
``failed`` → ``completed`` while the user was already told the
gateway is down (split-brain — the handoff actually delivered and
``switch_session`` re-pointed the session).
The gateway watcher itself fails its OWN claimed row unconditionally
(no ``only_states``) — it is the owner while the row is ``running``.
Returns True when a row was transitioned to ``failed``.
"""
def _do(conn):
if only_states:
placeholders = ", ".join("?" for _ in only_states)
cur = conn.execute(
"UPDATE sessions SET handoff_state = 'failed', "
f"handoff_error = ? WHERE id = ? AND handoff_state IN ({placeholders})",
(error[:500], session_id, *only_states),
)
else:
cur = conn.execute(
"UPDATE sessions SET handoff_state = 'failed', "
"handoff_error = ? WHERE id = ?",
(error[:500], session_id),
)
return cur.rowcount > 0
return bool(self._execute_write(_do))
def reclaim_stale_running_handoffs(self, error: str) -> List[str]:
"""Fail every handoff stuck in ``running``. Returns the ids reclaimed.
Only the gateway's watcher ever sets ``running``, and it does so for
the duration of a single in-process dispatch. So any row still in
``running`` when a watcher starts up belongs to a PREVIOUS gateway
that died mid-dispatch (crash, kill, machine reboot).
Such a row is unrecoverable *and* poisonous: ``request_handoff`` only
accepts a new request when the state is NULL/``completed``/``failed``,
so a stranded ``running`` row makes that session permanently unable to
hand off again — with no error surfaced anywhere.
Failing (rather than re-queueing as ``pending``) is deliberate: the
dead gateway may have already switched the session key and dispatched
the synthetic turn before dying, so a blind retry risks double
delivery. The user's CLI has long since timed out; the right outcome
is a clean terminal state they can retry from explicitly.
"""
def _do(conn):
cur = conn.execute(
"SELECT id FROM sessions WHERE handoff_state = 'running'"
)
ids = [r[0] for r in cur.fetchall()]
if ids:
conn.execute(
"UPDATE sessions SET handoff_state = 'failed', "
"handoff_error = ? WHERE handoff_state = 'running'",
(error[:500],),
)
return ids
try:
return self._execute_write(_do) or []
except Exception:
# Swallow but never silently: a persistently failing reclaim
# leaves poisonous 'running' rows in place (sessions that can
# never hand off again), so the operator needs a trace of it.
logger.warning(
"reclaim_stale_running_handoffs failed; stranded 'running' "
"handoff rows (if any) were left in place", exc_info=True,
)
return []
class AsyncSessionDB:
"""Async door onto SessionDB: offloads each call via asyncio.to_thread so a blocking SQLite call never freezes the event loop. Generic forwarder — the audit confirms no method returns a live cursor/generator."""
def __init__(self, db: "SessionDB") -> None:
self._db = db
def __getattr__(self, name: str):
attr = getattr(self._db, name)
if not callable(attr):
return attr
async def _offloaded(*args, **kwargs):
return await asyncio.to_thread(attr, *args, **kwargs)
return _offloaded
Reference in New Issue View Git Blame Copy Permalink