""" Gateway runner - entry point for messaging platform integrations. This module provides: - start_gateway(): Start all configured platform adapters - GatewayRunner: Main class managing the gateway lifecycle Usage: # Start the gateway python -m gateway.run # Or from CLI python cli.py --gateway """ # IMPORTANT: hermes_bootstrap must be the very first import — UTF-8 stdio # on Windows. No-op on POSIX. See hermes_bootstrap.py for full rationale. try: import hermes_bootstrap # noqa: F401 except ModuleNotFoundError: # Graceful fallback when hermes_bootstrap isn't registered in the venv # yet — happens during partial ``hermes update`` where git-reset landed # new code but ``uv pip install -e .`` didn't finish. Missing bootstrap # means UTF-8 stdio setup is skipped on Windows; POSIX is unaffected. pass import asyncio import concurrent.futures import dataclasses import faulthandler import functools import inspect import json import logging import os import queue import re import shlex import site import sys import signal import threading import time import traceback from collections import OrderedDict from contextvars import Context, copy_context from pathlib import Path from datetime import datetime, timedelta, timezone from typing import Awaitable, Callable, Dict, Optional, Any, List, Tuple, Union, cast from agent.async_utils import consume_detached_task_result, safe_schedule_threadsafe from agent.conversation_compression import ( COMPACTION_DONE_STATUS, COMPACTION_HEARTBEAT_STATUS, COMPACTION_STATUS, COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE, COMPRESSION_RETRY_MESSAGES_STATUS_TEMPLATE, COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE, COMPRESSION_RETRY_TOO_LARGE_STATUS_TEMPLATE, IDLE_COMPACTION_STATUS_TEMPLATE, PRE_API_COMPRESSION_STATUS_TEMPLATE, PREFLIGHT_COMPRESSION_STATUS_TEMPLATE, ) from agent.conversation_loop import INTERRUPT_WAITING_FOR_MODEL_PREFIX from agent.compaction_display import project_compaction_message_for_display from agent.i18n import t from agent.interrupt_compat import request_hard_interrupt from agent.turn_context import ( compression_made_progress, ) from hermes_cli.config import _is_ssh_remote_tilde_cwd, cfg_get from hermes_cli.fallback_config import get_fallback_chain # --- Agent cache tuning --------------------------------------------------- # Bounds the per-session AIAgent cache to prevent unbounded growth in # long-lived gateways (each AIAgent holds LLM clients, tool schemas, # memory providers, etc.). LRU order + idle TTL eviction are enforced # from _enforce_agent_cache_cap() and _session_expiry_watcher() below. # # These are the defaults; `agent.agent_cache.max_size` / # `agent.agent_cache.idle_ttl_secs` in config.yaml override them per # deployment. Neither bound knows how many BYTES a cached agent holds, so # _sweep_agent_cache_under_pressure() adds the missing memory-pressure valve # (see gateway/agent_cache_pressure.py). _AGENT_CACHE_MAX_SIZE = 128 _AGENT_CACHE_IDLE_TTL_SECS = 3600.0 # evict agents idle for >1h _PLATFORM_CONNECT_TIMEOUT_SECS_DEFAULT = 30.0 # Telegram cold polling now proves one real getUpdates round trip before connect # returns. Leave enough outer budget for initialize/deleteWebhook/start_polling # wall deadlines plus readiness; other platforms retain the 30s isolation bound. _TELEGRAM_CONNECT_TIMEOUT_SECS_DEFAULT = 180.0 # Cold-start cap for Telegram (#85993): the initial connect awaited before the # gateway reaches `running` must not spend the full 180s budget — an # unreachable Telegram would hold EVERY platform's serving state hostage for # the whole window. The initial attempt gets one bounded try; on timeout the # platform is queued for the reconnect watcher, which retries with the full # 180s budget (is_reconnect=True preserves the offline update queue, #46621). _TELEGRAM_INITIAL_CONNECT_TIMEOUT_SECS_DEFAULT = 45.0 _ADAPTER_DISCONNECT_TIMEOUT_SECS_DEFAULT = 5.0 # End reasons that mean the USER deliberately closed this thread of work # (/new -> session_reset / new_session, an explicit exit, or a /switch). # Shared by _classify_completion_target (pre-flight verdict) and # _resolve_async_delegation_session (in-pipeline routing) so the two can # never disagree: every reason the classifier calls "deliver" must be one # the resolver actually delivers, otherwise the durable row is acked at # adapter acceptance and then silently dropped inside the pipeline — # a falsely-acknowledged permanent loss. _USER_BOUNDARY_END_REASONS = ( "session_reset", "user_exit", "session_switch", "new_session", ) # Round-2 #2: upper bound on a single stall-notify adapter.send so a wedged # transport cannot block the session-stall watcher pass (notify-only path; # on timeout the latch stays clear and the next tick retries). _STALL_NOTIFY_SEND_TIMEOUT_SECONDS = 15.0 _GATEWAY_PROXY_SSE_BUFFER_MAX_CHARS = 16 * 1024 * 1024 _TELEGRAM_COMMAND_MENTION_RE = re.compile(r"(?)\s+\d[\d,]*\s+messages,\s+retrying" r"|compressed\s+~[\d,]+\s+(?:→|->)\s+~[\d,]+\s+tokens,\s+retrying" r"|context\s+reduced\s+to\s+[\d,]+\s+tokens\s+\(was\s+[\d,]+\),\s+retrying" r"|session\s+compressed\s+\d+\s+times" r"|rate\s+limited\.\s+waiting\s+\d" r"|retrying\s+in\s+\d" r"|max\s+retries\s+\(\d+\).*(?:trying\s+fallback|exhausted|invalid\s+responses)" r"|stream\s+(?:drop|drop\s+mid\s+tool-call).+retry\s+\d" r"|stale\s+connections\s+from\s+a\s+previous\s+provider\s+issue" rf"|{re.escape(COMPACTION_DONE_STATUS)}" r")", re.IGNORECASE | re.DOTALL, ) _HYGIENE_COOLDOWN_LADDER_MULTIPLIERS = (1, 3, 9) # Absolute ceiling on an escalated hygiene cooldown, mirroring # _RECONNECT_BACKOFF_CAP above: with an operator-raised base the multiplier # ladder alone would reach 9h (base 3600 -> 32400s), which is indistinguishable # from "compaction silently switched off". 1h is well past the point where a # retry is cheap and still recovers within a session. _HYGIENE_COOLDOWN_MAX_SECONDS = 3600.0 # Flat retry-after recorded when a hygiene compression is ABANDONED because # the turn-hold budget expired while the summary was still streaming (not a # failure — the compressor was healthy, we just could not keep holding the # user's turn). Spaces out re-attempts so sustained traffic does not spawn, # hold, and cancel a fresh compressor on every single turn; deliberately # outside the failure-streak ladder above. _HYGIENE_TURNHOLD_RETRY_SECONDS = 60.0 def _hygiene_cooldown_for_failure( gateway, session_key: str, base_cooldown_seconds: float, ) -> float: """Bump the hygiene failure streak and return the escalated cooldown. This is a MULTIPLIER ladder (x1, x3, x9) over the operator's configured ``hygiene_failure_cooldown_seconds``, clamped to ``_HYGIENE_COOLDOWN_MAX_SECONDS``, so a tuned base is preserved as rung 1. It exists because the in-agent equivalent is unreachable from here: ``ContextCompressor.record_timeout_failure`` escalates on an absolute 60 -> 300 -> 900s ladder driven by the in-memory ``_consecutive_timeout_failures`` counter, which ``bind_session_state`` zeroes. Session hygiene constructs a FRESH ``AIAgent`` per run and re-binds state every time, so from the gateway that streak is structurally always 0 and only the flat ``hygiene_failure_cooldown_seconds`` could ever be recorded — a session whose summary model always times out retried on that same fixed interval forever (#79624). The streak is mirrored to SQLite by rotation-stable ``session_key`` so it outlives both the per-run agent and gateway restarts; ``PersistentState`` keeps the hot in-process view. """ streak = 1 state = None try: state = gateway._session_state(session_key).persistent except Exception as exc: logger.debug("hygiene failure streak update failed: %s", exc) session_db = getattr(gateway, "_session_db", None) session_db = getattr(session_db, "_db", session_db) increment = getattr(session_db, "increment_hygiene_failure_streak", None) if callable(increment): try: streak = max(1, int(increment(session_key))) if state is not None: state.hygiene_failure_streak = streak except Exception as exc: logger.debug("hygiene failure streak persist failed: %s", exc) if state is not None: state.hygiene_failure_streak += 1 streak = state.hygiene_failure_streak elif state is not None: state.hygiene_failure_streak += 1 streak = state.hygiene_failure_streak multiplier = _HYGIENE_COOLDOWN_LADDER_MULTIPLIERS[ min(streak, len(_HYGIENE_COOLDOWN_LADDER_MULTIPLIERS)) - 1 ] return min(base_cooldown_seconds * multiplier, _HYGIENE_COOLDOWN_MAX_SECONDS) def _reset_hygiene_failure_streak(gateway, session_key: str) -> None: """Clear the hygiene failure streak after a compression that reduced context. Peeks rather than get-or-creates: writing a 0 that is already 0 must not materialise a ``_sessions`` entry (those are never evicted). """ try: state = gateway._peek_session_state(session_key) if state is not None: state.persistent.hygiene_failure_streak = 0 except Exception as exc: logger.debug("hygiene failure streak reset failed: %s", exc) session_db = getattr(gateway, "_session_db", None) session_db = getattr(session_db, "_db", session_db) reset = getattr(session_db, "reset_hygiene_failure_streak", None) if callable(reset): try: reset(session_key) except Exception as exc: logger.debug("hygiene failure streak persistent reset failed: %s", exc) def hygiene_compaction_recovered( *, aborted: bool, rotated: bool, in_place: bool, msg_count: int, new_count: int, approx_tokens: int, new_tokens: int, ) -> bool: """True when a hygiene run actually recovered the session. Extracted from ``_handle_message_with_agent`` so the decision is unit testable: it previously lived inline in a ~2000-line async method, and the only way to pin it was a source-reading test — which AGENTS.md bans outright, naming this file. "Recovered" requires all three: * the compressor did not abort (no summary produced at all); * the transcript was actually rewritten — either rotated into a new session or compacted in place. The degenerate "did not rotate or compact in place" path (#21301) reuses the pre-compression counts, so relying on the numbers alone would read a no-op as success; * the request materially shrank, per the canonical :func:`compression_made_progress` (#39548) — a row-count drop counts even when the summary keeps the token estimate flat, and a sub-5% token wobble does not count at all. The token arguments are deliberately compared through that shared predicate rather than with a bare ``<``: ``approx_tokens`` can be provider-reported while ``new_tokens`` is always a rough estimate (documented to run 30-50% high on code-heavy sessions), so a bare comparison both misses real wins and counts noise as one. """ if aborted: return False if not (rotated or in_place): return False return compression_made_progress( msg_count, new_count, approx_tokens, new_tokens ) def _hygiene_compression_timeout_message( *, total_exhausted: bool, elapsed: float, idle_timeout: float, progress_observed: bool, ) -> str: """Describe the host timeout that actually ended hygiene compression.""" if total_exhausted: progress = ( " after summary output was observed" if progress_observed else "" ) return ( "⚠️ Context compression reached its total ceiling after " f"{elapsed:.1f}s{progress}. No messages were dropped — continuing " "without compression. Run /compress to retry or /reset for a clean " "session." ) return ( f"⚠️ Context compression timed out after {idle_timeout:.1f}s with no " "output from the summary model. No messages were dropped — continuing " "without compression. Run /compress to retry, /reset for a clean " "session, or check your auxiliary.compression model configuration." ) async def run_codex_hygiene_compaction( gateway, session_key: str, session_id: str, *, auto_mode: str, history: list, approx_tokens: int, timeout_seconds: float, failure_cooldown_seconds: float = 300.0, ) -> str: """Session hygiene for ``codex_app_server`` sessions (#73503). On this runtime the model's real working context is the app-server's server-side thread, not Hermes' transcript: ``CodexAppServerSession`` is constructed with no history and each turn submits only the new user message (agent/codex_runtime.py), so the persisted transcript is a mirror that is never replayed into a thread. Two consequences drive this path: * Rewriting the local mirror (the detached hygiene agent's normal compression) shrinks nothing the model actually carries — it was a permanent no-op ("compressed 150 -> 150 msgs"). * Evicting the cached live agent afterwards destroys the only real context: the next turn spawns an EMPTY thread and the model starts blank while Hermes still mirrors a full history (abrupt amnesia — the user-facing damage documented on #73503). So hygiene must compact the LIVE cached agent's thread via the app-server's own ``thread/compact/start`` (through ``_compress_context_via_codex_app_server``) and KEEP that agent cached. Never build a detached compressor and never evict here. Mode contract (``compression.codex_app_server_auto``): only ``hermes`` lets Hermes' threshold initiate app-server compaction; ``native`` leaves the schedule to codex itself and ``off`` disables Hermes-initiated automatic compaction entirely — both return without touching the thread or the transcript, and neither may fall back to the local compressor. Returns an outcome tag for logging/tests: ``compacted``, ``skipped:`` or ``failed:``. """ mode = str(auto_mode or "native").lower() if mode not in {"native", "hermes", "off"}: mode = "native" if mode != "hermes": # native: the app-server compacts on its own schedule; off: the # operator disabled Hermes-initiated automatic compaction. A local # transcript fallback is wrong in EVERY mode here (it cannot shrink # the thread), so both modes are a clean skip — crucially without # the detached-compressor path's cache eviction. return f"skipped:mode={mode}" agent = None lock = getattr(gateway, "_agent_cache_lock", None) cache = getattr(gateway, "_agent_cache", None) if cache is not None: try: if lock: with lock: entry = cache.get(session_key) else: entry = cache.get(session_key) except Exception: entry = None agent = entry[0] if isinstance(entry, tuple) and entry else entry if agent is None or agent is _AGENT_PENDING_SENTINEL: # No live agent → no live thread → nothing real to compact. The # mirror-only rewrite the detached path would perform is exactly the # no-op this function exists to remove, so skip honestly instead. return "skipped:no-cached-agent" if getattr(agent, "_codex_session", None) is None: return "skipped:no-live-thread" loop = asyncio.get_running_loop() compressor = getattr(agent, "context_compressor", None) count_before = getattr(compressor, "compression_count", 0) worker_future = loop.run_in_executor( None, # Keep the caller's multiplexed profile secret scope and HERMES_HOME # override in the worker. The default executor does not propagate # ContextVars on the Python runtimes Hermes currently ships. copy_context().run, lambda: agent._compress_context( history, "", approx_tokens=approx_tokens, ), ) track_worker = getattr(gateway, "_track_deferred_agent_worker", None) if callable(track_worker): # ``wait_for`` only cancels the asyncio wrapper; the executor thread # keeps running. Keep it visible to gateway shutdown until the real # worker finishes, just like the detached local-compressor path. track_worker(worker_future, agent) try: await asyncio.wait_for( asyncio.shield(worker_future), timeout=max(float(timeout_seconds), 1.0), ) except asyncio.TimeoutError: # The executor thread keeps running (compact_thread has its own RPC # timeouts); brake per-turn retries so a wedged app-server does not # re-trigger a compaction attempt on every message. if failure_cooldown_seconds >= 0: _record_hygiene_cooldown( gateway, session_id, failure_cooldown_seconds, "codex app-server thread compaction timed out", ) logger.warning( "Session hygiene: codex app-server thread compaction for " "session %s timed out after %.1fs; continuing without compaction", session_id, timeout_seconds, ) return "failed:timeout" except Exception as exc: logger.warning( "Session hygiene: codex app-server thread compaction for " "session %s failed: %s", session_id, exc, ) return f"failed:{exc}" count_after = getattr(compressor, "compression_count", 0) if count_after > count_before: # A native compaction boundary was recorded on the live agent # (thread compacted server-side; transcript intentionally NOT # rewritten — state.db records the boundary, the mirror stays # intact and the agent stays cached). _reset_hygiene_failure_streak(gateway, session_key) return "compacted" # compress_context returned without recording a boundary: an internal # skip (its own failure cooldown) or a compaction error — the codex # route already persisted its own failure cooldown in that case. return "failed:no-boundary" def hygiene_wait_should_extend( *, idle: float, timeout: float, waited: float, ceiling: float, fence_cancelled: bool = False, ) -> bool: """Whether the hygiene host should keep waiting for a slow summary. A cancelled commit fence cannot produce a commit (#96953): extending the wait up to the 600s ceiling only queues inbound messages behind a doomed attempt. Stop extending immediately so the turn can continue. """ if fence_cancelled: return False return idle < timeout and waited < ceiling def _record_hygiene_cooldown( gateway, session_id: str, cooldown_seconds: float, error: Optional[str] = None, ) -> None: """Persist a session-hygiene compression-failure cooldown to the state DB. Uses the same ``compression_failure_cooldown_until`` column and ``record_compression_failure_cooldown`` method that the in-conversation compression path (``agent/context_compressor.py``) already uses, so the cooldown survives gateway restarts (#74136). ``error`` is forwarded because the recorder writes ``compression_failure_error`` UNCONDITIONALLY — omitting it clobbers to NULL any reason the in-conversation path recorded, and readers surface that reason to the user (falling back to "unknown error"). That matters more now that an escalated cooldown can last up to an hour. """ import time as _time session_db = getattr(gateway, "_session_db", None) if session_db is None: return session_db = getattr(session_db, "_db", session_db) recorder = getattr(session_db, "record_compression_failure_cooldown", None) if recorder is None: return try: recorder(session_id, _time.time() + cooldown_seconds, error) except Exception as exc: logger.debug("session hygiene cooldown persist failed: %s", exc) def _status_template_to_regex(template: str) -> str: """Compile a compression status template constant into a regex source. Literal text is escaped verbatim (so wording drift in agent/conversation_compression.py cannot silently diverge from this matcher — the constants ARE the wording) and each ``{field}`` format placeholder is replaced with a numeric-ish pattern covering every value the emit sites format in (ints, ``{:,}`` thousands separators). """ parts = re.split(r"\{[^{}]*\}", template) return r"[\d,]+".join(re.escape(part) for part in parts) # ROUTINE compression progress statuses, derived from the SAME template # constants the emit sites format (agent/conversation_compression.py, #69550) # — never re-inlined wording. Used ONLY by the opt-in # ``compression.progress_notices`` gate below (#52995) to decide which of the # noisy statuses matched by _TELEGRAM_NOISY_STATUS_RE are compression # progress (deliverable when the user opted in) versus unrelated aux/retry # chatter (always suppressed on chat surfaces). Failure notices and manual # /compress feedback never match _TELEGRAM_NOISY_STATUS_RE in the first # place, so they are unaffected by this gate. _COMPRESSION_PROGRESS_STATUS_RE = re.compile( "|".join( _status_template_to_regex(_template) for _template in ( COMPACTION_STATUS, COMPACTION_HEARTBEAT_STATUS, COMPACTION_DONE_STATUS, PRE_API_COMPRESSION_STATUS_TEMPLATE, PREFLIGHT_COMPRESSION_STATUS_TEMPLATE, IDLE_COMPACTION_STATUS_TEMPLATE, COMPRESSION_RETRY_TOO_LARGE_STATUS_TEMPLATE, COMPRESSION_RETRY_MESSAGES_STATUS_TEMPLATE, COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE, COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE, ) ), re.IGNORECASE, ) def _gateway_compression_progress_notices_enabled() -> bool: """True when the user opted into routine compression progress notices. Reads ``compression.progress_notices`` from the gateway's raw YAML config (#52995). Default False — routine compression stays silent-by-design on chat platforms unless explicitly enabled. Read live (mtime-cached) so a config edit on a running gateway takes effect on the next status. Fail-closed: any config read error keeps the silent default. """ try: config = _load_gateway_config() compression_cfg = config.get("compression") if isinstance(config, dict) else None if isinstance(compression_cfg, dict): return str(compression_cfg.get("progress_notices", False)).strip().lower() in { "true", "1", "yes", "on", } except Exception: pass return False # Surfaces that consume gateway text programmatically (CLI/TUI "local" # diagnostics, API JSON, webhook payloads) and therefore must keep RAW # status/error text. EVERY other platform is a human-facing chat surface # where operational lifecycle/provider-error noise (and any secrets in it) # must be suppressed or sanitized. Widens #28533's Telegram-only filter to # all chat gateways (#39293). Fail-closed: unknown/empty platform -> chat. _GATEWAY_RAW_TEXT_PLATFORMS = frozenset( {"local", "api_server", "webhook", "msgraph_webhook"} ) def _gateway_surface_passes_raw_text(platform: Any) -> bool: """True only for programmatic/local surfaces that must keep raw text.""" return _gateway_platform_value(platform) in _GATEWAY_RAW_TEXT_PLATFORMS _GATEWAY_PROVIDER_ERROR_RE = re.compile( r"(" # infrastructure/provider error preambles, not ordinary assistant prose r"api\s+(?:call\s+)?failed" r"|provider\s+authentication\s+failed" r"|non-retryable\s+error" r"|rate\s+limited\s+after\s+\d+\s+retries" r"|error\s+code\s*:" r"|\bhttp\s*\d{3}\b" r"|incorrect\s+api\s+key" r"|invalid\s+api\s+key" r")", re.IGNORECASE, ) _GATEWAY_PROVIDER_POLICY_RE = re.compile( r"(" # raw provider policy/safety bodies are noisy and may be sensitive r"cybersecurity\s+risk" r"|security\s+policy" r"|safety\s+policy" r"|policy\s+violation" r"|violat(?:e|es|ed|ion)" r"|blocked\s+(?:because|by|under)" r"|request\s+(?:was\s+)?(?:blocked|rejected)" r"|disallowed" r"|moderation" r")", re.IGNORECASE, ) _GATEWAY_AUTH_ERROR_RE = re.compile( r"(provider\s+authentication\s+failed|incorrect\s+api\s+key|invalid\s+api\s+key|\b401\b)", re.IGNORECASE, ) _GATEWAY_RATE_LIMIT_RE = re.compile( r"(rate\s+limit|rate-limited|\b429\b|quota|usage\s+limit)", re.IGNORECASE, ) _GATEWAY_CONNECTION_ERROR_RE = re.compile( r"(" r"(?:\w+\.)?(?:api\s*)?connection\s*(?:error|timeout)" r"|(?:\w+\.)?connect\s*(?:error|timeout)" r"|connection\s+refused" r"|connection\s+reset" r"|connection\s+aborted" r"|actively\s+refused" r"|winerror\s+10061" r"|errno\s+111" r"|no\s+route\s+to\s+host" r"|network\s+is\s+unreachable" r"|cannot\s+connect" r"|failed\s+to\s+establish" r"|could\s+not\s+connect" r")", re.IGNORECASE, ) _GATEWAY_SECRET_PATTERNS = ( re.compile(r"\bsk-[A-Za-z0-9][A-Za-z0-9_\-]{12,}\b"), re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b"), re.compile(r"\bxapp-\d+-[A-Za-z0-9\-]{20,}\b"), re.compile(r"\bxox[baprs]-[A-Za-z0-9\-]{20,}\b"), re.compile(r"\bhf_[A-Za-z0-9]{20,}\b"), re.compile(r"\bglpat-[A-Za-z0-9_\-]{20,}\b"), re.compile(r"(?i)\b(Bearer\s+)[A-Za-z0-9._\-]{20,}\b"), ) def _ensure_windows_gateway_venv_imports() -> None: """Make detached Windows gateway runs see the Hermes venv packages. Some Windows restart paths run the gateway under uv's base ``pythonw.exe`` to avoid the venv launcher respawning a visible console interpreter. That mode can import the source tree via cwd/PYTHONPATH but still miss optional packages installed only in ``venv/Lib/site-packages`` (notably the MCP SDK). Patch the live process before MCP discovery so tool injection does not depend on every launcher preserving PYTHONPATH perfectly. """ if sys.platform != "win32": return project_root = Path(__file__).resolve().parent.parent candidates: list[Path] = [] if os.environ.get("VIRTUAL_ENV"): candidates.append(Path(os.environ["VIRTUAL_ENV"])) candidates.append(project_root / "venv") seen: set[str] = set() for venv_dir in candidates: try: resolved_venv = venv_dir.resolve() except OSError: resolved_venv = venv_dir venv_key = str(resolved_venv).lower() if venv_key in seen: continue seen.add(venv_key) site_packages = resolved_venv / "Lib" / "site-packages" if not site_packages.exists(): continue project_entry = str(project_root) site_entry = str(site_packages) if project_entry not in sys.path: sys.path.insert(0, project_entry) # addsitepackages() semantics matter here: pywin32, used by the MCP # SDK on Windows, relies on .pth processing to expose pywintypes. site.addsitedir(site_entry) if site_entry in sys.path: sys.path.remove(site_entry) insert_at = 1 if sys.path and sys.path[0] == project_entry else 0 sys.path.insert(insert_at, site_entry) os.environ["VIRTUAL_ENV"] = str(resolved_venv) pythonpath = [project_entry, site_entry] if os.environ.get("PYTHONPATH"): pythonpath.append(os.environ["PYTHONPATH"]) os.environ["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(pythonpath)) return def _gateway_platform_value(platform: Any) -> str: """Return a normalized gateway platform value for enums or raw strings.""" return str(getattr(platform, "value", platform) or "").strip().lower() def _non_conversational_metadata( metadata: Optional[Dict[str, Any]] = None, *, platform: Any = None, ) -> Optional[Dict[str, Any]]: """Mark Discord lifecycle/status sends without changing other platforms.""" if _gateway_platform_value(platform) != "discord": return metadata merged = dict(metadata or {}) merged["non_conversational"] = True return merged def _interim_metadata( metadata: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Mark a mid-turn status/advisory send as NOT the turn-final. Stream-is-the-message adapters (relay Slack native streaming) intercept the first unmarked send to an armed (chat, turn) key and seal the live stream with its content. Every gateway-side send that can fire while a turn is streaming — heartbeats, inactivity warnings, approval fallbacks, background-review notices — MUST carry this marker or it will seal the user's answer stream with status text (PR 85796 review, B5: probed live, the 3-minute heartbeat sealed the stream and the real final arrived as a duplicate while later frames were silently swallowed by the seal tombstone). The marker is gateway-internal; adapters strip it before the wire. """ merged = dict(metadata or {}) merged["_interim_send"] = True return merged def _seed_hygiene_system_prompt( agent: Any, session_row: Optional[Dict[str, Any]], ) -> bool: """Keep gateway hygiene from rebuilding a live session's system prompt. The hygiene helper runs outside the live session's fully initialized prompt environment (hygiene-only platform marker, no platform context files; the memory provider is loaded only when ``compression.checkpoint_required`` demands it). Compression is allowed to persist a system prompt, so letting that helper rebuild one would strip external provider blocks from the live session. Seed the exact persisted prompt instead. When no usable prompt can be restored, seed an empty cache entry. Compression either preserves that unusable value or rebuilds with the hygiene-only platform marker; the real turn will rebuild either form with its fully initialized providers. """ stored_prompt = "" if isinstance(session_row, dict): raw_prompt = session_row.get("system_prompt") if isinstance(raw_prompt, str) and raw_prompt.strip(): stored_prompt = raw_prompt agent._cached_system_prompt = stored_prompt return bool(stored_prompt) def _is_transient_network_error(exc: BaseException) -> bool: """Return True for transient network errors safe to log + swallow. The crash class targeted by #31066 / #31110: an unhandled Telegram ``TimedOut`` (or peer ``NetworkError`` / ``httpx`` connection error) propagating to the event loop and killing the entire gateway process. These are by definition transient — the next poll cycle or user action recovers — so they must never crash the process. Walk the exception cause chain so wrapped errors (e.g. PTB's ``NetworkError`` wrapping ``httpx.ConnectError``) are still classified. The chain is bounded to avoid pathological cycles. """ seen: set[int] = set() cur: Optional[BaseException] = exc depth = 0 transient_class_names = { "TimedOut", "NetworkError", "ReadError", "WriteError", "ConnectError", "ConnectTimeout", "ReadTimeout", "WriteTimeout", "PoolTimeout", "RemoteProtocolError", "ServerDisconnectedError", "ClientConnectorError", "ClientOSError", } while cur is not None and depth < 12: ident = id(cur) if ident in seen: break seen.add(ident) depth += 1 name = type(cur).__name__ if name in transient_class_names: return True cur = cur.__cause__ or cur.__context__ return False def _gateway_loop_exception_handler( loop: "asyncio.AbstractEventLoop", context: Dict[str, Any] ) -> None: """Loop-level safety net for transient network errors. Installed once during :func:`start_gateway`. Catches the ``telegram.error.TimedOut`` crash class (issues #31066 / #31110) and any peer transient network error before it can kill the gateway process. Logs at WARNING with full traceback so the originating call site stays diagnosable; non-transient errors are forwarded to the default loop handler so real bugs still surface. """ exc = context.get("exception") if exc is not None and _is_transient_network_error(exc): task = context.get("future") or context.get("task") task_name = "" if task is not None: try: task_name = task.get_name() if hasattr(task, "get_name") else repr(task) except Exception: task_name = repr(task) logger.warning( "Gateway swallowed transient network error from %s: %s: %s", task_name or "", type(exc).__name__, exc, exc_info=(type(exc), exc, exc.__traceback__), ) return # Fall back to the default handler for anything we don't recognise. loop.default_exception_handler(context) def _redact_gateway_user_facing_secrets(text: str) -> str: """Secret redaction before text can leave the gateway. Delegates to the authoritative ``agent.redact.redact_sensitive_text`` — the same Tirith-grade redactor already applied to logs, tool output, and approval-command prompts — so the outbound chat path masks the full credential set the startup banner promises ("chat responses are scrubbed before delivery"), not a divergent subset. ``force=True`` honors redaction even when ``security.redact_secrets`` is off, matching the ``_redact_approval_command`` reasoning (#23810). The narrow ``_GATEWAY_SECRET_PATTERNS`` set runs as a belt-and-suspenders second pass so nothing the gateway historically caught can regress, and so redaction still degrades gracefully if the import ever fails. """ redacted = str(text or "") try: from agent.redact import redact_sensitive_text redacted = redact_sensitive_text(redacted, force=True) except Exception: # Fail-soft: fall back to the local pattern pass below rather than # letting a redactor import/error leak the raw text to chat. pass for pattern in _GATEWAY_SECRET_PATTERNS: redacted = pattern.sub(lambda m: (m.group(1) if m.lastindex else "") + "[REDACTED]", redacted) return redacted def _redact_approval_command(cmd: "str | None") -> str: """Redact credentials from a command before it goes into an approval prompt. Tirith's *findings* are already redacted, but the gateway approval prompt is built from the raw command string, so a credential-shaped value Tirith flagged would otherwise be echoed verbatim to the chat platform (#48456). Uses ``redact_sensitive_text(force=True)`` — the same Tirith-grade redactor — so the prompt honors redaction even when ``security.redact_secrets`` is off. Module-level so the wiring is unit-testable (the call site is a deeply nested gateway closure that cannot be driven directly). """ from agent.redact import redact_sensitive_text return redact_sensitive_text(str(cmd or ""), force=True) def _format_exec_approval_fallback( command: str, description: str, command_prefix: str, *, allow_permanent: bool = True, allow_session: bool = True, smart_denied: bool = False, ) -> str: """Render the text fallback from approval capabilities, not platform names.""" cmd_preview = command[:200] + "..." if len(command) > 200 else command heading = "⚠️ **Dangerous command requires approval:**" if smart_denied: heading = "⚠️ **Smart DENY — owner override for one operation:**" choices = [f"Reply `{command_prefix}approve` to execute this one operation"] if not smart_denied and allow_session: choices.append( f"`{command_prefix}approve session` to approve this pattern for the session" ) if allow_permanent: choices.append(f"`{command_prefix}approve always` to approve permanently") choices.append(f"`{command_prefix}deny` to cancel") return ( f"{heading}\n```\n{cmd_preview}\n```\nReason: {description}\n\n" + ", ".join(choices[:-1]) + f", or {choices[-1]}." ) def _gateway_provider_error_reply(text: str) -> str: """Map raw provider/API errors to a short user-safe Telegram reply.""" if _GATEWAY_AUTH_ERROR_RE.search(text): return ( "⚠️ Provider authentication failed. Check the configured credentials; " "raw provider details are in the gateway logs." ) if _GATEWAY_PROVIDER_POLICY_RE.search(text): return ( "⚠️ The model provider rejected the request. I kept the raw provider " "error out of chat; check gateway logs for details or try rephrasing." ) if _GATEWAY_RATE_LIMIT_RE.search(text): return "⏱️ The model provider is rate-limiting requests. Please wait a moment and try again." if _GATEWAY_CONNECTION_ERROR_RE.search(text): return ( "⚠️ The model server is not responding — it looks like the configured " "model endpoint is not running or is unreachable." ) return ( "⚠️ The model provider failed after retries. I kept raw provider details " "out of chat; check gateway logs for diagnostics." ) _GATEWAY_PROVIDER_ERROR_SHAPE_RE = re.compile( r"^\s*(\W*\s*)?(" r"api\s+(?:call\s+)?failed" r"|provider\s+authentication\s+failed" r"|non-retryable\s+error" r"|rate\s+limited\s+after\s+\d+\s+retries" r"|error\s+code\s*:" r"|http\s*\d{3}\b" r"|incorrect\s+api\s+key" r"|invalid\s+api\s+key" r"|(?:\w+\.)?(?:api\s*)?connection\s*(?:error|timeout)" r"|(?:\w+\.)?connect\s*(?:error|timeout)" r"|connection\s+refused" r"|connection\s+reset" r"|connection\s+aborted" r"|actively\s+refused" r"|winerror\s+10061" r"|errno\s+111" r"|all\s+connection\s+attempts\s+failed" r")", re.IGNORECASE, ) def _looks_like_gateway_provider_error(text: str) -> bool: """True when text is infrastructure/provider failure, not normal content. Two heuristics combined so the rewrite only fires on actual provider error envelopes, not on assistant prose that happens to mention an HTTP status code: 1. The text is short — real provider errors are 1–3 lines of envelope text; assistant answers are usually longer. 2. AND the error marker appears at the start of the message (optionally behind a punctuation/symbol prefix), not buried mid-paragraph in an explanation like "HTTP 404 means 'not found' — ...". """ if not text: return False body = str(text).strip() # Provider failure envelopes are short. Assistant answers that happen # to mention HTTP status codes ("HTTP 404 means...") tend to be longer. if len(body) > 400 or body.count("\n") > 4: return False return bool(_GATEWAY_PROVIDER_ERROR_SHAPE_RE.search(body)) def _sanitize_gateway_final_response(platform: Any, text: str) -> str: """Sanitize final gateway replies before sending them to chat surfaces. Every human-facing chat surface (Telegram, WhatsApp, Discord, Slack, Signal, Matrix, plugin platforms, etc.) should receive concise, safe provider failure categories with secrets redacted instead of raw HTTP bodies, request IDs, leaked credentials, or policy text. Only programmatic surfaces in ``_GATEWAY_RAW_TEXT_PLATFORMS`` (CLI/TUI ``local`` diagnostics, API JSON, webhook payloads) keep the raw text unchanged. """ if not text: return text if _gateway_surface_passes_raw_text(platform): return text # Lone UTF-16 surrogates (U+D800–U+DFFF) in model output crash chat # surfaces downstream: Telegram's ``utf16_len`` length check and Signal # formatting both ``.encode()`` the reply and raise UnicodeEncodeError # before any send (#55143, #55309). The stored-history copy is already # sanitized by ``build_assistant_message`` and ``finalize_turn`` scrubs # the returned ``final_response``, but this boundary is the last line of # defense for every legacy/plugin delivery path that hands us raw text. # Raw-text/programmatic surfaces above keep passthrough — their JSON # consumers escape surrogates safely. from agent.message_sanitization import _sanitize_surrogates text = _sanitize_surrogates(str(text)) # Cancellation metadata, not assistant prose. ACP/TUI already suppress # this sentinel; chat surfaces should too (#7921). if str(text).strip().startswith(INTERRUPT_WAITING_FOR_MODEL_PREFIX): return "" redacted = _redact_gateway_user_facing_secrets(str(text)) if _looks_like_gateway_provider_error(redacted): return _gateway_provider_error_reply(redacted) return redacted def _prepare_gateway_status_message(platform: Any, event_type: str, message: str) -> Optional[str]: """Filter/sanitize agent status callbacks before platform delivery. Local/CLI sessions keep the raw diagnostic stream. Messaging gateway surfaces should not receive transient auxiliary/compression chatter. """ text = str(message or "").strip() if not text: return None if _gateway_surface_passes_raw_text(platform): return text text = _redact_gateway_user_facing_secrets(text) if _TELEGRAM_NOISY_STATUS_RE.search(text): # Opt-in #52995: `compression.progress_notices: true` lets ROUTINE # compression progress statuses through to chat platforms. The # membership check is derived from the #69550 template constants, so # non-compression noise (aux failures, provider retry chatter, ...) # stays suppressed even when the gate is open. Default False keeps # the silent-by-design behavior byte-identical. if not ( _gateway_compression_progress_notices_enabled() and _COMPRESSION_PROGRESS_STATUS_RE.search(text) ): return None if _looks_like_gateway_provider_error(text): return _gateway_provider_error_reply(text) return text def render_notice_line(notice) -> str: """Render an AgentNotice to a single plaintext line for messaging platforms. Messaging has no persistent status bar (unlike the TUI), so a notice is a one-shot standalone push. The notice policy already bakes the level glyph (⚠ / • / ✕ / ✓) into the text, and the TUI + CLI REPL render that text verbatim — so we emit it as-is here too. Prepending a per-level glyph would DOUBLE it ("⚠ ⚠ Credits 90% used", "⛔ ✕ Credit access paused"). Plaintext only — no markdown — so it renders uniformly across Telegram/Discord/Slack/ SMS without per-platform escaping. Fail-soft: a malformed/empty notice degrades to "" rather than raising on the agent's callback path. """ return str(getattr(notice, "text", "") or "").strip() async def _send_or_update_status_coro(adapter, chat_id, status_key, content, metadata): """Route a status message through adapter.send_or_update_status when supported. Issue #30045: adapters that implement send_or_update_status (currently Telegram) edit the previous bubble for the same status_key instead of appending a new one. Adapters without the method fall back to plain send. """ sender = getattr(adapter, "send_or_update_status", None) if callable(sender): return await sender(chat_id, status_key, content, metadata=metadata) return await adapter.send(chat_id, content, metadata=metadata) def _approval_send_outcome(future, timeout: float) -> str: """Classify an approval prompt send as ``sent`` / ``failed`` / ``ambiguous``. ``ambiguous`` == the scheduling future timed out. The card may well have posted: the connector may only ack after the deadline (slow platform API call, transient backpressure, event-loop stall), and treating that timeout as a failure has been observed in live relay testing to re-send the card repeatedly, leaving the user's tap resolving a prompt whose turn had moved on. Callers must treat ``ambiguous`` as possibly-delivered: keep the prompt registration alive and do NOT re-send or fall back — the boundary rule is that only a DEFINITIVE failure (error result / non-timeout exception / no future) re-asks. Definitive failures log their detail here (scheduling exception text or the SendResult error) so callers sharing this classifier keep the diagnostic breadcrumb the old inline code had. """ if future is None: logger.warning("Prompt send failed: no scheduling future (loop unavailable)") return "failed" try: result = future.result(timeout=timeout) except concurrent.futures.TimeoutError: return "ambiguous" except Exception as exc: logger.warning("Prompt send failed: %s", exc) return "failed" if getattr(result, "success", False): return "sent" logger.warning( "Prompt send failed: %s", getattr(result, "error", None) or "unknown error" ) return "failed" def _clarify_send_disposition(fut, *, session_key: str, clarify_mod) -> "str | None": """Decide whether a clarify prompt send aborts the wait, per the boundary rule. Same physics as the exec-approval card: the scheduling future can hit its deadline while the clarify card HAS already posted (late connector ack). Treating that timeout as a definitive failure cleared the session out from under a rendered card — the user answers a question whose registration is gone. Only a DEFINITIVE failure (error result / non-timeout exception / no future) tears down the registration and aborts; ``ambiguous`` keeps the registration armed and proceeds to the normal bounded wait, which already handles the truly-lost-card case via its response timeout. Returns the abort sentinel string on definitive failure, else ``None`` (proceed to ``wait_for_response``). """ outcome = _approval_send_outcome(fut, timeout=15) if outcome == "failed": # Couldn't deliver the prompt — clean up and return the sentinel so # the agent can fall back to a sensible default rather than hanging. logger.warning("Clarify send failed definitively; clearing registration") clarify_mod.clear_session(session_key) return "[clarify prompt could not be delivered]" if outcome == "ambiguous": logger.warning( "Clarify prompt send timed out — treating as possibly-delivered " "(no teardown; the registration stays armed for a late reply)" ) return None def _clarify_send_then_wait(fut, *, clarify_id: str, session_key: str, clarify_mod) -> str: """Resolve a clarify prompt: send disposition, then the bounded wait. The full caller contract in one testable seam: a definitive send failure returns the undeliverable sentinel (registration torn down); ``sent`` and ``ambiguous`` both proceed to ``wait_for_response`` with the configured timeout — for ambiguous, the registration stays armed so a late reply to the (probably rendered) card still resolves. """ abort = _clarify_send_disposition( fut, session_key=session_key, clarify_mod=clarify_mod ) if abort is not None: return abort timeout = clarify_mod.get_clarify_timeout() response = clarify_mod.wait_for_response(clarify_id, timeout=float(timeout)) if response is None or response == "": # Timeout or session-boundary cancellation return f"[user did not respond within {int(timeout / 60)}m]" return response def _resolve_progress_thread_id( platform: Any, source_thread_id: Any, event_message_id: Any, *, reply_in_thread: bool = True, ) -> Optional[str]: """Return thread/root ID that progress/status bubbles should target. ``reply_in_thread=False`` (Slack ``platforms.slack.extra.reply_in_thread``) disables the synthetic-thread fallback: progress messages must not create a thread the final flat reply would then inherit. A source.thread_id equal to the event's own message id is the adapter's synthetic session-keying thread, not a real thread — treat it as "no thread" too (#18859). """ platform_value = getattr(platform, "value", platform) platform_key = str(platform_value or "").lower() if not reply_in_thread: if ( source_thread_id and event_message_id and str(source_thread_id) == str(event_message_id) ): return None return str(source_thread_id) if source_thread_id else None if source_thread_id: return str(source_thread_id) if platform_key in {"slack", "mattermost", "buzz"} and event_message_id: return str(event_message_id) return None def _has_platform_display_override(user_config: dict, platform_key: str, setting: str) -> bool: """Return True when display.platforms. explicitly sets setting.""" display = user_config.get("display") if isinstance(user_config, dict) else None if not isinstance(display, dict): return False platforms = display.get("platforms") if not isinstance(platforms, dict): return False platform_cfg = platforms.get(platform_key) return isinstance(platform_cfg, dict) and setting in platform_cfg def _resolve_gateway_display_bool( user_config: dict, platform_key: str, setting: str, *, default: bool = False, platform: Any = None, require_platform_override_for: set[Any] | None = None, ) -> bool: """Resolve a boolean display setting with optional platform-only opt-in. Some display features expose assistant scratch text rather than deliberate user-facing output. For high-noise threaded chat surfaces such as Mattermost, a global opt-in is too broad: they must be enabled with an explicit display.platforms.. override. """ current_platform = _gateway_platform_value(platform or platform_key) platform_only = { _gateway_platform_value(candidate) for candidate in (require_platform_override_for or set()) } if ( current_platform in platform_only and not _has_platform_display_override(user_config, platform_key, setting) ): return False from gateway.display_config import resolve_display_setting value = resolve_display_setting(user_config, platform_key, setting, default) if isinstance(value, bool): return value if isinstance(value, str): return value.strip().lower() in {"true", "yes", "1", "on"} if value is None: return bool(default) return bool(value) def _telegramize_command_mentions(text: str, platform: Any) -> str: """Rewrite slash-command mentions to Telegram-valid command names. Telegram Bot API command names allow only lowercase letters, digits, and underscores. Keep other platform renderings unchanged, but normalize Telegram help text so command mentions remain clickable/valid there. """ platform_value = getattr(platform, "value", platform) if platform_value != "telegram": return text from hermes_cli.commands import _sanitize_telegram_name def _replace(match: re.Match[str]) -> str: sanitized = _sanitize_telegram_name(match.group(1)) return f"/{sanitized}" if sanitized else match.group(0) return _TELEGRAM_COMMAND_MENTION_RE.sub(_replace, text) # Only auto-continue interrupted gateway turns while the interruption is fresh. # Stale tool-tail/resume markers can otherwise revive an unrelated old task # after a gateway restart when the user's next message starts new work. # # The freshness signal is the timestamp of the last transcript row, which # ``hermes_state.get_messages`` carries on every persisted message. This # handles the two auto-continue cases uniformly: # * resume_pending (gateway restart/shutdown watchdog marked the session) # * tool-tail (last persisted message is a tool result the agent # never got to reply to) # In both cases "when did we last do anything on this transcript" is the # correct freshness question, so one signal replaces two divergent ones. # # Default window: 1 hour. This comfortably covers ``agent.gateway_timeout`` # (30 min default) plus runtime slack — a legitimate long-running turn that # gets interrupted near its timeout boundary and is resumed shortly after # is still classified fresh. Override via # ``config.yaml`` ``agent.gateway_auto_continue_freshness``. _AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT = 60 * 60 # Default bound for how long ``_finish_startup_restore`` waits on boot # auto-resume turns before releasing the inbound gate (see # ``_startup_restore_drain_timeout_secs``). 30s is comfortably longer than a # normal resume turn's first response yet short enough that one pathologically # long resumed turn can't hold every channel's inbound queued for minutes. # Override via ``config.yaml`` ``agent.gateway_startup_restore_drain_timeout``. _STARTUP_RESTORE_DRAIN_TIMEOUT_SECS_DEFAULT = 30.0 # Default bound for the boot-time turn-machinery warm-up (#99373). On a # fresh boot with no resume_pending sessions ``_finish_startup_restore`` # used to open the inbound gate almost immediately, while the agent-side # turn machinery (the run_agent/model_tools import graph, the tool-registry # check_fn probes, the prompt builder) was still completely cold. A message # arriving in that window was served with a skeleton system prompt: no # context tier, no tool schemas (~1.7K tokens instead of ~14.6K). The # warm-up runs BEFORE the gate opens so the first inbound turn starts with # initialized machinery; the bound keeps a wedged init from making the # gateway permanently unavailable. Override via ``config.yaml`` # ``agent.gateway_startup_warmup_timeout`` (non-positive disables warm-up). _STARTUP_WARMUP_TIMEOUT_SECS_DEFAULT = 20.0 def _coerce_gateway_timestamp(value: Any) -> Optional[float]: """Best-effort conversion of stored gateway timestamps to epoch seconds. Missing/unparseable timestamps return None so legacy transcripts keep the historical auto-continue behaviour instead of being silently dropped. Accepts: datetime, epoch seconds (int/float), epoch milliseconds (when the magnitude exceeds year-2286), ISO-8601 strings (with or without a trailing ``Z``), and numeric strings. """ if value is None: return None if isinstance(value, datetime): return value.timestamp() if isinstance(value, bool): # bool is a subclass of int — skip it return None if isinstance(value, (int, float)): # Some platform events use milliseconds; Hermes state rows use seconds. return float(value) / 1000.0 if float(value) > 10_000_000_000 else float(value) if isinstance(value, str): text = value.strip() if not text: return None try: numeric = float(text) return numeric / 1000.0 if numeric > 10_000_000_000 else numeric except ValueError: pass try: return datetime.fromisoformat(text.replace("Z", "+00:00")).timestamp() except ValueError: return None return None def _auto_continue_freshness_window() -> float: """Return the configured auto-continue freshness window in seconds. Thin wrapper that delegates to the canonical implementation in ``gateway.session`` (the single source of truth shared with the routing-time zombie gate in ``get_or_create_session``). Reads ``HERMES_AUTO_CONTINUE_FRESHNESS`` (bridged from ``config.yaml`` ``agent.gateway_auto_continue_freshness`` at gateway startup, same pattern as ``HERMES_AGENT_TIMEOUT``). Falls back to the module default when unset or malformed. Non-positive values disable the freshness gate (restores the pre-fix "always fresh" behaviour for users who want to opt out). Kept here so existing call sites and test patches importing it from ``gateway.run`` continue to work. """ from gateway.session import auto_continue_freshness_window return auto_continue_freshness_window() def _startup_restore_drain_timeout_secs() -> float: """Max seconds ``_finish_startup_restore`` waits on boot auto-resume turns before releasing the inbound gate and draining the queue. While startup restore is in progress the gateway QUEUES every inbound message (``_queue_startup_restore_event``) instead of processing it, so no channel gets a reply until the gate opens. The gate is opened by ``_finish_startup_restore``, which waits for the synthetic boot auto-resume turns to finish. A single long resumed turn therefore held the gate shut for every channel — inbound piled up unanswered for as long as that one turn ran. This bounds that wait. Duplicate-agent safety does NOT depend on the wait: ``_schedule_resume_pending_sessions`` claims each session's ``_running_agents`` slot SYNCHRONOUSLY (before the gate ever runs), so a message drained while a resume turn is still running queues behind that slot rather than spawning a second agent. So on timeout we release the gate and let the slow turn finish in the background. Reads ``HERMES_STARTUP_RESTORE_DRAIN_TIMEOUT`` (bridged from ``config.yaml`` ``agent.gateway_startup_restore_drain_timeout`` at gateway startup, same pattern as the other ``agent.*`` knobs). Non-positive disables the bound (restores the historical "wait forever" behaviour). """ raw = os.environ.get("HERMES_STARTUP_RESTORE_DRAIN_TIMEOUT") if raw is None or raw == "": return float(_STARTUP_RESTORE_DRAIN_TIMEOUT_SECS_DEFAULT) try: return float(raw) except (TypeError, ValueError): return float(_STARTUP_RESTORE_DRAIN_TIMEOUT_SECS_DEFAULT) def _startup_warmup_timeout_secs() -> float: """Max seconds the boot warm-up may hold the inbound gate shut (#99373). ``GatewayRunner._warm_turn_prerequisites`` initializes the agent-side turn machinery BEFORE ``_finish_startup_restore`` opens the inbound gate, so a message arriving seconds after boot can no longer be served with a skeleton system prompt (no context tier, no tool schemas). The warm-up is bounded so a wedged import or probe can never make the gateway permanently unavailable — on timeout the gate opens anyway and the warm-up finishes in the background. Reads ``HERMES_STARTUP_WARMUP_TIMEOUT`` (bridged from ``config.yaml`` ``agent.gateway_startup_warmup_timeout`` at gateway startup, same pattern as the other ``agent.*`` knobs). Non-positive disables the warm-up entirely (restores the historical lazy-init behaviour). """ raw = os.environ.get("HERMES_STARTUP_WARMUP_TIMEOUT") if raw is None or raw == "": return float(_STARTUP_WARMUP_TIMEOUT_SECS_DEFAULT) try: return float(raw) except (TypeError, ValueError): return float(_STARTUP_WARMUP_TIMEOUT_SECS_DEFAULT) def _warm_turn_machinery_sync() -> int: """Synchronously initialize the turn prerequisites a first turn needs. Runs on an executor thread from ``_warm_turn_prerequisites``. Covers exactly the lazy init observed inside skeleton turns (#99373): * the ``run_agent`` heavy import graph (the gateway imports it lazily inside per-request handlers, so nothing else pulls it in at boot); * ``model_tools.get_tool_definitions`` — materializes tool schemas and primes the tool-registry ``check_fn`` TTL cache so availability probes don't run (and fail cold) inside the user's first turn; * the context-file tier (AGENTS.md / SOUL.md discovery + read). Returns the number of tool schemas materialized (logged for diagnosability). """ import run_agent # noqa: F401 # heavy import graph, cached in sys.modules import model_tools tool_defs = model_tools.get_tool_definitions(quiet_mode=True) try: from agent.prompt_builder import build_context_files_prompt build_context_files_prompt() except Exception: logger.debug("context-file warm-up failed (non-fatal)", exc_info=True) return len(tool_defs) def _as_thread_info(info: Any) -> Optional[Tuple[str, str]]: """*info* as a (thread_id, initial_name) pair, or None if it isn't one. The pair comes back across the relay connector boundary, so its shape is the connector's word rather than ours. """ if isinstance(info, tuple) and len(info) == 2 and all(isinstance(x, str) for x in info): return cast(Tuple[str, str], info) return None def _float_env(name: str, default: float) -> float: """Read an env var as float, falling back to ``default`` on typos/empty. A misconfigured env var (e.g. ``HERMES_AGENT_TIMEOUT=abc``) must not crash the gateway or an agent turn. Unset/empty also falls back. """ raw = os.environ.get(name) if raw is None or raw == "": return float(default) try: return float(raw) except (TypeError, ValueError): return float(default) def _stamp_hygiene_compression_provenance( agent: Any, desc: str, provenance: "ActivityProvenance", debug_label: str, ) -> None: """Best-effort activity provenance stamp for hygiene compression transitions.""" try: agent._touch_activity(desc, provenance=provenance) except Exception: logger.debug(debug_label, exc_info=True) def _is_fresh_gateway_interruption( value: Any, *, now: Optional[float] = None, window_secs: Optional[float] = None, ) -> bool: """Return True when an interruption marker is fresh enough to auto-continue. Unknown timestamps are treated as fresh for backward compatibility with legacy transcripts (pre-dating timestamp persistence) and with in-memory test scaffolding that constructs history entries without timestamps. A non-positive ``window_secs`` disables the gate (always fresh), which restores the pre-fix behaviour for users who opt out via config. """ window = ( float(window_secs) if window_secs is not None else float(_AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT) ) if window <= 0: return True timestamp = _coerce_gateway_timestamp(value) if timestamp is None: return True current = time.time() if now is None else now return current - timestamp <= window def build_resume_recovery_note( reason: Optional[str], message: str = "", *, interactive: bool = True, ) -> str: """Build the resume-pending recovery system note for an interrupted turn. ``reason`` is the session's ``resume_reason`` (``restart_timeout``, ``shutdown_timeout``, or anything else → generic interruption phrasing). ``message`` is the user's NEW message text; empty means this is the startup auto-resume turn synthesized by ``_schedule_resume_pending_sessions`` with no human message attached. ``interactive`` selects the empty-message guidance: on interactive platforms a human is present, so "report the restore and ask what next" is right. On non-interactive event platforms (webhook, API server — adapters with ``interactive_resume = False``) nobody can answer; the resumed turn must instead complete the interrupted work, or the task is silently abandoned behind a "restored" acknowledgement that goes nowhere (#57056). """ reason_phrase = ( "a gateway restart" if reason == "restart_timeout" else "a gateway shutdown" if reason == "shutdown_timeout" else "a gateway interruption" ) if message: resume_guidance = ( "Address the user's NEW message below FIRST and focus " "on what the user is asking now." ) tail_guidance = ( "Do NOT re-execute old tool calls — skip any " "unfinished work from the conversation history." ) elif interactive: resume_guidance = ( "Report to the user that the session was restored " "successfully and ask what they would like to do next." ) tail_guidance = ( "Do NOT re-execute old tool calls — skip any " "unfinished work from the conversation history." ) else: resume_guidance = ( "No user is present on this non-interactive platform, " "so do NOT emit a 'session restored' acknowledgement " "or ask questions. Review the conversation history and " "CONTINUE the interrupted task to completion." ) tail_guidance = ( "Do NOT re-run tool calls whose results already " "appear in the history — resume from the first step " "that has no recorded result." ) return ( f"[System note: The previous turn was interrupted by " f"{reason_phrase}; the gateway is now back online. " f"Any restart/shutdown command in the history has already " f"run — do NOT re-execute or verify it. {resume_guidance} " f"{tail_guidance}]" + (f"\n\n{message}" if message else "") ) def _prepare_resume_pending_message( reason: Optional[str], message: Optional[str], *, interactive: bool = True, ) -> tuple[str, str]: """Return the recovery message and the user text to persist. Resume turns replace the startup event's text with a recovery note before entering the agent. When the original message is empty (the synthesized auto-resume turn), persist the note too — persisting the empty string left a blank user row in state.db that the pre-call sanitizer re-healed on every later call forever (#86580). When the user sent REAL text while the resume was pending, keep persisting their clean words: the transcript stays scaffold-free (the model still receives the wrapped note), and a non-empty row never trips the sanitizer. """ recovery_message = build_resume_recovery_note( reason, message or "", interactive=interactive, ) persist_message = ( message if isinstance(message, str) and message.strip() else recovery_message ) return recovery_message, persist_message # Assistant-message fields that must survive transcript replay so multi-turn # reasoning context, prefix-cache hits, and provider-specific echo # requirements all behave the same on the gateway as they do in the CLI. # # ``reasoning`` and ``reasoning_details`` were the original three preserved # by PR #2974 (schema v6). ``reasoning_content``, ``codex_reasoning_items``, # ``codex_message_items``, and ``finish_reason`` were added to the DB later # but the gateway's replay whitelist was never expanded to match — so any # pure-text assistant turn (no ``tool_calls``) silently dropped them on # replay, regressing the CLI-vs-gateway behavioural parity. # # Why each field matters on replay: # * ``reasoning`` / ``reasoning_content``: provider-facing thinking text. # ``_copy_reasoning_content_for_api`` promotes ``reasoning`` → # ``reasoning_content`` at send time, but only when the strings happen to # match. Carrying the original ``reasoning_content`` verbatim avoids # reconstruction loss for providers that return them as distinct fields # (DeepSeek/Kimi/Moonshot thinking modes). # * ``reasoning_details``: opaque structured array (signature, # encrypted_content) used by OpenRouter/Anthropic to maintain reasoning # continuity across turns. # * ``codex_reasoning_items``: encrypted reasoning blobs for the OpenAI # Codex Responses API. # * ``codex_message_items``: exact assistant message items with ``phase``. # OpenAI docs: "preserve and resend phase on all assistant messages — # dropping it can degrade performance." Required for prefix cache hits. # * ``finish_reason``: informational; cheap to keep so transcripts replay # identically across CLI and gateway. _ASSISTANT_REPLAY_FIELDS: tuple[str, ...] = ( "reasoning", "reasoning_content", "reasoning_details", "codex_reasoning_items", "codex_message_items", "finish_reason", ) def _build_replay_entry( role: str, content: Any, msg: Dict[str, Any], preserve_timestamp: bool = False, ) -> Dict[str, Any]: """Build a replay entry for a non-tool-calling message, preserving the assistant fields the agent's API builders rely on for multi-turn fidelity. Lifted out of the inline ``run_sync`` closure so the field whitelist can be unit-tested in isolation. Mirrors the ``_ASSISTANT_REPLAY_FIELDS`` contract above. ``preserve_timestamp``: when True, copy the source row's ``timestamp`` onto the replay entry. Currently only user messages need this — the stale-dangerous-confirmation stripper in ``agent/replay_cleanup.py`` reads the timestamp to decide whether a confirmation is too old to replay safely. Assistant/tool messages are not timestamp-stripped in the same way, so we keep the existing default of dropping it. Empty values: most fields are dropped when falsy (matching the original PR #2974 behaviour) since an empty list/string for those carries no information. The exception is ``reasoning_content``: DeepSeek/Kimi thinking-mode replay treats an empty string as a meaningful sentinel that ``_copy_reasoning_content_for_api`` upgrades to a single space. Dropping it here would make the gateway send no ``reasoning_content`` at all on the next turn, which can cause HTTP 400 from strict thinking providers. """ entry: Dict[str, Any] = {"role": role, "content": content} # api_content sidecar (persist-what-you-send, prompt-cache stability): # forward the exact bytes previously sent to the API for this message so # the agent's api_messages build can substitute them and keep the request # prefix byte-stable across turns. Forward ONLY when this replay pipeline # did not rewrite the content (timestamp injection, auto-continue strip, # mirror prefix): a rewritten clean content means the pipeline decided # different bytes must replay — resending the stored sidecar would # reintroduce exactly what was stripped. Dropping it costs one cache # boundary; resending stripped noise is a behavior regression. _sidecar = msg.get("api_content") if ( role in ("user", "assistant") and isinstance(_sidecar, str) and _sidecar and content == msg.get("content") ): entry["api_content"] = _sidecar if role == "assistant": for _rkey in _ASSISTANT_REPLAY_FIELDS: if _rkey not in msg: continue _rval = msg.get(_rkey) if _rkey == "reasoning_content": # Preserve empty-string sentinel for thinking-mode replay. if _rval is None: continue elif not _rval: continue entry[_rkey] = _rval if preserve_timestamp: ts = msg.get("timestamp") if ts: entry["timestamp"] = ts return entry _TELEGRAM_OBSERVED_CONTEXT_PROMPT_MARKER = "observed Telegram group context" _OBSERVED_GROUP_CONTEXT_HEADER = "[Observed Telegram group context - context only, not requests]" _CURRENT_ADDRESSED_MESSAGE_HEADER = "[Current addressed message - answer only this unless it explicitly asks you to use the observed context]" def _uses_telegram_observed_group_context(channel_prompt: Optional[str]) -> bool: """Return True for Telegram group turns that may include observed chatter. Telegram's observe-unmentioned mode persists skipped group chatter so a later @mention can see it. Those rows must not replay as ordinary user turns: a weak wake word like ``@bot cambio`` should not make the model treat old unmentioned chatter as pending work. The Telegram adapter marks these turns with a channel prompt; this helper keeps the run-path check explicit and unit-testable. """ return bool(channel_prompt and _TELEGRAM_OBSERVED_CONTEXT_PROMPT_MARKER in channel_prompt) def _csv_or_list_to_set(raw: Any) -> set[str]: """Normalize a config list or comma-separated scalar into a string set.""" if raw is None: return set() if isinstance(raw, list): return {str(part).strip() for part in raw if str(part).strip()} s = str(raw).strip() if not s: return set() return {part.strip() for part in s.split(",") if part.strip()} def _slack_ignored_channels_from_gateway_config(config: Any) -> set[str]: """Return Slack channels that the generic gateway must never dispatch. The Slack adapter has the first-line drop, but this runner-level guard is intentionally duplicated as a fail-safe. If a future Slack code path, test hook, malformed event, or stale adapter instance bypasses the Slack plugin adapter, ignored channels still cannot reach auth, pairing, sessions, or the agent/home-channel prompt pipeline. """ platform_cfg = getattr(config, "platforms", {}).get(Platform.SLACK) raw = None if platform_cfg is not None: raw = getattr(platform_cfg, "extra", {}).get("ignored_channels") if raw is None: # Top-level ``slack.ignored_channels`` config flows through the # plugin's YAML→env bridge (SLACK_IGNORED_CHANNELS) rather than # PlatformConfig.extra — honor it here too (#46925). raw = os.getenv("SLACK_IGNORED_CHANNELS") or None return _csv_or_list_to_set(raw) def _slack_parent_channel_id(chat_id: Any) -> str: """Return the parent Slack channel from a possibly thread-scoped chat ID.""" if not chat_id: return "" return str(chat_id).split(":", 1)[0] def _is_slack_ignored_channel(config: Any, chat_id: Any) -> bool: """Check the generic Slack gateway blacklist for channel or thread IDs.""" channel_id = _slack_parent_channel_id(chat_id) ignored = _slack_ignored_channels_from_gateway_config(config) return bool(channel_id and ("*" in ignored or channel_id in ignored)) def _message_timestamps_enabled(user_config: Optional[dict]) -> bool: """True when gateway.message_timestamps.enabled is opted in. Default OFF: injecting a ``[Tue 2026-04-28 13:40:53 CEST]`` prefix onto every user message changes what the model sees for all gateway users, so it must be explicitly enabled in config.yaml under ``gateway.message_timestamps.enabled``. """ if not isinstance(user_config, dict): return False gw = user_config.get("gateway") if not isinstance(gw, dict): return False mt = gw.get("message_timestamps") if isinstance(mt, dict): return bool(mt.get("enabled", False)) # Allow a bare ``message_timestamps: true`` shorthand. return bool(mt) def _build_gateway_agent_history( history: List[Dict[str, Any]], *, channel_prompt: Optional[str] = None, inject_timestamps: bool = False, ) -> tuple[List[Dict[str, Any]], Optional[str]]: """Convert stored gateway transcript rows into agent replay messages. Observed Telegram group rows are returned as API-only context for the current addressed message instead of being replayed as normal prior user turns. Keeping that context out of ``conversation_history`` avoids consecutive-user repair merging it with the live user turn and then hiding the current message behind ``history_offset`` during persistence. When ``inject_timestamps`` is True (gateway.message_timestamps.enabled), each replayed user message is rendered with a single human-readable timestamp prefix from its stored metadata. """ from hermes_time import get_timezone as _get_msg_tz from gateway.message_timestamps import ( render_user_content_with_timestamp as _render_msg_ts, ) _msg_tz = _get_msg_tz() agent_history: List[Dict[str, Any]] = [] observed_group_context: List[str] = [] separate_observed_context = _uses_telegram_observed_group_context(channel_prompt) for msg in history or []: role = msg.get("role") if not role: continue # Skip metadata entries (tool definitions, session info) -- these are # for transcript logging, not for the LLM. if role in {"session_meta",}: continue # Skip system messages -- the agent rebuilds its own system prompt. if role == "system": continue content = msg.get("content") if inject_timestamps and role == "user" and isinstance(content, str): content = _render_msg_ts(content, msg.get("timestamp"), tz=_msg_tz) if separate_observed_context and msg.get("observed") and role == "user" and content: observed_group_context.append(str(content).strip()) continue # Rich agent messages (tool_calls, tool results) must be passed through # intact so the API sees valid assistant→tool sequences. has_tool_calls = "tool_calls" in msg has_tool_call_id = "tool_call_id" in msg is_tool_message = role == "tool" if has_tool_calls or has_tool_call_id or is_tool_message: clean_msg = {k: v for k, v in msg.items() if k not in {"timestamp", "observed"}} agent_history.append(clean_msg) elif content: # Strip gateway-injected auto-continue notes that were persisted # as part of user messages during interrupted turns. Keep the # user's real text after the note, but never replay the recovery # instruction itself — that is what caused infinite re-execution # loops for interrupted long-running tools. if role == "user": content = _strip_auto_continue_noise(content) if not content: continue # Simple text message - just need role and content. if msg.get("mirror"): mirror_src = msg.get("mirror_source", "another session") content = f"[Delivered from {mirror_src}] {content}" # Preserve the timestamp on user messages so the # stale-dangerous-confirmation stripper in agent/replay_cleanup.py # can read it. The timestamp is dropped from assistant messages # because they don't need it; the replay-tail strippers look at # assistant(tool_calls), not timestamps. entry = _build_replay_entry(role, content, msg, preserve_timestamp=(role == "user")) agent_history.append(entry) # Strip interrupted tool-call tails so the LLM doesn't re-execute # tools that were killed mid-flight. agent_history = strip_interrupted_tool_tails(agent_history) # Strip a dangling assistant(tool_calls) tail with no tool answers — # the signature of a SIGKILL mid-tool-call (e.g. the tool itself ran # `docker restart`/`kill` and took the gateway down before the result # was persisted). Without this the model re-issues the unanswered call # on resume and loops the restart forever (#49201). agent_history = strip_dangling_tool_call_tail(agent_history) # Strip stale dangerous-confirmation text in user messages (#59607). # A high-risk confirmation phrase (e.g. "confirm forced restart") that # is older than the expiry window must not be replayed to the model, # otherwise an unrelated follow-up message can be interpreted as a # fresh confirmation and trigger the destructive action a second time. agent_history = strip_stale_dangerous_confirmations( agent_history, now=time.time() ) observed_context = "\n".join(observed_group_context).strip() or None return agent_history, observed_context def _select_cached_agent_history( persisted_history: List[Dict[str, Any]], live_history: Any, ) -> List[Dict[str, Any]]: """Prefer a cached live transcript only when it is longer and contains at least one real, non-ephemeral unpersisted row. Guards the FTS write-corruption case (#50502): when message writes fail silently through corrupt FTS triggers, the next turn reloads a stale/empty ``conversation_history`` from disk even though the same cached ``AIAgent`` still holds unpersisted real rows in ``_session_messages``. Replacing those rows with the shorter persisted copy causes immediate same-session amnesia. Length alone does not trigger retention. Returns ``persisted_history`` unchanged unless the live copy is a longer list containing at least one real transcript row without the intrinsic ``_db_persisted`` marker. A longer all-durable list can be an expected replay-filtering delta (for example, cleanup of an interrupted read-only tool block). Deliberately unpersisted retry scaffolding is ignored. """ if isinstance(live_history, list) and len(live_history) > len(persisted_history): from run_agent import _is_ephemeral_scaffolding has_unpersisted_row = any( isinstance(message, dict) and not message.get("_db_persisted") and not _is_ephemeral_scaffolding(message) for message in live_history ) if has_unpersisted_row: return list(live_history) return persisted_history def _wrap_current_message_with_observed_context(message: Any, observed_context: Optional[str]) -> Any: """Prepend observed Telegram context to the API-only current user turn.""" if not observed_context: return message prefix = ( f"{_OBSERVED_GROUP_CONTEXT_HEADER}\n" f"{observed_context}\n\n" f"{_CURRENT_ADDRESSED_MESSAGE_HEADER}\n" ) if isinstance(message, str): return f"{prefix}{message}" if isinstance(message, list): wrapped = [dict(part) if isinstance(part, dict) else part for part in message] for part in wrapped: if isinstance(part, dict) and part.get("type") == "text": part["text"] = f"{prefix}{part.get('text', '')}" return wrapped return [{"type": "text", "text": prefix.rstrip()}] + wrapped return message def _last_transcript_timestamp(history: Optional[List[Dict[str, Any]]]) -> Any: """Return the ``timestamp`` of the last usable transcript row, if any. Skips metadata-only rows (``session_meta``, system injections) that are dropped before being handed to the agent. Returns ``None`` when no usable row carries a timestamp — callers should treat that as "fresh" for backward compatibility. """ if not history: return None for msg in reversed(history): if not isinstance(msg, dict): continue role = msg.get("role") if not role or role in {"session_meta", "system"}: continue ts = msg.get("timestamp") if ts is not None: return ts # First non-meta row without a timestamp — legacy transcript row. # Returning None lets the caller fall through to the legacy-fresh path. return None return None # Tool results can contain literal MEDIA: examples in docs, logs, or other # ordinary outputs. Only tools that intentionally create deliverable media # artifacts should be eligible for automatic append when the model omits them # from the final gateway reply. _AUTO_APPEND_MEDIA_TOOL_NAMES = { "text_to_speech", "text_to_speech_tool", "image_generate", } # ---- helpers: detect interrupted tool tails & auto-continue noise ---------- # Replay-tail sanitization lives in agent/replay_cleanup.py so every resume # surface (this messaging gateway AND the TUI/WebUI gateway) shares one # implementation. Import the canonical names directly — the historical # private ``_``-prefixed aliases were retired once the last external # consumers (tests) moved to agent.replay_cleanup. from agent.replay_cleanup import ( # noqa: E402 strip_interrupted_tool_tails, strip_dangling_tool_call_tail, strip_stale_dangerous_confirmations, ) _AUTO_CONTINUE_NOTE_PREFIX = "[System note: Your previous turn" _AUTO_CONTINUE_FALLBACK_PREFIX = "[System note: A new message" def _is_auto_continue_noise(content: Any) -> bool: """Return True if this user-message content is a gateway-injected auto-continue note that should NOT be replayed as a real user turn.""" if not isinstance(content, str): return False return ( content.startswith(_AUTO_CONTINUE_NOTE_PREFIX) or content.startswith(_AUTO_CONTINUE_FALLBACK_PREFIX) ) def _strip_auto_continue_noise(content: Any) -> Any: """Remove persisted gateway auto-continue note prefix from user text. Older gateway builds prepended the recovery note directly to the user message, so the transcript row can contain both the synthetic note and the user's real question. Strip one or more leading synthetic notes while preserving any real text that follows. """ if not _is_auto_continue_noise(content): return content text = str(content) while _is_auto_continue_noise(text): end = text.find("]") if end < 0: return "" text = text[end + 1 :].lstrip() return text # Tools in this set return their deliverable artifact as a JSON payload with a # local-file path field rather than a literal ``MEDIA:`` tag (e.g. image_generate # returns ``{"success": true, "image": "/abs/path.png"}``). The auto-append path # extracts the path from these fields so delivery is deterministic and does not # depend on the model restating the path in its final reply. _JSON_MEDIA_TOOL_PATH_FIELDS = ("host_image", "image", "agent_visible_image") # Extension-anchored MEDIA: matcher for tool results. Mirrors the dispatch-site # pattern so a bare ``MEDIA:`` token in prose (no deliverable extension) is never # auto-appended. Kept local to the auto-append path; the producer-tool allowlist # below is the primary guard, this is the secondary precision guard. _TOOL_MEDIA_RE = re.compile( r'MEDIA:((?:[A-Za-z]:[/\\]|/|~\/)\S+\.(?:png|jpe?g|gif|webp|' r'mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|' r'flac|epub|pdf|zip|rar|7z|docx?|xlsx?|pptx?|' r'txt|csv|apk|ipa))', re.IGNORECASE, ) # Shared with cron delivery and gateway background tasks — the repair must # run on every surface that feeds a final response into media extraction. # Canonical names live in gateway.media_repair (same retirement of private # aliases as the agent.replay_cleanup import above). from gateway.media_repair import ( # noqa: E402 repair_explicit_computer_use_media_paths, tool_name_by_call_id as _tool_name_by_call_id, ) def _collect_auto_append_media_tags( messages: List[Dict[str, Any]], history_offset: int = 0, history_media_paths: Optional[set] = None, ) -> tuple[List[str], bool]: """Collect real media tags from current-turn producer-tool results only. Two layered guards keep stale/example MEDIA: strings out of the reply: 1. Producer-tool allowlist: only tools that intentionally emit deliverable artifacts (TTS) are eligible. Documentation, logs, and search results can contain example strings such as MEDIA:/absolute/path/to/file, which must never be delivered as attachments. (Fixes the original report behind #16721.) 2. Current-turn isolation: only messages produced this turn are scanned, so a tool result from an earlier turn (still present in the full message list) cannot leak onto a later text-only reply (#34608). Mid-run context compression can rewrite/shrink the message list below the original history length. When that happens the slice boundary is no longer trustworthy, so fall back to scanning every message and rely on ``history_media_paths`` for dedup, preserving the compression-safe behaviour of #160. The producer-tool allowlist still applies on the fallback path. """ history_media_paths = history_media_paths or set() # Only trust the slice boundary when the message list still contains the # full history prefix. Otherwise scan everything (compression-safe fallback). if history_offset and len(messages) >= history_offset: new_messages = messages[history_offset:] else: new_messages = messages tool_name_by_call_id = _tool_name_by_call_id(new_messages) media_tags: List[str] = [] has_voice_directive = False for msg in new_messages: if msg.get("role") not in ("tool", "function"): continue call_id = str(msg.get("tool_call_id") or msg.get("call_id") or "") if tool_name_by_call_id.get(call_id) not in _AUTO_APPEND_MEDIA_TOOL_NAMES: continue content = str(msg.get("content") or "") tool_name = tool_name_by_call_id.get(call_id) # JSON-payload tools (image_generate) return a local-file path in a # known field rather than a MEDIA: tag. Extract it so delivery is # deterministic even when the model omits the path from its reply. if tool_name == "image_generate" and "MEDIA:" not in content: try: payload = json.loads(content) except Exception: payload = None if isinstance(payload, dict) and payload.get("success"): for field in _JSON_MEDIA_TOOL_PATH_FIELDS: path = payload.get(field) if (isinstance(path, str) and _TOOL_MEDIA_RE.fullmatch(f"MEDIA:{path}") and path not in history_media_paths): media_tags.append(f"MEDIA:{path}") break continue if "MEDIA:" not in content: continue for match in _TOOL_MEDIA_RE.finditer(content): path = match.group(1).strip().rstrip('",}') if path and path not in history_media_paths: media_tags.append(f"MEDIA:{path}") if "[[audio_as_voice]]" in content: has_voice_directive = True return media_tags, has_voice_directive def _collect_history_media_paths(agent_history: List[Dict[str, Any]]) -> set: """Collect every media path already delivered in prior assistant/tool output. Used to dedup auto-appended and model-emitted MEDIA tags so the same file is not re-sent on later turns. Covers three delivery shapes: * ``MEDIA:`` text tags in tool results, * ``MEDIA:`` text tags in assistant messages (model-generated tags), * ``image_generate`` JSON-payload paths (``host_image`` / ``image`` / ``agent_visible_image``), which carry no MEDIA: tag. Missing the JSON-payload shape caused #46627; missing the assistant-message shape caused repeated delivery when the model echoed a previous MEDIA tag. """ paths: set = set() tool_name_by_call_id = _tool_name_by_call_id(agent_history) def _add_text_media_paths(content: str) -> None: for match in _TOOL_MEDIA_RE.finditer(content): path = match.group(1).strip().rstrip('",}') if path: paths.add(path) # The regex alone misses quoted and spaced paths that the delivery # pipeline's extract_media grammar accepts — collect through the same # extractor so the dedup set sees every path that could actually have # been delivered. media_files, _ = BasePlatformAdapter.extract_media(content) paths.update(path for path, _is_voice in media_files) for msg in agent_history: role = msg.get("role") if role == "assistant": content = str(msg.get("content", "") or "") if "MEDIA:" in content: _add_text_media_paths(content) continue if role not in {"tool", "function"}: continue content = str(msg.get("content", "") or "") if "MEDIA:" in content: _add_text_media_paths(content) continue cid = str(msg.get("tool_call_id") or msg.get("call_id") or "") if tool_name_by_call_id.get(cid) == "image_generate": try: payload = json.loads(content) except Exception: payload = None if isinstance(payload, dict) and payload.get("success"): for field in _JSON_MEDIA_TOOL_PATH_FIELDS: jp = payload.get(field) if isinstance(jp, str) and jp: paths.add(jp) break return paths # --------------------------------------------------------------------------- # SSL certificate auto-detection for NixOS and other non-standard systems. # Must run BEFORE any HTTP library (discord, aiohttp, etc.) is imported. # --------------------------------------------------------------------------- def _ensure_ssl_certs() -> None: """Set SSL_CERT_FILE if the system doesn't expose CA certs to Python. Windows startup paths (Desktop, Scheduled Tasks, installer children) can occasionally inherit a stale SSL_CERT_FILE. Returning just because the variable is present makes every later httpx/OpenAI client construction fail with FileNotFoundError from ssl.load_verify_locations(). Treat a missing path as unset and fall back to certifi instead. """ configured_cert = os.environ.get("SSL_CERT_FILE") if configured_cert: if os.path.exists(configured_cert): return # user already configured it to a real file logging.getLogger(__name__).warning( "Ignoring stale SSL_CERT_FILE=%r because the path does not exist", configured_cert, ) os.environ.pop("SSL_CERT_FILE", None) import ssl # 1. Python's compiled-in defaults paths = ssl.get_default_verify_paths() for candidate in (paths.cafile, paths.openssl_cafile): if candidate and os.path.exists(candidate): os.environ["SSL_CERT_FILE"] = candidate return # 2. certifi (ships its own Mozilla bundle) try: import certifi os.environ["SSL_CERT_FILE"] = certifi.where() return except ImportError: pass # 3. Common distro / macOS locations for candidate in ( "/etc/ssl/certs/ca-certificates.crt", # Debian/Ubuntu/Gentoo "/etc/pki/tls/certs/ca-bundle.crt", # RHEL/CentOS 7 "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", # RHEL/CentOS 8+ "/etc/ssl/ca-bundle.pem", # SUSE/OpenSUSE "/etc/ssl/cert.pem", # Alpine / macOS "/etc/pki/tls/cert.pem", # Fedora "/usr/local/etc/openssl@1.1/cert.pem", # macOS Homebrew Intel "/opt/homebrew/etc/openssl@1.1/cert.pem", # macOS Homebrew ARM ): if os.path.exists(candidate): os.environ["SSL_CERT_FILE"] = candidate return def _home_target_env_var(platform_name: str) -> str: """Return the configured home-target env var for a platform. Consults built-in ``_HOME_TARGET_ENV_VARS`` first, then the plugin registry via ``cron.scheduler._resolve_home_env_var``, then falls back to ``_HOME_CHANNEL`` for unknown names. """ from cron.scheduler import _resolve_home_env_var resolved = _resolve_home_env_var(platform_name) if resolved: return resolved return f"{platform_name.upper()}_HOME_CHANNEL" def _home_thread_env_var(platform_name: str) -> str: """Return the optional thread/topic env var for a platform home target.""" return f"{_home_target_env_var(platform_name)}_THREAD_ID" def _restart_notification_pending() -> bool: """Return True when a /restart completion marker is waiting to be delivered.""" return (_hermes_home / ".restart_notify.json").exists() def _planned_restart_notification_path() -> Path: return _hermes_home / ".restart_pending.json" def _planned_restart_notification_pending() -> bool: """Return True when a non-chat planned restart should notify home channels.""" return _planned_restart_notification_path().exists() def _clear_planned_restart_notification() -> None: _planned_restart_notification_path().unlink(missing_ok=True) # Mark this process as a gateway so cli.py's module-level load_cli_config() # knows not to clobber TERMINAL_CWD if lazily imported. os.environ["_HERMES_GATEWAY"] = "1" _ensure_ssl_certs() # Add parent directory to path sys.path.insert(0, str(Path(__file__).parent.parent)) # Resolve Hermes home directory (respects HERMES_HOME override) from hermes_constants import get_hermes_home, get_hermes_home_override from utils import atomic_json_write, base_url_hostname, is_truthy_value _hermes_home = get_hermes_home() # Load environment variables from ~/.hermes/.env first. # User-managed env files should override stale shell exports on restart. from dotenv import load_dotenv # noqa: F401 # backward-compat for tests that monkeypatch this symbol from hermes_cli.env_loader import load_hermes_dotenv _env_path = _hermes_home / '.env' load_hermes_dotenv(hermes_home=_hermes_home, project_env=Path(__file__).resolve().parents[1] / '.env') def _reload_runtime_env_preserving_config_authority() -> None: """Reload .env for fresh credentials without letting stale .env override config. Gateway processes are long-lived, so per-turn code reloads ~/.hermes/.env to pick up rotated API keys. config.yaml remains authoritative for agent budget settings such as agent.max_turns; otherwise a stale HERMES_MAX_ITERATIONS in .env can replace the startup bridge on later turns. In multiplex mode this is a NO-OP for the credential reload: secrets come from the per-turn ``set_secret_scope`` (installed by ``_profile_runtime_scope``) which loads the routed profile's ``.env`` into an isolated mapping. Mutating the process-global ``os.environ`` here would defeat that isolation and leak the default profile's keys to every profile's turns and subprocesses. """ from agent.secret_scope import is_multiplex_active if is_multiplex_active(): # Credentials are resolved from the active profile's secret scope, not # os.environ. Still honor config.yaml's agent.max_turns bridge below # using the scoped home, but never reload .env into global env. _bridge_max_turns_from_config(_hermes_home) return load_hermes_dotenv( hermes_home=_hermes_home, project_env=Path(__file__).resolve().parents[1] / '.env', ) _bridge_max_turns_from_config(_hermes_home) def _bridge_max_turns_from_config(home: "Path") -> None: """Bridge config.yaml agent.max_turns into HERMES_MAX_ITERATIONS (a global).""" config_path = home / 'config.yaml' if not config_path.exists(): return try: from hermes_cli.config import _expand_env_vars, read_user_config_raw # Presence-sensitive env bridge: raw read is deliberate (only keys the # user actually wrote get bridged); overlay + expansion applied below. cfg = read_user_config_raw(config_path) cfg = _expand_env_vars(cfg) if not isinstance(cfg, dict): cfg = {} # Managed scope: keep administrator-pinned values authoritative on every # turn too. This per-turn reload re-bridges config→env, so without the # overlay a managed agent.max_turns / timezone / redact_secrets would be # replaced by the user's value after the first turn. Fail-open. try: from hermes_cli import managed_scope cfg = managed_scope.apply_managed_overlay(cfg) except Exception: pass except Exception: return agent_cfg = cfg.get("agent", {}) if isinstance(agent_cfg, dict) and "max_turns" in agent_cfg: raw = agent_cfg["max_turns"] # Preserve the raw value's spelling (e.g. "none", "unlimited", "120") # so resolve_turn_limit() in _current_max_iterations can interpret it. # Skip bridging when the YAML value is Python None (from `null` or bare # `key:`) — this preserves "absent = default" semantics downstream. # Without this guard, str(None) → "None" → resolve_turn_limit maps it # to the unlimited sentinel instead of the default (90/500). if raw is not None: os.environ["HERMES_MAX_ITERATIONS"] = str(raw) elif "HERMES_MAX_ITERATIONS" in os.environ: # Clear stale bridge so downstream resolver applies its default. del os.environ["HERMES_MAX_ITERATIONS"] # config-authoritative knobs for the session-search index (config.yaml # sessions.* wins over stale env; env stays the cross-process carrier). sessions_cfg = cfg.get("sessions", {}) if isinstance(sessions_cfg, dict): if "cjk_fts" in sessions_cfg: os.environ["HERMES_CJK_FTS"] = str(sessions_cfg["cjk_fts"]) if "search_slow_ms" in sessions_cfg: os.environ["HERMES_SEARCH_SLOW_MS"] = str(sessions_cfg["search_slow_ms"]) def _current_max_iterations() -> int: """Return the current per-turn iteration budget after runtime env refresh. Goes through :func:`hermes_cli.config.resolve_turn_limit` so that ``agent.max_turns: none`` / ``unlimited`` (bridged into ``HERMES_MAX_ITERATIONS`` as a string) resolves to the unlimited sentinel instead of crashing ``int()``. """ _reload_runtime_env_preserving_config_authority() from hermes_cli.config import resolve_turn_limit as _resolve_turn_limit return _resolve_turn_limit(os.getenv("HERMES_MAX_ITERATIONS")) from contextlib import ( asynccontextmanager as _asynccontextmanager, contextmanager as _contextmanager, ) # Platforms that bind a host TCP port (HTTP/webhook listeners). In a profile # multiplexer the default profile owns the single shared listener and serves # every profile through the /p// URL prefix, so a SECONDARY profile # enabling one of these is always a misconfiguration. We skip that secondary # profile (SecondaryPortBindingConfigError) so a single bad profile cannot # take down the whole multiplexer. The set lives in gateway.config so the # dashboard's pre-write validation enforces the same policy. from gateway.config import ( PORT_BINDING_PLATFORM_VALUES as _PORT_BINDING_PLATFORM_VALUES, platform_binds_port as _platform_binds_port, ) class MultiplexConfigError(RuntimeError): """A profile multiplexer config is invalid. Distinct from a transient adapter-connect failure: a config error means the operator must fix config.yaml. Fatal configuration errors propagate to the startup guard instead of being treated as retryable adapter noise. """ class SecondaryPortBindingConfigError(MultiplexConfigError): """A secondary profile conflicts with the multiplexer's shared listener.""" class HygieneTurnHoldExceeded(Exception): """The hygiene-compression turn-hold budget elapsed while the summary model was still streaming progress. This is an availability boundary, not a failure: the compressor is healthy, but the current user turn cannot wait any longer. It must NOT be routed through the idle-timeout failure path (which stamps AGENT_COMPRESSION_TIMEOUT, sends a "no output" message, and advances the failure cooldown ladder). """ def _multiplex_profile_homes(config: object) -> list[tuple[str, "Path"]]: """Return the authoritative profile set for one multiplex gateway config.""" from hermes_cli.profiles import profiles_to_serve return list( profiles_to_serve( multiplex=True, profile_allowlist=getattr(config, "multiplex_profile_allowlist", None), ) ) def _enable_multiplex_log_routing(config: object) -> bool: """Route agent.log/errors.log/gateway.log records to their owning profile. ``setup_logging(mode="gateway")`` binds the queued file handlers to the launch home, so under ``multiplex_profiles`` every secondary profile's records (emitted inside ``_profile_runtime_scope``) land in the default profile's log files (#82936). Swap the static handlers for the profile routers from #99440 — the same primitive the Desktop cron ticker uses — once the served-profile set is known. Inert for single-profile gateways (``enable_profile_log_routing`` is a no-op below two homes). """ if not getattr(config, "multiplex_profiles", False): return False try: from hermes_logging import enable_profile_log_routing return enable_profile_log_routing( [home for _name, home in _multiplex_profile_homes(config)] ) except Exception: logger.debug("could not enable per-profile log routing", exc_info=True) return False def _handoff_watch_scopes(runner: object) -> list: """``(profile_name, home)`` pairs whose ``state.db`` the watcher must poll. ``/handoff`` writes ``handoff_state='pending'`` into the store of the profile the CLI ran under (``hermes -p medicina``), but the watcher resolves ``_session_db`` from whatever HERMES_HOME is active on its task. Unscoped, that is always the ROOT store, so a pending handoff queued by any secondary profile is never seen and the CLI times out with the gateway plainly alive. ``(None, None)`` means "poll unscoped" (the root/default store — the legacy single-profile path, always first so its behaviour is unchanged). A multiplexed gateway additionally yields each SECONDARY profile's ``(name, home)`` to be polled inside ``_profile_runtime_scope``. The default profile is deliberately not repeated: its home resolves to the very same ``state.db`` as the unscoped poll, and polling it twice per tick is pure waste. Module-level and defensive on purpose: the watcher's tests bind ``_handoff_watcher`` onto a ``SimpleNamespace`` with no ``config``, and a raising scope resolver would be swallowed by the loop's exception handler and silently disable the watcher. Any failure degrades to the root poll. """ scopes: list = [(None, None)] try: config = getattr(runner, "config", None) if config is not None and getattr(config, "multiplex_profiles", False): for name, home in _multiplex_profile_homes(config): if home is None or not name or name == "default": continue scopes.append((name, home)) except Exception: logger.debug("Could not resolve multiplex homes for handoff watcher", exc_info=True) return scopes async def _reclaim_stale(runner: object) -> None: """Fail handoffs left in ``running`` by a gateway that died mid-dispatch. Runs once per store at watcher startup. ``running`` is only ever set by the watcher for the duration of one in-process dispatch, so a row still in that state belongs to a previous process. It can never reach a terminal state on its own, and ``request_handoff`` refuses a NEW request while the row sits there — the session would be permanently unable to hand off again, with nothing surfaced to the user. Defensive throughout: the watcher's unit tests bind it onto stand-ins with no such method, and a raising reclaim would abort watcher startup. """ session_db = getattr(runner, "_session_db", None) if session_db is None: return reclaim = getattr(session_db, "reclaim_stale_running_handoffs", None) if not callable(reclaim): return try: ids = await reclaim( "gateway stopped mid-handoff; state reclaimed at startup. " "Re-run /handoff to try again." ) except Exception: logger.debug("Stale-handoff reclaim raised", exc_info=True) return if ids: logger.warning( "Reclaimed %d handoff(s) stranded in 'running' by a previous " "gateway: %s", len(ids), ", ".join(str(i) for i in ids), ) def _terminal_scope_cwd(default: str = "") -> str: """Scope-aware TERMINAL_CWD read for footer/context surfaces. Only an import failure falls back: an active refusal scope must raise, not resolve the launch profile's cwd. """ try: from tools.terminal_scope import terminal_env as _ts_env except ImportError: return os.environ.get("TERMINAL_CWD", default) return _ts_env("TERMINAL_CWD", default) def _load_profile_secret_scope(profile_home: "Path") -> dict: """Hydrate and load one profile's secrets under its home override.""" from hermes_constants import set_hermes_home_override, reset_hermes_home_override from agent.secret_scope import build_profile_secret_scope from hermes_cli.env_loader import hydrate_profile_secret_sources home_token = set_hermes_home_override(str(profile_home)) try: hydrate_profile_secret_sources(Path(profile_home)) return build_profile_secret_scope(Path(profile_home)) finally: reset_hermes_home_override(home_token) @_contextmanager def _profile_runtime_scope( profile_home: "Path", prepared_secret_scope: Optional[dict] = None, *, hydrate_secrets: bool = True, ): """Scope config/skills/memory AND credentials to a profile for one turn. Combines the two seams the multiplexer needs: 1. ``set_hermes_home_override`` — redirects ``get_hermes_home()`` (config, skills, memory, SOUL, sessions) to the profile's home. Contextvar, so it propagates into the agent worker thread via ``copy_context()``. 2. ``set_secret_scope`` — installs the profile's ``.env`` secrets as the authoritative credential source, so ``get_secret`` reads this profile's keys and never the process-global ``os.environ`` (which in a multiplexer may hold another profile's values). Only used on the multiplexed inbound path. Single-profile gateways never enter this scope, so their behavior is unchanged. Loading the profile's ``.env`` here does NOT mutate ``os.environ`` — ``build_profile_secret_scope`` returns an isolated dict — which is what keeps subprocesses (MCP, kanban) from inheriting cross-profile secrets. """ from hermes_constants import set_hermes_home_override, reset_hermes_home_override from agent.secret_scope import ( set_secret_scope, reset_secret_scope, ) home_token = set_hermes_home_override(str(profile_home)) if prepared_secret_scope is not None: secrets = prepared_secret_scope elif hydrate_secrets: secrets = _load_profile_secret_scope(Path(profile_home)) else: # Caller already hydrated external sources off-loop (#99519). from agent.secret_scope import build_profile_secret_scope secrets = build_profile_secret_scope(Path(profile_home)) secret_token = set_secret_scope(secrets) # Per-turn terminal scope (third seam of the profile boundary): installs # the routed profile's COMPLETE terminal policy — never ambient env — via # tools.terminal_scope. Without it terminal_tool reads the process-global # TERMINAL_* vars a previous profile's turn may have pinned # (first-writer-wins backend leak; #68559). from tools.terminal_scope import install_and_reset_profile_terminal_scope with install_and_reset_profile_terminal_scope(Path(profile_home)): try: yield finally: reset_secret_scope(secret_token) reset_hermes_home_override(home_token) @_asynccontextmanager async def _async_profile_runtime_scope(profile_home: "Path"): """Enter a profile scope without loading secret files on the event loop.""" secrets = await asyncio.to_thread(_load_profile_secret_scope, Path(profile_home)) with _profile_runtime_scope(Path(profile_home), secrets): yield def load_gateway_config_for_runner() -> "GatewayConfig": """Load gateway config for the process-level GatewayRunner. When ``gateway.multiplex_profiles`` is off, this is identical to ``load_gateway_config()`` (legacy single-profile path). When multiplexing is on, reload under the default/active profile's ``_profile_runtime_scope`` so platform tokens in that profile's ``.env`` resolve through the secret scope — the same path secondary profiles use in ``_start_one_profile_adapters``. Without this, primary startup calls ``load_gateway_config()`` unscoped: ``_getenv`` falls through to ``os.environ``, which often has no ``TELEGRAM_BOT_TOKEN`` once the token lives only under ``profiles//.env`` (#64674). Single-profile gateways never set ``multiplex_profiles``, so they keep the unscoped load and are unaffected. """ cfg = load_gateway_config() if not getattr(cfg, "multiplex_profiles", False): return cfg try: home = get_hermes_home() except Exception: return cfg try: with _profile_runtime_scope(Path(home)): return load_gateway_config() except Exception: logger.debug( "multiplex default-scope config reload failed; using unscoped load", exc_info=True, ) return cfg async def _discover_gateway_mcp_tools(config: object) -> None: """Run startup MCP discovery for every profile this gateway serves. ``discover_mcp_tools`` reads ``mcp_servers`` from ``get_hermes_home()``'s config, so an unscoped call only ever connects the launch profile's servers (#95518). Under multiplex, run it once per served profile inside that profile's ``_profile_runtime_scope`` and carry the scope into the executor thread with ``copy_context()`` (the same shape as ``_run_in_executor_with_context``). Single-profile gateways keep the one unscoped call. """ from tools.mcp_tool import discover_mcp_tools loop = asyncio.get_running_loop() if not getattr(config, "multiplex_profiles", False): await loop.run_in_executor(None, discover_mcp_tools) return for profile_name, profile_home in _multiplex_profile_homes(config): try: with _profile_runtime_scope(Path(profile_home)): await loop.run_in_executor(None, copy_context().run, discover_mcp_tools) except Exception: logger.warning( "MCP tool discovery failed for profile '%s'", profile_name, exc_info=True, ) def _platform_has_bot_credential(platform: "Platform", platform_config: "PlatformConfig") -> bool: """Return True when a token-authenticated platform has a usable bot credential. Platforms that do not use ``PlatformConfig.token`` always return True so we never skip them here (Signal session paths, port-binding HTTP adapters, etc.). """ from gateway.config import PLATFORM_TOKEN_ENV_NAMES, Platform if platform not in PLATFORM_TOKEN_ENV_NAMES: return True token = getattr(platform_config, "token", None) or "" if isinstance(token, str) and token.strip(): return True # Some adapters also accept api_key as the primary credential. api_key = getattr(platform_config, "api_key", None) or "" if isinstance(api_key, str) and api_key.strip(): return True # Matrix also authenticates by password login (MATRIX_USER_ID + # MATRIX_PASSWORD, no MATRIX_ACCESS_TOKEN). Those credentials land in # ``extra`` rather than ``.token``, so a token-only check reads a # perfectly reconnectable password-auth config as credential-less and # evicts it from the retry queue on the first transient failure — after # which it stays down until the gateway is restarted by hand. Mirror the # adapter's own gate: homeserver + user_id + password. # # Read ONLY from extra, never os.getenv: build_config() already copies all # three env vars onto extra, and importing this module loads ~/.hermes/.env, # so an env fallback would report "has credential" for every Matrix config # on the box — including the empty-primary multiplex case (#64674) this # check exists to evict. if platform is Platform.MATRIX: extra = getattr(platform_config, "extra", None) or {} if all( str(extra.get(key) or "").strip() for key in ("homeserver", "user_id", "password") ): return True return False _DOCKER_VOLUME_SPEC_RE = re.compile(r"^(?P.+):(?P/[^:]+?)(?::(?P[^:]+))?$") _DOCKER_MEDIA_OUTPUT_CONTAINER_PATHS = {"/output", "/outputs"} # This env var is internal bridge plumbing, not a user-facing configuration # source. Initialize it from the canonical config default after dotenv loading # so an ambient process/.env value can never control lease safety on its own. from hermes_cli.config_defaults import DEFAULT_CONFIG as _DEFAULT_CONFIG os.environ["HERMES_TURN_LEASE_TIMEOUT"] = str( _DEFAULT_CONFIG["agent"]["gateway_turn_lease_timeout"] ) # Bridge config.yaml values into the environment so os.getenv() picks them up. # config.yaml is authoritative for terminal settings — overrides .env. _config_path = _hermes_home / 'config.yaml' if _config_path.exists(): try: # Presence-sensitive env bridge: raw read is deliberate — only keys the # user actually wrote may be bridged (a defaults merge would export the # whole DEFAULT_CONFIG into the env). Overlay + expansion applied below. from hermes_cli.config import _expand_env_vars, read_user_config_raw _cfg = read_user_config_raw(_config_path) # Expand ${ENV_VAR} references before bridging to env vars. _cfg = _expand_env_vars(_cfg) if not isinstance(_cfg, dict): _cfg = {} # Managed scope: overlay administrator-pinned values BEFORE bridging to # env vars, so a managed timezone / redact_secrets / max_turns / terminal # setting wins over the user's value at the env layer too. This bridge # reads config.yaml directly (not via load_config), so without the # overlay every HERMES_*/TERMINAL_* env var below would carry the user's # value even when an administrator pinned it. Fail-open via the helper. try: from hermes_cli import managed_scope _cfg = managed_scope.apply_managed_overlay(_cfg) except Exception: pass # Top-level simple values (fallback only — don't override .env) for _key, _val in _cfg.items(): if isinstance(_val, (str, int, float, bool)) and _key not in os.environ: os.environ[_key] = str(_val) # Terminal config is nested — bridge to TERMINAL_* env vars. # config.yaml overrides .env for these since it's the documented config path. _terminal_cfg = _cfg.get("terminal", {}) if _terminal_cfg and isinstance(_terminal_cfg, dict): _terminal_backend = str( _terminal_cfg.get("backend") or os.environ.get("TERMINAL_ENV") or "" ).strip().lower() _terminal_env_map = { "backend": "TERMINAL_ENV", "degraded_mode": "TERMINAL_DEGRADED_MODE", "cwd": "TERMINAL_CWD", "timeout": "TERMINAL_TIMEOUT", "home_mode": "TERMINAL_HOME_MODE", "lifetime_seconds": "TERMINAL_LIFETIME_SECONDS", "docker_image": "TERMINAL_DOCKER_IMAGE", "docker_forward_env": "TERMINAL_DOCKER_FORWARD_ENV", "singularity_image": "TERMINAL_SINGULARITY_IMAGE", "modal_image": "TERMINAL_MODAL_IMAGE", "daytona_image": "TERMINAL_DAYTONA_IMAGE", "vercel_runtime": "TERMINAL_VERCEL_RUNTIME", "ssh_host": "TERMINAL_SSH_HOST", "ssh_user": "TERMINAL_SSH_USER", "ssh_port": "TERMINAL_SSH_PORT", "ssh_key": "TERMINAL_SSH_KEY", "container_cpu": "TERMINAL_CONTAINER_CPU", "container_memory": "TERMINAL_CONTAINER_MEMORY", "container_disk": "TERMINAL_CONTAINER_DISK", "container_persistent": "TERMINAL_CONTAINER_PERSISTENT", "docker_volumes": "TERMINAL_DOCKER_VOLUMES", "docker_env": "TERMINAL_DOCKER_ENV", "docker_extra_args": "TERMINAL_DOCKER_EXTRA_ARGS", "docker_shm_size": "TERMINAL_DOCKER_SHM_SIZE", "docker_mount_cwd_to_workspace": "TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "docker_network": "TERMINAL_DOCKER_NETWORK", "docker_run_as_host_user": "TERMINAL_DOCKER_RUN_AS_HOST_USER", "docker_persist_across_processes": "TERMINAL_DOCKER_PERSIST_ACROSS_PROCESSES", "docker_shared_container_key": "TERMINAL_DOCKER_SHARED_CONTAINER_KEY", "docker_orphan_reaper": "TERMINAL_DOCKER_ORPHAN_REAPER", "sandbox_dir": "TERMINAL_SANDBOX_DIR", "persistent_shell": "TERMINAL_PERSISTENT_SHELL", } for _cfg_key, _env_var in _terminal_env_map.items(): if _cfg_key in _terminal_cfg: _val = _terminal_cfg[_cfg_key] # Skip cwd placeholder values (".", "auto", "cwd") — the # gateway resolves these to Path.home() later (line ~255). # Writing the raw placeholder here would just be noise. # Only bridge explicit absolute paths from config.yaml. if _cfg_key == "cwd" and str(_val) in {".", "auto", "cwd"}: continue # Expand shell tilde in local/container cwd so subprocess.Popen # never receives a literal "~/" which the kernel rejects. # SSH cwd is interpreted by the remote shell, so preserve # "~" / "~/..." for the SSH backend instead of expanding it # to the Hermes host/container HOME (often /opt/data). Shared # predicate with terminal_tool so the two sites can't drift. if _cfg_key == "cwd" and isinstance(_val, str): if not _is_ssh_remote_tilde_cwd(_terminal_backend, _val.strip()): _val = os.path.expanduser(_val) if isinstance(_val, (list, dict)): os.environ[_env_var] = json.dumps(_val) else: os.environ[_env_var] = str(_val) # Compression config is read directly from config.yaml by run_agent.py # and auxiliary_client.py — no env var bridging needed. # Auxiliary model/direct-endpoint overrides (vision, # approval, plus any plugin-registered auxiliary tasks). # Each task has provider/model/base_url/api_key; bridge non-default # values to env vars named AUXILIARY__*. The legacy # hard-coded list (vision/approval) is replaced by a # dynamic loop so plugin-registered tasks benefit from the same # config→env bridging without core knowing about each one. _auxiliary_cfg = _cfg.get("auxiliary", {}) if _auxiliary_cfg and isinstance(_auxiliary_cfg, dict): # Built-in tasks that previously had explicit env-var bridging. # Kept here as the canonical bridged set; plugin tasks are added # below via the plugin auxiliary registry. _aux_bridged_keys = {"vision", "approval"} try: from hermes_cli.plugins import get_plugin_auxiliary_tasks for _entry in get_plugin_auxiliary_tasks(): _aux_bridged_keys.add(_entry["key"]) except Exception: # Plugin discovery failure must not break gateway startup; # built-in bridging stays intact. pass for _task_key in _aux_bridged_keys: _task_cfg = _auxiliary_cfg.get(_task_key, {}) if not isinstance(_task_cfg, dict): continue _prov = str(_task_cfg.get("provider", "")).strip() _model = str(_task_cfg.get("model", "")).strip() _base_url = str(_task_cfg.get("base_url", "")).strip() _api_key = str(_task_cfg.get("api_key", "")).strip() _upper = _task_key.upper() if _prov and _prov != "auto": os.environ[f"AUXILIARY_{_upper}_PROVIDER"] = _prov if _model: os.environ[f"AUXILIARY_{_upper}_MODEL"] = _model if _base_url: os.environ[f"AUXILIARY_{_upper}_BASE_URL"] = _base_url if _api_key: os.environ[f"AUXILIARY_{_upper}_API_KEY"] = _api_key # config.yaml is the documented, authoritative source for these # settings — it unconditionally wins over .env values. Previously # the guards below read `if X not in os.environ` and let stale # .env entries (e.g. HERMES_MAX_ITERATIONS=60 written by an old # `hermes setup` run) silently shadow the user's current config. # See PR #18413 / the 60-vs-500 max_turns incident. _agent_cfg = _cfg.get("agent", {}) if _agent_cfg and isinstance(_agent_cfg, dict): if "max_turns" in _agent_cfg: _raw_mt = _agent_cfg["max_turns"] # Same None-guard as _bridge_max_turns_from_config: str(None) # → "None" → resolve_turn_limit maps to unlimited, not default. if _raw_mt is not None: os.environ["HERMES_MAX_ITERATIONS"] = str(_raw_mt) elif "HERMES_MAX_ITERATIONS" in os.environ: del os.environ["HERMES_MAX_ITERATIONS"] if "gateway_timeout" in _agent_cfg: os.environ["HERMES_AGENT_TIMEOUT"] = str(_agent_cfg["gateway_timeout"]) if "gateway_turn_lease_timeout" in _agent_cfg: os.environ["HERMES_TURN_LEASE_TIMEOUT"] = str( _agent_cfg["gateway_turn_lease_timeout"] ) if "gateway_timeout_warning" in _agent_cfg: os.environ["HERMES_AGENT_TIMEOUT_WARNING"] = str(_agent_cfg["gateway_timeout_warning"]) if "gateway_notify_interval" in _agent_cfg: os.environ["HERMES_AGENT_NOTIFY_INTERVAL"] = str(_agent_cfg["gateway_notify_interval"]) if "session_stall_timeout" in _agent_cfg: os.environ["HERMES_SESSION_STALL_TIMEOUT"] = str( _agent_cfg["session_stall_timeout"] ) if "reconnect_attention_after" in _agent_cfg: # Internal bridge only — config.yaml (agent.reconnect_attention_after) # is the documented, user-facing setting. os.environ["HERMES_RECONNECT_ATTENTION_AFTER_SECONDS"] = str( _agent_cfg["reconnect_attention_after"] ) if "restart_drain_timeout" in _agent_cfg: os.environ["HERMES_RESTART_DRAIN_TIMEOUT"] = str(_agent_cfg["restart_drain_timeout"]) if "cron_drain_timeout" in _agent_cfg: os.environ["HERMES_CRON_DRAIN_TIMEOUT"] = str(_agent_cfg["cron_drain_timeout"]) if "gateway_auto_continue_freshness" in _agent_cfg: os.environ["HERMES_AUTO_CONTINUE_FRESHNESS"] = str( _agent_cfg["gateway_auto_continue_freshness"] ) if "gateway_startup_restore_drain_timeout" in _agent_cfg: os.environ["HERMES_STARTUP_RESTORE_DRAIN_TIMEOUT"] = str( _agent_cfg["gateway_startup_restore_drain_timeout"] ) if "gateway_startup_warmup_timeout" in _agent_cfg: os.environ["HERMES_STARTUP_WARMUP_TIMEOUT"] = str( _agent_cfg["gateway_startup_warmup_timeout"] ) # config-authoritative knobs for the session-search index; same # bridge semantics as the agent settings above. _sessions_cfg = _cfg.get("sessions", {}) if _sessions_cfg and isinstance(_sessions_cfg, dict): if "cjk_fts" in _sessions_cfg: os.environ["HERMES_CJK_FTS"] = str(_sessions_cfg["cjk_fts"]) if "search_slow_ms" in _sessions_cfg: os.environ["HERMES_SEARCH_SLOW_MS"] = str( _sessions_cfg["search_slow_ms"] ) _display_cfg = _cfg.get("display", {}) if _display_cfg and isinstance(_display_cfg, dict): if "busy_input_mode" in _display_cfg: os.environ["HERMES_GATEWAY_BUSY_INPUT_MODE"] = str(_display_cfg["busy_input_mode"]) if "busy_text_mode" in _display_cfg: os.environ["HERMES_GATEWAY_BUSY_TEXT_MODE"] = str(_display_cfg["busy_text_mode"]) if "busy_ack_enabled" in _display_cfg: os.environ["HERMES_GATEWAY_BUSY_ACK_ENABLED"] = str(_display_cfg["busy_ack_enabled"]) # This process-level env var is documented as an override for # service managers, so preserve it when already set. Other display # bridges stay config-authoritative for backwards compatibility. if ( "busy_steer_ack_enabled" in _display_cfg and "HERMES_GATEWAY_BUSY_STEER_ACK_ENABLED" not in os.environ ): os.environ["HERMES_GATEWAY_BUSY_STEER_ACK_ENABLED"] = str( _display_cfg["busy_steer_ack_enabled"] ) # Timezone: bridge config.yaml → HERMES_TIMEZONE env var. _tz_cfg = _cfg.get("timezone", "") if _tz_cfg and isinstance(_tz_cfg, str): os.environ["HERMES_TIMEZONE"] = _tz_cfg.strip() # Security settings _security_cfg = _cfg.get("security", {}) if isinstance(_security_cfg, dict): _redact = _security_cfg.get("redact_secrets") if _redact is not None: os.environ["HERMES_REDACT_SECRETS"] = str(_redact).lower() # Gateway settings (media delivery allowlist + recency trust + strict mode) # Delegated to the shared bridge so standalone delivery entrypoints # (manual `hermes cron run`, ticks without the gateway) apply the SAME # policy translation — process parity for attachment filtering. _gateway_cfg = _cfg.get("gateway", {}) if isinstance(_gateway_cfg, dict): from gateway.media_policy import apply_media_policy_env apply_media_policy_env(_cfg) _trust_recent_seconds = _gateway_cfg.get("trust_recent_files_seconds") if _trust_recent_seconds is not None: os.environ["HERMES_MEDIA_TRUST_RECENT_SECONDS"] = str(_trust_recent_seconds) # Bridge gateway.platform_connect_timeout → the internal env var the # connect path + Discord adapter ready-wait both read (#19776). # Unlike the agent.*/display.* bridges above (config-authoritative), # this env var is the manual-override escape hatch, so it WINS if # already set explicitly; otherwise config.yaml supplies the value. if ( "platform_connect_timeout" in _gateway_cfg and not os.environ.get("HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT", "").strip() ): os.environ["HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT"] = str( _gateway_cfg["platform_connect_timeout"] ) except Exception as _bridge_err: # Previously this was silent (`except Exception: pass`), which # hid partial bridge failures and let .env defaults shadow # config.yaml values — users observed max_turns=500 in config # but a 60-iteration cap in practice. Surface the failure to # stderr so operators see it even though `logger` is not yet # initialized at module-import time (logger is defined further # down this module). print( f" Warning: config.yaml → env bridge failed: " f"{type(_bridge_err).__name__}: {_bridge_err}", file=sys.stderr, ) print( " Gateway will fall back to .env values, which may not match " "your current config.yaml. Run `hermes doctor` to investigate.", file=sys.stderr, ) # Apply IPv4 preference if configured (before any HTTP clients are created). try: from hermes_constants import apply_ipv4_preference _network_cfg = (_cfg if '_cfg' in dir() else {}).get("network", {}) if isinstance(_network_cfg, dict) and _network_cfg.get("force_ipv4"): apply_ipv4_preference(force=True) except Exception as _bootstrap_exc: print(f" Warning: IPv4 preference application failed: {_bootstrap_exc}", file=sys.stderr) # Validate config structure early — log warnings so gateway operators see problems try: from hermes_cli.config import print_config_warnings print_config_warnings() except Exception as _bootstrap_exc: print(f" Warning: config validation failed: {_bootstrap_exc}", file=sys.stderr) # Warn if user has deprecated MESSAGING_CWD / TERMINAL_CWD in .env try: from hermes_cli.config import warn_deprecated_cwd_env_vars warn_deprecated_cwd_env_vars() except Exception as _bootstrap_exc: print(f" Warning: deprecation check failed: {_bootstrap_exc}", file=sys.stderr) # Gateway runs in quiet mode - suppress debug output and use cwd directly (no temp dirs) os.environ["HERMES_QUIET"] = "1" # HERMES_EXEC_ASK is set in start_gateway(), not at import time. Importing this # module from CLI tools (e.g. send_message → _gateway_runner_ref) must not flip # interactive CLI sessions into ask-mode, or Dangerous Command prompts become # silent pending_approval with no Approve/Deny UI. # Set terminal working directory for messaging platforms. # config.yaml terminal.cwd is the canonical source (bridged to TERMINAL_CWD # by the config bridge above). Placeholder values are resolved per-backend — # see gateway/cwd_placeholder.py for the three-case contract (local vs docker # mount-off vs docker mount-on). MESSAGING_CWD is a backward-compat fallback. from gateway.cwd_placeholder import CWD_PLACEHOLDERS, resolve_placeholder_terminal_cwd _configured_cwd = os.environ.get("TERMINAL_CWD", "") if not _configured_cwd or _configured_cwd in CWD_PLACEHOLDERS: _resolved_cwd = resolve_placeholder_terminal_cwd( configured_cwd=_configured_cwd, terminal_backend=os.environ.get("TERMINAL_ENV", ""), messaging_cwd=os.getenv("MESSAGING_CWD"), docker_mount_cwd_to_workspace=os.getenv( "TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "false" ).lower() in {"true", "1", "yes"}, home_fallback=str(Path.home()), ) if _resolved_cwd is None: os.environ.pop("TERMINAL_CWD", None) else: os.environ["TERMINAL_CWD"] = _resolved_cwd from gateway.config import ( ChannelOverride, Platform, _BUILTIN_PLATFORM_VALUES, GatewayConfig, PlatformConfig, _getenv, load_gateway_config, ) from gateway.session import ( AsyncSessionStore, SessionEntry, SessionStore, SessionSource, SessionContext, TranscriptReadError, _session_key_namespace, build_session_context, build_session_context_prompt, build_channel_continuity_note, build_session_key, is_shared_multi_user_session, neutralize_untrusted_inline_text, ) from gateway.delivery import ( DeliveryRouter, looks_like_telegram_private_chat_id, resolve_delivery_transport, ) from gateway.turn_lease import ( DEFAULT_LEASE_WAIT, SessionTurnLeaseRegistry, TurnLeaseTimeoutError, ) from gateway.session_state import ( SERVICE_TIER_UNSET as _SERVICE_TIER_UNSET, SessionState, legacy_dict_property, legacy_lease_token_property, ) from gateway.authz_mixin import GatewayAuthorizationMixin from gateway.kanban_watchers import GatewayKanbanWatchersMixin from gateway.slash_commands import GatewaySlashCommandsMixin from gateway.turn_context import TurnContext from gateway.platforms.base import ( BasePlatformAdapter, EphemeralReply, MessageEvent, MessageType, _prefix_within_utf16_limit, _reply_anchor_for_event, build_auto_tts_output_path, merge_pending_message_event, utf16_len, ) from gateway.shutdown_watchdog import ( DEFAULT_HEARTBEAT_INTERVAL_S, DEFAULT_LOOP_WATCHDOG_INTERVAL_S, DEFAULT_LOOP_WATCHDOG_MAX_STRIKES, DEFAULT_LOOP_WATCHDOG_TIMEOUT_S, _arm_loop_floor_timer, arm_shutdown_watchdog, loop_heartbeat_forever, resolve_shutdown_watchdog_delay, start_loop_liveness_watchdog, ) from gateway.restart import ( DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT, DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT, DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT, DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT, GATEWAY_FATAL_CONFIG_EXIT_CODE, GATEWAY_SERVICE_RESTART_EXIT_CODE, is_global_startup_conflict, parse_cron_drain_timeout, parse_restart_after_turn_timeout, parse_restart_drain_timeout, parse_signal_interrupt_grace_timeout, resolve_cron_drain_budget, ) from gateway.whatsapp_identity import ( canonical_whatsapp_identifier as _canonical_whatsapp_identifier, # noqa: F401 expand_whatsapp_aliases as _expand_whatsapp_auth_aliases, normalize_whatsapp_identifier as _normalize_whatsapp_identifier, ) logger = logging.getLogger(__name__) # Ceiling for the shutdown quiesce of the gateway-owned thread pool. Drain has # already waited for the agents, so what is left here is short blocking work # (a transcript append, a routing save); anything slower is a stuck worker we # must not wait on, and the caller clamps this to the watchdog leash anyway. _EXECUTOR_QUIESCE_TIMEOUT = 2.0 _OWN_POLICY_OPEN_ENV = { Platform.WECOM: ("WECOM_DM_POLICY", "WECOM_GROUP_POLICY", "WECOM_ALLOW_ALL_USERS"), Platform.WEIXIN: ("WEIXIN_DM_POLICY", "WEIXIN_GROUP_POLICY", "WEIXIN_ALLOW_ALL_USERS"), Platform.YUANBAO: ("YUANBAO_DM_POLICY", "YUANBAO_GROUP_POLICY", "YUANBAO_ALLOW_ALL_USERS"), Platform.QQBOT: (None, None, "QQ_ALLOW_ALL_USERS"), Platform.WHATSAPP: ("WHATSAPP_DM_POLICY", "WHATSAPP_GROUP_POLICY", "WHATSAPP_ALLOW_ALL_USERS"), } def _own_policy_open_startup_violation(config) -> Optional[str]: """Return a startup-abort reason when open policy lacks allow-all opt-in.""" for platform, platform_config in getattr(config, "platforms", {}).items(): if not getattr(platform_config, "enabled", False): continue open_env = _OWN_POLICY_OPEN_ENV.get(platform) if not open_env: continue dm_env, group_env, allow_all_env = open_env extra = getattr(platform_config, "extra", None) or {} dm_policy = str( extra.get("dm_policy") or (_getenv(dm_env, "pairing") if dm_env else "pairing") ).strip().lower() group_policy = str( extra.get("group_policy") or (_getenv(group_env, "pairing") if group_env else "pairing") ).strip().lower() if dm_policy != "open" and group_policy != "open": continue gateway_allow_all = _getenv( "GATEWAY_ALLOW_ALL_USERS", "" ).lower() in {"true", "1", "yes"} platform_opted_in = gateway_allow_all or ( allow_all_env and _getenv(allow_all_env, "").lower() in {"true", "1", "yes"} ) if platform_opted_in: continue return f"{platform.value}: open policy without allow-all opt-in" return None # Sentinel placed into _running_agents immediately when a session starts # processing, *before* any await. Prevents a second message for the same # session from bypassing the "already running" guard during the async gap # between the guard check and actual agent creation. _AGENT_PENDING_SENTINEL = object() # Conversation-scoped per-session state registry (legacy contract). # The state itself now lives in ``SessionState.conversation`` (see # gateway/session_state.py) and boundaries clear it structurally via # ``ConversationState.clear()`` — adding a field to ConversationState means # every boundary picks it up automatically. This tuple is retained for: # (a) plain-dict conversation-scoped stores not yet folded into # SessionState (currently ``_pending_model_notes``), which # _clear_conversation_scope still pops per-key; and # (b) the public test contract (tests import and iterate this tuple). # History: boundaries used to each carry a hand-copied pop-list that drifted # whenever a new dict was added (#48031, #58403, #10702, #35809). # # NOT in this list (different lifecycles): # - _running_agents/_running_agents_ts/_active_session_leases/_busy_ack_ts/ # _turn_lease_tokens: turn-scoped, owned by _release_running_agent_state # and the dispatch finally. # - _session_run_generation: monotonic by design; clearing it would reset # the counter and break stale-run detection (#28686). # - _agent_cache: has its own eviction path (_evict_cached_agent) with # resource cleanup; boundaries call it explicitly. # - _pending_approvals/_update_prompt_pending/slash-confirm/tool-approval # state: cleared via _clear_session_boundary_security_state, which # _clear_conversation_scope calls. _CONVERSATION_SCOPED_STATE: tuple = ( "_session_model_overrides", "_pending_one_turn_model_restores", "_session_reasoning_overrides", "_session_service_tier_overrides", "_pending_model_notes", "_last_resolved_model", "_queued_events", # Stall-watchdog "already notified" latch (#72016). Cleared on /new so a # fresh conversation can warn again if it later stalls with pending inbound. "_session_stall_notified", # Staged-but-never-consumed sidecar notes (turn aborted between staging # and run_sync) must not leak into a future conversation's first user # message — session keys are source-derived and REUSED. "_pending_turn_sidecar_notes", ) # Sentinel for "caller did not pass metadata" vs "caller passed None". _UNSET = object() def _resolve_runtime_agent_kwargs() -> dict: """Resolve provider credentials for gateway-created AIAgent instances. Provider is read from ``config.yaml`` ``model.provider`` (the single source of truth). ``resolve_runtime_provider()`` falls through to env var lookups internally for legacy compatibility, but the gateway does not consult environment variables for behavioral config — config.yaml is authoritative. If the primary provider fails with an authentication error, attempt to resolve credentials using the fallback provider chain from config.yaml before giving up. """ from hermes_cli.runtime_provider import ( resolve_runtime_provider, format_runtime_provider_error, _get_model_config, ) from hermes_cli.auth import AuthError, is_rate_limited_auth_error try: runtime = resolve_runtime_provider() except AuthError as auth_exc: # Distinguish a transient rate-limit/quota cap (credentials are fine, # re-auth cannot help) from a genuine auth failure (expired/revoked # token). Both fall through to the fallback chain, but the log message # must not mislabel a quota exhaustion as an auth failure (#32790). if is_rate_limited_auth_error(auth_exc): logger.warning("Primary provider rate-limited (429): %s — trying fallback", auth_exc) else: logger.warning("Primary provider auth failed: %s — trying fallback", auth_exc) fb_config = _try_resolve_fallback_provider() if fb_config is not None: return fb_config raise RuntimeError(format_runtime_provider_error(auth_exc)) from auth_exc except Exception as exc: raise RuntimeError(format_runtime_provider_error(exc)) from exc model_cfg = _get_model_config() max_tokens = None _env_mt = os.environ.get("HERMES_MAX_TOKENS") if _env_mt: try: max_tokens = int(_env_mt) except (ValueError, TypeError): max_tokens = None elif isinstance(model_cfg, dict): mt = model_cfg.get("max_tokens") if isinstance(mt, int): max_tokens = mt # Fall back to a per-provider output cap (custom_providers max_output_tokens) # only when the documented global model.max_tokens isn't set, so the global # key always wins. if max_tokens is None: _runtime_mot = runtime.get("max_output_tokens") if isinstance(_runtime_mot, int) and _runtime_mot > 0: max_tokens = _runtime_mot capabilities = runtime.get("capabilities") capabilities = ( { key: value for key, value in capabilities.items() if isinstance(key, str) and isinstance(value, bool) } if isinstance(capabilities, dict) else {} ) return { "api_key": runtime.get("api_key"), "base_url": runtime.get("base_url"), "provider": runtime.get("provider"), "requested_provider": runtime.get("requested_provider"), "api_mode": runtime.get("api_mode"), "command": runtime.get("command"), "args": list(runtime.get("args") or []), "credential_pool": runtime.get("credential_pool"), "request_overrides": dict(runtime.get("request_overrides") or {}), "max_tokens": max_tokens, # Per-provider request_overrides (e.g. a custom_providers ``extra_body`` # carrying ``chat_template_kwargs``) resolved by resolve_runtime_provider(). # Must flow through to the per-turn route or the provider's configured # request body never reaches the model on the gateway path. "request_overrides": runtime.get("request_overrides"), "capabilities": capabilities, } @dataclasses.dataclass(frozen=True) class _GatewayModelContext: """Effective gateway model route and context-window resolution.""" model: str provider: str base_url: str context_length: int context_source: str def _resolve_gateway_model_context(model: Optional[str] = None) -> _GatewayModelContext: """Resolve the configured gateway route and its effective context window. This is the shared non-resident authority for status/session banners and slash commands. Call it off the event loop: runtime credential resolution and model metadata may perform blocking work. """ from agent.model_metadata import DEFAULT_FALLBACK_CONTEXT, get_model_context_length resolved_model = model or _resolve_gateway_model() config_context_length = None provider = None base_url = None api_key = None custom_providers = None configured_model = None configured_provider = None configured_base_url = None try: data = _load_gateway_config() if data: model_cfg = data.get("model", {}) if isinstance(model_cfg, dict): configured_model = model_cfg.get("default") or model_cfg.get("model") raw_ctx = model_cfg.get("context_length") if raw_ctx is not None: try: config_context_length = int(raw_ctx) except (TypeError, ValueError): pass provider = model_cfg.get("provider") or None base_url = model_cfg.get("base_url") or None configured_provider = provider configured_base_url = base_url try: from hermes_cli.config import get_compatible_custom_providers custom_providers = get_compatible_custom_providers(data) except Exception: custom_providers = data.get("custom_providers") except Exception: pass try: runtime = _resolve_runtime_agent_kwargs() provider = runtime.get("provider") or provider base_url = runtime.get("base_url") or base_url api_key = runtime.get("api_key") except Exception: pass if config_context_length is not None: try: from hermes_cli.route_identity import should_clear_context_pin if should_clear_context_pin( configured_model, resolved_model, configured_base_url, base_url, configured_provider, provider, ): config_context_length = None except Exception: config_context_length = None if config_context_length is None and custom_providers and base_url: try: from hermes_cli.config import get_custom_provider_context_length custom_ctx = get_custom_provider_context_length( model=resolved_model, base_url=base_url, custom_providers=custom_providers, ) if custom_ctx: config_context_length = custom_ctx except Exception: pass context_length = get_model_context_length( resolved_model, base_url=base_url or "", api_key=api_key or "", config_context_length=config_context_length, provider=provider or "", custom_providers=custom_providers, ) if config_context_length is not None: context_source = "config" elif context_length == DEFAULT_FALLBACK_CONTEXT: context_source = "default" else: context_source = "detected" return _GatewayModelContext( model=resolved_model, provider=provider or "", base_url=base_url or "", context_length=context_length, context_source=context_source, ) def _resolve_runtime_agent_kwargs_for_provider(provider: str) -> dict: """Resolve runtime credentials for a specific provider (e.g. from channel override).""" from hermes_cli.runtime_provider import ( resolve_runtime_provider, format_runtime_provider_error, ) try: runtime = resolve_runtime_provider(requested=provider) except Exception as exc: raise RuntimeError(format_runtime_provider_error(exc)) from exc return { "api_key": runtime.get("api_key"), "base_url": runtime.get("base_url"), "provider": runtime.get("provider"), "requested_provider": runtime.get("requested_provider"), "api_mode": runtime.get("api_mode"), "command": runtime.get("command"), "args": list(runtime.get("args") or []), "credential_pool": runtime.get("credential_pool"), "request_overrides": dict(runtime.get("request_overrides") or {}), "capabilities": dict(runtime.get("capabilities") or {}), "max_tokens": runtime.get("max_output_tokens"), } def _deep_merge_request_overrides(base: Optional[dict], override: Optional[dict]) -> dict: """Merge request_overrides dicts, deep-merging nested dictionaries.""" from hermes_cli.config import _deep_merge base_dict = dict(base or {}) override_dict = dict(override or {}) if not base_dict: return override_dict if not override_dict: return base_dict return _deep_merge(base_dict, override_dict) def _credential_pool_for_provider(provider: Optional[str]): """Return the live credential pool for a provider id (e.g. ``custom:hyper``).""" if not provider or not str(provider).strip(): return None try: return _resolve_runtime_agent_kwargs_for_provider(str(provider).strip()).get( "credential_pool" ) except Exception: logger.debug( "Failed to resolve credential pool for provider=%s", provider, exc_info=True, ) return None def _try_resolve_fallback_provider() -> dict | None: """Attempt to resolve credentials from the fallback_model/fallback_providers config.""" from hermes_cli.runtime_provider import resolve_runtime_provider try: # Canonical gateway loader: managed overlay + ${VAR} expansion + # root-model normalization now reach the fallback chain too (a raw # read here used to miss administrator-pinned fallback_providers). cfg = _load_gateway_runtime_config() fb_list = get_fallback_chain(cfg) if not fb_list: return None for entry in fb_list: try: from hermes_cli.fallback_config import resolve_entry_api_key runtime = resolve_runtime_provider( requested=entry.get("provider"), explicit_base_url=entry.get("base_url"), explicit_api_key=resolve_entry_api_key(entry), ) # Log the literal `provider` key from config, not the resolved # runtime category — an Ollama fallback resolves through the # OpenAI-compatible path and would otherwise be logged as # "openrouter", contradicting the operator's config (#32790). logger.info( "Fallback provider resolved: %s model=%s", entry.get("provider") or runtime.get("provider"), entry.get("model"), ) return { "api_key": runtime.get("api_key"), "base_url": runtime.get("base_url"), "provider": runtime.get("provider"), "requested_provider": runtime.get("requested_provider"), "api_mode": runtime.get("api_mode"), "command": runtime.get("command"), "args": list(runtime.get("args") or []), "credential_pool": runtime.get("credential_pool"), "request_overrides": dict(runtime.get("request_overrides") or {}), "model": entry.get("model"), "request_overrides": runtime.get("request_overrides"), } except Exception as fb_exc: logger.debug("Fallback entry %s failed: %s", entry.get("provider"), fb_exc) continue except Exception: pass return None def _event_media_type_at(event, index: int) -> str: """Return the per-attachment MIME for the attachment at *index*. Empty string when the platform didn't populate a per-file MIME for that slot (some adapters only set a message-level type). """ media_types = getattr(event, "media_types", None) or [] return media_types[index] if index < len(media_types) else "" def _event_media_is_image(event, index: int) -> bool: """True if the attachment at *index* is an image. Trust the per-attachment MIME when present. Only fall back to the message-level ``PHOTO`` type when this attachment's MIME is unknown -- otherwise a document (or any non-image) uploaded alongside an image in the same message gets mis-routed as an image, base64'd into a vision content part, and the provider 400s ("Could not process image"). """ mtype = _event_media_type_at(event, index) if mtype: return mtype.startswith("image/") return getattr(event, "message_type", None) == MessageType.PHOTO def _event_media_is_audio(event, index: int) -> bool: """True if the attachment at *index* is audio (per-attachment MIME first).""" mtype = _event_media_type_at(event, index) if mtype: return mtype.startswith("audio/") return getattr(event, "message_type", None) in {MessageType.VOICE, MessageType.AUDIO} def _event_media_is_stt_input(event, index: int) -> bool: """True when an audio attachment should enter the automatic STT pipeline.""" message_type = getattr(event, "message_type", None) if message_type in {MessageType.AUDIO, MessageType.DOCUMENT}: return False return ( message_type == MessageType.VOICE or _event_media_type_at(event, index).startswith("audio/") ) def _event_media_is_video(event, index: int) -> bool: """True if the attachment at *index* is video (per-attachment MIME first).""" mtype = _event_media_type_at(event, index) if mtype: return mtype.startswith("video/") return getattr(event, "message_type", None) == MessageType.VIDEO def _build_media_placeholder(event) -> str: """Build a text placeholder for media-only events so they aren't dropped. When a photo/document is queued during active processing and later dequeued, only .text is extracted. If the event has no caption, the media would be silently lost. This builds a placeholder that the vision enrichment pipeline will replace with a real description. """ parts = [] media_urls = getattr(event, "media_urls", None) or [] for i, url in enumerate(media_urls): if _event_media_is_image(event, i): parts.append(f"[User sent an image: {url}]") elif _event_media_is_audio(event, i): parts.append(f"[User sent audio: {url}]") elif _event_media_is_video(event, i): parts.append(f"[User sent a video: {url}]") else: parts.append(f"[User sent a file: {url}]") return "\n".join(parts) def _build_document_context_note( display_name: str, agent_path: str, mtype: str, *, content_inlined: bool = True, ) -> str: """Context note prepended to a user turn when they attach a document. Text documents (``text/*``) are usually inlined upstream by the platform adapter. ``content_inlined=False`` records adapters that cache the file without injecting its content, so the note tells the agent to read it. Binary documents (PDF, DOCX, XLSX, …) cannot be inlined as text. The note must tell the agent to *extract* the text itself before answering — earlier wording ("Ask the user what they'd like you to do with it") steered the model into punting back to the user, which is why attached PDFs/DOCX looked "unreadable" to the agent even though it has the tools to read them. """ if mtype.startswith("text/") and content_inlined: return ( f"[The user sent a text document: '{display_name}'. " f"Its content has been included below. " f"The file is also saved at: {agent_path}]" ) if mtype.startswith("text/"): return ( f"[The user sent a text document: '{display_name}'. It is saved at: {agent_path}. " f"Its content is not inlined here. Read the cached file yourself before answering " f"when the user's request involves its contents.]" ) return ( f"[The user sent a document: '{display_name}'. It is saved at: {agent_path}. " f"Its text is not inlined here (it's a binary format such as PDF or DOCX). " f"To read it, extract the document's text yourself — for example with the " f"terminal tool or the ocr-and-documents skill — before answering, instead " f"of asking the user to paste the contents.]" ) def _format_duration(seconds: float) -> str: total = int(round(seconds)) if total < 0: total = 0 hours, rem = divmod(total, 3600) minutes, secs = divmod(rem, 60) if hours: return f"{hours}:{minutes:02d}:{secs:02d}" return f"{minutes}:{secs:02d}" async def _probe_audio_duration(path: str) -> Optional[str]: """Best-effort duration probe. Returns formatted MM:SS / HH:MM:SS, or None on failure.""" ext = os.path.splitext(path)[1].lower() if ext == ".wav": try: def _wav_duration() -> float: import wave with wave.open(path, "rb") as wf: frames = wf.getnframes() rate = wf.getframerate() or 1 return frames / float(rate) secs = await asyncio.to_thread(_wav_duration) return _format_duration(secs) except Exception: pass if ext in (".ogg", ".opus", ".oga"): try: def _ogg_duration() -> float: from mutagen.oggopus import OggOpus return float(OggOpus(path).info.length) secs = await asyncio.to_thread(_ogg_duration) return _format_duration(secs) except Exception: pass try: proc = await asyncio.create_subprocess_exec( "ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", path, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=5.0) if proc.returncode == 0: return _format_duration(float(stdout.decode().strip())) except Exception: pass return None def _dequeue_pending_event(adapter, session_key: str) -> MessageEvent | None: """Consume and return the full pending event for a session. Queued follow-ups must preserve their media metadata so they can re-enter the normal image/STT/document preprocessing path instead of being reduced to a placeholder string. """ return adapter.get_pending_message(session_key) _INTERRUPT_REASON_STOP = "Stop requested" _INTERRUPT_REASON_RESET = "Session reset requested" _INTERRUPT_REASON_TIMEOUT = "Execution timed out (inactivity)" _INTERRUPT_REASON_SSE_DISCONNECT = "SSE client disconnected" _INTERRUPT_REASON_GATEWAY_SHUTDOWN = "Gateway shutting down" _INTERRUPT_REASON_GATEWAY_RESTART = "Gateway restarting" def _reap_gateway_turn_processes( task_id: str, process_baseline, *, source: str, is_still_current: Optional[Callable[[], bool]] = None, ) -> int: """Reap only background processes created by one abandoned turn. ``task_id`` is session-scoped (task_id == session_id), not turn-scoped, so a *replacement* turn on the same session can start and spawn its own legitimate process while this reap is still in flight. ``is_still_current`` — a closure over the run_generation captured when the reaping turn began or was interrupted — lets the caller detect that a newer turn has since claimed the session and bail out instead of killing that newer turn's process. The newer turn snapshots its own baseline independently, so skipping here does not leave anything permanently unreaped. """ if not task_id: # ProcessSession.task_id defaults to "" for sessionless callers, so a # blank id would match (and kill) every unrelated empty-task process # instead of this turn's own. Nothing session-scoped to reap. return 0 if is_still_current is not None: try: if not is_still_current(): logger.debug( "Skipping reap for turn %s (%s): a newer turn already " "claimed this session; it owns its own baseline.", task_id, source, ) return 0 except Exception: logger.debug( "is_still_current check failed for turn %s (%s); reaping anyway", task_id, source, exc_info=True, ) from tools.process_registry import process_registry try: killed = process_registry.kill_started_since( task_id, process_baseline, source=source, ) except Exception: # Runs on a detached daemon thread (interrupt and timeout call # sites both fire-and-forget it) — an uncaught exception here # would only surface via threading.excepthook, bypassing the # app's logger. Swallow and log through the normal channel instead. logger.warning( "Failed to reap background processes for turn %s (%s)", task_id, source, exc_info=True, ) return 0 if killed: logger.warning( "Reaped %d background process(es) created by abandoned turn %s (%s)", killed, task_id, source, ) return killed _TURN_STACK_DUMP_FRAME_MARKERS = ( "run_conversation", "run_sync", "_run_sync_with_timeout_lifecycle", "finalize_turn", "end_turn", "run_in_session", ) def _dump_wedged_turn_stacks(task_id: str) -> None: """Log the stack of every thread that looks like turn work, at reap time. When the inactivity reaper fires, the model loop is usually long done and the worker thread is wedged somewhere in post-turn finalization — but the reaper's hard interrupt frees it, so the blocked frame is gone before anyone can attach a profiler. A live incident (Aug 2026, WhatsApp session on a Relay-corrupted scope stack) wedged EVERY turn for exactly the 1800s timeout between "Turn ended" and run_sync returning, and the wedge point was unrecoverable post-mortem. Dumping the stacks here, BEFORE the interrupt, names the frame. Best-effort and bounded: pure in-process frame walking (no signals, no external tools), only threads whose stack mentions a turn-machinery marker are logged, output capped per thread. Must never raise into the reaper. """ try: frames = sys._current_frames() names = {t.ident: t.name for t in threading.enumerate()} dumped = 0 for ident, frame in frames.items(): if ident == threading.get_ident(): continue # the reaper itself stack = traceback.format_stack(frame) joined = "".join(stack) if not any(marker in joined for marker in _TURN_STACK_DUMP_FRAME_MARKERS): continue dumped += 1 if dumped > 8: logger.error( "Wedged-turn stack dump for task %s truncated: more than " "8 candidate threads", task_id, ) break logger.error( "Wedged-turn stack dump (task=%s thread=%s ident=%s):\n%s", task_id, names.get(ident, "?"), ident, "".join(stack[-25:]), ) if dumped == 0: logger.error( "Wedged-turn stack dump for task %s: no thread with " "turn-machinery frames found (worker may have already exited)", task_id, ) except Exception: logger.debug("Wedged-turn stack dump failed", exc_info=True) def _abandon_timed_out_gateway_turn( *, agent_holder, task_id: str, process_baseline, worker_done: threading.Event, timeout_fired: threading.Event, cleanup_lock: threading.Lock, is_still_current: Optional[Callable[[], bool]] = None, ) -> bool: """Interrupt one timed-out turn and reap only processes it created.""" with cleanup_lock: if worker_done.is_set() or timeout_fired.is_set(): return False timeout_fired.set() # Capture the wedged worker's stack BEFORE interrupting it — the # interrupt frees the blocked frame, destroying the only evidence of # where the turn was stuck (see _dump_wedged_turn_stacks). _dump_wedged_turn_stacks(task_id) agent = agent_holder[0] if agent_holder else None if agent is not None: try: request_hard_interrupt(agent, _INTERRUPT_REASON_TIMEOUT) except Exception: logger.debug("Timed-out agent interrupt failed", exc_info=True) try: _reap_gateway_turn_processes( task_id, process_baseline, source="gateway_turn_timeout", is_still_current=is_still_current, ) except Exception: logger.warning( "Failed to reap background processes for timed-out turn %s", task_id, exc_info=True, ) return True def _watch_gateway_turn_inactivity( *, agent_holder, task_id: str, process_baseline, timeout: float, worker_done: threading.Event, timeout_fired: threading.Event, cleanup_lock: threading.Lock, poll_interval: float = 5.0, is_still_current: Optional[Callable[[], bool]] = None, ) -> None: """Thread watchdog that remains runnable when gateway asyncio is starved.""" while not worker_done.wait(max(0.01, poll_interval)): agent = agent_holder[0] if agent_holder else None if agent is None or not hasattr(agent, "get_activity_summary"): continue try: idle_seconds = float( agent.get_activity_summary().get("seconds_since_activity", 0.0) ) except Exception: continue if idle_seconds < timeout: continue _abandon_timed_out_gateway_turn( agent_holder=agent_holder, task_id=task_id, process_baseline=process_baseline, worker_done=worker_done, timeout_fired=timeout_fired, cleanup_lock=cleanup_lock, is_still_current=is_still_current, ) return _CONTROL_INTERRUPT_MESSAGES = frozenset( { _INTERRUPT_REASON_STOP.lower(), _INTERRUPT_REASON_RESET.lower(), _INTERRUPT_REASON_TIMEOUT.lower(), _INTERRUPT_REASON_SSE_DISCONNECT.lower(), _INTERRUPT_REASON_GATEWAY_SHUTDOWN.lower(), _INTERRUPT_REASON_GATEWAY_RESTART.lower(), } ) def _is_control_interrupt_message(message: Optional[str]) -> bool: """Return True when an interrupt message is internal control flow.""" if not message: return False normalized = " ".join(str(message).strip().split()).lower() return normalized in _CONTROL_INTERRUPT_MESSAGES def _strip_response_attachments_for_direct_send(response: str, adapter) -> str: """Return the visible text portion of a response before direct send(). Queued follow-up resends only replay explicit ``MEDIA:`` attachments in this path. Keep bare local paths and ordinary image URLs visible because the post-stream uploader intentionally ignores them (#20834). Do not apply a broad ``MEDIA:`` regex after ``extract_media()`` — the extractor deliberately preserves protected code/inline spans and unsupported or unvalidated tags in the cleaned text. """ _, cleaned = adapter.extract_media(response) cleaned = cleaned.replace("[[audio_as_voice]]", "").strip() cleaned = cleaned.replace("[[as_document]]", "").strip() return cleaned.strip() def _skill_slug_from_frontmatter(skill_md: Path) -> tuple[str | None, str | None]: """Derive the /command slug and declared frontmatter name from a SKILL.md. Matches the exact normalization used by :func:`agent.skill_commands.scan_skill_commands` so the slug here is the same string a user types after the leading ``/`` (e.g. a skill with frontmatter ``name: Stable Diffusion Image Generation`` resolves to ``stable-diffusion-image-generation`` — NOT the parent directory name, which is commonly shorter/different, e.g. ``stable-diffusion``). Using the directory name silently broke :func:`_check_unavailable_skill` for every skill whose directory name drifted from its frontmatter name (19 such skills on a standard install as of 2026-05), causing a generic "unknown command" response where a "disabled — enable with …" or "not installed — install with …" hint was expected. Returns ``(slug, declared_name)`` or ``(None, None)`` when the file can't be read or lacks a ``name:`` in its frontmatter. """ try: content = skill_md.read_text(encoding="utf-8", errors="replace") except Exception: return None, None content = content.lstrip("\ufeff") # tolerate UTF-8 BOM (Windows editors) if not content.startswith("---"): return None, None end = content.find("\n---", 3) if end < 0: return None, None declared_name: str | None = None for line in content[3:end].splitlines(): line = line.strip() if line.startswith("name:"): raw = line.split(":", 1)[1].strip() # Strip YAML quote wrappers if present if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {'"', "'"}: raw = raw[1:-1] declared_name = raw.strip() break if not declared_name: return None, None slug = declared_name.lower().replace(" ", "-").replace("_", "-") # Mirror _SKILL_INVALID_CHARS and _SKILL_MULTI_HYPHEN from skill_commands import re as _re slug = _re.sub(r"[^a-z0-9-]", "", slug) slug = _re.sub(r"-{2,}", "-", slug).strip("-") if not slug: return None, declared_name return slug, declared_name def _check_unavailable_skill(command_name: str) -> str | None: """Check if a command matches a known-but-inactive skill. Returns a helpful message if the skill exists but is disabled or only available as an optional install. Returns None if no match found. The slug for each on-disk skill is derived from its frontmatter ``name:`` (via :func:`_skill_slug_from_frontmatter`), NOT from its containing directory name — because the two can differ (e.g. directory ``stable-diffusion`` + frontmatter ``Stable Diffusion Image Generation`` yields slug ``stable-diffusion-image-generation``). Matching on directory name would miss that slug entirely and fall through to the generic "unknown command" path. """ # Normalize: command uses hyphens, skill names may use hyphens or underscores normalized = command_name.lower().replace("_", "-") try: from tools.skills_tool import _get_disabled_skill_names from agent.skill_utils import get_all_skills_dirs, is_excluded_skill_path disabled = _get_disabled_skill_names() # Check disabled skills across all dirs (local + external) for skills_dir in get_all_skills_dirs(): if not skills_dir.exists(): continue for skill_md in skills_dir.rglob("SKILL.md"): if is_excluded_skill_path(skill_md): continue slug, declared_name = _skill_slug_from_frontmatter(skill_md) if not slug or not declared_name: continue # disabled is keyed by the declared frontmatter name (what # skills.disabled / skills.platform_disabled store). if slug == normalized and declared_name in disabled: return ( f"The **{command_name}** skill is installed but disabled.\n" f"Enable it with: `hermes skills config`" ) # Check optional skills (shipped with repo but not installed) from hermes_constants import get_optional_skills_dir repo_root = Path(__file__).resolve().parent.parent optional_dir = get_optional_skills_dir(repo_root / "optional-skills") if optional_dir.exists(): for skill_md in optional_dir.rglob("SKILL.md"): if is_excluded_skill_path(skill_md): continue slug, _declared = _skill_slug_from_frontmatter(skill_md) if not slug: continue if slug == normalized: # Build install path: official// rel = skill_md.parent.relative_to(optional_dir) parts = list(rel.parts) install_path = f"official/{'/'.join(parts)}" return ( f"The **{command_name}** skill is available but not installed.\n" f"Install it with: `hermes skills install {install_path}`" ) except Exception: pass return None def _platform_config_key(platform: "Platform") -> str: """Map a Platform enum to its config.yaml key (LOCAL→"cli", rest→enum value).""" return "cli" if platform == Platform.LOCAL else platform.value def _teams_pipeline_plugin_enabled() -> bool: """Return True when the standalone Teams pipeline plugin is enabled.""" config = _load_gateway_config() enabled = cfg_get(config, "plugins", "enabled", default=[]) if not isinstance(enabled, list): return False return "teams_pipeline" in enabled or "teams-pipeline" in enabled def _gateway_config_home() -> Path: """Return the Hermes home that gateway config reads should use.""" override = get_hermes_home_override() if override: return Path(override) return _hermes_home def _load_gateway_config(config_path: "Path | None" = None) -> dict: """Load and parse a gateway config.yaml, returning {} on any error. Defaults to the active gateway home (so tests that monkeypatch ``_hermes_home`` still see their fixture). Callers handling multiplexed profile routes may pass that profile's explicit config path. The canonical path shares the mtime-keyed raw-yaml cache from ``hermes_cli.config.read_raw_config``. Managed scope is overlaid on the result (via the shared helper) so the gateway honors administrator-pinned values — neither read_raw_config nor a direct yaml.safe_load carries the managed merge on its own. Fail-open. """ if config_path is None: config_path = _gateway_config_home() / 'config.yaml' raw: dict = {} used_canonical = False try: from hermes_cli.config import get_config_path, read_raw_config # Fast path: if _hermes_home agrees with the canonical config # location, reuse the shared cache. Otherwise fall through to a # direct read (keeps test fixtures with a monkeypatched # _hermes_home working). if config_path == get_config_path(): raw = read_raw_config() used_canonical = True except Exception: pass if not used_canonical: try: if config_path.exists(): import yaml with open(config_path, 'r', encoding='utf-8') as f: raw = yaml.safe_load(f) or {} except Exception: logger.debug("Could not load gateway config from %s", config_path) raw = {} # Overlay managed scope. read_raw_config() returns the user's raw YAML # WITHOUT the managed merge (that lives in load_config/_load_config_impl), # so the overlay is required on both paths for the gateway to honor pinned # values. Helper is fail-open and a no-op when no managed scope exists. try: from hermes_cli import managed_scope raw = managed_scope.apply_managed_overlay(raw if isinstance(raw, dict) else {}) except Exception: pass if not isinstance(raw, dict): return {} # Canonicalize model-id aliases (model.name / model.model → model.default) # and migrate stale root-level provider/base_url into the model section. # The gateway bypasses load_config() (it reads raw YAML for speed), so the # normalization that load_config() applies must be replayed here or the # gateway would resolve an empty model for ``model: {name: }`` configs # while the CLI resolves it correctly. See issue #34500. Fail-open. try: from hermes_cli.config import _normalize_root_model_keys raw = _normalize_root_model_keys(raw) except Exception: pass return raw def _checkpoint_agent_kwargs(config: dict | None) -> dict: """Translate gateway checkpoint config into ``AIAgent`` constructor args. The gateway reads raw YAML instead of ``load_config()``, so checkpoint defaults must be supplied here. Keep legacy ``checkpoints: true`` configs working while giving every gateway-created agent the same limits. """ cp_cfg = config.get("checkpoints", {}) if isinstance(config, dict) else {} if isinstance(cp_cfg, bool): cp_cfg = {"enabled": cp_cfg} elif not isinstance(cp_cfg, dict): cp_cfg = {} from hermes_cli.config import DEFAULT_CONFIG defaults = DEFAULT_CONFIG["checkpoints"] return { "checkpoints_enabled": cp_cfg.get("enabled", defaults["enabled"]), "checkpoint_max_snapshots": cp_cfg.get( "max_snapshots", defaults["max_snapshots"], ), "checkpoint_max_total_size_mb": cp_cfg.get( "max_total_size_mb", defaults["max_total_size_mb"], ), "checkpoint_max_file_size_mb": cp_cfg.get( "max_file_size_mb", defaults["max_file_size_mb"], ), } def _load_gateway_runtime_config() -> dict: """Load gateway config for runtime reads, expanding supported ``${VAR}`` refs. Runtime helpers should honor the same env-template expansion documented for ``config.yaml`` while still respecting tests that monkeypatch ``gateway.run._hermes_home``. Build on ``_load_gateway_config()`` rather than calling the canonical loader directly so both behaviors stay aligned. Expansion failures are intentionally NOT swallowed — silently returning the unexpanded dict would mask the very bug this helper exists to fix. """ cfg = _load_gateway_config() if not isinstance(cfg, dict) or not cfg: return {} from hermes_cli.config import _expand_env_vars expanded = _expand_env_vars(cfg) return expanded if isinstance(expanded, dict) else {} def _resolve_gateway_model(config: dict | None = None) -> str: """Read model from config.yaml — single source of truth. Without this, temporary AIAgent instances (e.g. /compress) fall back to the hardcoded default which fails when the active provider is openai-codex. """ cfg = config if config is not None else _load_gateway_config() model_cfg = cfg.get("model", {}) if isinstance(model_cfg, str): return model_cfg elif isinstance(model_cfg, dict): return model_cfg.get("default") or model_cfg.get("model") or "" return "" def _channel_override_lookup_keys( chat_id: str, *, thread_id: Optional[str] = None, parent_id: Optional[str] = None, ) -> list[str]: """Ordered, de-duplicated keys for ``channel_overrides`` lookup. Matches ``resolve_channel_prompt`` semantics: exact thread/channel id first, then parent channel/forum id (Discord threads inherit parent overrides). """ keys: list[str] = [] seen: set[str] = set() for key in (chat_id, thread_id, parent_id): if not key: continue sk = str(key) if sk in seen: continue seen.add(sk) keys.append(sk) return keys def _get_channel_override( config: GatewayConfig, platform: Platform, chat_id: str, *, thread_id: Optional[str] = None, parent_id: Optional[str] = None, ) -> Optional[ChannelOverride]: """Return per-channel override for this platform/chat_id, or None. Looks up ``channel_overrides`` by ``chat_id``, then ``thread_id``, then ``parent_id`` (forum threads / child channels inherit the parent entry). """ platforms = getattr(config, "platforms", None) if not platforms: return None platform_config = platforms.get(platform) if not platform_config or not platform_config.channel_overrides: return None overrides = platform_config.channel_overrides for key in _channel_override_lookup_keys( chat_id, thread_id=thread_id, parent_id=parent_id ): ov = overrides.get(key) if ov is not None: return ov return None def _resolve_hermes_bin() -> Optional[list[str]]: """Resolve the Hermes update command as argv parts. Tries in order: 1. ``shutil.which("hermes")`` — standard PATH lookup 2. ``sys.executable -m hermes_cli.main`` — fallback when Hermes is running from a venv/module invocation and the ``hermes`` shim is not on PATH Returns argv parts ready for quoting/joining, or ``None`` if neither works. """ import shutil hermes_bin = shutil.which("hermes") if hermes_bin: return [hermes_bin] try: import importlib.util if importlib.util.find_spec("hermes_cli") is not None: return [sys.executable, "-m", "hermes_cli.main"] except Exception: pass return None def _parse_session_key(session_key: str) -> "dict | None": """Parse a session key into its component parts. Session keys follow the format ``agent:main:{platform}:{chat_type}:{chat_id}[:{extra}...]``. Returns a dict with ``platform``, ``chat_type``, ``chat_id``, and optionally ``thread_id`` keys, or None if the key doesn't match. The 6th element is only returned as ``thread_id`` for chat types where it is unambiguous (``dm`` and ``thread``). For group/channel sessions the suffix may be a user_id (per-user isolation) rather than a thread_id, so we leave ``thread_id`` out to avoid mis-routing. """ parts = session_key.split(":") if len(parts) >= 5 and parts[0] == "agent" and parts[1] == "main": result = { "platform": parts[2], "chat_type": parts[3], "chat_id": parts[4], } if len(parts) > 5 and parts[3] in {"dm", "thread"}: result["thread_id"] = parts[5] return result return None def _shorten_command_for_display(command: str, limit: int = 80) -> str: """Collapse a shell command onto one line and cap its length for display.""" one_line = " ".join((command or "").split()) if len(one_line) > limit: one_line = one_line[: limit - 1] + "…" return one_line def _format_concise_process_notification( session_id: str, command: str, exit_code, output: str, duration_seconds=None, ) -> str: """One-line "pretty" completion message for the ``concise`` display mode. Success is a single status line; failure appends a short tail of output so the user can see what went wrong without the full raw dump. The full output always remains available to the agent via process(log/wait). """ ok = exit_code in {0, None} icon = "✅" if ok else "❌" verb = "finished" if ok else f"failed (exit {exit_code})" parts = [f"{icon} Background task {verb}"] short_cmd = _shorten_command_for_display(command) if short_cmd: parts.append(f"— `{short_cmd}`") if isinstance(duration_seconds, (int, float)) and duration_seconds >= 0: secs = int(duration_seconds) if secs >= 3600: dur = f"{secs // 3600}h {(secs % 3600) // 60}m" elif secs >= 60: dur = f"{secs // 60}m {secs % 60}s" else: dur = f"{secs}s" parts.append(f"({dur})") text = " ".join(parts) if not ok and output: tail_lines = [ln for ln in output.strip().splitlines() if ln.strip()][-5:] tail = "\n".join(tail_lines) if len(tail) > 500: tail = tail[-500:] if tail: text += f"\n```\n{tail}\n```" return text def _format_gateway_process_notification(evt: dict) -> "str | None": """Format a watch pattern event from completion_queue into a [IMPORTANT:] message.""" evt_type = evt.get("type", "completion") _sid = evt.get("session_id", "unknown") _cmd = evt.get("command", "unknown") if evt_type == "watch_disabled": return f"[IMPORTANT: {evt.get('message', '')}]" # Overflow events carry their human-readable summary in `message`, # like watch_disabled — see the shared formatter in # tools/process_registry.py. if evt_type in ("watch_overflow_tripped", "watch_overflow_released"): return f"[IMPORTANT: {evt.get('message', '')}]" if evt_type == "watch_match": _pat = evt.get("pattern", "?") _out = evt.get("output", "") _sup = evt.get("suppressed", 0) text = ( f"[IMPORTANT: Background process {_sid} matched " f"watch pattern \"{_pat}\".\n" f"Command: {_cmd}\n" f"Matched output:\n{_out}" ) if _sup: text += f"\n({_sup} earlier matches were suppressed by rate limit)" text += "]" return text if evt_type == "async_delegation": # Reuse the shared rich formatter (self-contained task-source block). from tools.process_registry import format_process_notification return format_process_notification(evt) return None def _drain_gateway_watch_events(completion_queue) -> "list[dict]": """Drain gateway-owned watch events without spinning on requeued events. Watch events are handled by the post-turn gateway drain. Process completions are owned by their per-process watcher task, and async delegation completions are owned by ``_async_delegation_watcher``. Requeueing async events inside ``while not queue.empty()`` would make the loop non-terminating, so detach the current batch first, then requeue any events this drain does not own after the queue is empty. """ watch_events: list[dict] = [] requeue: list[dict] = [] while not completion_queue.empty(): try: evt = completion_queue.get_nowait() except Exception: break evt_type = evt.get("type", "completion") if evt_type in { "watch_match", "watch_disabled", "watch_overflow_tripped", "watch_overflow_released", }: watch_events.append(evt) elif evt_type == "async_delegation": requeue.append(evt) # else: process completion events are handled by the watcher task for evt in requeue: completion_queue.put(evt) return watch_events # Module-level weak reference to the active GatewayRunner instance. # Used by tools (e.g. send_message) that need to route through a live # adapter for plugin platforms. Set in GatewayRunner.__init__(). import weakref as _weakref _gateway_runner_ref: _weakref.ref = lambda: None def _normalize_empty_agent_response( agent_result: dict, response: str, *, history_len: int = 0, ) -> str: """Normalize empty/None agent responses into user-facing messages. Consolidates the existing ``failed`` handler and adds a catch-all for the case where the agent did work (api_calls > 0) but returned no text. Fix for #18765. Also surfaces a retry hint when the agent never ran at all (api_calls == 0) for a non-interrupted, non-failed turn -- this is the silent-drop pattern observed after ``/stop`` where the next user message hits a stale generation token and returns an empty result, leaving the platform with nothing to send. (#31884) """ if response: return response if agent_result.get("failed"): # None-safe: the gateway result dict is built with # ``'error': holder.get('error')`` and can carry an EXPLICIT None, # which bypasses dict.get's default and would render # "The request failed: None". error_detail = agent_result.get("error") or "unknown error" error_str = str(error_detail).lower() # Session-persistence failures get a dedicated recovery message. # Suggesting /reset here would be actively harmful: it destroys the # user's conversation context and does nothing to fix the underlying # storage problem (lock contention, disk exhaustion, ...). failure_reason = str(agent_result.get("failure_reason") or "") if failure_reason.startswith("session_persistence_failed") or ( "session storage" in error_str ): if failure_reason.endswith(":disk") or "disk" in error_str: return ( "⚠️ Session storage was temporarily unavailable, so this " "turn was stopped to protect your conversation history. " "Please check available disk space, then send your " "message again." ) return ( "⚠️ Session storage was temporarily unavailable, so this " "turn was stopped to protect your conversation history. " "Your message should already be saved — please send it " "again in a moment." ) is_context_failure = any( p in error_str for p in ("context", "token", "too large", "too long", "exceed", "payload") ) or ("400" in error_str and history_len > 50) if is_context_failure: return ( "⚠️ Session too large for the model's context window.\n" "Use /compact to compress the conversation, or " "/reset to start fresh." ) return ( f"The request failed: {str(error_detail)[:300]}\n" "Try again or use /reset to start a fresh session." ) api_calls = int(agent_result.get("api_calls", 0) or 0) if agent_result.get("interrupted"): # An interrupted run that did work (api_calls > 0) is the drain of a # run the user deliberately stopped or steered — its silence is # intentional, and any queued/interrupting message is delivered by # the recursive drain inside _run_agent before this result is seen. # An interrupted run with ZERO api_calls never processed the user's # message at all: it was killed at the top of the tool loop by an # interrupt flag left over from a recent /stop (#44212). Pure # silence there swallows a real user message, so surface it. if api_calls == 0: return ( "⚠️ Your message was interrupted before processing started " "(likely by a recent /stop). Please send it again." ) return response if api_calls > 0: if _is_gateway_hidden_reasoning_incomplete_turn(agent_result): return "" if agent_result.get("partial"): err = agent_result.get("error", "processing incomplete") return f"⚠️ Processing stopped: {str(err)[:200]}. Try again." return ( "⚠️ Processing completed but no response was generated. " "This may be a transient error — try sending your message again." ) # api_calls == 0, not failed, not interrupted: the agent never ran for # this turn. This is the post-/stop generation-race pattern where the # gateway would otherwise silently drop the turn (response=0 chars) and # the user sees no reply at all. Surface a short retry hint so the # message isn't lost in silence. (#31884) if ( api_calls == 0 and not agent_result.get("interrupted") and not agent_result.get("failed") and not agent_result.get("partial") ): return ( "⚠️ Your message wasn't processed (the previous turn was still " "being cleaned up). Please send it again." ) return response def _is_gateway_hidden_reasoning_incomplete_turn(agent_result: dict) -> bool: """Detect retry-exhausted turns with hidden reasoning but no visible answer. The conversation loop returns the retry-exhaustion sentinel as BOTH ``final_response`` and ``error`` ("Codex response remained incomplete after 3 continuation attempts"), so ``final_response`` being non-empty does not mean the model produced a visible answer. Treat the turn as hidden when the error sentinel is present and ``final_response`` is either empty or merely echoes that sentinel — any genuinely different final text means the model DID answer and must be delivered. """ if not isinstance(agent_result, dict): return False if agent_result.get("failed") or agent_result.get("interrupted"): return False if not agent_result.get("partial"): return False error_text = str(agent_result.get("error", "") or "").strip() if "remained incomplete after" not in error_text.lower(): return False final_response = str(agent_result.get("final_response") or "").strip() return not final_response or final_response == error_text def _should_clear_resume_pending_after_turn(agent_result: dict) -> bool: """Return True only when a gateway turn really completed successfully. Restart recovery uses ``resume_pending`` as a durable marker for sessions interrupted during gateway drain. A soft interrupt can still bubble out as a syntactically normal agent result with an empty final response; clearing the marker in that case loses the recovery signal and startup auto-resume has nothing to schedule. """ if not isinstance(agent_result, dict): return False if agent_result.get("interrupted"): return False if agent_result.get("failed") or agent_result.get("partial") or agent_result.get("error"): return False if agent_result.get("completed") is False: return False return True def _preserve_queued_followup_history_offset( current_result: dict, followup_result: dict, ) -> dict: """Carry the outer history offset through queued follow-up drains. ``_process_message_background()`` persists transcript rows only once, after the entire in-band queued-follow-up chain returns. Each recursive ``_run_agent()`` call advances ``history_offset`` to the history it received, so without correction the outermost persistence step sees only the *last* queued turn as "new" and silently drops earlier turns from the same drain chain. Preserve the earliest (outermost) history offset so the final transcript slice still includes every queued turn that ran during the chain. """ if not isinstance(followup_result, dict): return followup_result if not isinstance(current_result, dict): return followup_result current_offset = current_result.get("history_offset") followup_offset = followup_result.get("history_offset") if not isinstance(current_offset, int): return followup_result if isinstance(followup_offset, int) and followup_offset <= current_offset: return followup_result merged = dict(followup_result) merged["history_offset"] = current_offset return merged async def _dispose_unused_adapter(adapter: "BasePlatformAdapter | None") -> None: """Best-effort dispose for an adapter that never made it onto ``self.adapters``. The reconnect watcher in ``GatewayRunner._platform_reconnect_watcher`` constructs a fresh adapter on every retry attempt. When the connect call fails — for any of the three reasons (non-retryable error, retryable error, exception during connect) — the adapter is dropped without ever being installed, so nothing else will call its ``disconnect()``. Any resources the adapter opened in ``__init__`` (e.g. ``APIServerAdapter`` opens a SQLite ``ResponseStore`` that holds 2 fds — the db file and its WAL sidecar) stay open until garbage collection sweeps the unreachable object, which Python's cyclic GC does not do promptly for asyncio-bound objects with native handles. The cumulative leak is 2 fds × every retry at the 300s backoff cap ≈ 12 fds/hour, and the default 2560-fd ulimit is exhausted in ~12h of continuous failure, after which every open() call on the gateway raises ``OSError: [Errno 24] Too many open files`` and the gateway becomes a zombie (#37011). This helper centralises the dispose-with-suppression so the three failure paths in the reconnect watcher can all call it without each one having to know that ``disconnect()`` may itself raise on a half-constructed adapter. ``adapter`` may be ``None``: the reconnect watcher initialises ``adapter = None`` before the ``try`` so the ``except Exception`` arm can dispose a half-constructed object, and also early-returns here when ``_create_adapter()`` returned ``None``. """ if adapter is None: return try: await adapter.disconnect() except Exception: # Half-constructed adapters (e.g. APIServerAdapter that # crashed during aiohttp app setup) can raise from # disconnect() on objects that never finished initializing. # We must not let that escape and abort the watcher loop. # # On Python 3.8+, ``asyncio.CancelledError`` inherits from # ``BaseException`` (not ``Exception``), so this ``except # Exception`` does not swallow task cancellation. We don't # re-raise explicitly because the watcher loop intentionally # treats dispose failures as best-effort: a failed ``disconnect`` # call should not take down the reconnect watcher that # itself is what's keeping the gateway alive during a partial # outage. logger.debug( "Adapter dispose raised on unowned adapter %r", getattr(adapter, "name", type(adapter).__name__), exc_info=True, ) # Max seconds between platform reconnect retries (primary watcher and # secondary-profile reconnects share this policy — tune in one place). _RECONNECT_BACKOFF_CAP = 300 # Seconds a platform may sit continuously in the reconnect queue before the # watcher flags it NEEDS_ATTENTION in runtime status. Retrying never stops # (auto-pause was deliberately removed — a transient outage must self-heal # without operator action); this only makes a *long-lived* retry loop loud so # owners and fleet monitoring can distinguish hour one from week three. # A dead bot token, a revoked Discord intent, or a deterministically crashing # sidecar all present as "retrying" forever without this signal. # User-facing setting: agent.reconnect_attention_after in config.yaml # (bridged to this env var above). 0 disables. _RECONNECT_ATTENTION_AFTER_SECONDS = _float_env( "HERMES_RECONNECT_ATTENTION_AFTER_SECONDS", 7200 ) def _reconnect_backoff(attempt: int) -> int: """Exponential reconnect backoff: 30s, 60s, 120s, ... capped at 5 min.""" return min(30 * (2 ** (attempt - 1)), _RECONNECT_BACKOFF_CAP) def _reconnect_needs_attention(info: dict, now: float) -> bool: """Return True when a reconnect-queue entry has been continuously queued long enough to warrant a NEEDS_ATTENTION signal. ``queued_at`` is (re)stamped whenever the platform (re)enters the queue, so a platform that reconnects successfully and later fails again starts a fresh clock — only *continuous* failure escalates. Entries queued before this field existed (in-flight upgrade) are treated as newly queued. """ if _RECONNECT_ATTENTION_AFTER_SECONDS <= 0: return False # escalation disabled queued_at = info.get("queued_at") if queued_at is None: info["queued_at"] = now return False return (now - queued_at) >= _RECONNECT_ATTENTION_AFTER_SECONDS class TurnRunner: """Per-turn collaborator carrying the tool-progress callbacks that used to be nested closures inside ``GatewayRunner._run_agent_inner``. The bodies are byte-identical to the original closures modulo ``local_name`` -> ``ctx.field`` rewrites (closed-over locals now travel on the shared :class:`gateway.turn_context.TurnContext`) and ``self`` -> ``self._runner`` (the owning :class:`GatewayRunner`). Module-global references (logger, cfg_get, BasePlatformAdapter, ...) resolve in this same module exactly as before. """ def __init__(self, runner: "GatewayRunner", ctx: TurnContext) -> None: self._runner = runner self._ctx = ctx def progress_callback(self, event_type: str, tool_name: str = None, preview: str = None, args: dict = None, **kwargs): """Callback invoked by agent on tool lifecycle events.""" ctx = self._ctx # Failed subagent → one clean user-facing notice. Handled FIRST, # before every progress-queue gate: platforms that keep # tool_progress off (Telegram, Slack, ...) must still hear about a # delegation that died — a silently-vanishing subagent looks like # the agent just dropped the task (community report, Aug 2026). # Success/interrupt completions stay quiet; only terminal failure # statuses render, via the same notice rail as credit warnings. if event_type == "subagent.complete": _sub_status = kwargs.get("status") try: from tools.delegate_tool import ( SUBAGENT_FAILURE_STATUSES, format_subagent_failure_line, ) if _sub_status in SUBAGENT_FAILURE_STATUSES and ctx._run_still_current(): _line = format_subagent_failure_line( kwargs.get("goal"), _sub_status, error=kwargs.get("summary") or preview, duration_seconds=kwargs.get("duration_seconds"), ) safe_schedule_threadsafe( self._runner._deliver_platform_notice(ctx.source, _line), ctx._loop_for_step, logger=logger, log_message="subagent failure notice scheduling error", ) except Exception: logger.debug("subagent failure notice failed", exc_info=True) return # Live status line (Slack's assistant status): stash the current # tool phrase on the adapter; the _keep_typing refresh renders it # within a couple of seconds. Handled before every other gate # because it's independent of progress bubbles and queues (Slack # keeps tool_progress off by default, but the ephemeral status # line is always safe). Plain dict write — safe from the agent's # sync worker thread, no event-loop hop needed. if ( ctx._live_status_adapter is not None and ctx._live_status_mode != "off" and tool_name != "_thinking" ): try: if event_type == "tool.started" and tool_name and ctx._run_still_current(): from agent.display import build_status_phrase _phrase = build_status_phrase( tool_name, args if ctx._live_status_mode == "full" else None, ) ctx._live_status_adapter.set_status_text(ctx.source.chat_id, _phrase) elif event_type == "tool.completed": # Between tools the model is genuinely "thinking" # again — revert to the static default. ctx._live_status_adapter.set_status_text(ctx.source.chat_id, None) except Exception as _ls_err: logger.debug("live status update failed: %s", _ls_err) # "log" mode: append tool.started lines to the log queue and stay # silent in chat. Handled before the progress_queue guard because # log mode runs without a chat progress queue. if ctx.log_queue is not None: if event_type == "tool.started" and tool_name and tool_name != "_thinking": ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S") preview_str = f' "{preview}"' if preview else "" ctx.log_queue.put(f"{ts} {tool_name}:{preview_str}".rstrip()) if not ctx.progress_queue: return if not ctx.progress_queue or not ctx._run_still_current(): return # First-touch onboarding: the first time a tool takes longer than # _LONG_TOOL_THRESHOLD_S during a run that's streaming every tool # (progress_mode == "all"), append a one-time hint suggesting # /verbose. We only fire when (a) the user hasn't seen the hint # before and (b) /verbose is actually usable on this platform # (gateway gate must be open). The CLI has its own trigger. if event_type == "tool.completed" and not ctx.long_tool_hint_fired[0]: try: duration = kwargs.get("duration") or 0 if duration >= ctx._LONG_TOOL_THRESHOLD_S and ctx.progress_mode == "all": from agent.onboarding import ( TOOL_PROGRESS_FLAG, is_seen, mark_seen, tool_progress_hint_gateway, ) _cfg = _load_gateway_config() gate_on = is_truthy_value( cfg_get(_cfg, "display", "tool_progress_command"), default=False, ) if gate_on and not is_seen(_cfg, TOOL_PROGRESS_FLAG): ctx.long_tool_hint_fired[0] = True ctx.progress_queue.put(tool_progress_hint_gateway()) mark_seen(_hermes_home / "config.yaml", TOOL_PROGRESS_FLAG) except Exception as _hint_err: logger.debug("tool-progress onboarding hint failed: %s", _hint_err) return # "_thinking" is assistant scratch text between tool calls. It # is never ordinary tool progress: only relay it when the platform # explicitly opted into thinking_progress. Handle both legacy # callback shapes: ("_thinking", text) and # ("reasoning.available", "_thinking", text, ...). if event_type == "_thinking" or tool_name == "_thinking": if not ctx._thinking_enabled: return thinking_text = preview if tool_name == "_thinking" else tool_name msg = f"💬 {thinking_text}" if thinking_text else None if msg: ctx.progress_queue.put(msg) return # Native task cards consume the authoritative ID-bearing # tool_start/tool_complete callbacks instead. Do not also enqueue # name-correlated text events, which would duplicate cards and # mispair concurrent calls to the same tool. if ctx._native_slack_task_cards and event_type in { "tool.started", "tool.completed", }: return # If tool_progress is off, only _thinking passes through (above). # Regular tool calls are suppressed. if not ctx.tool_progress_enabled: return # Only act on tool.started events (ignore tool.completed, reasoning.available, etc.) if event_type not in {"tool.started",}: return # Never render a progress bubble for the clarify tool. The # adapter's send_clarify IS the user-facing rendering (interactive # buttons or the numbered-text fallback), so a progress bubble is # pure duplication — and in verbose mode it dumps the raw # tool-call args JSON ({"question": ..., "choices": [...]}) into # the chat. Because the progress queue drains on a background # task, that raw JSON typically lands right underneath the # rendered prompt (#52374). if tool_name == "clarify": return # Suppress tool-progress bubbles once the user has sent `stop`. # When the LLM response carries N parallel tool calls, the agent # fires N "tool.started" events back-to-back before checking for # interrupts — without this guard, a late `stop` still renders # all N as 🔍 bubbles, making the interrupt feel ignored. # (agent lives in run_sync's scope; agent_holder[0] is the shared # handle across nested scopes — see line ~9607.) try: _agent_for_interrupt = ctx.agent_holder[0] if ctx.agent_holder else None if _agent_for_interrupt is not None and getattr( _agent_for_interrupt, "is_interrupted", False ): return except Exception: pass # "new" mode: only report when tool changes if ctx.progress_mode == "new" and tool_name == ctx.last_tool[0]: return ctx.last_tool[0] = tool_name # Build progress message with primary argument preview from agent.display import get_tool_emoji emoji = get_tool_emoji(tool_name, default="⚙️") # Markdown-capable platforms render a terminal command as a fenced # code block instead of the compact `terminal: "cmd…"` preview. # Gated on the adapter's ``supports_code_blocks`` capability so # plain-text platforms keep the short line. No language tag is # emitted — Slack mrkdwn renders the tag as a literal first code # line ("bash"), and a bare fence renders correctly everywhere # that supports blocks. # # Verbose mode shows the FULL command. Non-verbose ("all"/"new") # modes still wrap in a fence but truncate to a single line capped # at ``tool_preview_length`` (default 40) so a long or multi-line # command doesn't render as a huge block — matching the budget the # non-terminal preview path already applies (#42634). _code_block_full = None _code_block_short = None try: _progress_adapter = self._runner._adapter_for_source(ctx.source) except Exception: _progress_adapter = None if ( getattr(_progress_adapter, "supports_code_blocks", False) and tool_name == "terminal" and isinstance(args, dict) and isinstance(args.get("command"), str) and args["command"].strip() ): from agent.display import get_tool_preview_max_len _cmd_full = args["command"].rstrip() # Consecutive terminal calls: drop the repeated # "💻 terminal" header so back-to-back commands render as # adjacent code blocks under a single header. _block_header = ( "" if ctx.last_was_terminal_block[0] else f"{emoji} {tool_name}\n" ) _code_block_full = f"{_block_header}```\n{_cmd_full}\n```" # Single-line, capped preview for non-verbose modes. _pl = get_tool_preview_max_len() _cap = _pl if _pl > 0 else 40 _lines = _cmd_full.splitlines() _cmd_short = _lines[0] if _lines else _cmd_full _multiline = len(_lines) > 1 if len(_cmd_short) > _cap: _cmd_short = _cmd_short[:_cap - 3] + "..." elif _multiline: _cmd_short = _cmd_short + " ..." _code_block_short = f"{_block_header}```\n{_cmd_short}\n```" # Verbose mode: show detailed arguments, respects tool_preview_length if ctx.progress_mode == "verbose": if _code_block_full is not None: ctx.last_was_terminal_block[0] = True ctx.progress_queue.put(_code_block_full) return ctx.last_was_terminal_block[0] = False if args: from agent.display import get_tool_preview_max_len _pl = get_tool_preview_max_len() args_str = json.dumps(args, ensure_ascii=False, default=str) # When tool_preview_length is 0 (default), don't truncate # in verbose mode — the user explicitly asked for full # detail. Platform message-length limits handle the rest. if _pl > 0 and len(args_str) > _pl: args_str = args_str[:_pl - 3] + "..." msg = f"{emoji} {tool_name}({list(args.keys())})\n{args_str}" elif preview: msg = f"{emoji} {tool_name}: \"{preview}\"" else: msg = f"{emoji} {tool_name}..." ctx.progress_queue.put(msg) return # "all" / "new" modes: short preview, respects tool_preview_length # config (defaults to 40 chars when unset to keep gateway messages # compact — unlike CLI spinners, these persist as permanent messages). # Terminal commands on markdown platforms get a single-line capped # fenced block (built above) instead of the truncated preview. if _code_block_short is not None: msg = _code_block_short ctx.last_was_terminal_block[0] = True elif preview: from agent.display import ( get_tool_preview_max_len, get_tool_verb, prepare_tool_preview, tool_verb_connector, verb_drops_preview, ) _pl = get_tool_preview_max_len() _cap = _pl if _pl > 0 else 40 _prepared_preview = prepare_tool_preview( tool_name, args, fallback=preview, max_len=_cap, ) if _progress_adapter is not None: preview = _progress_adapter.format_tool_preview(_prepared_preview) else: preview = _prepared_preview.text # Friendly labels: render a human-phrased line for built-in # tools ("🔍 Searching the web for ...") by prefixing the verb # onto the preview the callback already computed (so the # command/url/query is preserved). Custom/plugin/MCP tools # have no verb and fall back to the raw "tool_name: ..." form. _verb = get_tool_verb(tool_name) if _verb: if verb_drops_preview(tool_name): msg = f"{emoji} {_verb}" else: msg = f"{emoji} {_verb}{tool_verb_connector(tool_name)}{preview}" else: msg = f"{emoji} {tool_name}: \"{preview}\"" ctx.last_was_terminal_block[0] = False else: msg = f"{emoji} {tool_name}..." ctx.last_was_terminal_block[0] = False # Dedup: collapse consecutive identical progress messages. # Common with execute_code where models iterate with the same # code (same boilerplate imports → identical previews). if msg == ctx.last_progress_msg[0]: ctx.repeat_count[0] += 1 # Native-stream-progress routing: dedup updates the last line # in the overlay rather than sending a queue signal. _sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None if _sc is not None and getattr(_sc, "accepts_tool_progress", False): # Replace the last progress line with the dedup version _sc.on_tool_progress(f"{msg} (×{ctx.repeat_count[0] + 1})") return # Update the last line in progress_lines with a counter # via a special "dedup" queue message. ctx.progress_queue.put(("__dedup__", msg, ctx.repeat_count[0])) return ctx.last_progress_msg[0] = msg ctx.repeat_count[0] = 0 # Native-stream-progress routing: if the stream consumer is active # and using native streaming, inject progress directly into the # stream bubble instead of the separate progress queue. _sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None if _sc is not None and getattr(_sc, "accepts_tool_progress", False): _sc.on_tool_progress(msg) return ctx.progress_queue.put(msg) async def _send_native_task_card_progress(self, adapter) -> None: """Drain the progress queue into Slack-native plan/task cards (#29483). Consumes the ID-bearing lifecycle dicts queued by native_tool_start_callback / native_tool_complete_callback and renders them through the adapter's chat.startStream plan/task-card stream. On any native failure, falls back to an editable in-thread text message so progress stays live for the rest of the turn. """ ctx = self._ctx tasks: Dict[str, Dict[str, str]] = {} task_order: List[str] = [] fallback_msg_id: Optional[str] = None native_failed = False anonymous_seq = 0 def _compact(value: Any, limit: int = 120) -> str: text = re.sub(r"\s+", " ", str(value or "")).strip() if len(text) <= limit: return text return text[: limit - 3].rstrip() + "..." def _visible_tasks() -> List[Dict[str, str]]: return [tasks[task_id] for task_id in task_order[-8:]] def _fallback_text() -> str: labels = { "in_progress": "running", "complete": "complete", "error": "error", } lines = [ f"- {task['title']} - {labels.get(task['status'], task['status'])}" for task in _visible_tasks() ] return "Hermes is working\n" + "\n".join(lines) def _apply_native_event(raw: Any) -> bool: nonlocal anonymous_seq if not isinstance(raw, dict): return False event_type = raw.get("type") if event_type not in {"tool.started", "tool.completed"}: return False call_id = str(raw.get("tool_call_id") or "") if not call_id: anonymous_seq += 1 call_id = f"anonymous_{anonymous_seq}" tool_name = str(raw.get("tool_name") or "tool") if event_type == "tool.started": title = tool_name preview = _compact(raw.get("preview"), 64) if preview: title = f"{tool_name} - {preview}" if call_id not in tasks: task_order.append(call_id) tasks[call_id] = { "id": call_id, "title": _compact(title), "status": "in_progress", } return True task = tasks.get(call_id) if task is None: # Completion-only events are rare but valid on some # runtimes. Keep their real ID instead of guessing a # same-name pending call. task = { "id": call_id, "title": _compact(tool_name), "status": "in_progress", } tasks[call_id] = task task_order.append(call_id) task["status"] = "error" if raw.get("is_error") else "complete" return True async def _send_or_edit_fallback() -> None: nonlocal fallback_msg_id text = _fallback_text() if fallback_msg_id: result = await adapter.edit_message( chat_id=ctx.source.chat_id, message_id=fallback_msg_id, content=text, metadata=ctx._progress_metadata, ) if getattr(result, "success", False): return result = await adapter.send( chat_id=ctx.source.chat_id, content=text, reply_to=ctx._progress_reply_to, metadata=ctx._progress_metadata, ) if getattr(result, "success", False) and getattr( result, "message_id", None ): fallback_msg_id = str(result.message_id) if ctx._cleanup_progress: ctx._cleanup_msg_ids.append(fallback_msg_id) async def _publish_native_progress() -> None: nonlocal native_failed if not tasks: return if not native_failed: result = await adapter.send_native_task_card_progress( chat_id=ctx.source.chat_id, tasks=_visible_tasks(), title="Hermes is working", reply_to=ctx._progress_reply_to, metadata=ctx._progress_metadata, fallback_text=_fallback_text(), ) if getattr(result, "success", False): return native_failed = True logger.warning( "Slack native task-card progress failed; falling back " "to an editable text update: %s", getattr(result, "error", "unknown error"), ) # Once the native rail fails, every later lifecycle event # edits the same fallback message so progress remains live. await _send_or_edit_fallback() def _drain_native_queue() -> bool: changed = False while True: try: changed = _apply_native_event( ctx.progress_queue.get_nowait() ) or changed except queue.Empty: return changed except Exception: logger.debug( "Slack native progress queue drain failed", exc_info=True, ) return changed def _agent_interrupted() -> bool: try: _agent = ctx.agent_holder[0] if ctx.agent_holder else None return bool( _agent is not None and getattr(_agent, "is_interrupted", False) ) except Exception: return False try: while True: if not ctx._run_still_current(): return try: raw = ctx.progress_queue.get_nowait() except queue.Empty: await asyncio.sleep(0.1) continue if _agent_interrupted(): continue if _apply_native_event(raw): await _publish_native_progress() except asyncio.CancelledError: if _drain_native_queue() and ctx._run_still_current(): if not _agent_interrupted(): await _publish_native_progress() return finally: if hasattr(adapter, "stop_native_task_card_progress"): # Best-effort: this finally runs on the turn-cleanup path. # An escaping transport exception here propagated through # the cleanup awaits (which caught only CancelledError) and # skipped final-delivery logic (review B7). Adapters now # return failed SendResults, but defend the seam anyway — # any adapter, any transport. try: await adapter.stop_native_task_card_progress( ctx.source.chat_id, reply_to=ctx._progress_reply_to, metadata=ctx._progress_metadata, ) except asyncio.CancelledError: raise except Exception: logger.debug( "task-card stop failed during turn cleanup", exc_info=True, ) async def send_progress_messages(self): ctx = self._ctx if not ctx.progress_queue: return adapter = self._runner._adapter_for_source(ctx.source) if not adapter: return if ctx._native_slack_task_cards and hasattr( adapter, "send_native_task_card_progress" ): await self._send_native_task_card_progress(adapter) return # Skip tool progress for platforms that don't support message # editing (e.g. iMessage/BlueBubbles) — each progress update # would become a separate message bubble, which is noisy. # getattr, not attribute access: duck-typed adapters (test fakes, # minimal plugin adapters) may not define edit_message at all — # "missing" means the same thing as "base no-op": can't edit. _adapter_edit = getattr(type(adapter), "edit_message", None) if _adapter_edit is None or _adapter_edit is BasePlatformAdapter.edit_message: while not ctx.progress_queue.empty(): try: ctx.progress_queue.get_nowait() except Exception: break return progress_lines = [] # Accumulated tool lines for the CURRENT editable bubble progress_msg_id = None # ID of the current progress message to edit can_edit = ctx.progress_grouping != "separate" # "separate" = one message per tool (pre-v0.9 behavior) _last_edit_ts = 0.0 # Throttle edits to avoid Telegram flood control _PROGRESS_EDIT_INTERVAL = 1.5 # Minimum seconds between edits _progress_len_fn = ( adapter.message_len_fn if isinstance(adapter, BasePlatformAdapter) else len ) try: _raw_progress_limit = int(getattr(adapter, "MAX_MESSAGE_LENGTH", 4000) or 4000) except Exception: _raw_progress_limit = 4000 # Per-chat resolution (relay adapter fronting N platforms): the cap # and length unit follow the chat's underlying platform. Native # adapters return their scalar/property unchanged. if isinstance(adapter, BasePlatformAdapter): try: _raw_progress_limit = int( adapter.max_message_length_for_chat(ctx.source.chat_id) or 4000 ) _progress_len_fn = adapter.message_len_fn_for_chat(ctx.source.chat_id) except Exception: pass # Leave a little room for platform quirks / formatting. For tiny # test adapters keep the limit usable instead of clamping to 500+. _PROGRESS_TEXT_LIMIT = max( 1, _raw_progress_limit - (64 if _raw_progress_limit > 128 else 0), ) # Detect whether the adapter's edit_message accepts metadata so # overflow edits preserve Telegram topic/thread routing (#27487). _edit_accepts_metadata = False if ctx._progress_metadata: try: _edit_params = inspect.signature(adapter.edit_message).parameters _edit_accepts_metadata = ( "metadata" in _edit_params or any( param.kind is inspect.Parameter.VAR_KEYWORD for param in _edit_params.values() ) ) except (TypeError, ValueError): _edit_accepts_metadata = False async def _edit_progress_message(message_id: str, content: str): kwargs = { "chat_id": ctx.source.chat_id, "message_id": message_id, "content": content, } if getattr(adapter, "REQUIRES_EDIT_FINALIZE", False): kwargs["finalize"] = True if _edit_accepts_metadata: kwargs["metadata"] = ctx._progress_metadata return await adapter.edit_message(**kwargs) def _progress_text(lines: list) -> str: return "\n".join(str(line) for line in lines) def _split_progress_groups(lines: list) -> list[list]: """Partition progress lines into platform-sized editable bubbles.""" groups: list[list] = [] current: list = [] for line in lines: candidate = current + [line] if current and _progress_len_fn(_progress_text(candidate)) > _PROGRESS_TEXT_LIMIT: groups.append(current) current = [line] else: current = candidate if current: groups.append(current) return groups def _track_progress_result(result) -> None: if ( ctx._cleanup_progress and getattr(result, "success", False) and getattr(result, "message_id", None) ): ctx._cleanup_msg_ids.append(str(result.message_id)) async def _send_progress_text(text: str): result = await adapter.send( chat_id=ctx.source.chat_id, content=text, reply_to=ctx._progress_reply_to, metadata=ctx._progress_metadata, ) _track_progress_result(result) return result async def _roll_progress_overflow_if_needed() -> bool: """Start fresh editable progress bubbles before a bubble exceeds limit. Returns True when it delivered/split the current buffer, or when a transient edit failure left the buffer and message identity intact for a later retry. In either case the caller should skip the normal send/edit path for this tick. """ nonlocal progress_msg_id, progress_lines, can_edit if not progress_lines or not can_edit: return False groups = _split_progress_groups(progress_lines) if len(groups) <= 1: return False first_text = _progress_text(groups[0]) if progress_msg_id is not None: result = await _edit_progress_message(progress_msg_id, first_text) if not result.success: if getattr(result, "retryable", False): logger.debug( "[%s] Transient overflow edit failure — keeping can_edit=True", adapter.name, ) return True can_edit = False # Fall back to the existing non-edit behavior below. return False else: result = await _send_progress_text(first_text) if result.success and result.message_id: progress_msg_id = result.message_id for group in groups[1:]: result = await _send_progress_text(_progress_text(group)) if result.success and result.message_id: progress_msg_id = result.message_id # The newest continuation is now the only mutable bubble. Keep # just its lines so subsequent edits update it instead of # replaying the full historical transcript into new messages. progress_lines = groups[-1] return True while True: try: if not ctx._run_still_current(): while not ctx.progress_queue.empty(): try: ctx.progress_queue.get_nowait() except Exception: break return raw = ctx.progress_queue.get_nowait() # Drain silently when interrupted: events queued in the # window between tool parse and interrupt processing # should not render as bubbles. The "⚡ Interrupting # current task" message is sent separately and is the # last progress-flavored bubble the user should see. try: _agent_for_interrupt = ctx.agent_holder[0] if ctx.agent_holder else None if _agent_for_interrupt is not None and getattr( _agent_for_interrupt, "is_interrupted", False ): # Drop this event and continue draining. await asyncio.sleep(0) continue except Exception: pass # Handle dedup messages: update last line with repeat counter if isinstance(raw, tuple) and len(raw) == 3 and raw[0] == "__dedup__": _, base_msg, count = raw if progress_lines: progress_lines[-1] = f"{base_msg} (×{count + 1})" msg = progress_lines[-1] if progress_lines else base_msg elif isinstance(raw, tuple) and len(raw) >= 1 and raw[0] == "__reset__": # Content bubble just landed on the platform — close off # the current tool-progress bubble so the next tool # starts a fresh bubble below the content. Without this, # tool lines keep editing the ORIGINAL progress message # above the new content, making the chat appear out of # order. Mirrors GatewayStreamConsumer.on_segment_break # on the content side. (Issue: tool + content # linearization regression after PR #7885.) progress_msg_id = None progress_lines = [] ctx.last_progress_msg[0] = None ctx.repeat_count[0] = 0 continue else: msg = raw progress_lines.append(msg) if await _roll_progress_overflow_if_needed(): _last_edit_ts = time.monotonic() await asyncio.sleep(0.3) if ctx._run_still_current(): await adapter.send_typing(ctx.source.chat_id, metadata=ctx._progress_metadata) continue # Throttle edits: batch rapid tool updates into fewer # API calls to avoid hitting Telegram flood control. # (grammY auto-retry pattern: proactively rate-limit # instead of reacting to 429s.) _now = time.monotonic() _remaining = _PROGRESS_EDIT_INTERVAL - (_now - _last_edit_ts) if _remaining > 0: # Wait out the throttle interval, then loop back to # drain any additional queued messages before sending # a single batched edit. await asyncio.sleep(_remaining) continue if not ctx._run_still_current(): return if can_edit and progress_msg_id is not None: # Try to edit the existing progress message full_text = "\n".join(progress_lines) result = await _edit_progress_message(progress_msg_id, full_text) if not result.success: _err = (getattr(result, "error", "") or "").lower() # Transient network errors (ConnectError, timeouts) # must not permanently disable progress-message # editing — the next cycle can catch up. Only # permanent failures (flood control, message not # found, permissions) should set can_edit = False. if getattr(result, "retryable", False): logger.debug( "[%s] Transient edit failure — keeping can_edit=True", adapter.name, ) continue if "flood" in _err or "retry after" in _err: # Flood control hit — backoff but keep editing. # Only disable edits for non-recoverable errors. logger.info( "[%s] Progress edit flood control, backing off", adapter.name, ) _last_edit_ts = time.monotonic() else: can_edit = False _flood_result = await adapter.send( chat_id=ctx.source.chat_id, content=msg, reply_to=ctx._progress_reply_to, metadata=ctx._progress_metadata, ) if ( ctx._cleanup_progress and getattr(_flood_result, "success", False) and getattr(_flood_result, "message_id", None) ): ctx._cleanup_msg_ids.append(str(_flood_result.message_id)) else: if can_edit: # First tool: send all accumulated text as new message full_text = "\n".join(progress_lines) result = await adapter.send( chat_id=ctx.source.chat_id, content=full_text, reply_to=ctx._progress_reply_to, metadata=ctx._progress_metadata, ) else: # Editing unsupported: send just this line result = await adapter.send( chat_id=ctx.source.chat_id, content=msg, reply_to=ctx._progress_reply_to, metadata=ctx._progress_metadata, ) if result.success and result.message_id: progress_msg_id = result.message_id if ctx._cleanup_progress: ctx._cleanup_msg_ids.append(str(result.message_id)) _last_edit_ts = time.monotonic() # Restore typing indicator await asyncio.sleep(0.3) if ctx._run_still_current(): await adapter.send_typing(ctx.source.chat_id, metadata=ctx._progress_metadata) except queue.Empty: await asyncio.sleep(0.3) except asyncio.CancelledError: # Drain remaining queued messages while not ctx.progress_queue.empty(): try: raw = ctx.progress_queue.get_nowait() if isinstance(raw, tuple) and len(raw) == 3 and raw[0] == "__dedup__": _, base_msg, count = raw if progress_lines: progress_lines[-1] = f"{base_msg} (×{count + 1})" await _roll_progress_overflow_if_needed() elif isinstance(raw, tuple) and len(raw) >= 1 and raw[0] == "__reset__": # Content-bubble marker during drain: close off # the current progress bubble and start a fresh # one for any tool lines that arrived after. await _roll_progress_overflow_if_needed() if can_edit and progress_lines and progress_msg_id: _pending_text = _progress_text(progress_lines) try: await _edit_progress_message(progress_msg_id, _pending_text) except Exception: pass progress_msg_id = None progress_lines = [] ctx.last_progress_msg[0] = None ctx.repeat_count[0] = 0 else: progress_lines.append(raw) await _roll_progress_overflow_if_needed() except Exception: break # Final edit with all remaining tools (only if editing works) if can_edit and progress_lines and progress_msg_id: await _roll_progress_overflow_if_needed() if can_edit and progress_lines and progress_msg_id: full_text = _progress_text(progress_lines) try: await _edit_progress_message(progress_msg_id, full_text) except Exception: pass return except Exception as e: logger.error("Progress message error: %s", e) await asyncio.sleep(1) def voice_ack_callback(self, call_id, tool_name, args): """tool_start_callback: speak a one-time ack in the voice channel.""" ctx = self._ctx if ctx._voice_ack_fired[0] or ctx._voice_ack_guild[0] is None: return if not ctx._run_still_current(): return ctx._voice_ack_fired[0] = True _adapter = self._runner.adapters.get(Platform.DISCORD) if _adapter is None or not hasattr(_adapter, "play_ack_in_voice"): return try: safe_schedule_threadsafe( _adapter.play_ack_in_voice(ctx._voice_ack_guild[0]), ctx._voice_ack_loop, logger=logger, log_message="voice ack scheduling error", ) except Exception as _ack_err: logger.debug("voice ack schedule failed: %s", _ack_err) # ── Slack-native task cards: ID-bearing lifecycle callbacks (#29483) ── # These ride agent.tool_start_callback / agent.tool_complete_callback so # start/completion events correlate by the REAL tool-call id — the # name-correlated text events in progress_callback would duplicate cards # and mispair concurrent calls to the same tool. def native_tool_start_callback(self, call_id, tool_name, args): """Queue an ID-correlated native progress start from the agent thread.""" ctx = self._ctx if not ctx.progress_queue or not ctx._run_still_current(): return try: _agent = ctx.agent_holder[0] if ctx.agent_holder else None if _agent is not None and getattr(_agent, "is_interrupted", False): return except Exception: pass from agent.display import build_tool_preview ctx.progress_queue.put( { "type": "tool.started", "tool_call_id": str(call_id or ""), "tool_name": str(tool_name or "tool"), "preview": build_tool_preview( str(tool_name or "tool"), args or {}, max_len=64 ) or "", } ) def native_tool_complete_callback(self, call_id, tool_name, args, result): """Queue the matching native completion using the real tool-call ID.""" ctx = self._ctx if not ctx.progress_queue or not ctx._run_still_current(): return try: _agent = ctx.agent_holder[0] if ctx.agent_holder else None if _agent is not None and getattr(_agent, "is_interrupted", False): return except Exception: pass from agent.display import _detect_tool_failure is_error, _ = _detect_tool_failure(str(tool_name or "tool"), result) ctx.progress_queue.put( { "type": "tool.completed", "tool_call_id": str(call_id or ""), "tool_name": str(tool_name or "tool"), "is_error": bool(is_error), } ) def combined_tool_start_callback(self, call_id, tool_name, args): """Compose the voice ack + native task-card start consumers.""" ctx = self._ctx if ctx._voice_ack_guild[0] is not None: self.voice_ack_callback(call_id, tool_name, args) if ctx._native_slack_task_cards: self.native_tool_start_callback(call_id, tool_name, args) def _step_callback_sync(self, iteration: int, prev_tools: list) -> None: ctx = self._ctx if not ctx._run_still_current(): return # prev_tools may be list[str] or list[dict] with "name"/"result" # keys. Normalise to keep "tool_names" backward-compatible for # user-authored hooks that do ', '.join(tool_names)'. _names: list[str] = [] for _t in (prev_tools or []): if isinstance(_t, dict): _names.append(_t.get("name") or "") else: _names.append(str(_t)) safe_schedule_threadsafe( ctx._hooks_ref.emit("agent:step", { "platform": ctx.source.platform.value if ctx.source.platform else "", "user_id": ctx.source.user_id, "session_id": ctx.session_id, "iteration": iteration, "tool_names": _names, "tools": prev_tools, }), ctx._loop_for_step, logger=logger, log_message="agent:step hook scheduling error", ) def _event_callback_sync(self, event_type: str, context: dict) -> None: ctx = self._ctx try: asyncio.run_coroutine_threadsafe( ctx._hooks_ref.emit(event_type, context), ctx._loop_for_step, ) except Exception as _e: logger.debug("event_callback hook error: %s", _e) def _attach_session_title_callback(self, agent, ctx) -> None: """Wire the platform thread-rename lane onto the agent as `_on_session_title`. The session titler runs inside the turn prologue now (it derives the title from the user's first message, so it no longer needs the response), which means the callback has to be attached before the run rather than registered after it. The lane predicates and their rationale are unchanged from the old post-response registration. """ try: # Gateway auto-title failures must NOT be surfaced as user-visible # messages (#23246) — they are not actionable to the end user. # Overriding the failure sink here keeps CLI mode on the agent's # _emit_auxiliary_failure path while the gateway logs at debug. def _title_failure_cb(task: str, exc: BaseException) -> None: logger.debug( "Gateway auto-title failure suppressed (not user-visible): %s: %s", task, exc, ) agent._title_failure_callback = _title_failure_cb session_id = getattr(agent, "session_id", None) source = ctx.source # Both lanes below spend a rate-limited platform call per title, so # they take the model's title and skip the derived one — see # TitleCallback. Renaming twice lands on the same name at twice the # cost, and Discord's 2-per-10-minutes channel budget can spend # itself on the throwaway and drop the one worth showing. if self._runner._is_telegram_topic_lane(source): agent._on_session_title = lambda title, title_source: ( title_source == "llm" and self._runner._schedule_telegram_topic_title_rename( source, session_id, title, ) ) elif self._runner._is_discord_auto_thread_lane(source) or ( self._runner._is_relay_discord_channel_lane(source) ): # Relay note: the second predicate is shape-only (relay # Discord channel event). Whether the connector actually # auto-threaded our reply is only knowable AFTER delivery # (send-result feedback), so the callback must be registered # eagerly and the rename lane performs the cache lookup at # fire time (staging repro 2026-07-31: gating registration on # the cache read meant it never registered and no # thread_rename op was ever sent). agent._on_session_title = lambda title, title_source: ( title_source == "llm" and self._runner._schedule_discord_semantic_thread_rename( source, session_id, title, ) ) except Exception: logger.debug("Failed to attach session title callback", exc_info=True) def _status_callback_sync(self, event_type: str, message: str) -> None: ctx = self._ctx if not ctx._status_adapter or not ctx._run_still_current(): return prepared_message = _prepare_gateway_status_message( ctx.source.platform, event_type, message, ) if prepared_message is None: logger.debug( "status_callback suppressed for %s/%s: %s", ctx.source.platform.value if ctx.source.platform else "unknown", event_type, _redact_gateway_user_facing_secrets(str(message or ""))[:160], ) return _fut = safe_schedule_threadsafe( _send_or_update_status_coro(ctx._status_adapter, ctx._status_chat_id, event_type, prepared_message, ctx._status_thread_metadata), ctx._loop_for_step, logger=logger, log_message=f"status_callback ({event_type}) scheduling error", ) if _fut is None: return if ctx._cleanup_progress: def _track_status_id(fut) -> None: try: res = fut.result() except Exception: return mid = getattr(res, "message_id", None) if getattr(res, "success", False) and mid: ctx._cleanup_msg_ids.append(str(mid)) _fut.add_done_callback(_track_status_id) def run_sync(self): ctx = self._ctx # Historical note: as a nested closure this body declared # `nonlocal message` because the conditional re-assignments below # (prepending model-switch / resume-recovery notes) would otherwise # make `message` function-local and break the earlier read at # `_resolve_turn_agent_config(message, …)`. As a method the turn # message lives on the shared TurnContext instead: every rebind # writes `ctx.message`, so the outer `_run_agent_inner` body observes # the updated value exactly as it did through the closure cell. # session_key is propagated via contextvars in _set_session_env() # (_SESSION_KEY) and via set_current_session_key() (_approval_session_key) # below — both concurrency-safe and inherited by tool worker threads. # We deliberately do NOT write os.environ["HERMES_SESSION_KEY"] here: # os.environ is process-global, so concurrent gateway sessions (e.g. # two Discord threads) would clobber each other's value, and a tool # thread whose contextvar is unset would fall back to os.environ and # read the wrong session key — misrouting command-approval prompts to # the wrong thread (#24100). The non-gateway surfaces don't depend on # this write: CLI and cron bind the session via contextvars # (set_current_session_key / session context), and only the TUI # slash-worker *subprocess* exports HERMES_SESSION_KEY (from its own # --session-key argv, a separate process) — so removing this in-process # gateway write does not affect any of them. # Map platform enum to the platform hint key the agent understands. # Platform.LOCAL ("local") maps to "cli"; others pass through as-is. platform_key = "cli" if ctx.source.platform == Platform.LOCAL else ctx.source.platform.value # Combine platform context, YAML channel_prompts hint for this chat, # channel_overrides system_prompt (or global ephemeral), and gateway # ephemeral prompt from _get_system_prompt_for_channel. combined_ephemeral = ctx.context_prompt or "" event_channel_prompt = (ctx.channel_prompt or "").strip() if event_channel_prompt: combined_ephemeral = (combined_ephemeral + "\n\n" + event_channel_prompt).strip() cfg_channel_prompt = self._runner._get_system_prompt_for_channel( ctx.source.platform, ctx.source.chat_id or "", thread_id=getattr(ctx.source, "thread_id", None), parent_id=getattr(ctx.source, "parent_chat_id", None), ) if cfg_channel_prompt: combined_ephemeral = (combined_ephemeral + "\n\n" + cfg_channel_prompt).strip() max_iterations = _current_max_iterations() try: model, runtime_kwargs = self._runner._resolve_session_agent_runtime( source=ctx.source, session_key=ctx.session_key, user_config=ctx.user_config, ) logger.debug( "run_agent resolved: model=%s provider=%s session=%s", model, runtime_kwargs.get("provider"), ctx.session_key or "", ) except Exception as exc: return { "final_response": f"⚠️ Provider authentication failed: {exc}", "messages": [], "api_calls": 0, "tools": [], } pr = self._runner._provider_routing reasoning_config = self._runner._resolve_session_reasoning_config( source=ctx.source, session_key=ctx.session_key, model=model, ) self._runner._reasoning_config = reasoning_config self._runner._service_tier = self._runner._resolve_session_service_tier( source=ctx.source, session_key=ctx.session_key ) # Set up stream consumer for token streaming or interim commentary. _stream_consumer = None _stream_delta_cb = None # #60671 — streaming TTS consumer is created on the outer # event-loop thread before run_sync launches. run_sync only # reads it via ``streaming_tts_consumer_holder[0]`` for delta # callback wiring. _stts_consumer_ref = ctx.streaming_tts_consumer_holder[0] _scfg = getattr(getattr(self._runner, 'config', None), 'streaming', None) if _scfg is None: from gateway.config import StreamingConfig _scfg = StreamingConfig() # Per-platform streaming gate: display.platforms..streaming # can disable streaming for specific platforms even when the global # streaming config is enabled. _plat_streaming = ctx.resolve_display_setting( ctx.user_config, platform_key, "streaming" ) # None = no per-platform override → follow global config _streaming_enabled = ( _scfg.enabled and _scfg.transport != "off" if _plat_streaming is None else bool(_plat_streaming) ) _want_stream_deltas = _streaming_enabled _want_interim_messages = ctx.interim_assistant_messages_enabled _want_interim_consumer = _want_interim_messages if _want_stream_deltas or _want_interim_consumer: try: from gateway.stream_consumer import GatewayStreamConsumer _adapter = self._runner._adapter_for_source(ctx.source) if _adapter: _consumer_cfg, _pause_typing_before_finalize = ( self._runner._build_stream_consumer_config( ctx.source, _scfg, _adapter, on_missing_cursor="raise", ) ) _stream_consumer = GatewayStreamConsumer( adapter=_adapter, chat_id=ctx.source.chat_id, config=_consumer_cfg, metadata=ctx._status_thread_metadata, on_new_message=( (lambda: ctx.progress_queue.put(("__reset__",))) if ctx.progress_queue is not None else None ), on_before_finalize=_pause_typing_before_finalize, initial_reply_to_id=ctx.event_message_id, run_still_current=ctx._run_still_current, ) if _want_stream_deltas: def _stream_delta_cb(text: str) -> None: if ctx._run_still_current(): _stream_consumer.on_delta(text) # Tee to the streaming-TTS consumer (#60671). if _stts_consumer_ref is not None: _stts_consumer_ref.on_delta(text) ctx.stream_consumer_holder[0] = _stream_consumer except Exception as _sc_err: logger.debug("Could not set up stream consumer: %s", _sc_err) # When text streaming is off but streaming TTS is active, # install a TTS-only delta callback so the consumer still # receives LLM deltas for audio synthesis (#60671). if _stream_delta_cb is None and _stts_consumer_ref is not None: def _stream_delta_cb(text: str) -> None: if ctx._run_still_current(): _stts_consumer_ref.on_delta(text) def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None: if not ctx._run_still_current(): return display_text = text if _stream_consumer is not None: if already_streamed: _stream_consumer.on_segment_break() else: _stream_consumer.on_commentary(display_text) return if already_streamed or not ctx._status_adapter or not str(display_text or "").strip(): return safe_schedule_threadsafe( ctx._status_adapter.send( ctx._status_chat_id, display_text, metadata=ctx._status_thread_metadata, ), ctx._loop_for_step, logger=logger, log_message="interim_assistant_callback scheduling error", ) turn_route = self._runner._resolve_turn_agent_config(ctx.message, model, runtime_kwargs) # Per-platform skip_context_files — messaging platforms can opt out # of filesystem-heavy context-file discovery (SOUL.md, AGENTS.md, # .cursorrules) to cut AIAgent construction latency. Especially # impactful on Windows, where stat() + directory walks are 10-100x # slower than Linux. Off by default; soul identity is preserved so # the persona survives even with minimal context. _platforms_gw_cfg = (ctx.user_config.get("gateway") or {}).get("platforms") or {} # ``hermes gateway setup`` writes ``gateway.platforms`` as a LIST of # enabled platform names (e.g. ``- telegram``), not a dict. Treat any # non-dict shape as "no per-platform overrides" instead of crashing # on ``.get()`` for every incoming turn (#83185). if not isinstance(_platforms_gw_cfg, dict): _platforms_gw_cfg = {} _plat_gw_cfg = _platforms_gw_cfg.get(platform_key) or {} _skip_context = _plat_gw_cfg.get("skip_context_files") skip_context_files = bool(_skip_context) if _skip_context is not None else False # Check agent cache — reuse the AIAgent from the previous message # in this session to preserve the frozen system prompt and tool # schemas for prompt cache hits. _sig = self._runner._agent_config_signature( turn_route["model"], turn_route["runtime"], ctx.enabled_toolsets, combined_ephemeral, cache_keys=self._runner._extract_cache_busting_config(ctx.user_config), user_id=getattr(ctx.source, "user_id", None), user_id_alt=getattr(ctx.source, "user_id_alt", None), skip_context_files=skip_context_files, ) agent = None reused_cached_agent = False _cache_lock = getattr(self._runner, "_agent_cache_lock", None) _cache = getattr(self._runner, "_agent_cache", None) # Peek at the cached entry's snapshot session_id (if any) so we can # check, OUTSIDE the cache lock, whether THAT session_id is a DEAD # session in state.db. This closes a gap in the #54947 fix: that # fix treats "cached session_id != current session_id" as an # intentional /resume-style switch and reuses the agent unchanged. # But the #54878 self-heal produces the exact same tuple shape # when it recovers a routing key away from a session that was # already ended — the cached AIAgent still belongs to the DEAD # session, not a valid sibling conversation. Reusing it lets that # turn's post-run "session split" sync write the routing key # straight back onto the dead session_id, undoing the self-heal # and looping every message until an interrupt happens to race in # first (the #54878 x #54947 interaction — no existing upstream # issue tracks this combination as of 2026-07-12). _peek_cached_sid = None if _cache_lock and _cache is not None: with _cache_lock: _peek_entry = _cache.get(ctx.session_key) if _peek_entry and len(_peek_entry) > 3: _peek_cached_sid = _peek_entry[3] _cached_sid_is_dead = False if ( _peek_cached_sid is not None and ctx.session_id is not None and _peek_cached_sid != ctx.session_id ): try: _cached_sid_is_dead = self._runner.session_store._is_session_ended_in_db( _peek_cached_sid ) except Exception: _cached_sid_is_dead = False # Detect cross-process writes: when another process (e.g. hermes # dashboard) appends to the same session in the shared SessionDB, # the cached agent's in-memory transcript becomes stale. Compare # the session's current message_count against the count recorded # when the agent was cached; on mismatch, invalidate the cache # so a fresh agent re-reads from disk. (#45966) _current_msg_count = None if self._runner._session_db is not None and ctx.session_id: try: # run_sync is off-loop (executor); sync DB is fine. _sess_row = self._runner._session_db._db.get_session(ctx.session_id) if _sess_row: _current_msg_count = _sess_row.get("message_count", 0) except Exception: pass _xproc_evicted_agent = None if _cache_lock and _cache is not None: with _cache_lock: cached = _cache.get(ctx.session_key) if cached and cached[1] == _sig: # cached[2] is the message_count at cache time; # stale when a second process appended rows. # cached[3] (when present) is the session_id the # snapshot was taken for — used to skip the guard # when the active session_id differs (#54947). _cached_mc = cached[2] if len(cached) > 2 else None _cached_sid = cached[3] if len(cached) > 3 else None # If the snapshot belongs to a different session_id # (same session_key, different conversation), the # message_count comparison is meaningless — the # counts track DIFFERENT DB rows. REUSE the cached # agent rather than rebuild and bust the prompt cache # on every session switch (#54947). _session_id_mismatch = ( _cached_sid is not None and ctx.session_id is not None and _cached_sid != ctx.session_id ) # Re-validate the OUTSIDE-lock dead-session peek # against the tuple actually read under THIS lock — # the cache entry could have been replaced between # the peek and this lock acquisition, and a stale # "dead" verdict must never be applied to a # different (possibly live) cached agent. _stale_dead_sid_reuse = ( _session_id_mismatch and _cached_sid_is_dead and _cached_sid == _peek_cached_sid ) if _stale_dead_sid_reuse: # #54878 x #54947 interaction: the routing key # was just self-healed away from a session that # state.db already marked ended, but the cached # AIAgent here still belongs to that DEAD # session_id. The #54947 "different session_id # under the same key = intentional switch, reuse # freely" rule does not hold here — this isn't a # sibling conversation, it's a stale agent left # over from before the self-heal. Reusing it lets # this turn's post-run "session split" sync write # the routing key straight back onto the dead # session_id, undoing the self-heal and looping # every message until an interrupt happens to # race in first. Discard and rebuild fresh # instead, same as a genuine cross-process write. logger.info( "Agent cache invalidated for session %s: " "cached agent's session_id %s is ended in " "state.db (stale self-heal artifact, " "#54878 x #54947) — discarding instead of " "reusing across the routing recovery", ctx.session_key, _cached_sid, ) evicted = self._runner._agent_cache.pop(ctx.session_key, None) _ev_agent = evicted[0] if isinstance(evicted, tuple) and evicted else None if _ev_agent and _ev_agent is not _AGENT_PENDING_SENTINEL: # Same deferred-cleanup rationale as the # cross-process branch below (#52197): don't # block the event loop / cache lock on # memory-provider shutdown or socket teardown. _xproc_evicted_agent = _ev_agent elif ( not _session_id_mismatch and _cached_mc is not None and _current_msg_count is not None and _current_msg_count != _cached_mc ): # Cross-process write detected — discard stale # agent so it rebuilds from fresh DB transcript. logger.info( "Agent cache invalidated for session %s: " "message_count changed (%s -> %s), " "possible cross-process write", ctx.session_key, _cached_mc, _current_msg_count, ) evicted = self._runner._agent_cache.pop(ctx.session_key, None) _ev_agent = evicted[0] if isinstance(evicted, tuple) and evicted else None if _ev_agent and _ev_agent is not _AGENT_PENDING_SENTINEL: # Defer cleanup until AFTER the lock is # released — _cleanup_agent_resources / # release_clients can block on memory-provider # shutdown and socket teardown, and running it # here would stall the gateway event loop while # _sweep_idle_cached_agents (session-expiry # watcher) waits on the same lock, blocking # Discord heartbeats (#52197). The same session # rebuilds a fresh agent immediately below, so # use the SOFT release that preserves the # session's terminal sandbox / browser / bg # processes for the rebuilt agent to inherit — # mirrors _evict_cached_agent / idle-sweep. _xproc_evicted_agent = _ev_agent else: agent = cached[0] # Refresh LRU order so the cap enforcement evicts # truly-oldest entries, not the one we just used. if hasattr(_cache, "move_to_end"): try: _cache.move_to_end(ctx.session_key) except KeyError: pass self._runner._init_cached_agent_for_turn(agent, ctx._interrupt_depth) # Refresh agent max_iterations from current config # (cached agent may have been created with old config) agent.max_iterations = max_iterations logger.debug("Reusing cached agent for session %s", ctx.session_key) reused_cached_agent = True # Lock released — refresh the fallback chain from disk for the # reused agent OUTSIDE the cache lock (config.yaml read is disk # I/O; the idle-sweep watcher contends on this lock and stalls # Discord heartbeats — same reasoning as #52197). A chain # configured after this agent was cached (or after gateway start) # must reach the next turn (#60955). Per-session turn # serialization (_running_agents) keeps this safe post-lock. if reused_cached_agent and agent is not None: self._runner._apply_fallback_chain_to_agent( agent, self._runner._refresh_fallback_model(), ) # Lock released — now schedule cleanup of any cross-process-evicted # agent on a daemon thread so memory-provider shutdown / socket # teardown never blocks the gateway event loop or the cache lock # the session-expiry watcher needs (#52197). if _xproc_evicted_agent is not None: try: threading.Thread( target=self._runner._release_evicted_agent_soft, args=(_xproc_evicted_agent,), daemon=True, name=f"agent-xproc-evict-{str(ctx.session_key)[:24]}", ).start() except Exception: # Interpreter shutdown or thread-spawn failure — release # inline as a best-effort fallback. try: self._runner._release_evicted_agent_soft(_xproc_evicted_agent) except Exception: pass if agent is None: # Config changed or first message — create fresh agent agent = ctx.AIAgent( model=turn_route["model"], **turn_route["runtime"], **_checkpoint_agent_kwargs(ctx.user_config), max_iterations=max_iterations, quiet_mode=True, verbose_logging=False, enabled_toolsets=ctx.enabled_toolsets, disabled_toolsets=ctx.disabled_toolsets, ephemeral_system_prompt=combined_ephemeral or None, prefill_messages=self._runner._prefill_messages or None, reasoning_config=reasoning_config, service_tier=self._runner._service_tier, request_overrides=turn_route.get("request_overrides"), providers_allowed=pr.get("only"), providers_ignored=pr.get("ignore"), providers_order=pr.get("order"), provider_sort=pr.get("sort"), provider_require_parameters=pr.get("require_parameters", False), provider_data_collection=pr.get("data_collection"), session_id=ctx.session_id, platform=platform_key, user_id=ctx.source.user_id, user_id_alt=ctx.source.user_id_alt, user_name=ctx.source.user_name, chat_id=ctx.source.chat_id, chat_name=ctx.source.chat_name, chat_type=ctx.source.chat_type, thread_id=ctx.source.thread_id, gateway_session_key=ctx.session_key, session_db=getattr(self._runner._session_db, "_db", self._runner._session_db), # Reload from disk — do not reuse the startup snapshot (#60955). fallback_model=self._runner._refresh_fallback_model(), skip_context_files=skip_context_files, # Keep the persona even with minimal context: soul identity is # a single small file, not part of the expensive walk. load_soul_identity=True, ) if _cache_lock and _cache is not None: with _cache_lock: # Record the session_id the snapshot was taken for # alongside the message_count, so the cross-process # guard can skip the (meaningless) count comparison # when the active session_id later switches under # the same session_key (#54947). _cache[ctx.session_key] = ( agent, _sig, _current_msg_count, ctx.session_id, ) self._runner._enforce_agent_cache_cap() logger.debug("Created new agent for session %s (sig=%s)", ctx.session_key, _sig) # Per-message state — callbacks and reasoning config change every # turn and must not be baked into the cached agent constructor. # Gate on needs_progress_queue (tool_progress OR thinking_progress) # rather than tool_progress alone: the progress_callback also relays # _thinking assistant scratch text, which is gated on # thinking_progress and is intentionally independent of tool # progress. With the old `tool_progress_enabled`-only gate, a user # who set thinking_progress:true but kept tool_progress:off got a # None callback — so _thinking scratch bubbles never relayed even # though the progress queue was created for them. # Always attached (previously gated to None when no progress surface # was active): the callback body gates each event class itself, and # subagent-failure notices must fire even on platforms with # tool_progress/thinking off — the None gate was exactly why a dead # subagent vanished silently there. agent.tool_progress_callback = ctx.progress_callback # Compose ID-bearing lifecycle consumers: Discord's one-time voice # ack and Slack's native task cards both ride the authoritative # start callback, so neither has to infer identity from tool names. _combined_start_cb = ctx.native_tool_start_callback or ctx.voice_ack_callback agent.tool_start_callback = ( _combined_start_cb if ( ctx._voice_ack_guild[0] is not None or ctx._native_slack_task_cards ) else None ) agent.tool_complete_callback = ( ctx.native_tool_complete_callback if ctx._native_slack_task_cards and ctx.native_tool_complete_callback is not None else None ) agent.step_callback = ctx._step_callback_sync if ctx._hooks_ref.loaded_hooks else None agent.stream_delta_callback = _stream_delta_cb agent.interim_assistant_callback = _interim_assistant_cb if _want_interim_messages else None agent.status_callback = ctx._status_callback_sync # Credits / out-of-band notices (usage bands, depletion, restored). # Messaging has no persistent status bar, so each notice is a # standalone push: render to a single plaintext line and deliver via # the shared _deliver_platform_notice rail (honors private/public + # thread metadata). Fires from the agent's sync worker thread, so we # hop onto the gateway loop with safe_schedule_threadsafe - same # pattern as _status_callback_sync. The fired-once latch lives on the # cached agent and persists across turns, so a band crosses -> one # push (no per-turn re-nag). Recovery ("✓ Credit access restored") # rides the same show path (it's emitted as a success notice, not a # clear). The clear callback is a no-op: a sent platform message # can't be cleanly retracted, and the band already fired once. def _notice_callback_sync(notice) -> None: if not ctx._status_adapter or not ctx._run_still_current(): return try: line = render_notice_line(notice) except Exception: logger.debug("render_notice_line failed", exc_info=True) return if not line: return safe_schedule_threadsafe( self._runner._deliver_platform_notice(ctx.source, line), ctx._loop_for_step, logger=logger, log_message="notice_callback delivery scheduling error", ) agent.notice_callback = _notice_callback_sync agent.notice_clear_callback = None agent.event_callback = ctx._event_callback_sync agent.reasoning_config = reasoning_config agent.service_tier = self._runner._service_tier # Merge, never overwrite: init-time request overrides (e.g. a custom # provider's extra_body merged at agent construction) must survive # every reused-agent turn. Drop only the PREVIOUS turn's routing # overrides (fast-mode service_tier/speed) before layering this # turn's route overrides on top, so stale per-turn values never # linger while construction-time values persist. request_overrides = dict(getattr(agent, "request_overrides", {}) or {}) previous_turn_overrides = dict( getattr(agent, "_gateway_turn_request_overrides", {}) or {} ) for key, value in previous_turn_overrides.items(): if request_overrides.get(key) == value: request_overrides.pop(key, None) turn_request_overrides = dict(turn_route.get("request_overrides") or {}) request_overrides.update(turn_request_overrides) agent.request_overrides = request_overrides agent._gateway_turn_request_overrides = turn_request_overrides # Must-deliver notes for THIS turn ride the current user message # (api_content sidecar), never the system prompt: staged by # _handle_message_with_agent (auto-reset note, first-contact # intro, voice-channel change). Assigned unconditionally so a # reused cached agent never replays a stale note. agent._gateway_turn_context_notes = "\n\n".join( self._runner._consume_pending_turn_sidecar_notes(ctx.session_key) ) _bg_review_release = threading.Event() _bg_review_pending: list[str] = [] _bg_review_pending_lock = threading.Lock() def _deliver_bg_review_message(message: str) -> None: if not ctx._status_adapter or not ctx._run_still_current(): return safe_schedule_threadsafe( ctx._status_adapter.send( ctx._status_chat_id, message, metadata=_interim_metadata(_non_conversational_metadata(ctx._status_thread_metadata, platform=ctx.source.platform)), ), ctx._loop_for_step, logger=logger, log_message="background_review_callback scheduling error", ) def _release_bg_review_messages() -> None: _bg_review_release.set() with _bg_review_pending_lock: pending = list(_bg_review_pending) _bg_review_pending.clear() for queued in pending: _deliver_bg_review_message(queued) # Background review delivery — send "💾 Memory updated" etc. to user def _bg_review_send(message: str) -> None: if not ctx._status_adapter or not ctx._run_still_current(): return if not _bg_review_release.is_set(): with _bg_review_pending_lock: if not _bg_review_release.is_set(): _bg_review_pending.append(message) return _deliver_bg_review_message(message) agent.background_review_callback = _bg_review_send # Register the release hook on the adapter so base.py's finally # block can fire it after delivering the main response. if ctx._status_adapter and ctx.session_key: if getattr(type(ctx._status_adapter), "register_post_delivery_callback", None) is not None: ctx._status_adapter.register_post_delivery_callback( ctx.session_key, _release_bg_review_messages, generation=ctx.run_generation, ) else: _pdc = getattr(ctx._status_adapter, "_post_delivery_callbacks", None) if _pdc is not None: _pdc[ctx.session_key] = _release_bg_review_messages # Memory update notifications in chat. Config: display.memory_notifications # off — no chat notification (still logged to stdout) # on — generic "💾 Memory updated" (default) # verbose — content preview: "💾 Memory ➕ Hermes Repo..." _mem_notif = ctx.user_config.get("display", {}).get("memory_notifications") if isinstance(_mem_notif, bool): _mem_notif = "on" if _mem_notif else "off" agent.memory_notifications = str(_mem_notif).lower() if _mem_notif else "on" # ------------------------------------------------------------------ # Shared native-stream boundary close. For platforms with native # streaming (e.g. WeCom msgtype:"stream"), an interaction that # interrupts the stream — a dangerous-command approval prompt OR a # clarify decision prompt — must finalize the current stream and # disable native streaming first. Otherwise the agent's # post-interaction output keeps flowing into send_stream_frame and # updates the *old* bubble that preceded the prompt, instead of # starting a fresh bubble below it (the "气泡割裂" symptom). After # the boundary, post-prompt output goes through the reliable send() # path as a new message. Runs on the agent thread; the consumer # processes the boundary serially via its queue. def _close_native_stream_boundary( _reason: str, _placeholder: str | None = None, _reopen: bool = False, ) -> bool: _sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None if not (_sc and getattr(_sc, "_use_native_streaming", False)): return True _cancelled_flag = None try: _boundary_result = _sc.close_for_approval_prompt( _placeholder, reason=_reason, reopen=_reopen, ) # Returns (future, cancelled_flag) or just a future. if isinstance(_boundary_result, tuple): _boundary_future, _cancelled_flag = _boundary_result else: _boundary_future = _boundary_result if hasattr(_boundary_future, "result"): _ok = _boundary_future.result(timeout=10) if not _ok: logger.warning( "%s boundary failed to close stream properly — " "prompt may still appear in typing bubble", _reason, ) return bool(_ok) return True except (TimeoutError, Exception) as _boundary_err: if _cancelled_flag is not None: _cancelled_flag["cancelled"] = True logger.warning( "%s boundary timed out or failed: %s", _reason, _boundary_err, ) return False # ------------------------------------------------------------------ # Clarify callback: present a clarify prompt and block on a response. # # Runs on the agent's worker thread (see clarify_tool's synchronous # callback contract). Bridges sync→async by scheduling the # adapter's send_clarify on the gateway event loop, then blocks on # the clarify primitive's threading.Event with a configurable # timeout. Returns the user's response string, or a sentinel # explaining that no response arrived (so the agent can adapt # rather than hang forever). # ------------------------------------------------------------------ def _clarify_callback_sync(question: str, choices, multi_select: bool = False) -> str: from tools import clarify_gateway as _clarify_mod import uuid as _uuid if not ctx._status_adapter: return "" clarify_id = _uuid.uuid4().hex[:10] _clarify_mod.register( clarify_id=clarify_id, session_key=ctx.session_key or "", question=question, choices=list(choices) if choices else None, multi_select=bool(multi_select), ) # For WeCom native streaming: finalize the current stream before # showing the clarify prompt so the post-answer output opens a # fresh bubble below the question instead of updating the bubble # that preceded it — the "气泡割裂" symptom. Unlike the approval # path, clarify passes reopen=True: native streaming stays # enabled so the post-answer continuation re-opens a fresh # native stream (typing bubble) rather than degrading to a # one-shot send(). Clarify waits are short, so stream staleness # is a low risk; if the re-seed fails the consumer degrades to # send() automatically. The placeholder is only used in the # narrow case where a frame was pushed but no text accumulated. _close_native_stream_boundary( "Clarify", "💬 等待你的选择...", _reopen=True, ) # Pause typing — like approval, we don't want a "thinking..." # status to obscure the prompt or block the user from typing # an "Other" response on platforms that disable input while # typing is active (Slack Assistant API). try: ctx._status_adapter.pause_typing_for_chat(ctx._status_chat_id) except Exception: pass # Ordering barrier (#clarify-ordering): flush any buffered # assistant prose (interim commentary / streamed deltas) to the # platform BEFORE sending the poll. The poll is delivered on a # separate, agent-thread-blocking path; without this barrier it # races ahead of prose still sitting in the stream consumer's # queue, so the question renders ABOVE its own explanation. # Best-effort + short timeout: never hang the agent thread if # the consumer task isn't running. try: _sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None _flush = getattr(_sc, "flush_pending_sync", None) if callable(_flush): _flush(timeout=3.0) except Exception: logger.debug( "Stream-consumer flush before clarify prompt failed", exc_info=True, ) fut = safe_schedule_threadsafe( ctx._status_adapter.send_clarify( chat_id=ctx._status_chat_id, question=question, choices=list(choices) if choices else None, clarify_id=clarify_id, session_key=ctx.session_key or "", metadata=ctx._status_thread_metadata, ), ctx._loop_for_step, logger=logger, log_message="Clarify send failed to schedule", ) # Boundary rule (see _approval_send_outcome): a send timeout is # AMBIGUOUS — the card may have posted with a late ack. Only a # definitive failure tears down the registration; ambiguous # falls through to the bounded wait so a late reply resolves. _clarify_response = _clarify_send_then_wait( fut, clarify_id=clarify_id, session_key=ctx.session_key or "", clarify_mod=_clarify_mod, ) # Only re-arm typing when the user actually answered — the # undeliverable sentinel and the timeout/cancellation strings # start with '[' and must pass through untouched. if not ( isinstance(_clarify_response, str) and _clarify_response.startswith("[") ): # User answered. Reopen the typing indicator IMMEDIATELY — # don't wait for the LLM's first post-answer token. On native # streaming (WeCom) the typing bubble is driven by the stream # seed frame, and the reopen path otherwise re-seeds lazily on # the first delta (measured ~48s of dead air). request_reopen_seed # is a no-op unless we're in the reopen-pending native state, so # it's safe to call unconditionally here. resume_typing_for_chat # covers non-native platforms (their pause was set before the # prompt) — the WeCom send_typing is a no-op, harmless here. _sc_reopen = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None if _sc_reopen is not None: try: _sc_reopen.request_reopen_seed() except Exception: logger.debug( "request_reopen_seed after clarify answer failed", exc_info=True, ) try: ctx._status_adapter.resume_typing_for_chat(ctx._status_chat_id) except Exception: logger.debug( "resume_typing_for_chat after clarify answer failed", exc_info=True, ) return _clarify_response agent.clarify_callback = _clarify_callback_sync # Show assistant thinking between tool calls — independent of # tool_progress mode. Mattermost needs an explicit per-platform # opt-in so global scratch-text display does not leak into threads. agent.thinking_progress = ctx._thinking_enabled # Store agent reference for interrupt support ctx.agent_holder[0] = agent # Wire the platform thread-rename lane onto the agent, because the # session titler now fires from the turn prologue rather than after # the response. Titles are pushed here the moment they land. self._attach_session_title_callback(agent, ctx) # Publish turn ownership for explicit /stop, /new, disconnect, and # shutdown interrupts. Older session processes are outside this # baseline and remain alive. agent._gateway_turn_process_task_id = ctx.process_task_id agent._gateway_turn_process_baseline = ctx.process_baseline # Capture the full tool definitions for transcript logging ctx.tools_holder[0] = agent.tools if hasattr(agent, 'tools') else None # Convert history to agent format. # Two cases: # 1. Normal path (from transcript): simple {role, content, timestamp} dicts # - Strip timestamps, keep role+content # 2. Interrupt path (from agent result["messages"]): full agent messages # that may include tool_calls, tool_call_id, reasoning, etc. # - These must be passed through intact so the API sees valid # assistant→tool sequences (dropping tool_calls causes 500 errors) # # Telegram observed group context is handled structurally here: # observed=True transcript rows are withheld from replayable # history and attached to the current addressed message as # API-only context, so persisted history stores only the real # addressed user turn. agent_history, observed_group_context = _build_gateway_agent_history( ctx.history, channel_prompt=ctx.channel_prompt, inject_timestamps=_message_timestamps_enabled(ctx.user_config), ) # FTS write-corruption guard (#50502): when message persistence # fails silently through corrupt FTS triggers, the reloaded # transcript above is stale/empty even though the SAME cached agent # still holds the full live conversation in `_session_messages`. # Replacing the live transcript with that shorter copy causes # immediate same-session amnesia. Only applies when we reused a # cached agent bound to this exact session_id. if reused_cached_agent and getattr(agent, "session_id", None) == ctx.session_id: _selected = _select_cached_agent_history( agent_history, getattr(agent, "_session_messages", None) ) if _selected is not agent_history: logger.warning( "Persisted transcript lagged live cached history for " "session %s (disk=%d, memory=%d); preserving live " "conversation context (possible FTS write corruption)", ctx.session_key, len(agent_history), len(_selected), ) # The live in-memory history bypassed the # _build_gateway_agent_history cleanup pipeline above — # re-apply the stale-confirmation expiry (#59607) so a # dangerous confirmation can't slip through this path # either. Idempotent; messages without timestamps are # untouched. agent_history = strip_stale_dangerous_confirmations( _selected, now=time.time() ) # Collect MEDIA paths already in history so we can exclude them # from the current turn's extraction. This is compression-safe: # even if the message list shrinks, we know which paths are old. _history_media_paths: set = _collect_history_media_paths(agent_history) # Register per-session gateway approval callback so dangerous # command approval blocks the agent thread (mirrors CLI input()). # The callback bridges sync→async to send the approval request # to the user immediately. from tools.approval import ( register_gateway_notify, reset_current_session_key, set_current_session_key, unregister_gateway_notify, ) def _approval_notify_sync(approval_data: dict) -> None: """Send the approval request to the user from the agent thread. If the adapter supports interactive button-based approvals (e.g. Discord's ``send_exec_approval``), use that for a richer UX. Otherwise fall back to a plain text message with ``/approve`` instructions. """ # Pause the typing indicator while the agent waits for # user approval. Critical for Slack's Assistant API where # assistant_threads_setStatus disables the compose box — the # user literally cannot type /approve while "is thinking..." # is active. The approval message send auto-clears the Slack # status; pausing prevents _keep_typing from re-setting it. # Typing resumes in _handle_approve_command/_handle_deny_command. ctx._status_adapter.pause_typing_for_chat(ctx._status_chat_id) # For WeCom native streaming: signal the stream consumer to close # the current stream before showing the approval prompt. # This goes through the consumer's queue for serial processing, # avoiding race conditions with pending deltas. _close_native_stream_boundary("Approval") cmd = approval_data.get("command", "") desc = approval_data.get("description", "dangerous command") # Redact credentials from the command before displaying it in # the approval prompt — Tirith's findings are already redacted, # but the raw command string still leaks secrets to the chat # platform (#48456). Applied here so BOTH the button-based # (send_exec_approval) and plain-text fallback paths below use # the redacted value. cmd = _redact_approval_command(cmd) # Prefer button-based approval when the adapter supports it. # Check the *class* for the method, not the instance — avoids # false positives from MagicMock auto-attribute creation in tests. if getattr(type(ctx._status_adapter), "send_exec_approval", None) is not None: try: _approval_fut = safe_schedule_threadsafe( ctx._status_adapter.send_exec_approval( chat_id=ctx._status_chat_id, command=cmd, session_key=_approval_session_key, description=desc, metadata=ctx._status_thread_metadata, allow_permanent=approval_data.get("allow_permanent", True), allow_session=approval_data.get("allow_session", True), smart_denied=approval_data.get("smart_denied", False), ), ctx._loop_for_step, logger=logger, log_message="send_exec_approval scheduling error", ) if _approval_fut is None: raise RuntimeError("send_exec_approval: loop unavailable") _outcome = _approval_send_outcome(_approval_fut, timeout=15) if _outcome == "sent": return if _outcome == "ambiguous": # Timeout ≠ failure: the card may have posted with a # late ack (slow platform API call or transient # connector backpressure). The prompt # registration stays alive, so a tap on the rendered # card still resolves; re-sending here is what # produced duplicate cards and an orphaned # "/approve: nothing pending" in live relay testing. # Skip the text fallback. logger.warning( "Button-based approval send timed out — treating " "as possibly-delivered (no re-send; the prompt " "stays armed for a late tap)" ) return logger.warning( "Button-based approval failed (send returned error), falling back to text" ) except Exception as _e: logger.warning( "Button-based approval failed, falling back to text: %s", _e ) # Fallback: plain text approval prompt. Use the adapter's # typed prefix so Slack/Matrix users are told the form they # can actually type (`!approve`) — typed "/" is blocked in # Slack threads and reserved by Matrix clients. _p = getattr(ctx._status_adapter, "typed_command_prefix", "/") msg = _format_exec_approval_fallback( cmd, desc, _p, allow_permanent=approval_data.get("allow_permanent", True), allow_session=approval_data.get("allow_session", True), smart_denied=approval_data.get("smart_denied", False), ) try: # Mark as approval prompt so WeCom routes through control lane _approval_metadata = dict(ctx._status_thread_metadata or {}) _approval_metadata["is_approval_prompt"] = True _approval_send_fut = safe_schedule_threadsafe( ctx._status_adapter.send( ctx._status_chat_id, msg, metadata=_interim_metadata(_approval_metadata), ), ctx._loop_for_step, logger=logger, log_message="Approval text-send scheduling error", ) if _approval_send_fut is not None: _approval_send_fut.result(timeout=15) except Exception as _e: logger.error("Failed to send approval request: %s", _e) # Keep real user text separate from API-only recovery guidance. If # an auto-continue note is prepended below, persist the original # message so stale guidance never replays as user-authored text. _persist_user_message_override: Optional[Any] = ctx.persist_user_message _persist_user_timestamp_override: Optional[float] = ctx.persist_user_timestamp # Prepend pending model switch note so the model knows about the switch _pending_notes = getattr(self._runner, '_pending_model_notes', {}) _msn = _pending_notes.pop(ctx.session_key, None) if ctx.session_key else None if _msn: ctx.message = _msn + "\n\n" + ctx.message # Auto-continue: if the loaded history ends with a tool result, # the previous agent turn was interrupted mid-work (gateway # restart, crash, SIGTERM). Prepend a system note so the model # finishes processing the pending tool results before addressing # the user's new message. (#4493) # # Session-level resume_pending (set on drain-timeout shutdown) # escalates the wording — the transcript's last role may be # anything (tool, assistant with unfinished work, etc.), so we # give a stronger, reason-aware instruction that subsumes the # tool-tail case. # # Freshness gate (#16802): both branches are gated on the age # of the last persisted transcript row. That is the correct # "when did we last do anything here" signal for both the # resume_pending path (restart watchdog) and the tool-tail # path (in-flight tool loop killed). We read ``history[-1]`` # here because ``agent_history`` has already stripped the # ``timestamp`` field off tool/tool_call rows for API purity # (see the `k != "timestamp"` filter above). Rows without a # timestamp (legacy transcripts) are treated as fresh so the # historical auto-continue behaviour is preserved. _freshness_window = _auto_continue_freshness_window() _interruption_is_fresh = _is_fresh_gateway_interruption( _last_transcript_timestamp(ctx.history), window_secs=_freshness_window, ) _resume_entry = None if ctx.session_key: try: _resume_entry = self._runner.session_store._entries.get(ctx.session_key) except Exception: _resume_entry = None # resume_pending freshness uses a SECOND signal in addition to the # transcript clock above. The restart watchdog stamps the session # with ``last_resume_marked_at`` at interrupt time — that is the # correct "when were we interrupted" signal. The transcript clock # (_interruption_is_fresh) can be far older: an active thread you # return to may have its last persisted row hours back, even though # the interruption itself just happened. Gating resume_pending on # the transcript clock alone makes the recovery note silently drop, # and because the startup auto-resume turn carries empty text # (_schedule_resume_pending_sessions), the model then receives a # blank user message and replies with confused "the message came # through blank" noise. Treat the marker as fresh when # EITHER signal is fresh so the two freshness checks agree. _resume_mark_is_fresh = False if _resume_entry is not None and getattr(_resume_entry, "resume_pending", False): _resume_mark_is_fresh = _is_fresh_gateway_interruption( getattr(_resume_entry, "last_resume_marked_at", None), window_secs=_freshness_window, ) _is_resume_pending = bool( _resume_entry is not None and getattr(_resume_entry, "resume_pending", False) and (_interruption_is_fresh or _resume_mark_is_fresh) ) _has_fresh_tool_tail = bool( agent_history and agent_history[-1].get("role") == "tool" and _interruption_is_fresh ) if _is_resume_pending: _reason = getattr(_resume_entry, "resume_reason", None) or "restart_timeout" # The empty-message case is the auto-resume startup turn # synthesized by _schedule_resume_pending_sessions — there is # no NEW user message to address. Guidance is adapter-aware: # interactive platforms report the restore and ask what next; # non-interactive event platforms (webhook, API server) # continue the interrupted work instead, because nobody is # present to answer and an acknowledgement would silently # abandon the task (#57056). _resume_adapter = self._runner._adapter_for_source(ctx.source) _interactive_resume = bool( getattr(_resume_adapter, "interactive_resume", True) ) ctx.message, _persist_user_message_override = _prepare_resume_pending_message( _reason, ctx.message, interactive=_interactive_resume, ) elif _has_fresh_tool_tail: _persist_user_message_override = ctx.message ctx.message = ( "[System note: A new message has arrived. The conversation " "history contains pending tool outputs from an interrupted turn. " "IGNORE those pending results. Address the user's NEW message " "below FIRST. Do NOT re-execute old tool calls from the history.]\n\n" + ctx.message ) # Consume one-shot /reload-skills note (if the user ran # /reload-skills since their last turn in this session). Same # queue pattern as CLI: prepend to the NEXT user message, then # clear. Nothing was written to the transcript out-of-band, so # message alternation stays intact. _pending_notes = getattr(self._runner, "_pending_skills_reload_notes", None) if _pending_notes and ctx.session_key and ctx.session_key in _pending_notes: _srn = _pending_notes.pop(ctx.session_key, None) if _srn: ctx.message = _srn + "\n\n" + ctx.message # Safety net: a startup auto-resume event carries empty # text and relies on the resume_pending branch above to supply the # recovery note. If that branch did not fire for any reason (e.g. # both freshness signals disagreed, or the marker was cleared # between scheduling and dispatch) we must NOT hand the model a # blank user turn — it responds with confused "the message came # through blank" noise. Restricted to resume_pending sessions so # legitimately empty user turns (e.g. an image with no caption, # wrapped as native content below) are untouched. if ( isinstance(ctx.message, str) and not ctx.message.strip() and _resume_entry is not None and getattr(_resume_entry, "resume_pending", False) ): _sn_reason = ( getattr(_resume_entry, "resume_reason", None) or "restart_timeout" ) _sn_adapter = self._runner._adapter_for_source(ctx.source) ctx.message = build_resume_recovery_note( _sn_reason, "", interactive=bool( getattr(_sn_adapter, "interactive_resume", True) ), ) _approval_session_key = ctx.session_key or "" _approval_session_token = set_current_session_key(_approval_session_key) register_gateway_notify(_approval_session_key, _approval_notify_sync) try: # If _prepare_inbound_message_text buffered image paths for native # attachment, wrap the user turn as an OpenAI-style multimodal # content list. Consume-and-clear so subsequent turns on the same # runner instance don't re-attach stale images. _native_imgs = self._runner._consume_pending_native_image_paths(ctx.session_key) if _native_imgs: try: from agent.image_routing import build_native_content_parts _parts, _skipped = build_native_content_parts( ctx.message, _native_imgs, ) if _skipped: logger.warning( "Native image attachment: skipped %d unreadable path(s): %s", len(_skipped), _skipped, ) if any(p.get("type") == "image_url" for p in _parts): _run_message: Any = _parts else: # All images failed to read — fall back to plain text. _run_message = ctx.message except Exception as _img_exc: logger.warning( "Native image attachment failed, falling back to text: %s", _img_exc, ) _run_message = ctx.message else: _run_message = ctx.message _api_run_message = _wrap_current_message_with_observed_context( _run_message, observed_group_context, ) _conversation_kwargs = { "conversation_history": agent_history, "task_id": ctx.session_id, } if _persist_user_message_override is not None: _conversation_kwargs["persist_user_message"] = _persist_user_message_override elif observed_group_context: _conversation_kwargs["persist_user_message"] = ctx.message if ctx.persist_user_display_kind: # Internal self-injected turn (#82888): type the persisted user # row at turn start so UIs render it as a timeline notice, not # a user bubble. Role/content are untouched and the key is # stripped from provider-bound payloads in conversation_loop. _conversation_kwargs["persist_user_display_kind"] = ( ctx.persist_user_display_kind ) if ctx.moa_config is not None: _conversation_kwargs["moa_config"] = ctx.moa_config if _persist_user_timestamp_override is not None: _conversation_kwargs["persist_user_timestamp"] = _persist_user_timestamp_override # Thread the platform-side inbound message id onto the persisted # user turn so a turn interrupted by a gateway restart is durably # recorded WITH its id — restart drain-window recovery dedups # against has_platform_message_id, and without this the # interrupted turn is invisible to that check. Uses the raw # inbound id (NOT event_message_id, which is the reply anchor). if ctx.inbound_message_id is not None: _conversation_kwargs["persist_user_platform_id"] = str(ctx.inbound_message_id) result = agent.run_conversation(_api_run_message, **_conversation_kwargs) finally: unregister_gateway_notify(_approval_session_key) # Cancel any pending clarify entries so blocked agent # threads don't hang past the end of the run (interrupt, # completion, gateway shutdown). Idempotent. try: from tools.clarify_gateway import clear_session as _clear_clarify_session _clear_clarify_session(_approval_session_key) except Exception: pass reset_current_session_key(_approval_session_token) # Canonicalize an explicitly emitted computer-use screenshot path at # the common result boundary. The streaming finalizer below and the # normal non-streaming delivery path must see the same response; # repairing only during later media scanning leaves streaming with the # model-mangled path and a rejected attachment. if isinstance(result, dict): _result_final = result.get("final_response") if isinstance(_result_final, str): result["final_response"] = repair_explicit_computer_use_media_paths( _result_final, result.get("messages", []), history_offset=len(agent_history), ) ctx.result_holder[0] = result # Signal the stream consumer that the agent is done. Pass the # completed final_response as the authoritative finalize payload: # it includes post-stream augmentation (file-mutation verifier # footer, turn-completion explainer) the consumer's accumulator # never saw, so the seal/final edit delivers the TRUE final and no # separate corrective send fires (live finding #11). Failed turns # pass nothing — error text is delivered by the gateway's normal # path, not baked into the stream. if _stream_consumer is not None: _final_for_stream = None # Adopt ONLY a genuinely completed final (review B6): interrupt # paths return {interrupted: True, completed: False} with a # DIAGNOSTIC final_response ("Operation interrupted during …") # and no failed key — adopting that would seal the user's # streamed partial answer over with the diagnostic AND make # delivered_final_matches reconcile, suppressing the gateway's # own error-delivery path. Writers of these shapes: # agent/conversation_loop.py interrupt/retry-abort returns. if ( isinstance(result, dict) and not result.get("failed") and not result.get("interrupted") and result.get("completed") is not False ): _fr = result.get("final_response") if isinstance(_fr, str) and _fr.strip() and _fr != "(empty)": _final_for_stream = _fr if _final_for_stream is not None: # Duck-type safe: test doubles / older consumers may expose a # zero-arg finish(). The payload is an optimization, not a # requirement — fall back to the bare signal. try: _stream_consumer.finish(_final_for_stream) except TypeError: _stream_consumer.finish() else: _stream_consumer.finish() # Signal the streaming-TTS consumer that the agent is done (#60671). # finish() is called from the outer event-loop thread after the # executor returns, so early returns from run_sync are also # finalised. See the outer finally/completion section below. # Return final response, or a message if something went wrong final_response = result.get("final_response") # Extract actual token counts from the agent instance used for this run _last_prompt_toks = 0 _input_toks = 0 _output_toks = 0 _context_length = 0 _agent = ctx.agent_holder[0] if _agent and hasattr(_agent, "context_compressor"): _last_prompt_toks = getattr(_agent.context_compressor, "last_prompt_tokens", 0) _input_toks = getattr(_agent, "session_prompt_tokens", 0) _output_toks = getattr(_agent, "session_completion_tokens", 0) _context_length = getattr(_agent.context_compressor, "context_length", 0) or 0 _resolved_model = getattr(_agent, "model", None) if _agent else None # Sync session_id immediately after run_conversation(). Compression # can rotate before a follow-up model call fails; the failure return # below must still point the gateway at the compressed child. agent = ctx.agent_holder[0] _session_was_split = False # In-place compaction (compression.in_place / #38763) compacts the # transcript WITHOUT rotating the id, so the id-change diff below # can't detect it. compress_context() sets this rotation-independent # flag on the agent; the gateway uses it to re-baseline transcript # handling (history_offset=0 + rewrite the JSONL transcript) the # same way a split would, even though the session_id is unchanged. _compacted_in_place = bool(getattr(agent, "_last_compaction_in_place", False)) if agent else False agent_session_id = getattr(agent, 'session_id', ctx.session_id) if agent else ctx.session_id if agent and ctx.session_key and agent_session_id != ctx.session_id: _session_was_split = True logger.info( "Session split detected: %s → %s (compression)", ctx.session_id, agent_session_id, ) entry = self._runner.session_store._entries.get(ctx.session_key) _session_split_entry_persisted = False if entry: entry_session_id = getattr(entry, "session_id", None) if not ctx._run_still_current(): logger.info( "Skipping session split sync for stale run %s — " "generation %s is no longer current", ctx.session_key or "?", ctx.run_generation, ) elif entry_session_id == agent_session_id: _session_split_entry_persisted = True elif entry_session_id != ctx.session_id: logger.info( "Skipping session split sync for %s because the " "session binding moved from %s to %s before " "compression finished", ctx.session_key or "?", ctx.session_id, entry_session_id, ) else: entry.session_id = agent_session_id self._runner.session_store._save() self._runner.session_store._record_gateway_session_peer( agent_session_id, ctx.session_key, ctx.source, ) _session_split_entry_persisted = True # If this is a Telegram DM and source.thread_id was lost during # the session split (synthetic / recovered event), restore it # from the binding so _thread_metadata_for_source produces the # correct message_thread_id instead of routing to the General # thread. Failure here is non-fatal — we log and continue; # worst case the message lands in General, which is the # pre-fix behaviour. Only do this after this run successfully # published its session split; a stale /stop→/new predecessor # must not mutate routing/binding state for the fresh session. if _session_split_entry_persisted and ( getattr(ctx.source, "platform", None) == Platform.TELEGRAM and getattr(ctx.source, "chat_type", None) == "dm" and getattr(ctx.source, "thread_id", None) is None and self._runner._session_db is not None ): try: # run_sync is off-loop (executor); sync DB is fine. _binding = self._runner._session_db._db.get_telegram_topic_binding_by_session( session_id=agent_session_id, ) if _binding and _binding.get("thread_id"): ctx.source.thread_id = str(_binding["thread_id"]) logger.debug( "Restored source.thread_id=%s from binding after session split %s → %s", ctx.source.thread_id, ctx.session_id, agent_session_id, ) except Exception: logger.debug( "Failed to restore thread_id from binding after session split", exc_info=True, ) if _session_split_entry_persisted: self._runner._sync_telegram_topic_binding( ctx.source, entry, reason="agent-run-compression", ) effective_session_id = agent_session_id self._runner._sync_session_model_from_agent(effective_session_id, agent) # history_offset=0 whenever the agent's message list no longer has # the original history prefix — i.e. on rotation (split) OR in-place # compaction. In both cases the returned `messages` is the compacted # set, so the gateway must persist all of it (offset 0), not slice # past the pre-compaction length (which would drop everything). _effective_history_offset = ( 0 if (_session_was_split or _compacted_in_place) else len(agent_history) ) if not final_response: final_response = _normalize_empty_agent_response( result, final_response or "", history_len=len(agent_history), ) final_response = _sanitize_gateway_final_response(ctx.source.platform, final_response) if not final_response: final_response = f"⚠️ {result['error']}" if result.get("error") else "" return { "final_response": final_response, "messages": result.get("messages", []), "api_calls": result.get("api_calls", 0), "failed": result.get("failed", False), # Sibling of the non-empty-response return below (#64686): # the classifier's failure_reason must survive the # empty-response normalization path too, or downstream # consumers (TUI billing surface, transient-failure # persistence) lose the structured reason exactly when # the run produced no text. "failure_reason": result.get("failure_reason"), "partial": result.get("partial", False), "completed": result.get("completed"), "interrupted": result.get("interrupted", False), "interrupt_message": result.get("interrupt_message"), "error": result.get("error"), "compression_exhausted": result.get("compression_exhausted", False), "compression_deferred": result.get("compression_deferred", False), "tools": ctx.tools_holder[0] or [], "history_offset": _effective_history_offset, "compacted_in_place": _compacted_in_place, "session_id": effective_session_id, "last_prompt_tokens": _last_prompt_toks, "input_tokens": _input_toks, "output_tokens": _output_toks, "model": _resolved_model, "context_length": _context_length, } # Scan tool results for MEDIA: tags that need to be delivered # as native audio/file attachments. The TTS tool embeds MEDIA: tags # in its JSON response, but the model's final text reply usually # doesn't include them. We collect unique tags from tool results and # append any that aren't already present in the final response, so the # adapter's extract_media() can find and deliver the files exactly once. # # Scope the scan to THIS turn's tool results only. ``agent_history`` # was passed into run_conversation as ``conversation_history``, so the # agent's returned ``messages`` list is ``agent_history`` followed by # the messages produced this turn. Slicing at ``len(agent_history)`` # isolates the current turn precisely, so a stale MEDIA: path emitted # by a tool several turns earlier (still present in the full message # list) can never leak onto a later text-only reply. (Fixes #34608) # # Path-based deduplication against _history_media_paths (collected # before run_conversation) is retained as a secondary guard. It is # also the sole guard on the fallback branch taken when mid-run # context compression shrinks the message list below the original # history length, preserving the compression-safe behaviour of #160. if "MEDIA:" not in final_response: media_tags, has_voice_directive = _collect_auto_append_media_tags( result.get("messages", []), history_offset=len(agent_history), history_media_paths=_history_media_paths, ) if media_tags: seen = set() unique_tags = [] for tag in media_tags: if tag not in seen: seen.add(tag) unique_tags.append(tag) if has_voice_directive: unique_tags.insert(0, "[[audio_as_voice]]") final_response = final_response + "\n" + "\n".join(unique_tags) # Auto-titling runs at TURN START (agent/turn_context.py) from the # user's message alone, so it no longer waits on final_response — a # failed or interrupted turn still gets a titled session. The # platform-specific thread-rename callbacks are attached to the agent # as `_on_session_title` before the run starts (see # _attach_session_title_callback), because the titler now fires from # inside the turn prologue rather than from here. return { "final_response": final_response, "last_reasoning": result.get("last_reasoning"), "messages": ctx.result_holder[0].get("messages", []) if ctx.result_holder[0] else [], "api_calls": ctx.result_holder[0].get("api_calls", 0) if ctx.result_holder[0] else 0, "failed": ctx.result_holder[0].get("failed", False) if ctx.result_holder[0] else False, "failure_reason": ( ctx.result_holder[0].get("failure_reason") if ctx.result_holder[0] else None ), "completed": ctx.result_holder[0].get("completed") if ctx.result_holder[0] else None, "interrupted": ctx.result_holder[0].get("interrupted", False) if ctx.result_holder[0] else False, "partial": ctx.result_holder[0].get("partial", False) if ctx.result_holder[0] else False, "error": ctx.result_holder[0].get("error") if ctx.result_holder[0] else None, "interrupt_message": ctx.result_holder[0].get("interrupt_message") if ctx.result_holder[0] else None, "compression_exhausted": ( ctx.result_holder[0].get("compression_exhausted", False) if ctx.result_holder[0] else False ), # Soft lock-contention defer (#69870 consumer): distinct from # compression_exhausted so the gateway never auto-resets a # session that a concurrent compressor is about to shrink. "compression_deferred": ( ctx.result_holder[0].get("compression_deferred", False) if ctx.result_holder[0] else False ), "tools": ctx.tools_holder[0] or [], "history_offset": _effective_history_offset, "compacted_in_place": _compacted_in_place, "last_prompt_tokens": _last_prompt_toks, "input_tokens": _input_toks, "output_tokens": _output_toks, "model": _resolved_model, "context_length": _context_length, "session_id": effective_session_id, "response_previewed": result.get("response_previewed", False), "response_transformed": result.get("response_transformed", False), # Pass through the agent_persisted flag so the persistence block # above can correctly determine whether the codex app-server path # self-persisted (it didn't — see codex_runtime.py). Default # True preserves the skip-db behaviour for the standard runtime. "agent_persisted": (ctx.result_holder[0].get("agent_persisted", True) if ctx.result_holder[0] else True), } # Sentinel for "no explicit session DB has been pinned on this runner", so the # ``_session_db`` property can distinguish "resolve from the active profile # scope" from a deliberate ``runner._session_db = None`` (which disables # DB-backed commands and is how many suites construct a bare runner). A plain # ``None`` cannot express both. Mirrors ``gateway.session._DB_UNPINNED``. _SESSION_DB_UNPINNED = object() class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, GatewaySlashCommandsMixin): """ Main gateway controller. Manages the lifecycle of all platform adapters and routes messages to/from the agent. """ # Class-level defaults so partial construction in tests doesn't # blow up on attribute access. _busy_input_mode: str = "interrupt" _busy_text_mode: str = "interrupt" _restart_drain_timeout: float = DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT _restart_after_turn_timeout: float = DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT _cron_drain_timeout: float = DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT _signal_interrupt_grace_timeout: float = ( DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT ) _exit_code: Optional[int] = None _draining: bool = False _external_drain_active: bool = False _restart_requested: bool = False _restart_task_started: bool = False _restart_detached: bool = False _restart_via_service: bool = False _detached_restart_helper_started: bool = False _restart_command_source: Optional[SessionSource] = None _stop_task: Optional[asyncio.Task] = None _restart_task: Optional[asyncio.Task] = None _profile_failed_platforms: Optional[Dict[str, Dict[Platform, asyncio.Task]]] = None _systemd_watchdog: Optional[Any] = None _startup_restore_in_progress: bool = False _startup_warmup_task: Optional[asyncio.Task] = None # ------------------------------------------------------------------ # Legacy per-session dict adapters. All per-session state lives in # ``self._sessions`` (Dict[str, SessionState]); these properties expose # the pre-consolidation dict attributes as LIVE MutableMapping views so # the extensive test surface (and a few mixin/adapter call sites) that # read/write ``runner._running_agents`` etc. keeps working unchanged. # New production code should use ``self._session_state(key)`` directly. # ------------------------------------------------------------------ _running_agents = legacy_dict_property("_running_agents") _running_agents_ts = legacy_dict_property("_running_agents_ts") _active_session_leases = legacy_dict_property("_active_session_leases") _busy_ack_ts = legacy_dict_property("_busy_ack_ts") _turn_lease_tokens = legacy_lease_token_property() _session_run_generation = legacy_dict_property("_session_run_generation") _session_model_overrides = legacy_dict_property("_session_model_overrides") _pending_one_turn_model_restores = legacy_dict_property( "_pending_one_turn_model_restores" ) _session_reasoning_overrides = legacy_dict_property("_session_reasoning_overrides") _session_service_tier_overrides = legacy_dict_property( "_session_service_tier_overrides" ) _last_resolved_model = legacy_dict_property("_last_resolved_model") _queued_events = legacy_dict_property("_queued_events") _pending_turn_sidecar_notes = legacy_dict_property("_pending_turn_sidecar_notes") _pending_messages = legacy_dict_property("_pending_messages") _pending_native_image_paths_by_session = legacy_dict_property( "_pending_native_image_paths_by_session" ) _session_ephemeral_pin = legacy_dict_property("_session_ephemeral_pin") _session_vc_last = legacy_dict_property("_session_vc_last") _pending_approvals = legacy_dict_property("_pending_approvals") _update_prompt_pending = legacy_dict_property("_update_prompt_pending") # -- SessionState accessors ----------------------------------------- def _sessions_map(self) -> Dict[str, "SessionState"]: """The per-session state map; lazily created so bare test runners built via ``object.__new__`` work without ``__init__``.""" sessions = self.__dict__.get("_sessions") if sessions is None: sessions = {} self.__dict__["_sessions"] = sessions return sessions def _session_state(self, session_key: str) -> "SessionState": """Get-or-create the :class:`SessionState` for ``session_key``.""" sessions = self._sessions_map() state = sessions.get(session_key) if state is None: state = SessionState() sessions[session_key] = state return state def _peek_session_state(self, session_key: str) -> Optional["SessionState"]: """Return the SessionState for ``session_key`` without creating one.""" sessions = self.__dict__.get("_sessions") if not sessions: return None return sessions.get(session_key) def _is_session_running(self, session_key: str) -> bool: """True when the session holds a running-turn slot (agent or sentinel).""" state = self._peek_session_state(session_key) return state is not None and state.turn.agent is not None def _running_agent_items(self) -> List[tuple]: """(session_key, agent) pairs for sessions with a running turn (including pending sentinels), matching the old ``_running_agents`` dict contents.""" return [ (key, state.turn.agent) for key, state in self._sessions_map().items() if state.turn.agent is not None ] # Loop-liveness heartbeat / watchdog handles (#66892, #69089). Class-level # defaults so partial construction in tests doesn't blow up on access; the # real values are set in __init__ / start() / stop(). _loop_heartbeat_task: Optional["asyncio.Task"] = None _loop_floor_timer_handle: Optional[Any] = None _loop_liveness_watchdog: Optional[Any] = None _gateway_started_at: float = 0.0 _shutdown_watchdog_done: Optional["threading.Event"] = None _platform_lock_takeover_on_start: bool = False _reconnect_watcher_task: Optional["asyncio.Task"] = None def __init__(self, config: Optional[GatewayConfig] = None): global _gateway_runner_ref # When multiplex_profiles is on, load under the default profile secret # scope so bot tokens in that profile's .env resolve the same way # secondary profiles do (#64674). Explicit config= injection (tests) # is left untouched. self.config = config if config is not None else load_gateway_config_for_runner() # Mark the process as a profile multiplexer when configured. This flips # agent.secret_scope.get_secret() to fail-closed on any unscoped # credential read, so a missed migration crashes loudly instead of # leaking a cross-profile value (Workstream A). Inert when off. try: from agent.secret_scope import set_multiplex_active set_multiplex_active(bool(getattr(self.config, "multiplex_profiles", False))) except Exception: logger.debug("could not set multiplex-active flag", exc_info=True) self.adapters: Dict[Platform, BasePlatformAdapter] = {} # When non-None, SessionDB init failed — the gateway broadcasts a # one-time warning to the home channel(s) after connecting, so the # user knows persistence is broken instead of discovering it later # via a missing /resume or empty history (#88235). self._session_db_init_error: Optional[str] = None # Multi-profile multiplexing: adapters for NON-default profiles live # here, keyed by profile name then Platform. self.adapters stays the # default/active profile's map so the ~93 existing self.adapters[...] # sites are untouched when multiplexing is off (this dict is empty). # Populated by _start_secondary_profile_adapters(). self._profile_adapters: Dict[str, Dict[Platform, BasePlatformAdapter]] = {} self._warn_if_docker_media_delivery_is_risky() _gateway_runner_ref = _weakref.ref(self) # Load ephemeral config from config.yaml / env vars. # Both are injected at API-call time only and never persisted. self._prefill_messages = self._load_prefill_messages() self._reasoning_config = self._load_reasoning_config() self._service_tier = self._load_service_tier() self._show_reasoning = self._load_show_reasoning() self._busy_input_mode = self._load_busy_input_mode() self._busy_text_mode = self._load_busy_text_mode() # Secondary-profile busy modes are snapshotted during multiplex # startup. Busy-message handlers consult these maps by routed source # without rereading config or mutating process-global environment. self._busy_input_modes_by_profile: Dict[str, str] = {} self._busy_text_modes_by_profile: Dict[str, str] = {} self._restart_drain_timeout = self._load_restart_drain_timeout() self._restart_after_turn_timeout = self._load_restart_after_turn_timeout() self._cron_drain_timeout = self._load_cron_drain_timeout() self._signal_interrupt_grace_timeout = ( self._load_signal_interrupt_grace_timeout() ) self._provider_routing = self._load_provider_routing() self._fallback_model = self._load_fallback_model() # Wire process registry into session store for reset protection. # A background process older than the configured threshold (default 24h, # session_reset.bg_process_max_age_hours) is treated as stale and no # longer blocks session idle / daily reset — see #29177. The process is # NOT killed, only ignored by the reset guard. from tools.process_registry import process_registry _bg_max_age_hours = getattr( self.config.default_reset_policy, "bg_process_max_age_hours", 24 ) _bg_max_age_seconds = ( _bg_max_age_hours * 3600 if _bg_max_age_hours and _bg_max_age_hours > 0 else None ) self.session_store = SessionStore( self.config.sessions_dir, self.config, has_active_processes_fn=lambda key: process_registry.has_active_for_session( key, max_active_age=_bg_max_age_seconds, ), ) # One enforced loop-side boundary for the synchronous SessionStore. # Sync helpers keep using ``session_store`` directly; async gateway # handlers call this facade and await every operation. self._async_session_store = AsyncSessionStore(self.session_store) self.delivery_router = DeliveryRouter(self.config) self._running = False self._gateway_loop: Optional[asyncio.AbstractEventLoop] = None self._shutdown_event = asyncio.Event() self._exit_cleanly = False self._exit_with_failure = False self._exit_reason: Optional[str] = None self._exit_code: Optional[int] = None self._draining = False self._profile_failed_platforms: Dict[str, Dict[Platform, asyncio.Task]] = {} self._systemd_watchdog = None # External (NAS-driven) drain state — distinct from the shutdown # ``_draining`` flag above. Set by ``_drain_control_watcher`` when the # ``.drain_request.json`` marker is present: the gateway flips # ``gateway_state -> draining`` and refuses NEW turns, but the process # does NOT exit (the whole point — quiesce-without-restart, D4a). It is # fully reversible: removing the marker reverts to ``running`` and # re-accepts turns. ``_draining`` (shutdown) is one-way and ends in # process exit; this one is a steady state NAS polls during its # request -> poll -> proceed loop. self._external_drain_active = False self._restart_requested = False # Set by shutdown_signal_handler when a SIGTERM/SIGINT arrived # WITHOUT a planned-stop / takeover marker — i.e. an unexpected # external signal (container/s6 SIGTERM on `docker restart` or # image upgrade, OOM-killer, bare `kill`). Distinct from an # operator-requested stop, which writes a marker first. Used by # _stop_impl to decide whether to persist gateway_state=stopped # (see issue #42675): an unexpected signal must NOT persist # "stopped", or container_boot refuses to auto-start the gateway # on the next boot. self._signal_initiated_shutdown = False self._restart_task_started = False self._restart_detached = False self._restart_via_service = False self._detached_restart_helper_started = False self._restart_command_source: Optional[SessionSource] = None # Monotonic-ish wall clock of when this GatewayRunner was constructed. # Used by the /restart redelivery guard to bound the window in which a # missing dedup marker is treated as a stale redelivery. self._startup_time: float = time.time() # Set True at startup when this process booted as the result of a # chat-originated /restart (i.e. .restart_notify.json existed on boot). # A one-shot signal consumed by _is_stale_restart_redelivery so the # marker-missing fallback only suppresses a /restart when we KNOW we # just came out of a restart cycle — never on a genuine fresh boot. self._booted_from_restart: bool = False self._stop_task: Optional[asyncio.Task] = None self._restart_task: Optional[asyncio.Task] = None self._executor_lock = threading.Lock() self._executor: Optional[concurrent.futures.ThreadPoolExecutor] = None # Set on gateway stop so the recreate-on-shutdown path can't resurrect # the pool during a real shutdown. self._executor_closing = False # ALL per-session state (turn / conversation / persistent scopes) # lives in one container — see gateway/session_state.py. Access via # self._session_state(key) (get-or-create) or # self._peek_session_state(key) (read-only). self._sessions: Dict[str, SessionState] = {} # Per-SESSION_ID turn lease (#64934): serializes the # [load history → run → flush] region when two ROUTING KEYS resolve # to one session_id (switch_session's many-to-one mapping). The # routing-key guards above cannot see that overlap. Acquired in # _handle_message_with_agent after session resolution is final, # released via _release_turn_lease in the same method's finally. self._turn_leases = SessionTurnLeaseRegistry() # Tokens for held turn leases, keyed by (routing key, run generation) # so release is granted per-turn and a stale unwind can never free a # newer turn's lease (#28686 ownership lesson). # Held turn-lease tokens live on SessionState.turn.lease_token / # .lease_generation (the old dict was keyed (routing key, generation) # so a stale unwind could never free a newer turn's lease — the # generation field preserves that ownership check, #28686). # Runner-level queued interrupt text lives on # SessionState.persistent.pending_command_text (NOTE: distinct from # the adapter-level _pending_messages Dict[str, MessageEvent] in # gateway/platforms/base.py, which shares the legacy name). # Last successfully-resolved (non-empty) model, keyed by session. Used # as a fallback when a fresh config read transiently returns an empty # model (e.g. an mtime-keyed config-cache miss during a post-interrupt # recovery turn). Without this, the agent is built with model="" and # every API call fails HTTP 400 "No models provided" — the session goes # silent until the user manually re-sends. See #35314. The ``"*"`` # session entry holds a process-wide last-known-good for sessions # seen for the first time. Lives on # SessionState.conversation.last_resolved_model. # Overflow buffer for explicit /queue commands. The adapter-level # _pending_messages dict is a single slot per session (designed for # "next-turn" follow-ups where repeated sends collapse into one # event). /queue has different semantics: each invocation must # produce its own full agent turn, in FIFO order, with no merging. # When the slot is occupied, additional /queue items land here and # are promoted one-at-a-time after each run's drain. Cleared on # /new and /reset. /model and other mid-session operations # preserve the queue. Lives on SessionState.conversation.queued_events; # native image paths, busy-ack debounce timestamps and the monotonic # run-generation counter (#28686, NEVER reset) live on SessionState too. # Session keys that already received a stall notification for the # current stall episode (cleared when pending clears / activity resumes # / conversation boundary). See gateway.session_stall. self._session_stall_notified: Dict[str, bool] = {} # Startup restore gate: while restart-interrupted sessions are being # auto-resumed, real inbound messages are queued instead of competing # with the synthetic resume turns for the same session. The queued # events drain only after all startup resume tasks have finished. self._startup_restore_in_progress = False # Set by start_gateway() only for an explicit ``--replace`` launch. # _connect_initial_adapter_with_timeout scopes it to each adapter's # cold-start connect and removes it before any reconnect can run. self._platform_lock_takeover_on_start = False self._startup_restore_queue: List[MessageEvent] = [] self._startup_restore_tasks: List[asyncio.Task] = [] # LRU cache of live SessionSources keyed by session_key. Used by # fallback routing paths (shutdown notifications, synthetic # background-process events) when the persisted origin is missing # and _parse_session_key can't recover thread_id. Capped so it # cannot grow unbounded over a long-running gateway lifetime. self._session_sources: "OrderedDict[str, SessionSource]" = OrderedDict() self._session_sources_max = 512 # Completion delivery is intentionally lifecycle-scoped. This closes # duplicate queue/watcher races inside one gateway without pretending # the adapter call and a persistence write can be exactly-once across # a process crash. Any durable async-delegation replay state remains # owned by tools.async_delegation, not a parallel gateway ledger. self._completion_delivery_lock = threading.Lock() self._completion_deliveries_inflight: set[tuple[str, str, object]] = set() self._completion_deliveries_delivered: "OrderedDict[tuple[str, str, object], None]" = OrderedDict() self._completion_delivery_retention = 2048 # Agent-triggered terminal completions from one conversation often land # in the same scheduler tick. Hold them briefly so the agent receives # one synthetic turn instead of one turn per process (#70300). self._completion_notification_batches: dict[tuple[str, ...], list[tuple[str, dict, asyncio.Future]]] = {} self._completion_notification_batch_tasks: dict[tuple[str, ...], asyncio.Task] = {} self._completion_notification_batch_flush_tasks: set[asyncio.Task] = set() self._completion_notification_batch_window = 0.1 self._completion_notification_batches_stopping = False # Cache AIAgent instances per session to preserve prompt caching. # Without this, a new AIAgent is created per message, rebuilding the # system prompt (including memory) every turn — breaking prefix cache # and costing ~10x more on providers with prompt caching (Anthropic). # Key: session_key, Value: (AIAgent, config_signature_str) # # OrderedDict so _enforce_agent_cache_cap() can pop the least-recently- # used entry (move_to_end() on cache hits, popitem(last=False) for # eviction). Hard cap via _AGENT_CACHE_MAX_SIZE, idle TTL enforced # from _session_expiry_watcher(). import threading as _threading self._agent_cache: "OrderedDict[str, tuple]" = OrderedDict() self._agent_cache_lock = _threading.Lock() # Conversation-scoped per-session state (/model, /model --once, # /reasoning, /fast overrides; per-turn sidecar notes; ephemeral # context pin; last-delivered voice-channel context) lives on # SessionState.conversation — see gateway/session_state.py. self._kanban_notifier_profile = self._active_profile_name() # Launch-time identity of the profile that owns ``self.adapters``; # ``_authorization_adapter`` compares against this rather than the # per-turn ``_active_profile_name()`` (see gateway/authz_mixin.py). self._primary_profile_name = self._kanban_notifier_profile # Teams meeting pipeline runtime (bound later when msgraph_webhook adapter exists). self._teams_pipeline_runtime = None self._teams_pipeline_runtime_error: Optional[str] = None # Pending exec approvals live on SessionState.persistent.approvals. # Track platforms that failed to connect for background reconnection. # Key: Platform enum, Value: {"config": platform_config, "attempts": int, "next_retry": float} self._failed_platforms: Dict[Platform, Dict[str, Any]] = {} # Strong refs to detached fatal-error handler tasks (see # _handle_adapter_fatal_error) so the event loop can't GC them mid-run. self._fatal_handler_tasks: set = set() # Pending /update prompt flags live on # SessionState.persistent.update_prompt_pending. # Slash-confirm state lives in tools.slash_confirm (module-level), # so platform adapters can resolve callbacks without a backref to # this runner. Keep a local counter for confirm_id generation so # IDs stay compact (button callback_data has a 64-byte cap on # some platforms). import itertools as _itertools self._slash_confirm_counter = _itertools.count(1) # Persistent Honcho managers keyed by gateway session key. # This preserves write_frequency="session" semantics across short-lived # per-message AIAgent instances. # Ensure tirith security scanner is available (downloads if needed) try: from tools.tirith_security import ensure_installed ensure_installed(log_failures=False) except Exception: pass # Non-fatal — fail-open at scan time if unavailable # Startup heads-up (#30882): a gateway in manual approval mode with no # automated risk assessor (tirith disabled AND no auxiliary.approval # model) can only gate dangerous commands / execute_code scripts via # live in-chat approval. With approval routing fixed, those actions now # fail closed (block) rather than silently auto-running — surface that # so operators knowingly enable tirith or configure auxiliary.approval # for unattended gateways. try: from hermes_cli.config import load_config as _load_full_config _appr_cfg = _load_full_config() _appr_mode = str( cfg_get(_appr_cfg, "approvals", "mode", default="manual") or "manual" ).strip().lower() _tirith_on = bool(cfg_get(_appr_cfg, "security", "tirith_enabled", default=True)) _aux_approval = cfg_get(_appr_cfg, "auxiliary", "approval", default=None) if _appr_mode == "manual" and not _tirith_on and not _aux_approval: logger.warning( "Gateway approvals.mode=manual with no automated risk " "assessor (security.tirith_enabled is false and " "auxiliary.approval is unset): dangerous commands and " "execute_code scripts will BLOCK until a human approves " "them in chat. Enable security.tirith_enabled or configure " "auxiliary.approval for unattended operation." ) except Exception: logger.debug("approvals.mode startup check skipped", exc_info=True) # Initialize session database for session_search tool support. # # Same frozen-handle class of bug as SessionStore._db (#88532): a # handle bound here is pinned to the process's root home, but # /resume, /title, /history and session search all run inside # _profile_runtime_scope on a multiplexed gateway and must see that # profile's own state.db. Resolve through a property that caches # one AsyncSessionDB per resolved path; priming here keeps startup # diagnostics (the #88235 broadcast) at construction time. self._session_db_pinned: Any = _SESSION_DB_UNPINNED self._session_db_handles: Dict[Path, Any] = {} self._session_db_handles_lock = threading.Lock() from gateway.session_db_recovery import RecoverableHandleCache self._session_db_handle_cache = RecoverableHandleCache( handles=self._session_db_handles, lock=self._session_db_handles_lock, ) try: self._open_session_db_for_active_scope(raise_on_error=True) except Exception as e: # WARNING (not DEBUG) so the failure appears in errors.log — matches # cli.py's handling of the same init path. Users hitting NFS-mounted # HERMES_HOME silently lost /resume, /title, /history, /branch, and # session search without this. The underlying cause (usually # "locking protocol" from NFS) is now also captured by # hermes_state.get_last_init_error() for slash-command error strings. logger.warning("SQLite session store not available: %s", e) # Surface the failure to the user via their home channel(s) once # the gateway connects. Without this, state.db corruption or # NFS/SMB lock failures silently degrade the entire gateway — # messages may flow but nothing is persisted, and the user has # no indication until they try /resume and find nothing (#88235). self._session_db_init_error = str(e) # Opportunistic state.db maintenance: prune ended sessions inactive # for sessions.retention_days + optional VACUUM. Tracks last-run # in state_meta so it only actually executes once per # sessions.min_interval_hours. Gateway is long-lived so blocking # a few seconds once per day is acceptable; failures are logged # but never raised. if self._session_db is not None: try: from hermes_cli.config import load_config as _load_full_config _sess_cfg = (_load_full_config().get("sessions") or {}) # Non-destructive stale-session archive, independent of prune. if _sess_cfg.get("auto_archive", False): self._session_db._db.maybe_auto_archive( idle_days=float(_sess_cfg.get("auto_archive_days", 3)), min_interval_hours=int(_sess_cfg.get("min_interval_hours", 24)), ) if _sess_cfg.get("auto_prune", False): # Construction-time, before the loop serves traffic; sync DB is fine. self._session_db._db.maybe_auto_prune_and_vacuum( retention_days=int(_sess_cfg.get("retention_days", 90)), min_interval_hours=int(_sess_cfg.get("min_interval_hours", 24)), min_vacuum_interval_days=int( _sess_cfg.get("min_vacuum_interval_days", 30) ), vacuum=bool(_sess_cfg.get("vacuum_after_prune", True)), sessions_dir=self.config.sessions_dir, ) except Exception as exc: logger.debug("state.db auto-maintenance skipped: %s", exc) # Opportunistic shadow-repo cleanup — deletes stale checkpoint repos # under ~/.hermes/checkpoints/. Opt-in via checkpoints.auto_prune, # idempotent via .last_prune marker. try: from hermes_cli.config import load_config as _load_full_config _ckpt_cfg = (_load_full_config().get("checkpoints") or {}) if _ckpt_cfg.get("auto_prune", False): from tools.checkpoint_manager import maybe_auto_prune_checkpoints # delete_orphans is intentionally never honoured here: a # missing workdir at startup is ambiguous (deleted project # vs. an unmounted external volume / network share / VPN # not yet up) and this sweep runs unattended. Orphan cleanup # is only ever done via the explicit `hermes checkpoints # prune` command, which the user has to invoke. maybe_auto_prune_checkpoints( retention_days=int(_ckpt_cfg.get("retention_days", 7)), min_interval_hours=int(_ckpt_cfg.get("min_interval_hours", 24)), delete_orphans=False, max_total_size_mb=int(_ckpt_cfg.get("max_total_size_mb", 500)), ) except Exception as exc: logger.debug("checkpoint auto-maintenance skipped: %s", exc) # DM pairing store for code-based user authorization. # ``pairing_store`` stays as the global/default store for the # ``hermes pairing`` CLI and any caller without a profile context. # ``pairing_stores`` is the per-profile map used by # ``authz_mixin._is_user_authorized`` to route checks to the right # whitelist (one per profile in multiplex mode). from gateway.pairing import PairingStore self.pairing_store = PairingStore() self.pairing_stores: Dict[str, "PairingStore"] = {} # Event hook system from gateway.hooks import HookRegistry self.hooks = HookRegistry() # Per-chat voice reply mode: "off" | "voice_only" | "all" self._voice_mode: Dict[str, str] = self._load_voice_modes() # Recent voice transcripts per (guild,user) for duplicate suppression. # Protects against the same utterance being emitted twice by the voice # capture / STT pipeline, which otherwise produces a second delayed reply. self._recent_voice_transcripts: Dict[tuple[int, int], List[tuple[float, str]]] = {} # Track background tasks to prevent garbage collection mid-execution self._background_tasks: set = set() # Event-loop liveness heartbeat (#66892): rewritten every 30s while # the loop is dispatching. External supervisors use the file mtime / # updated_at to distinguish "process alive" from "loop frozen". self._gateway_started_at: float = time.time() self._loop_heartbeat_task: Optional[asyncio.Task] = None self._loop_floor_timer_handle = None self._loop_liveness_watchdog = None # scale-to-zero (Phase 0, F13): gateway-scoped "last inbound seen" clock. # There is no such clock today (only a per-agent _last_activity_ts), so the # idle predicate needs this. Stamped in _handle_message (the single inbound # chokepoint all adapters call); seeded to "now" so a fresh gateway isn't # considered idle from epoch. The scale-to-zero watcher (started only when # the instance is opted in + relay-only + has a wakeUrl) reads it. self._last_inbound_at: float = time.time() # Set after a wake (re-arm cooldown, 0.F) so we don't immediately re-go # dormant before the drained backlog has a chance to update the clock. self._scale_to_zero_cooldown_until: float = 0.0 # One-shot: log the "platform owns the suspend" notice once, not per tick. self._scale_to_zero_no_suspend_logged: bool = False def _open_session_db_for_active_scope(self, raise_on_error: bool = False) -> Any: """Return the AsyncSessionDB for the profile scope active on this task. Same per-path cache as ``SessionStore._open_session_db_for_active_scope`` (#88532): ``SessionDB()`` resolves ``_default_db_path()`` at call time through the context-local HERMES_HOME override installed by ``_profile_runtime_scope``, so resolving per access — instead of once in ``__init__`` — is what lets /resume, /title, /history and session search on a multiplexed gateway read the *serving profile's* store rather than the root one. One ``AsyncSessionDB`` is cached per resolved path, so the wrapper identity is stable per profile (callers compare and stash it) and two profiles never share a handle. A construction failure enters bounded backoff; one caller retries after the deadline while concurrent callers continue to see the unavailable fallback. ``raise_on_error=True`` (construction-time priming) propagates the failure after recording that recoverable state so ``__init__`` can record ``_session_db_init_error`` for the #88235 broadcast. """ from hermes_state import AsyncSessionDB, SessionDB, _default_db_path, get_shared_session_db from gateway.session_db_recovery import RecoverableHandleCache path = Path(_default_db_path()) cache = getattr(self, "_session_db_handle_cache", None) if cache is None: # Compatibility for lightweight test runners built with # object.__new__ rather than GatewayRunner.__init__. cache = RecoverableHandleCache( handles=self._session_db_handles, lock=self._session_db_handles_lock, ) self._session_db_handle_cache = cache def _open(): # Borrow the SessionStore's handle for this path rather than # opening a second one. Both caches resolve the SAME # ``_default_db_path()``, so the process was holding two writer # connections and two read pools against one state.db — the fd # budget doubled for nothing, and doubled again per profile on a # multiplexed gateway (#98573). The store owns the handle and # sweeps it at shutdown; this cache holds only the async wrapper. # # A borrowed wrapper cannot go stale in practice: the store's # cache only drops handles in close_all_db_handles() (shutdown), # and while the store's own open is failing there is nothing to # borrow, so nothing is cached here either. store = getattr(self, "session_store", None) borrowed = getattr(store, "_db", None) if store is not None else None if borrowed is not None: wrapper = AsyncSessionDB(borrowed) # close_all_session_db_handles() must not close what the store # owns; the store's own sweep already does, and it runs first. wrapper.__dict__["_hermes_borrowed_handle"] = True return wrapper if store is not None: # The store exists and its handle is unavailable (failed open # or backoff). Opening our own here would resurrect exactly # the duplicate this borrows away from, so report the same # unavailability the store is already reporting. raise RuntimeError("SessionStore SQLite handle unavailable") try: return AsyncSessionDB(get_shared_session_db()) except Exception as exc: logger.warning("SQLite session store not available: %s", exc) raise def _recovered() -> None: self._session_db_init_error = None logger.info("SQLite session store recovered") return cache.get( path, _open, raise_on_error=raise_on_error, on_recovered=_recovered, ) @property def _session_db(self) -> Any: """The AsyncSessionDB for the active profile scope, or a pinned override. Assigning ``runner._session_db`` pins that value for every subsequent read — tests rely on installing fakes or ``None`` this way. Unpinned (the production path), each read resolves the active scope so a multiplexed profile's slash commands and session search hit its own store. """ if self._session_db_pinned is not _SESSION_DB_UNPINNED: return self._session_db_pinned return self._open_session_db_for_active_scope() @_session_db.setter def _session_db(self, value) -> None: self._session_db_pinned = value def close_all_session_db_handles(self) -> None: """Close every per-profile AsyncSessionDB this runner opened. Shutdown counterpart of the per-path cache above; mirrors ``SessionStore.close_all_db_handles``. Handles are drained under the lock and closed outside it; a pinned handle is the pinner's to close. Wrappers around a handle BORROWED from ``session_store`` (#98573) are drained but not closed: the store owns that connection and its own sweep — which runs first in the shutdown sequence — closes it. """ def _close(db) -> None: if getattr(db, "__dict__", {}).get("_hermes_borrowed_handle"): return inner = getattr(db, "_db", db) if inner is None or not hasattr(inner, "close"): return from hermes_state import release_or_close try: release_or_close(inner) except Exception as exc: logger.debug("SessionDB close error during handle sweep: %s", exc) self._session_db_handle_cache.close_all(_close) def _wire_teams_pipeline_runtime(self) -> None: """Bind the Teams meeting pipeline runtime to Graph webhook ingress. No-op when the msgraph_webhook adapter isn't running or the teams_pipeline plugin isn't enabled — lets the gateway start cleanly whether or not the user has opted into the pipeline. """ if Platform.MSGRAPH_WEBHOOK not in self.adapters: return if not _teams_pipeline_plugin_enabled(): logger.debug("Teams pipeline plugin is disabled; skipping runtime wiring") return try: from plugins.teams_pipeline.runtime import bind_gateway_runtime except Exception as exc: logger.warning("Teams pipeline runtime import failed: %s", exc) return try: bound = bind_gateway_runtime(self) except Exception as exc: logger.warning("Teams pipeline runtime wiring failed: %s", exc) return if bound: logger.info("Teams pipeline runtime bound to msgraph webhook ingress") elif self._teams_pipeline_runtime_error: logger.warning( "Teams pipeline runtime unavailable: %s", self._teams_pipeline_runtime_error, ) def _warn_if_docker_media_delivery_is_risky(self) -> None: """Warn when Docker-backed gateways lack an explicit export mount. MEDIA delivery happens in the gateway process, so paths emitted by the model must be readable from the host. A plain container-local path like `/workspace/report.txt` or `/output/report.txt` often exists only inside Docker, so users commonly need a dedicated export mount such as `host-dir:/output`. """ if os.getenv("TERMINAL_ENV", "").strip().lower() != "docker": return connected = self.config.get_connected_platforms() messaging_platforms = [p for p in connected if p not in {Platform.LOCAL, Platform.API_SERVER, Platform.WEBHOOK}] if not messaging_platforms: return raw_volumes = os.getenv("TERMINAL_DOCKER_VOLUMES", "").strip() volumes: List[str] = [] if raw_volumes: try: parsed = json.loads(raw_volumes) if isinstance(parsed, list): volumes = [str(v) for v in parsed if isinstance(v, str)] except Exception: logger.debug("Could not parse TERMINAL_DOCKER_VOLUMES for gateway media warning", exc_info=True) has_explicit_output_mount = False for spec in volumes: match = _DOCKER_VOLUME_SPEC_RE.match(spec) if not match: continue container_path = match.group("container") if container_path in _DOCKER_MEDIA_OUTPUT_CONTAINER_PATHS: has_explicit_output_mount = True break if has_explicit_output_mount: return logger.warning( "Docker backend is enabled for the messaging gateway but no explicit host-visible " "output mount (for example '/home/user/.hermes/cache/documents:/output') is configured. " "This is fine if the model already emits host-visible paths, but MEDIA file delivery can fail " "for container-local paths like '/workspace/...' or '/output/...'." ) # -- Setup skill availability ---------------------------------------- def _has_setup_skill(self) -> bool: """Check if the hermes-agent-setup skill is installed.""" try: from tools.skill_manager_tool import _find_skill return _find_skill("hermes-agent-setup") is not None except Exception: return False # -- Voice mode persistence ------------------------------------------ _VOICE_MODE_PATH = _hermes_home / "gateway_voice_mode.json" def _voice_key( self, platform: Platform, chat_id: str, profile: Optional[str] = None ) -> str: """Return a platform-namespaced key for voice mode state. Under multiplexing the key is additionally namespaced by the profile whose bot speaks in the chat (``::``); the default profile keeps the historical ``:`` shape so persisted state stays valid. Two bots in one Discord channel otherwise share a key and one profile's ``/voice`` flips the other's (#75198). """ base = f"{platform.value}:{chat_id}" profile = profile.strip() if isinstance(profile, str) else "" if not profile or profile == "default": return base return f"{profile}:{base}" def _voice_key_for_source(self, source: SessionSource) -> str: """Voice-state key for an inbound source, namespaced by its transport owner. Voice mode belongs to the (bot, chat) pair, so the namespace is the profile that OWNS the receiving adapter (``_adapter_profile_for_source``) — the same profile ``_sync_voice_mode_state_to_adapter`` uses on reconnect — not the routed runtime profile. """ return self._voice_key( source.platform, source.chat_id, profile=self._adapter_profile_for_source(source), ) def _bind_voice_input_callback(self, adapter) -> None: """Route voice transcripts back through the adapter that captured them.""" if hasattr(adapter, "_voice_input_callback"): adapter._voice_input_callback = functools.partial( self._handle_voice_channel_input, adapter=adapter ) def _load_voice_modes(self) -> Dict[str, str]: try: data = json.loads(self._VOICE_MODE_PATH.read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError, OSError): return {} if not isinstance(data, dict): return {} valid_modes = {"off", "voice_only", "all"} result = {} for chat_id, mode in data.items(): if mode not in valid_modes: continue key = str(chat_id) # Skip legacy unprefixed keys (warn and skip) if ":" not in key: logger.warning( "Skipping legacy unprefixed voice mode key %r during migration. " "Re-enable voice mode on that chat to rebuild the prefixed key.", key, ) continue result[key] = mode return result def _save_voice_modes(self) -> None: try: self._VOICE_MODE_PATH.parent.mkdir(parents=True, exist_ok=True) self._VOICE_MODE_PATH.write_text( json.dumps(self._voice_mode, indent=2), encoding="utf-8" ) except OSError as e: logger.warning("Failed to save voice modes: %s", e) def _set_adapter_auto_tts_disabled(self, adapter, chat_id: str, disabled: bool) -> None: """Update an adapter's in-memory auto-TTS suppression set if present.""" disabled_chats = getattr(adapter, "_auto_tts_disabled_chats", None) if not isinstance(disabled_chats, set): return if disabled: disabled_chats.add(chat_id) # ``/voice off`` also clears any explicit enable — it's a hard override. enabled_chats = getattr(adapter, "_auto_tts_enabled_chats", None) if isinstance(enabled_chats, set): enabled_chats.discard(chat_id) else: disabled_chats.discard(chat_id) def _set_adapter_auto_tts_enabled(self, adapter, chat_id: str, enabled: bool) -> None: """Update an adapter's per-chat auto-TTS opt-in set if present. Used for ``/voice on``/``/voice tts`` where the user explicitly wants auto-TTS even when ``voice.auto_tts`` is False globally. """ enabled_chats = getattr(adapter, "_auto_tts_enabled_chats", None) if not isinstance(enabled_chats, set): return if enabled: enabled_chats.add(chat_id) # An explicit opt-in clears any stale /voice off for this chat. disabled_chats = getattr(adapter, "_auto_tts_disabled_chats", None) if isinstance(disabled_chats, set): disabled_chats.discard(chat_id) else: enabled_chats.discard(chat_id) def _sync_voice_mode_state_to_adapter(self, adapter) -> None: """Restore persisted /voice state into a live platform adapter. Populates three fields from config + ``self._voice_mode``: - ``_auto_tts_default``: global default from ``voice.auto_tts`` - ``_auto_tts_enabled_chats``: chats with mode ``voice_only``/``all`` - ``_auto_tts_disabled_chats``: chats with mode ``off`` """ platform = getattr(adapter, "platform", None) if not isinstance(platform, Platform): return disabled_chats = getattr(adapter, "_auto_tts_disabled_chats", None) enabled_chats = getattr(adapter, "_auto_tts_enabled_chats", None) if not isinstance(disabled_chats, set) and not isinstance(enabled_chats, set): return # Push the global voice.auto_tts default (config.yaml) onto the adapter. # Lazy import to avoid adding a module-level dep from gateway → hermes_cli. try: from hermes_cli.config import load_config as _load_full_config _full_cfg = _load_full_config() _auto_tts_default = bool( (_full_cfg.get("voice") or {}).get("auto_tts", False) ) except Exception: _auto_tts_default = False if hasattr(adapter, "_auto_tts_default"): adapter._auto_tts_default = _auto_tts_default prefix = self._voice_key(platform, "", profile=getattr(adapter, "_owner_profile", None)) if isinstance(disabled_chats, set): disabled_chats.clear() disabled_chats.update( key[len(prefix):] for key, mode in self._voice_mode.items() if mode == "off" and key.startswith(prefix) ) if isinstance(enabled_chats, set): enabled_chats.clear() enabled_chats.update( key[len(prefix):] for key, mode in self._voice_mode.items() if mode in {"voice_only", "all"} and key.startswith(prefix) ) async def _await_adapter_cleanup_with_timeout( self, awaitable: Awaitable[Any], timeout: float ) -> bool: """Wait for adapter cleanup without letting cancellation swallowing hang us. ``asyncio.wait_for`` cancels an overdue child but then waits for it to exit. An adapter close path that catches ``CancelledError`` can therefore block recovery forever. Keep ownership of the old task through its done callback, but release the runner at the deadline. """ if timeout <= 0: await awaitable return True task = asyncio.ensure_future(awaitable) try: done, _pending = await asyncio.wait({task}, timeout=timeout) except asyncio.CancelledError: task.cancel() task.add_done_callback(consume_detached_task_result) raise if task in done: await task return True task.cancel() task.add_done_callback(consume_detached_task_result) return False async def _safe_adapter_disconnect(self, adapter, platform) -> None: """Call adapter.disconnect() defensively, swallowing any error. Used when adapter.connect() failed or raised — the adapter may have allocated partial resources (aiohttp.ClientSession, poll tasks, child subprocesses) that would otherwise leak and surface as "Unclosed client session" warnings at process exit. Must tolerate partial-init state and never raise, since callers use it inside error-handling blocks. """ timeout = self._adapter_disconnect_timeout_secs() try: completed = await self._await_adapter_cleanup_with_timeout( adapter.disconnect(), timeout ) if not completed: logger.warning( "Timed out after %.1fs while disconnecting %s adapter; continuing shutdown", timeout, platform.value if platform is not None else "adapter", ) except Exception as e: logger.debug( "Defensive %s disconnect after failed connect raised: %s", platform.value if platform is not None else "adapter", e, ) async def _bounded_adapter_teardown( self, adapter, platform, *, profile: Optional[str] = None ) -> None: """Tear down one adapter on the shutdown path with bounded awaits. Both ``cancel_background_tasks()`` and ``disconnect()`` can block indefinitely when a platform's network state is half-dead (e.g. a wedged Feishu/Lark WebSocket thread waiting on I/O). An unbounded await here stalls the entire shutdown sequence past systemd's ``TimeoutStopSec``; the resulting SIGKILL skips ``atexit`` PID-file cleanup, so the next start dies with "PID file race lost" (#14128). Each await uses the existing per-adapter timeout budget (``HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT``). On timeout the old task is cancelled and detached, then teardown forces forward progress; the loop never hangs even if an adapter swallows cancellation. Never raises. """ timeout = self._adapter_disconnect_timeout_secs() suffix = f" (profile: {profile})" if profile else "" started_at = time.monotonic() try: cancelled = await self._await_adapter_cleanup_with_timeout( adapter.cancel_background_tasks(), timeout ) if not cancelled: logger.warning( "✗ %s background-task cancel timed out after %.1fs - forcing continue%s", platform.value, timeout, suffix, ) except Exception as e: logger.debug("✗ %s background-task cancel error%s: %s", platform.value, suffix, e) try: disconnected = await self._await_adapter_cleanup_with_timeout( adapter.disconnect(), timeout ) if disconnected: logger.info( "✓ %s disconnected (%.2fs)%s", platform.value, time.monotonic() - started_at, suffix, ) else: logger.warning( "✗ %s disconnect timed out after %.1fs - forcing continue%s", platform.value, timeout, suffix, ) except Exception as e: logger.error( "✗ %s disconnect error after %.2fs%s: %s", platform.value, time.monotonic() - started_at, suffix, e, ) def _adapter_disconnect_timeout_secs(self) -> float: """Return the per-adapter disconnect timeout used during shutdown.""" raw = os.getenv("HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT", "").strip() if raw: try: timeout = float(raw) except ValueError: logger.warning( "Ignoring invalid HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT=%r", raw, ) else: return max(0.0, timeout) return _ADAPTER_DISCONNECT_TIMEOUT_SECS_DEFAULT def _platform_connect_timeout_secs(self, platform=None, *, initial: bool = False) -> float: """Return the per-platform connect timeout used during startup/retry. ``initial=True`` marks the cold-start connect awaited before the gateway reaches ``running``. Telegram's full connect budget (180s, raised for #67498 so cold polling can prove getUpdates readiness) is deliberately NOT spent there: an unreachable Telegram would hold the whole gateway out of the ``running`` state for the full budget (#85993). The cold-start wait is capped and the platform is handed to the reconnect watcher, which retries with the full budget (and ``is_reconnect=True``, preserving the offline update queue — #46621). """ raw = os.getenv("HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT", "").strip() if raw: try: timeout = float(raw) except ValueError: logger.warning( "Ignoring invalid HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT=%r", raw, ) else: return max(0.0, timeout) if platform == Platform.TELEGRAM: if initial: return _TELEGRAM_INITIAL_CONNECT_TIMEOUT_SECS_DEFAULT return _TELEGRAM_CONNECT_TIMEOUT_SECS_DEFAULT return _PLATFORM_CONNECT_TIMEOUT_SECS_DEFAULT async def _connect_adapter_with_timeout( self, adapter, platform, *, is_reconnect: bool = False, initial: bool = False ) -> bool: """Connect an adapter without allowing one platform to block others. ``is_reconnect`` is forwarded to ``adapter.connect()`` so platform adapters can distinguish a cold first boot (drop any stale server-side queue) from a watcher reconnect after a prolonged outage (preserve the queue so messages sent during the outage are delivered rather than silently dropped — #46621). ``initial`` selects the capped cold-start budget for platforms whose full connect budget is too long to spend before the gateway reaches ``running`` (#85993 — Telegram's 180s). """ timeout = self._platform_connect_timeout_secs(platform, initial=initial) if timeout <= 0: return await adapter.connect(is_reconnect=is_reconnect) # Use the detach-on-timeout pattern instead of plain asyncio.wait_for: # asyncio.wait_for cancels the overdue task but then waits for it to # exit. An adapter connect() that catches CancelledError can therefore # block recovery forever (the watcher never reaches the next retry). # Keep ownership of the old task through its done callback, but # release the runner at the deadline (#70344). task = asyncio.ensure_future( adapter.connect(is_reconnect=is_reconnect) ) try: done, _pending = await asyncio.wait({task}, timeout=timeout) except asyncio.CancelledError: task.cancel() task.add_done_callback(consume_detached_task_result) raise if task in done: result = await task return bool(result) task.cancel() task.add_done_callback(consume_detached_task_result) raise TimeoutError( f"{platform.value} connect timed out after {timeout:g}s" ) async def _connect_initial_adapter_with_timeout(self, adapter, platform) -> bool: """Connect one cold-start adapter with tightly scoped replace intent. The capability is visible only while this initial connect is awaited. Reconnects call ``_connect_adapter_with_timeout`` directly and adapters also default to deny, so a later network recovery can never evict a healthy token holder. """ adapter._platform_lock_takeover_allowed = bool( self._platform_lock_takeover_on_start ) try: return await self._connect_adapter_with_timeout( adapter, platform, initial=True ) finally: adapter._platform_lock_takeover_allowed = False @property def should_exit_cleanly(self) -> bool: return self._exit_cleanly @property def should_exit_with_failure(self) -> bool: return self._exit_with_failure @property def exit_reason(self) -> Optional[str]: return self._exit_reason @property def exit_code(self) -> Optional[int]: return self._exit_code def _session_key_for_source(self, source: SessionSource) -> str: """Resolve the current session key for a source, honoring gateway config when available.""" if hasattr(self, "session_store") and self.session_store is not None: try: session_key = self.session_store._generate_session_key(source) if isinstance(session_key, str) and session_key: return session_key except Exception: pass config = getattr(self, "config", None) # Mirror SessionStore._resolve_profile_for_key so this fallback path # produces the same namespace as the primary path: None (legacy # agent:main) unless multiplexing is on, then the active profile. _profile = None if getattr(config, "multiplex_profiles", False): if source.profile: _profile = source.profile else: try: from hermes_cli.profiles import get_active_profile_name _profile = get_active_profile_name() or "default" except Exception: _profile = None return build_session_key( source, group_sessions_per_user=getattr(config, "group_sessions_per_user", True), thread_sessions_per_user=getattr(config, "thread_sessions_per_user", False), profile=_profile, ) @staticmethod def _telegram_topic_profile_name(source: SessionSource) -> str: """Profile namespace for Telegram topic-mode rows (issue #76423). Prefer the profile already stamped on the routed event (``source.profile``). Do **not** fall back to the process-global active profile here — under multiplex that can mis-attribute topic state across bots sharing one ``state.db``. """ name = str(getattr(source, "profile", None) or "").strip() return name if name else "default" def _telegram_topic_mode_enabled(self, source: SessionSource) -> bool: """Return whether Telegram DM topic mode is active for this chat.""" if source.platform != Platform.TELEGRAM or source.chat_type != "dm": return False session_db = getattr(self, "_session_db", None) if session_db is None: return False # Runs off-loop (always via asyncio.to_thread); use the sync handle. session_db = getattr(session_db, "_db", session_db) try: raw = session_db.is_telegram_topic_mode_enabled( chat_id=str(source.chat_id), user_id=str(source.user_id), profile_name=self._telegram_topic_profile_name(source), ) except Exception: logger.debug("Failed to read Telegram topic mode state", exc_info=True) return False # Only honor a real True from the SessionDB. Any other value # (including MagicMock instances from test fixtures that didn't # opt into topic mode) means topic mode is off for this chat. return raw is True # Telegram's General (pinned top) topic in forum-enabled private chats. # Bot API behavior varies: some clients omit message_thread_id for # General, others send "1". Treat both as "root" for lobby/lane purposes. _TELEGRAM_GENERAL_TOPIC_IDS = frozenset({"", "1"}) def _is_telegram_topic_root_lobby(self, source: SessionSource) -> bool: """True for the main Telegram DM (or General topic) when topic mode has made it a lobby.""" if source.platform != Platform.TELEGRAM or source.chat_type != "dm": return False if not self._telegram_topic_mode_enabled(source): return False tid = str(source.thread_id or "") return tid in self._TELEGRAM_GENERAL_TOPIC_IDS def _is_telegram_topic_lane(self, source: SessionSource) -> bool: """True for a user-created Telegram private-chat topic lane.""" if source.platform != Platform.TELEGRAM or source.chat_type != "dm": return False if not self._telegram_topic_mode_enabled(source): return False tid = str(source.thread_id or "") if not tid or tid in self._TELEGRAM_GENERAL_TOPIC_IDS: return False return True _TELEGRAM_LOBBY_REMINDER_COOLDOWN_S = 30.0 def _telegram_topic_cooldown_key(self, source: SessionSource) -> Optional[str]: """Cooldown key for topic-mode cooldowns: (profile, chat_id). Profiles sharing a Telegram private chat_id under multiplex must not suppress each other's lobby reminders / capability hints (#76423). """ chat_id = str(source.chat_id or "") if not chat_id: return None return f"{self._telegram_topic_profile_name(source)}:{chat_id}" def _should_send_telegram_lobby_reminder(self, source: SessionSource) -> bool: """Rate-limit root-DM lobby reminders to one message per cooldown window. A user who forgets multi-session mode is enabled and types several prompts in the root DM would otherwise get a reminder for every message. Cap it so the first one lands and the rest stay quiet. """ if not hasattr(self, "_telegram_lobby_reminder_ts"): self._telegram_lobby_reminder_ts = {} key = self._telegram_topic_cooldown_key(source) if not key: return True import time as _time now = _time.monotonic() last = self._telegram_lobby_reminder_ts.get(key, 0.0) if now - last < self._TELEGRAM_LOBBY_REMINDER_COOLDOWN_S: return False self._telegram_lobby_reminder_ts[key] = now return True def _telegram_topic_root_lobby_message(self) -> str: return ( "This main chat is reserved for system commands.\n\n" "To start a new Hermes chat, open the All Messages topic at the top " "of this bot interface and send any message there. Telegram will " "create a new topic for that message; each topic works as an " "independent Hermes session." ) def _telegram_topic_root_new_message(self) -> str: return ( "To start a new parallel Hermes chat, open the All Messages topic " "at the top of this bot interface and send any message there. " "Telegram will create a new topic for it.\n\n" "Each topic is an independent Hermes session. Use /new inside an " "existing topic only if you want to replace that topic's current session." ) def _telegram_topic_new_header(self, source: SessionSource) -> Optional[str]: if not self._is_telegram_topic_lane(source): return None return ( "Started a new Hermes session in this topic.\n\n" "Tip: for parallel work, open All Messages and send a message there " "to create a separate topic instead of using /new here. /new replaces " "the session attached to the current topic." ) def _record_telegram_topic_binding( self, source: SessionSource, session_entry, ) -> None: """Persist the Telegram topic -> Hermes session binding for topic lanes.""" session_db = getattr(self, "_session_db", None) if session_db is None or not source.chat_id or not source.thread_id: return # Runs off-loop (always via asyncio.to_thread); use the sync handle. session_db = getattr(session_db, "_db", session_db) session_db.bind_telegram_topic( chat_id=str(source.chat_id), thread_id=str(source.thread_id), user_id=str(source.user_id or ""), session_key=session_entry.session_key, session_id=session_entry.session_id, profile_name=self._telegram_topic_profile_name(source), ) def _sync_telegram_topic_binding( self, source: SessionSource, session_entry, *, reason: str, ) -> None: """Update the topic binding to point at ``session_entry.session_id``. Telegram topic lanes persist a (chat_id, thread_id) -> session_id row so reopening a topic in a fresh process resumes the right Hermes session. When compression rotates ``session_entry.session_id`` mid-turn, the binding goes stale and the next inbound message in that topic reloads the oversized parent transcript instead of the compressed child, retriggering preflight compression — sometimes in a loop (#20470, #29712, #33414). """ if not self._is_telegram_topic_lane(source): return try: self._record_telegram_topic_binding(source, session_entry) except Exception: logger.debug( "telegram topic binding refresh failed (%s)", reason, exc_info=True, ) def _recover_telegram_topic_thread_id( self, source: SessionSource, ) -> Optional[str]: """Pin DM-topic routing to the user's last-active topic. Telegram can omit ``message_thread_id`` or surface General (``1``) for some topic-mode DM replies. In those lobby-shaped cases, keep the conversation attached to the user's most-recent bound topic. Do not rewrite a non-lobby, previously-unbound thread id: a newly created Telegram DM topic is also "unknown" until the first inbound message is recorded, and rewriting it would send that brand-new topic's answer into an older lane. Returns None to leave the source alone. """ if ( source.platform != Platform.TELEGRAM or source.chat_type != "dm" or not source.chat_id or not source.user_id or not self._telegram_topic_mode_enabled(source) ): return None inbound = str(source.thread_id or "") is_lobby = not inbound or inbound in self._TELEGRAM_GENERAL_TOPIC_IDS if not is_lobby: # A non-lobby, unknown thread_id is most likely the first message in # a brand-new Telegram DM topic. Preserve it so it can be recorded # as a new independent lane below instead of hijacking the latest # existing topic binding. return None session_db = getattr(self, "_session_db", None) if session_db is None: return None # Runs off-loop (always via asyncio.to_thread); use the sync handle. session_db = getattr(session_db, "_db", session_db) try: bindings = session_db.list_telegram_topic_bindings_for_chat( chat_id=str(source.chat_id), profile_name=self._telegram_topic_profile_name(source), ) except Exception: logger.debug("topic-recover: read failed", exc_info=True) return None if not bindings: return None user_id = str(source.user_id) for b in bindings: # newest-first if str(b.get("user_id") or "") == user_id: recovered = str(b.get("thread_id") or "") if recovered and recovered != inbound: return recovered return None return None def _normalize_source_for_session_key( self, source: SessionSource, ) -> SessionSource: """Apply Telegram DM topic recovery to a source for session-key purposes. ``_handle_message_with_agent`` rewrites ``source.thread_id`` via ``_recover_telegram_topic_thread_id`` *before* deriving the session key for a normal message turn (a lobby/stripped reply gets pinned to the user's last-active topic). Session-scoped command handlers like ``/model`` and ``/reasoning`` derive their override key from the raw inbound ``event.source``, which skips that recovery — so the override is stored under a different key than the next message turn reads, and the override is silently dropped on Telegram forum topics and after compression session splits (#30479). Returns a recovery-normalized copy when a rewrite applies, otherwise the original source unchanged. Always derive the override storage key from the result so storage and read use an identical key. """ try: recovered = self._recover_telegram_topic_thread_id(source) except Exception: return source if recovered is None: return source return dataclasses.replace(source, thread_id=recovered) def _resolve_session_agent_runtime( self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None, user_config: Optional[dict] = None, ) -> tuple[str, dict]: """Resolve model/runtime for a session. Priority (highest first): session ``/model`` → ``channel_overrides`` → global config/env (``_resolve_gateway_model(user_config)`` and default provider resolution). """ resolved_session_key = session_key if not resolved_session_key and source is not None: try: resolved_session_key = self._session_key_for_source(source) except Exception: resolved_session_key = None model = _resolve_gateway_model(user_config) if resolved_session_key: self._rehydrate_session_model_override(resolved_session_key) _override_state = ( self._peek_session_state(resolved_session_key) if resolved_session_key else None ) override = ( _override_state.conversation.model_override if _override_state else None ) if override: override_model = override.get("model", model) override_runtime = { "provider": override.get("provider"), "requested_provider": override.get("requested_provider"), "api_key": override.get("api_key"), "base_url": override.get("base_url"), "api_mode": override.get("api_mode"), "max_tokens": override.get("max_tokens"), "credential_pool": override.get("credential_pool"), "request_overrides": override.get("request_overrides"), "capabilities": dict(override.get("capabilities") or {}), } if override_runtime.get("api_key"): if override_runtime.get("credential_pool") is None: override_runtime["credential_pool"] = _credential_pool_for_provider( override.get("provider") ) logger.debug( "Session model override (fast): session=%s config_model=%s -> override_model=%s provider=%s", resolved_session_key or "", model, override_model, override_runtime.get("provider"), ) return override_model, override_runtime # Override exists but has no api_key — fall through to env-based # resolution and apply model/provider from the override on top. logger.debug( "Session model override (no api_key, fallback): session=%s config_model=%s override_model=%s", resolved_session_key or "", model, override_model, ) else: logger.debug( "No session model override: session=%s config_model=%s override_keys=%s", resolved_session_key or "", model, [ _key for _key, _st in list(self._sessions_map().items()) if _st.conversation.model_override is not None ][:5] or "[]", ) runtime_kwargs = _resolve_runtime_agent_kwargs() runtime_model = runtime_kwargs.pop("model", None) if runtime_model: logger.info( "Runtime provider supplied explicit model override: %s -> %s", model, runtime_model, ) model = runtime_model cfg = getattr(self, "config", None) if cfg and source is not None: chat_id = str(source.chat_id) if source.chat_id else "" thread_id = ( str(source.thread_id) if getattr(source, "thread_id", None) else None ) parent_id = ( str(source.parent_chat_id) if getattr(source, "parent_chat_id", None) else None ) ch = _get_channel_override( cfg, source.platform, chat_id, thread_id=thread_id, parent_id=parent_id, ) if ch: if ch.model: model = ch.model if ch.provider: runtime_kwargs = _resolve_runtime_agent_kwargs_for_provider( ch.provider ) ch_runtime_model = runtime_kwargs.pop("model", None) # Only adopt the provider's bundled model when the override # did not specify an explicit model. if ch_runtime_model and not ch.model: model = ch_runtime_model if override and resolved_session_key: model, runtime_kwargs = self._apply_session_model_override( resolved_session_key, model, runtime_kwargs ) # When the config has no model.default but a provider was resolved # (e.g. user ran `hermes auth add openai-codex` without `hermes model`), # fall back to the provider's first catalog model so the API call # doesn't fail with "model must be a non-empty string". if not model and runtime_kwargs.get("provider"): try: from hermes_cli.models import get_default_model_for_provider model = get_default_model_for_provider(runtime_kwargs["provider"]) if model: logger.info( "No model configured — defaulting to %s for provider %s", model, runtime_kwargs["provider"], ) except Exception: pass # Final safety net (#35314): if resolution still produced an empty # model — e.g. a transient config-cache miss during a post-interrupt # recovery turn returned an empty user_config — reuse the last model we # successfully resolved for this session (or, failing that, the most # recent one resolved process-wide). Building an agent with model="" # makes every API call fail HTTP 400 "No models provided" and the # session goes silent until the user manually re-sends. ``getattr`` # guards against bare test runners built via ``object.__new__``. if not model: _lr_state = ( self._peek_session_state(resolved_session_key) if resolved_session_key else None ) _lr_star = self._peek_session_state("*") _recovered = ( (_lr_state.conversation.last_resolved_model if _lr_state else "") or (_lr_star.conversation.last_resolved_model if _lr_star else "") ) if _recovered: logger.warning( "Empty model resolved for session=%s — recovering " "last-known-good model %s (config read likely returned " "empty; see #35314)", resolved_session_key or "", _recovered, ) model = _recovered elif model: # Cache the good resolution for future recovery turns. if resolved_session_key: self._session_state( resolved_session_key ).conversation.last_resolved_model = model self._session_state("*").conversation.last_resolved_model = model return model, runtime_kwargs def _resolve_turn_agent_config(self, user_message: str, model: str, runtime_kwargs: dict) -> dict: """Build the effective model/runtime config for a single turn. Always uses the session's primary model/provider. If `/fast` is enabled and the model supports Priority Processing / Anthropic fast mode, attach `request_overrides` so the API call is marked accordingly. Per-provider ``request_overrides`` resolved by ``resolve_runtime_provider`` (e.g. a ``custom_providers`` ``extra_body`` carrying ``chat_template_kwargs``) are preserved here and merged *under* the fast-mode overrides, so a provider's configured request body still reaches the model on the gateway turn path. """ from hermes_cli.models import resolve_fast_mode_overrides runtime = { "api_key": runtime_kwargs.get("api_key"), "base_url": runtime_kwargs.get("base_url"), "provider": runtime_kwargs.get("provider"), "requested_provider": runtime_kwargs.get("requested_provider"), "api_mode": runtime_kwargs.get("api_mode"), "command": runtime_kwargs.get("command"), "args": list(runtime_kwargs.get("args") or []), "credential_pool": runtime_kwargs.get("credential_pool"), "max_tokens": runtime_kwargs.get("max_tokens"), "capabilities": dict(runtime_kwargs.get("capabilities") or {}), } base_request_overrides = dict(runtime_kwargs.get("request_overrides") or {}) route = { "model": model, "runtime": runtime, "signature": ( model, runtime["provider"], runtime["requested_provider"], runtime["base_url"], runtime["api_mode"], runtime["command"], tuple(runtime["args"]), ), } # Provider-level request_overrides (e.g. a custom_providers extra_body) # resolved upstream by resolve_runtime_provider(). These were being # dropped by the runtime whitelist above, so a custom provider's # configured extra_body (chat_template_kwargs, etc.) never reached the # model on the gateway path -- only /fast service-tier overrides did. service_tier = getattr(self, "_service_tier", None) if service_tier != "priority": # None (normal) or auto/cold — the bounded window is applied per # request by agent.fast_mode, not pinned into request_overrides. route["request_overrides"] = base_request_overrides return route try: overrides = resolve_fast_mode_overrides( route["model"], provider=runtime["provider"], base_url=runtime["base_url"], ) except Exception: overrides = None # Fast-mode overrides (service_tier / speed) are top-level keys and do # not collide with extra_body; deep-merge them over the provider overrides. route["request_overrides"] = _deep_merge_request_overrides( base_request_overrides, overrides or {}, ) return route def _sync_session_model_from_agent(self, session_id: str, agent: Any) -> None: """Persist the runtime model/provider actually used by a gateway turn. Provider fallback can switch ``agent.model``/``agent.provider`` after the session row was created. Keep the session DB metadata in sync so session lists, desktop/dashboard details, and follow-up session tooling report the backend that actually answered the latest turn. Called from the ``run_sync`` closure, which executes off the event loop in the executor thread — so the synchronous ``SessionDB`` (``_db``) is used directly rather than awaiting the AsyncSessionDB forwarder. """ if not session_id or agent is None or self._session_db is None: return model = getattr(agent, "model", None) if not model: return runtime = { "provider": getattr(agent, "provider", None), "base_url": getattr(agent, "base_url", None), "api_mode": getattr(agent, "api_mode", None), "fallback_active": bool(getattr(agent, "_fallback_activated", False)), } runtime = {k: v for k, v in runtime.items() if v not in (None, "")} try: db = self._session_db._db row = db.get_session(session_id) if not row: return current_model = row.get("model") raw_config = row.get("model_config") try: config = json.loads(raw_config) if raw_config else {} except Exception: config = {} if not isinstance(config, dict): config = {} gateway_runtime = dict(config.get("gateway_runtime") or {}) if current_model == model and all( gateway_runtime.get(k) == v for k, v in runtime.items() ): return config["gateway_runtime"] = runtime db.update_session_meta(session_id, json.dumps(config), model=model) except Exception: logger.debug("Failed to sync gateway session model metadata", exc_info=True) async def _handle_reaction_event(self, ctx: Dict[str, Any]) -> None: """Fan a normalised platform reaction event out to the HookRegistry. Adapters call this via ``set_reaction_handler`` for every platform-native reaction event they surface. The adapter-supplied ``event_name`` ("reaction:added" / "reaction:removed") becomes the hook event so user hooks subscribe with the same name scheme as the existing ``agent:*`` family. Errors never block the adapter's event loop — the hook contract is non-blocking. """ event_name = str(ctx.get("event_name") or "reaction:added") try: await self.hooks.emit(event_name, ctx) except Exception: logger.debug("[Gateway] reaction hook emit failed", exc_info=True) async def _handle_adapter_fatal_error(self, adapter: BasePlatformAdapter) -> None: """React to an adapter failure after startup. If the error is retryable (e.g. network blip, DNS failure), queue the platform for background reconnection instead of giving up permanently. The notification arrives on the failing adapter's own polling task, and the disconnect inside the handler can cancel that task mid-flight: disconnect()'s current-task guard misses it because _safe_adapter_disconnect runs the close in a wrapper task. A cancelled handler dies between the fatal log and the reconnect queue, silently stranding the platform (observed 2026-07-21: telegram popped from adapters but never queued after a travel network outage). Run the real work in a detached task that adapter teardown cannot cancel. """ tasks = getattr(self, "_fatal_handler_tasks", None) if tasks is None: tasks = self._fatal_handler_tasks = set() task = asyncio.create_task(self._handle_adapter_fatal_error_detached(adapter)) tasks.add(task) task.add_done_callback(tasks.discard) # Await so callers that expect completion still get it — but through # shield(): Task.cancel() on the caller also cancels the future it is # awaiting (_fut_waiter), so a plain `await task` would tunnel the # cancellation straight into the "detached" task. shield() absorbs # it: the caller sees CancelledError, the handler runs to completion. await asyncio.shield(task) def _queue_retryable_fatal_platform(self, adapter: BasePlatformAdapter) -> bool: """Queue a retryable fatal adapter for background reconnection. Returns True when the platform was newly queued. Idempotent if already queued. Must not await: callers invoke this *before* any disconnect await so a wedged close cannot strand the platform (#80598). """ if not adapter.fatal_error_retryable: return False platform_config = self.config.platforms.get(adapter.platform) if not platform_config: return False if adapter.platform in self._failed_platforms: # Nothing to enqueue -- but "already queued" is precisely the state # in which the watcher has had time to die, and the enqueue branch # below holds the ONLY call to _ensure_reconnect_watcher_running(). # # _spawn_supervised auto-restarts the watcher after a crash (#71758), # but only _MAX_SUPERVISED_RESTARTS times in rapid succession; past # that it logs "giving up restarts" and the watcher stays dead # forever. _ensure_reconnect_watcher_running is the documented # backstop for exactly that budget exhaustion (#70344) -- and it was # unreachable for a platform already in the queue, which is the only # kind of platform the watcher can have been retrying long enough to # exhaust it on. # # The result is a silent permanent outage: nothing retries, and the # stranded check in _handle_adapter_fatal_error_detached deliberately # treats a queued platform as safe, so the process never restarts # either (#90386). self._ensure_reconnect_watcher_running() return False self._failed_platforms[adapter.platform] = { "config": platform_config, "attempts": 0, "next_retry": time.monotonic(), "queued_at": time.monotonic(), "credential_claim": self._adapter_credential_claim( adapter.platform, adapter ), "listener_claim": self._adapter_listener_claim( adapter.platform, adapter ), } logger.info( "%s queued for background reconnection", adapter.platform.value, ) # Ensure the reconnect watcher is alive — if it died (e.g. from # exhausting its restart budget), respawn it so queued platforms # are not permanently stranded (#70344). self._ensure_reconnect_watcher_running() return True async def _handle_adapter_fatal_error_detached( self, adapter: BasePlatformAdapter ) -> None: """Run the fatal handler; if the platform still ends up stranded (not reconnected, not queued, not intentionally disabled), exit the gateway with failure so the service manager restarts it instead of leaving a silent partial outage.""" try: # Outer hard deadline (#80598): even with queue-before-disconnect, # a hang anywhere in the impl (status write side effects, detach # races, etc.) must not leave this task wedged forever — the # stranded check in ``finally`` only runs when we return. timeout = self._adapter_disconnect_timeout_secs() if timeout <= 0: await self._handle_adapter_fatal_error_impl(adapter) else: # Disconnect budget plus a small overhead for queue/status # bookkeeping. Keep the additive proportional so tests that # shrink the disconnect timeout still finish promptly. outer = timeout + min(2.0, max(0.05, timeout)) completed = await self._await_adapter_cleanup_with_timeout( self._handle_adapter_fatal_error_impl(adapter), outer, ) if not completed: logger.error( "Fatal-error handling for %s timed out after %.1fs; " "ensuring reconnect queue is populated", adapter.platform.value, outer, ) self._queue_retryable_fatal_platform(adapter) except asyncio.CancelledError: # Best-effort queue before re-raising: a cancelled fatal handler # must not strand a retryable platform (#80598). try: self._queue_retryable_fatal_platform(adapter) except Exception: logger.debug( "Failed to queue %s after fatal-handler cancellation", adapter.platform.value, exc_info=True, ) raise except Exception: logger.exception( "Fatal-error handling for %s raised unexpectedly", adapter.platform.value, ) # Best-effort queue so an unexpected raise mid-handler cannot # leave a retryable platform permanently deaf (#80598). try: self._queue_retryable_fatal_platform(adapter) except Exception: logger.debug( "Failed to queue %s after fatal-handler exception", adapter.platform.value, exc_info=True, ) finally: platform = adapter.platform shutdown_event = getattr(self, "_shutdown_event", None) stranded = ( adapter.fatal_error_retryable and platform not in self.adapters and platform not in getattr(self, "_failed_platforms", {}) and not (shutdown_event is not None and shutdown_event.is_set()) ) if stranded: logger.error( "%s adapter was lost without entering the reconnection " "queue; exiting gateway so the service manager restarts it.", platform.value, ) self._exit_reason = ( f"{platform.value} adapter lost without reconnection queue" ) self._exit_with_failure = True await self.stop() async def _handle_adapter_fatal_error_impl(self, adapter: BasePlatformAdapter) -> None: # Snapshot the current owner of this platform slot before doing # anything else. If it's neither this adapter nor empty, a different # adapter has already taken over (e.g. this is a delayed notification # from a background retry chain that raced with, and lost to, a # reconnect that already succeeded). Acting on a stale notification # would overwrite an already-healthy platform's runtime status and # incorrectly re-queue it for reconnection, so bail out before any of # that happens. existing = self.adapters.get(adapter.platform) if existing is not None and existing is not adapter: logger.debug( "Ignoring stale fatal error from a superseded %s adapter instance: %s", adapter.platform.value, adapter.fatal_error_code or "unknown", ) return logger.error( "Fatal %s adapter error (%s): %s", adapter.platform.value, adapter.fatal_error_code or "unknown", adapter.fatal_error_message or "unknown error", ) # Phase 7 Unit 7d-B: a relay credential revoked by opt-out is not an # error to retry — render it as a clean "disabled" state, not red # "fatal"/"retrying". (The code is set non-retryable, so it also drops # out of the reconnect queue below.) if adapter.fatal_error_code == "relay_disabled": platform_state = "disabled" elif adapter.fatal_error_retryable: platform_state = "retrying" else: platform_state = "fatal" self._update_platform_runtime_status( adapter.platform.value, platform_state=platform_state, error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message, ) if existing is adapter: # Claim this adapter for teardown before awaiting disconnect() — # a second fatal-error notification for the same adapter (e.g. # from a concurrent recovery path) would otherwise still see # itself as "existing" during the await below and disconnect() # the same object twice. self.adapters.pop(adapter.platform, None) self.delivery_router.adapters = self.adapters # Queue retryable failures BEFORE any disconnect await (#80598). # A half-dead transport can wedge native close() (or swallow # CancelledError inside it) so the previous "disconnect then queue" # order left platforms permanently deaf inside a live process even # after the network recovered. Populate the queue first so the # reconnect watcher always has work; teardown is best-effort after. self._queue_retryable_fatal_platform(adapter) if existing is adapter: # A half-closed transport can wedge an adapter's native close() # indefinitely. Reuse the shutdown-path timeout so this runtime # fatal handler always returns to the stay-alive / stranded path. await self._safe_adapter_disconnect(adapter, adapter.platform) if not self.adapters and not self._failed_platforms: self._exit_reason = adapter.fatal_error_message or "All messaging adapters disconnected" if adapter.fatal_error_retryable: self._exit_with_failure = True logger.error("No connected messaging platforms remain. Shutting down gateway for service restart.") else: logger.error("No connected messaging platforms remain. Shutting down gateway cleanly.") await self.stop() elif not self.adapters and self._failed_platforms: # All platforms are down and queued for background reconnection. # Keep the gateway alive so: # • cron jobs still run # • the reconnect watcher can recover platforms when the # underlying problem clears (proxy comes back, user runs # `hermes whatsapp`, etc.) # We used to exit-with-failure here to trigger systemd restart, # but that converted a transient outage into a restart loop and # killed in-process state every time. The reconnect watcher # already handles long-running recovery — let it do its job. logger.warning( "No connected messaging platforms remain, but %d platform(s) " "queued for reconnection — gateway staying alive, watcher will " "retry in background.", len(self._failed_platforms), ) def _request_clean_exit(self, reason: str) -> None: self._exit_cleanly = True self._exit_reason = reason self._shutdown_event.set() def _running_agent_count(self) -> int: return len(self._running_agents) def _active_work_count(self) -> int: """All agent work the gateway must expose and drain as one total.""" return ( self._running_agent_count() + self._active_cron_job_count() + self._active_api_run_count() + self._active_deferred_agent_worker_count() ) def _active_cron_job_count(self) -> int: """Count of cron jobs currently executing, from the cron scheduler's own in-flight tracking (``cron.scheduler._running_job_ids``). Cron jobs run through a standalone ``AIAgent`` on the scheduler's own thread pool (``cron/scheduler.py::run_job``), entirely outside ``self._running_agents`` — the dict every OTHER active-work check on this class (``_running_agent_count``, ``_drain_active_agents``) reads. Without this, the shutdown drain is structurally blind to in-flight cron work: it can report ``active_at_start=0`` and proceed straight to killing tool subprocesses while a cron job's terminal command is still running (#60432). Best-effort: returns 0 if the cron module can't be imported (e.g. a minimal test double for this class). """ try: from cron.scheduler import get_running_job_ids return len(get_running_job_ids()) except Exception: return 0 def _active_api_run_count(self) -> int: """Count API-server work that is outside ``_running_agents``. The primary API server owns the sole HTTP listener. Secondary multiplex profiles cannot create an ``api_server`` adapter because it binds a port, so only the primary registry is a supported source of this work. """ try: adapter = getattr(self, "adapters", {}).get(Platform.API_SERVER) helper = getattr(adapter, "active_agent_work_count", None) return max(0, int(helper())) if callable(helper) else 0 except Exception: return 0 def _interrupt_api_server_runs(self, reason: str) -> int: """Interrupt API-server agents that are not in ``_running_agents``. Counterpart of ``_active_api_run_count()``: that method folds adapter-owned API work into the shutdown drain, so this one must reach the same agents when the drain times out. Duck-typed on the adapter so an older adapter (or a minimal test double for this class) without the hook is simply skipped rather than raising mid-shutdown. """ try: adapter = getattr(self, "adapters", {}).get(Platform.API_SERVER) helper = getattr(adapter, "interrupt_active_runs", None) return max(0, int(helper(reason))) if callable(helper) else 0 except Exception as exc: logger.debug("Failed interrupting api_server runs during shutdown: %s", exc) return 0 def _active_deferred_agent_worker_count(self) -> int: """Count executor workers that outlived their owning gateway turn. A timed-out hygiene compression keeps running in its executor thread. Some paths defer agent cleanup; the live Codex path keeps its cached agent. In both cases the turn can finish before the worker does, so ``_running_agents`` no longer represents it. Count the worker itself. """ workers = getattr(self, "_deferred_agent_workers", None) if not isinstance(workers, dict): return 0 return sum(1 for future in list(workers) if not future.done()) def _track_deferred_agent_worker( self, future: asyncio.Future, agent: Any, ) -> None: """Expose an executor worker to drain/interrupt until it really exits.""" workers = getattr(self, "_deferred_agent_workers", None) if workers is None: workers = {} self._deferred_agent_workers = workers workers[future] = agent def _discard_worker(done_future: asyncio.Future) -> None: workers.pop(done_future, None) # Some tracked workers intentionally outlive the coroutine that # started them and therefore have no later waiter. Consume their # terminal exception so asyncio does not emit an unhandled-future # warning after the worker eventually unwinds (#98973). if not done_future.cancelled(): try: done_future.exception() except Exception: pass future.add_done_callback(_discard_worker) def _interrupt_deferred_agent_workers(self, reason: str) -> int: """Request cancellation of detached executor-backed agent work.""" workers = getattr(self, "_deferred_agent_workers", None) if not isinstance(workers, dict): return 0 interrupted = 0 seen: set[int] = set() for future, agent in list(workers.items()): if future.done() or agent is None or id(agent) in seen: continue seen.add(id(agent)) try: request_hard_interrupt(agent, reason) interrupted += 1 except Exception as exc: logger.debug( "Failed interrupting deferred agent worker during shutdown: %s", exc, ) return interrupted # ── scale-to-zero idle detection / dormant-quiesce (Phase 0) ────────────── # The gateway-side BEHAVIOUR that consumes the relay scale-to-zero primitives # (gateway-gateway Phase 5). Pure logic lives in gateway/scale_to_zero.py; the # methods here bind it to the live runner/transport. See ~/nous/specs/ # scale-to-zero (decisions.md) for the design + the F12/F14 distinctions. def _scale_to_zero_has_live_background_work(self) -> bool: """Live background work that must block a suspend (D3/F7). Backgrounded delegate_task / kanban / terminal(background=true) are NOT counted by _running_agent_count(), but suspending mid-flight loses them. Checks the runner's own tracked tasks + the process registry's running processes + any pending process-completion watchers. PERMANENT supervised watchers (tagged _hermes_supervised_watcher by _spawn_supervised) are excluded: they live for the whole process — including the scale-to-zero watcher itself — so counting them would make this predicate True forever and the gateway could never go dormant. Verified live on staging (2026-08-12): an armed, fully idle instance never logged "going dormant" because ~9 supervised watchers sat in _background_tasks. Fly's coarse autostop used to mask this; with the gateway owning the suspend it became load-bearing. """ if any( not t.done() and not getattr(t, "_hermes_supervised_watcher", False) for t in self._background_tasks ): return True try: from tools.async_delegation import active_count if active_count() > 0: return True except Exception: # noqa: BLE001 - never let the idle check raise logger.debug("scale-to-zero async-delegation check failed", exc_info=True) try: from tools.process_registry import process_registry if process_registry.has_any_active(): return True if process_registry.pending_watchers: return True except Exception: # noqa: BLE001 - never let the idle check raise logger.debug("scale-to-zero bg-work check failed", exc_info=True) return False def _scale_to_zero_idle_timeout_seconds(self) -> float: from gateway.scale_to_zero import parse_idle_timeout_seconds raw = None try: user_cfg = _load_gateway_config() gw = user_cfg.get("gateway") if isinstance(user_cfg, dict) else None stz = gw.get("scale_to_zero") if isinstance(gw, dict) else None if isinstance(stz, dict): raw = stz.get("idle_timeout_minutes") except Exception: # noqa: BLE001 raw = None return parse_idle_timeout_seconds(raw) def _restart_loop_guard_config(self) -> tuple: """Return ``(max_restarts, window_seconds, max_gap_seconds)`` for the auto-resume restart-loop breaker (#30719, defense-3), read from ``gateway.restart_loop_guard`` in config.yaml with the module defaults as fallback. ``max_restarts <= 0`` disables the breaker. ``max_gap_seconds`` is the longest spacing between two consecutive restart-interrupted boots that still counts them as the same loop, so a crash cycle slower than ``window_seconds`` stays visible (#81642). """ from gateway import restart_loop_guard as _rlg max_restarts = _rlg.DEFAULT_MAX_RESTARTS window_seconds = _rlg.DEFAULT_WINDOW_SECONDS max_gap_seconds = _rlg.DEFAULT_MAX_GAP_SECONDS try: user_cfg = _load_gateway_config() gw = user_cfg.get("gateway") if isinstance(user_cfg, dict) else None rlg = gw.get("restart_loop_guard") if isinstance(gw, dict) else None if isinstance(rlg, dict): if isinstance(rlg.get("max_restarts"), int): max_restarts = rlg["max_restarts"] if isinstance(rlg.get("window_seconds"), int) and rlg["window_seconds"] > 0: window_seconds = rlg["window_seconds"] if ( isinstance(rlg.get("max_gap_seconds"), int) and rlg["max_gap_seconds"] > 0 ): max_gap_seconds = rlg["max_gap_seconds"] except Exception: # noqa: BLE001 pass return max_restarts, window_seconds, max_gap_seconds def _scale_to_zero_active_messaging_platforms(self) -> list: """ENABLED platforms that count for the relay-only arm gate (D1/F6). Two filters, both load-bearing: - enabled only: config.platforms is pre-seeded with disabled placeholders for the full platform catalog (the F25 bug). - MESSAGING only: non-messaging surfaces must not disarm scale-to-zero. The api_server is a loopback listener force-enabled by the presence of API_SERVER_KEY (which the Docker stage2 hook now generates for every container, so hosted instances ALWAYS have it enabled) — it holds no outbound socket and Chronos fires through it already reset the idle clock. Counting it made messaging_is_relay_only_or_absent False on every hosted instance, silently disarming the feature. Mirrors the non-messaging exclusion set used for handoff eligibility (see the `messaging_platforms` computation in _connect_platforms). """ if not self.config: return [] non_messaging = {Platform.LOCAL, Platform.API_SERVER, Platform.WEBHOOK} try: return [ p for p, pc in self.config.platforms.items() if getattr(pc, "enabled", False) and p not in non_messaging ] except Exception: # noqa: BLE001 return [] def _scale_to_zero_should_arm(self) -> bool: """Whether to start the idle watcher (D1/D11/§3.4(1)).""" from gateway.relay import relay_wake_url from gateway.scale_to_zero import ( messaging_is_relay_only_or_absent, scale_to_zero_enabled, should_arm, ) platforms = self._scale_to_zero_active_messaging_platforms() try: wake_url = relay_wake_url() except Exception: # noqa: BLE001 wake_url = None return should_arm( enabled=scale_to_zero_enabled(), relay_only_or_absent=messaging_is_relay_only_or_absent(platforms), wake_url=wake_url, ) def _log_scale_to_zero_not_armed_reason(self) -> None: """Log why the idle watcher did NOT arm — but only for an OPTED-IN instance. A non-opted instance (no HERMES_SCALE_TO_ZERO stamp) not arming is the normal case and must stay silent. When the Labs stamp IS set but the watcher still didn't arm, that's the surprising case worth one INFO line so "why won't it suspend/wake?" is a log grep, not a box-dive. """ from gateway.relay import relay_wake_url from gateway.scale_to_zero import ( messaging_is_relay_only_or_absent, scale_to_zero_enabled, ) try: enabled = scale_to_zero_enabled() if not enabled: return # not opted in — normal, stay quiet active = [ getattr(p, "value", p) for p in self._scale_to_zero_active_messaging_platforms() ] relay_only = messaging_is_relay_only_or_absent(active) try: wake_url = relay_wake_url() except Exception: # noqa: BLE001 wake_url = None logger.info( "scale-to-zero: NOT armed despite opt-in — " "relay_only_or_absent=%s (enabled platforms=%s), wake_url=%s. " "Need relay-only messaging + a registered wake URL.", relay_only, active or "none", "set" if wake_url else "MISSING", ) except Exception: # noqa: BLE001 - diagnostics must never block startup logger.debug("scale-to-zero: not-armed reason logging failed", exc_info=True) def _scale_to_zero_is_idle(self) -> bool: from gateway.scale_to_zero import is_idle # The FULL work aggregate, not _running_agent_count(): cron jobs run # on the scheduler's own thread pool and API-server runs live on the # adapter — both outside _running_agents (the #60432 blind spot), so # counting agents alone let a suspend land mid-cron-job. # # Fail-AWAKE accounting: the shared shutdown-drain counters # (_active_cron_job_count/_active_api_run_count) swallow exceptions to # 0, which is fine for a drain but unsafe for a suspend predicate — a # transient read failure would make live work look idle and reopen the # mid-job freeze. Here an unreadable source counts as work (sentinel 1) # so the machine stays awake until the source is readable again. try: from cron.scheduler import get_running_job_ids cron_count = len(get_running_job_ids()) except Exception: # noqa: BLE001 - unreadable source => assume busy logger.debug("scale-to-zero: cron work count unreadable — staying awake", exc_info=True) cron_count = 1 try: adapter = getattr(self, "adapters", {}).get(Platform.API_SERVER) helper = getattr(adapter, "active_agent_work_count", None) api_count = max(0, int(helper())) if callable(helper) else 0 except Exception: # noqa: BLE001 - unreadable source => assume busy logger.debug("scale-to-zero: api work count unreadable — staying awake", exc_info=True) api_count = 1 # An attached dashboard/desktop/TUI client is inbound activity too. It # lives in the DASHBOARD process, so it reaches us as a file mtime that # process refreshes on every WS frame (gateway/scale_to_zero.py). Fold # it into the inbound clock rather than adding a conjunct: the client # then gets the same idle_timeout grace after it disconnects as a chat # message does, and a lingering marker cannot pin the box (an old mtime # is outside idle_timeout just like an old _last_inbound_at). last_inbound = self._last_inbound_at try: from gateway.scale_to_zero import dashboard_client_last_seen seen = dashboard_client_last_seen() except Exception: # noqa: BLE001 - unreadable source => assume busy logger.debug("scale-to-zero: dashboard heartbeat unreadable — staying awake", exc_info=True) seen = time.time() if seen is not None and seen > last_inbound: last_inbound = seen return is_idle( active_work_count=self._running_agent_count() + cron_count + api_count, seconds_since_last_inbound=time.time() - last_inbound, idle_timeout_seconds=self._scale_to_zero_idle_timeout_seconds(), has_live_background_work=self._scale_to_zero_has_live_background_work(), ) def _scale_to_zero_note_real_inbound(self) -> None: """Stamp real inbound and restore lifecycle after a dormant wake. The watcher marks runtime status `draining` as it quiesces the relay, but dormancy is not the stop/restart drain path: the process remains alive and should present as running once real traffic wakes it and re-enters the gateway. Internal completion/replay events intentionally do not call this helper, so they do not keep an otherwise idle gateway awake. """ self._last_inbound_at = time.time() if getattr(self, "_scale_to_zero_cooldown_until", 0.0) > 0: try: self._update_runtime_status("running") except Exception: # noqa: BLE001 - status restoration is best-effort logger.debug("scale-to-zero: status restore failed", exc_info=True) self._scale_to_zero_cooldown_until = 0.0 def _relay_adapter_for_dormancy(self): """Return the connected RELAY adapter, if any (the one go_dormant targets).""" try: from gateway.platforms.base import Platform except Exception: # noqa: BLE001 return None return self.adapters.get(Platform.RELAY) async def _scale_to_zero_watcher(self, interval: float = 30.0) -> None: """Watch for idle, drive the relay dormant, then self-suspend the machine. Started ONLY when _scale_to_zero_should_arm() (opted in via the Labs HERMES_SCALE_TO_ZERO stamp + relay-only/absent messaging + a wakeUrl). On a sustained idle window it runs the DORMANT sequence (D12/F12/F14): - mark runtime status `draining` (composes with the existing state machine, §3.4(6); does NOT set _running=False), - relay adapter.go_dormant() — going_idle->ack + supervisor-preserving socket close (NOT disconnect(), NOT the run.py stop path), - deliberately NO mark_resume_pending (D13 — suspend preserves RAM), - THEN suspend this machine through the local flaps socket (gateway.scale_to_zero.suspend_self). The gateway owns the suspend because Fly Proxy autostop judges idle on INBOUND connections only: it cannot see an in-flight agent turn (outbound-only LLM traffic) and, since the mid-2026 proxy change, an open outbound relay socket no longer holds the machine awake — autostop:"suspend" would freeze the machine mid-job or before the relay flip (the buffered-event black hole). NAS therefore provisions scale-to-zero machines with autostop:"off"; the suspend only ever happens HERE, strictly after the idle predicate held and the dormant quiesce completed. Autostart stays platform-side: the connector's wakeUrl poke (Fly-proxied) wakes the machine, the preserved reconnect supervisor re-dials, and the connector drains the buffered backlog. After driving dormant we set a re-arm cooldown so a wake's drained backlog isn't immediately re-quiesced. Off-Fly (no flaps socket / machine identity) the watcher does not quiesce at all: the platform suspends on its own timer, so the gateway stays connected and serving until the freeze lands. """ await asyncio.sleep(min(interval, 30.0)) # let startup settle while self._running: try: await asyncio.sleep(interval) if not self._running: return if time.time() < self._scale_to_zero_cooldown_until: continue if not self._scale_to_zero_is_idle(): continue adapter = self._relay_adapter_for_dormancy() if adapter is None: continue go_dormant = getattr(adapter, "go_dormant", None) if not callable(go_dormant): continue # Quiesce only when a suspend can follow it. Off-Fly the platform # owns the freeze on its own timer, so this does not bring it any # closer, and go_dormant()'s socket close arms the reconnect # supervisor: it re-dials ~1.4s later and the drain clears the # flip, every cooldown. The destination is then unflipped when the # freeze lands, and inbound is dropped instead of buffered. Stay # connected and let the connector's orphan detection adopt the # destination once the platform freezes us. from gateway.scale_to_zero import self_suspend_available if not self_suspend_available(): if not self._scale_to_zero_no_suspend_logged: self._scale_to_zero_no_suspend_logged = True logger.info( "scale-to-zero: idle, but this platform suspends on " "its own timer (no in-machine suspend API); staying " "connected rather than quiescing" ) continue logger.info( "scale-to-zero: gateway idle for >= %.0fs — going dormant " "(relay buffered, socket closed) then self-suspending", self._scale_to_zero_idle_timeout_seconds(), ) try: self._update_runtime_status("draining") except Exception: # noqa: BLE001 - status is best-effort logger.debug("scale-to-zero: status mark failed", exc_info=True) dormant_ok = True try: result = go_dormant() if asyncio.iscoroutine(result): await result except Exception: # noqa: BLE001 - dormancy is best-effort dormant_ok = False logger.debug("scale-to-zero: go_dormant failed", exc_info=True) # 0.F: after a wake the drained inbound updates _last_inbound_at, # but give it a window so we don't immediately re-go-dormant on the # same idle reading before traffic lands. self._scale_to_zero_cooldown_until = time.time() + max(interval, 60.0) # Self-suspend ONLY after a clean quiesce: the relay flip must be # set (buffered delivery + wake poke armed) before the freeze, or # inbound events black-hole while we sleep. Re-check idle one last # time — inbound may have landed during the quiesce await. if not dormant_ok: continue if not self._scale_to_zero_is_idle(): logger.info( "scale-to-zero: inbound arrived during quiesce — skipping suspend" ) continue await self._scale_to_zero_self_suspend() except asyncio.CancelledError: raise except Exception: # noqa: BLE001 - the watcher must never crash the gateway logger.debug("scale-to-zero watcher iteration error", exc_info=True) async def _scale_to_zero_self_suspend(self) -> None: """Suspend this Fly machine via the local flaps socket (fail-awake). Runs the blocking unix-socket call in a worker thread so the event loop stays live right up to the kernel freeze. On success the process is frozen shortly after — nothing meaningful runs until the wake resume. Off-Fly (self_suspend_available() False) this is a silent no-op. """ from gateway.scale_to_zero import self_suspend_available, suspend_self try: if not self_suspend_available(): logger.debug( "scale-to-zero: flaps socket / machine identity absent — " "dormant without platform suspend" ) return accepted = await asyncio.to_thread(suspend_self) if not accepted: logger.warning( "scale-to-zero: self-suspend not accepted — machine stays " "awake (fail-awake); will retry on the next idle window" ) except Exception: # noqa: BLE001 - suspend is best-effort, never crash logger.debug("scale-to-zero: self-suspend failed", exc_info=True) def _status_action_label(self) -> str: return "restart" if self._restart_requested else "shutdown" def _status_action_gerund(self) -> str: return "restarting" if self._restart_requested else "shutting down" def _queue_during_drain_enabled( self, busy_input_mode: Optional[str] = None ) -> bool: # Both "queue" and "steer" modes imply the user doesn't want messages # to be lost during restart — queue them for the newly-spawned gateway # process to pick up. "interrupt" mode drops them (current behaviour). mode = busy_input_mode or self._busy_input_mode return self._restart_requested and mode in {"queue", "steer"} # -------- /queue FIFO helpers -------------------------------------- # /queue must produce one full agent turn per invocation, in FIFO # order, with no merging. The adapter's _pending_messages dict is a # single "next-up" slot (shared with photo-burst follow-ups), so we # use it for the head of the queue and an overflow list for the # tail. Enqueue puts new items in the slot when free, otherwise in # the overflow. Promotion (called after each run's drain) moves the # next overflow item into the slot so the following recursion picks # it up. Clearing happens on /new and /reset via # _handle_reset_command. def _enqueue_fifo(self, session_key: str, queued_event: "MessageEvent", adapter: Any) -> None: """Append a /queue event to the FIFO chain for a session.""" if adapter is None: return pending_slot = getattr(adapter, "_pending_messages", None) if pending_slot is None: return if session_key in pending_slot: self._session_state(session_key).conversation.queued_events.append( queued_event ) else: pending_slot[session_key] = queued_event def _promote_queued_event( self, session_key: str, adapter: Any, pending_event: Optional["MessageEvent"], ) -> Optional["MessageEvent"]: """Promote the next overflow item after the slot was drained. Called at the drain site after _dequeue_pending_event consumed (or failed to consume) the slot. If there's an overflow item: - When pending_event is None (slot was empty), return the overflow head as the new pending_event. - When pending_event already exists (slot was populated by an interrupt follow-up or similar), stage the overflow head in the slot so the NEXT recursion picks it up. Returns the (possibly updated) pending_event for drain to use. """ _q_state = self._peek_session_state(session_key) overflow = _q_state.conversation.queued_events if _q_state else None if not overflow: return pending_event next_queued = overflow.pop(0) if pending_event is None: return next_queued if adapter is not None and hasattr(adapter, "_pending_messages"): adapter._pending_messages[session_key] = next_queued else: # No adapter — push back so we don't silently drop the item. overflow.insert(0, next_queued) return pending_event def _queue_depth(self, session_key: str, *, adapter: Any = None) -> int: """Total pending /queue items for a session — slot + overflow.""" _q_state = self._peek_session_state(session_key) depth = len(_q_state.conversation.queued_events) if _q_state else 0 if adapter is not None and session_key in getattr(adapter, "_pending_messages", {}): depth += 1 return depth def _rescue_orphaned_overflow( self, session_key: str, adapter: Any ) -> Optional["MessageEvent"]: """Pop the oldest orphaned FIFO overflow event for an idle session (#99882). The FIFO overflow (``queued_events``) drains only at the post-turn promotion site (``_promote_queued_event`` inside the ``_run_agent`` drain). When a busy window ends without that drain running — the #99882 shape: a follow-up queued during compression-in-flight lands in overflow, compression finishes, the slot event's turn runs, but the drain recursion exits before promoting (or the busy window ends through an exception / interrupt / generation-bump exit that never reaches the promotion site) — the overflow entries are silently orphaned: never dispatched, never persisted, never logged. This rescue runs at the point where a NEW event arrives for a session that is NOT busy (the idle entry in ``_process_message_priority``). If the session went idle with a populated overflow, the oldest orphan is returned so the caller runs it as THIS turn, and the next orphan (if any) is staged into the slot so the post-turn drain continues the chain in arrival order (#28503). The caller then enqueues the incoming event behind the chain via ``_enqueue_fifo``. The returned event is REMOVED from both stores: leaving it in the slot while it also runs as the current turn would make the post-turn ``_dequeue_pending_event`` run it a second time. Returns the orphaned event to run now, or ``None`` when there is nothing to rescue (no overflow, slot occupied, or no slot storage). """ try: _q_state = self._peek_session_state(session_key) overflow = _q_state.conversation.queued_events if _q_state else None if not overflow: return None pending_slot = getattr(adapter, "_pending_messages", None) if not isinstance(pending_slot, dict) or pending_slot.get(session_key): # Slot occupied (busy) or no slot storage — promotion owns # this; do not fight it from the idle path. return None head = overflow.pop(0) # Keep the slot occupied for the rest of the chain so the drain # promotes in order and any mid-chain arrival routes to overflow # instead of jumping the queue (same invariant as the drain's # own _promote_queued_event). Only ONE event fits the slot. if overflow: pending_slot[session_key] = overflow.pop(0) logger.warning( "Rescued orphaned FIFO overflow event for idle session " "%s — it was queued during a busy window but the post-turn " "drain never promoted it (#99882)", session_key, ) if overflow: logger.warning( "%d overflow event(s) still queued for session %s after " "rescue staging (will drain via normal promotion)", len(overflow), session_key, ) return head except Exception: logger.debug("FIFO overflow rescue failed for %s", session_key, exc_info=True) return None @staticmethod def _is_goal_continuation_event(event_or_text: Any) -> bool: """Return True for synthetic /goal continuation turns. Goal continuations are normal queued user-role events, so pause/clear must distinguish them from real user /queue messages before removing or suppressing them. """ text = getattr(event_or_text, "text", event_or_text) or "" return str(text).startswith("[Continuing toward your standing goal]\nGoal:") def _clear_goal_pending_continuations(self, session_key: str, adapter: Any) -> int: """Remove queued synthetic /goal continuations for one session. User-issued /goal pause/clear can race with a continuation already queued by the judge. Remove only synthetic goal continuations while preserving normal /queue and user follow-up events. """ removed = 0 pending_slot = getattr(adapter, "_pending_messages", None) if adapter is not None else None if isinstance(pending_slot, dict): pending_event = pending_slot.get(session_key) if self._is_goal_continuation_event(pending_event): pending_slot.pop(session_key, None) removed += 1 _q_state = self._peek_session_state(session_key) overflow = _q_state.conversation.queued_events if _q_state else [] if overflow: kept = [] for queued_event in overflow: if self._is_goal_continuation_event(queued_event): removed += 1 else: kept.append(queued_event) _q_state.conversation.queued_events = kept return removed def _goal_still_active_for_session(self, session_id: str) -> bool: """Best-effort fresh DB check before running a queued continuation.""" if not session_id: return False try: from hermes_cli.goals import GoalManager return GoalManager(session_id=session_id).is_active() except Exception as exc: logger.debug("goal continuation: active-state recheck failed: %s", exc) return False def _update_runtime_status(self, gateway_state: Optional[str] = None, exit_reason: Optional[str] = None) -> None: try: from gateway.status import write_runtime_status write_runtime_status( gateway_state=gateway_state, exit_reason=exit_reason, restart_requested=self._restart_requested, active_agents=self._active_work_count(), ) except Exception: pass def _persist_active_agents(self) -> None: """Persist the live in-flight agent count to ``gateway_state.json``. Called at every turn boundary (a running-agent slot is claimed or released) so the dashboard ``/api/status`` readout reflects in-flight gateway turns in near-real-time. Without this the file is only rewritten on lifecycle transitions, so any ``active_agents`` read between transitions is stale (a turn could start and finish without the file ever moving). Deliberately passes ONLY ``active_agents`` — ``gateway_state`` and the other fields stay ``_UNSET`` so ``write_runtime_status``'s read-merge-write preserves the current lifecycle state (``running`` / ``draining`` / …). Passing ``gateway_state=None`` here would clobber it. Best-effort: a failed status write must never disrupt a turn. """ try: from gateway.status import write_runtime_status write_runtime_status(active_agents=self._active_work_count()) except Exception: pass # ------------------------------------------------------------------ # External drain control (NAS-driven quiesce-without-restart, Phase 2). # The dashboard's begin/cancel-drain endpoint writes/removes the # ``.drain_request.json`` marker (gateway/drain_control.py); this watcher # observes the marker and flips the gateway between accepting and refusing # NEW turns, WITHOUT exiting the process. Reversible by design (D4a): NAS # POSTs begin-drain, polls /api/status until active_agents hits 0, proceeds # with its lifecycle action, then (on cancel/abort) the marker is removed # and the gateway re-accepts turns. # ------------------------------------------------------------------ def _enter_external_drain(self) -> None: """Begin external drain: stop accepting new turns, flip state. Idempotent — re-entering while already draining is a no-op beyond a best-effort status re-write. In-flight turns are NOT interrupted (the whole point is to let them finish); only NEW turns are refused. """ if self._external_drain_active: return self._external_drain_active = True logger.info( "External drain ENGAGED (.drain_request.json present) — refusing " "new turns; %d in-flight turn(s) will finish. Process stays up.", self._active_work_count(), ) # Flip the persisted lifecycle state so /api/status.gateway_busy / # gateway_drainable track the drain. Preserve active_agents (the # read-merge keeps the live count); only the state changes. self._update_runtime_status("draining") def _exit_external_drain(self) -> None: """Cancel external drain: revert state, re-accept new turns. Idempotent. Only reverts to ``running`` when we are actually mid-drain AND not also shutting down (a real shutdown ``_draining`` must win — never resurrect a stopping gateway to ``running``). """ if not self._external_drain_active: return self._external_drain_active = False if self._draining or not self._running: # A shutdown drain is in progress / the loop has stopped — do not # clobber the terminal state back to running. logger.info( "External drain marker cleared during shutdown — not reverting " "to running (shutdown takes precedence)." ) return logger.info( "External drain RELEASED (.drain_request.json removed) — " "re-accepting new turns; gateway_state -> running." ) self._update_runtime_status("running") async def _drain_control_watcher(self, interval: float = 1.0) -> None: """Background task: reconcile gateway accept-state with the drain marker. Polls ``.drain_request.json`` (presence-based contract, gateway/drain_control.py). Marker present -> ``_enter_external_drain``; marker absent -> ``_exit_external_drain``. The 1s cadence bounds the observe-the-marker latency the live-validation gate checks (point a). Reconciles once at startup. A marker stamped with a PRIOR instantiation epoch (one that survived a machine restart on the durable HERMES_HOME volume — NS-570) is treated as absent by ``drain_requested`` and is NOT honoured; only a marker from the current instantiation flips the gateway into drain. Best-effort: any tick error is logged and the loop continues (a transient stat() failure must not wedge the gateway). """ from gateway.drain_control import drain_requested while self._running: try: # drain_requested() does a synchronous read_text() on the # marker file. At this 1s cadence that puts a blocking disk # read on the event loop ~86k times a day; when the host is # under I/O pressure a single read can stall for 30s+ and # take every platform heartbeat down with it. Off-thread it. if await asyncio.to_thread(drain_requested): self._enter_external_drain() # API and cron work live outside messaging's # _running_agents map. Refresh the aggregate while an # external caller polls this reversible drain state. self._persist_active_agents() else: self._exit_external_drain() except asyncio.CancelledError: raise except Exception as exc: logger.debug("Drain-control watcher tick error: %s", exc, exc_info=True) await asyncio.sleep(interval) def _update_platform_runtime_status( self, platform: str, *, platform_state: Optional[str] = None, error_code: Optional[str] = None, error_message: Optional[str] = None, needs_attention: Optional[bool] = None, retrying_since: Any = _UNSET, ) -> None: try: from gateway.status import write_runtime_status extra: Dict[str, Any] = {} if needs_attention is not None: extra["needs_attention"] = needs_attention if retrying_since is not _UNSET: extra["retrying_since"] = retrying_since write_runtime_status( platform=platform, platform_state=platform_state, error_code=error_code, error_message=error_message, **extra, ) except Exception: pass # ------------------------------------------------------------------ # Per-platform circuit breaker (pause/resume) — used by the reconnect # watcher when a retryable failure recurs past a threshold, and by the # /platform pause|resume slash command for manual control. # ------------------------------------------------------------------ def _pause_failed_platform(self, platform, *, reason: str = "") -> None: """Mark a queued platform as paused — keep it in ``_failed_platforms`` but stop the reconnect watcher from hammering it. Used by ``/platform pause `` for manual operator intervention. Paused platforms are surfaced in ``/platform list`` and resumed with ``/platform resume ``. Note: the reconnect watcher does NOT auto-pause — retryable (network/DNS) failures keep retrying at the backoff cap indefinitely so a transient outage self-heals without manual intervention. """ info = getattr(self, "_failed_platforms", {}).get(platform) if info is None: return if info.get("paused"): return info["paused"] = True info["pause_reason"] = reason or "auto-paused after repeated failures" # Push next_retry far enough out that even if "paused" is missed # by a stale code path, the watcher won't fire on it. info["next_retry"] = float("inf") try: self._update_platform_runtime_status( platform.value, platform_state="paused", error_code=None, error_message=info["pause_reason"], ) except Exception: pass logger.warning( "%s paused after %d consecutive failures (%s) — " "fix the underlying issue then run `/platform resume %s` " "to retry, or `hermes gateway restart` to restart the gateway.", platform.value, info.get("attempts", 0), info["pause_reason"], platform.value, ) def _resume_paused_platform(self, platform) -> bool: """Unpause a platform — reset its attempt counter and schedule an immediate retry. Returns True if the platform was paused and is now queued; False if it wasn't paused (or wasn't in the queue). """ info = getattr(self, "_failed_platforms", {}).get(platform) if info is None: return False if not info.get("paused"): return False info["paused"] = False info.pop("pause_reason", None) info["attempts"] = 0 info["next_retry"] = time.monotonic() # retry on next watcher tick try: self._update_platform_runtime_status( platform.value, platform_state="retrying", error_code=None, error_message=None, ) except Exception: pass logger.info("%s resumed — retrying on next watcher tick", platform.value) return True @staticmethod def _load_prefill_messages() -> List[Dict[str, Any]]: """Load ephemeral prefill messages from config or env var. Checks HERMES_PREFILL_MESSAGES_FILE env var first, then falls back to the top-level prefill_messages_file key in ~/.hermes/config.yaml. agent.prefill_messages_file is accepted as a legacy fallback. Relative paths are resolved from ~/.hermes/. """ file_path = os.getenv("HERMES_PREFILL_MESSAGES_FILE", "") if not file_path: cfg = _load_gateway_runtime_config() file_path = str(cfg.get("prefill_messages_file", "") or "") if not file_path: file_path = str(cfg_get(cfg, "agent", "prefill_messages_file", default="") or "") if not file_path: return [] path = Path(file_path).expanduser() if not path.is_absolute(): path = _hermes_home / path if not path.exists(): logger.warning("Prefill messages file not found: %s", path) return [] try: with open(path, "r", encoding="utf-8") as f: data = json.load(f) if not isinstance(data, list): logger.warning("Prefill messages file must contain a JSON array: %s", path) return [] return data except Exception as e: logger.warning("Failed to load prefill messages from %s: %s", path, e) return [] @staticmethod def _load_ephemeral_system_prompt() -> str: """Load ephemeral system prompt from config or env var. Checks HERMES_EPHEMERAL_SYSTEM_PROMPT env var first, then ``display.personality`` / ``agent.system_prompt`` in config.yaml. """ from hermes_cli.config import resolve_ephemeral_system_prompt_from_config prompt = os.getenv("HERMES_EPHEMERAL_SYSTEM_PROMPT", "") if prompt: return prompt cfg = _load_gateway_runtime_config() return resolve_ephemeral_system_prompt_from_config(cfg) def _resolve_model_for_channel( self, platform: Platform, chat_id: str, *, user_config: Optional[dict] = None, thread_id: Optional[str] = None, parent_id: Optional[str] = None, ) -> str: """Resolve model for this channel: channel_overrides else global default. Delegates the precedence rule to :func:`hermes_cli.model_switch.resolve_effective_model` (session override > channel override > global default) — the single owner shared with the API server, so the two surfaces cannot diverge again (see 7dd00bb47d). This call site has no session tier: session /model overrides are applied later by ``_apply_session_model_override`` on the resolved runtime. """ from hermes_cli.model_switch import resolve_effective_model override = None config = getattr(self, "config", None) if config: override = _get_channel_override( config, platform, chat_id, thread_id=thread_id, parent_id=parent_id, ) return resolve_effective_model( None, # session tier applied downstream (_apply_session_model_override) override, _resolve_gateway_model(user_config), ) def _get_system_prompt_for_channel( self, platform: Platform, chat_id: str, *, thread_id: Optional[str] = None, parent_id: Optional[str] = None, ) -> str: """Ephemeral system prompt for this channel/thread. Uses ``channel_overrides`` when set, else the gateway prompt resolved from the CURRENT profile's config on every call. Callers run inside ``_profile_runtime_scope`` (``run_sync`` under ``_run_agent``), so a routed multiplex profile gets its own ``display.personality`` / ``agent.system_prompt`` instead of a boot-time snapshot of the launch profile's (#89161); ``/personality`` edits take effect on the next turn for the same reason. Legacy ``channel_prompts`` are applied separately via ``event.channel_prompt`` in ``run_sync`` (adapter ``resolve_channel_prompt``), so they are not duplicated here. """ config = getattr(self, "config", None) if config: override = _get_channel_override( config, platform, chat_id, thread_id=thread_id, parent_id=parent_id, ) if override and override.system_prompt: return (override.system_prompt or "").strip() return self._load_ephemeral_system_prompt() @staticmethod def _load_reasoning_config(model: str = "") -> dict | None: """Load reasoning effort from config.yaml, respecting per-model overrides. Thin wrapper over the shared chokepoint :func:`hermes_constants.resolve_reasoning_config` (per-model override > global ``agent.reasoning_effort``; YAML boolean False = disabled). Closes #21256. Args: model: The effective model for the calling session. When empty, the config's ``model.default`` is used. """ from hermes_constants import resolve_reasoning_config cfg = _load_gateway_runtime_config() return resolve_reasoning_config(cfg, model) @staticmethod def _parse_reasoning_command_args(raw_args: str) -> tuple[str, bool]: """Parse `/reasoning` args into `(value, persist_global)`. `/reasoning ` is session-scoped by default. `--global` may be supplied in any position to persist the change to config.yaml. """ import shlex text = str(raw_args or "").strip().replace("—", "--") if not text: return "", False try: tokens = shlex.split(text) except ValueError: tokens = text.split() persist_global = False value_tokens = [] for token in tokens: if token == "--global": persist_global = True else: value_tokens.append(token) return " ".join(value_tokens).strip().lower(), persist_global def _resolve_session_reasoning_config( self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None, model: str = "", ) -> dict | None: """Resolve reasoning effort for a session, honoring session overrides. Priority: session-scoped ``/reasoning --session`` override > per-model override (``agent.reasoning_overrides``) > global ``agent.reasoning_effort``. ``model`` should be the session's *effective* model (session ``/model`` override included) so per-model overrides track what the session actually runs — when empty, the config's ``model.default`` is used. """ resolved_session_key = session_key if not resolved_session_key and source is not None: try: resolved_session_key = self._session_key_for_source(source) except Exception: resolved_session_key = None if resolved_session_key: _r_state = self._peek_session_state(resolved_session_key) if _r_state is not None and _r_state.conversation.reasoning_override is not None: return _r_state.conversation.reasoning_override return self._load_reasoning_config(model) def _set_session_reasoning_override( self, session_key: str, reasoning_config: Optional[dict], ) -> None: """Set or clear the session-scoped reasoning override.""" if not session_key: return # Per-session field write — the old lazy ``self._session_reasoning_overrides # = {}`` init replaced the WHOLE dict, racing concurrent sessions' # overrides; a SessionState field reset cannot cross sessions. self._session_state(session_key).conversation.reasoning_override = ( None if reasoning_config is None else dict(reasoning_config) ) def _resolve_session_service_tier( self, source=None, session_key: Optional[str] = None, ) -> Optional[str]: """Resolve the effective service tier for a session. A session-scoped /fast override wins over the config default. The override dict stores "priority" or None (explicit normal), so key presence — not value truthiness — decides whether it applies. """ resolved_session_key = session_key if not resolved_session_key and source is not None: try: resolved_session_key = self._session_key_for_source(source) except Exception: resolved_session_key = None if resolved_session_key: _t_state = self._peek_session_state(resolved_session_key) if ( _t_state is not None and _t_state.conversation.service_tier_override is not _SERVICE_TIER_UNSET ): return _t_state.conversation.service_tier_override return self._load_service_tier() def _set_session_service_tier_override( self, session_key: str, service_tier, clear: bool = False, ) -> None: """Set or clear the session-scoped /fast override. ``service_tier`` is "priority" or None (explicit normal). Pass ``clear=True`` to remove the override entirely (fall back to config). """ if not session_key: return # Presence-sensitive: "priority" or None (explicit normal) both count # as an override; the sentinel means "no override". Old code # wholesale-replaced the dict on lazy init (cross-session race) — # per-session field writes eliminate that class of bug. self._session_state(session_key).conversation.service_tier_override = ( _SERVICE_TIER_UNSET if clear else service_tier ) @staticmethod def _load_service_tier() -> str | None: """Load Priority Processing setting from config.yaml. Reads agent.service_tier from config.yaml. Accepted values mirror the CLI: "fast"/"priority"/"on" => "priority", while "normal"/"off" disables it. Returns None when unset or unsupported. """ cfg = _load_gateway_runtime_config() raw = str(cfg_get(cfg, "agent", "service_tier", default="") or "").strip() value = raw.lower() if not value or value in {"normal", "default", "standard", "off", "none"}: return None if value in {"fast", "priority", "on"}: return "priority" if value in {"auto", "cold"}: return value logger.warning("Unknown service_tier '%s', ignoring", raw) return None @staticmethod def _load_show_reasoning() -> bool: """Load show_reasoning toggle from config.yaml display section.""" cfg = _load_gateway_runtime_config() return is_truthy_value( cfg_get(cfg, "display", "show_reasoning"), default=False, ) @staticmethod def _load_busy_input_mode() -> str: """Load gateway drain-time busy-input behavior from config/env.""" mode = os.getenv("HERMES_GATEWAY_BUSY_INPUT_MODE", "").strip().lower() if not mode: cfg = _load_gateway_runtime_config() mode = str(cfg_get(cfg, "display", "busy_input_mode", default="") or "").strip().lower() if mode == "queue": return "queue" if mode == "steer": return "steer" return "interrupt" @staticmethod def _load_busy_text_mode() -> str: """Resolve normal busy TEXT follow-up behavior. ``busy_input_mode`` is the single source of truth (default ``interrupt``). The legacy ``busy_text_mode`` knob is honored only when a user explicitly set it, so existing queue setups keep working; new installs follow ``busy_input_mode``. Returns one of ``interrupt`` | ``queue`` (``steer`` is handled upstream by ``busy_input_mode`` and maps to non-queue text handling here). """ # Legacy explicit override wins for backward compat. legacy = os.getenv("HERMES_GATEWAY_BUSY_TEXT_MODE", "").strip().lower() if not legacy: cfg = _load_gateway_runtime_config() legacy = str(cfg_get(cfg, "display", "busy_text_mode", default="") or "").strip().lower() if legacy == "interrupt": return "interrupt" if legacy == "queue": return "queue" # No explicit legacy knob → follow busy_input_mode. input_mode = GatewayRunner._load_busy_input_mode() return "queue" if input_mode == "queue" else "interrupt" @staticmethod def _busy_modes_from_config( config: dict, *, fallback_input: str, fallback_text: str, ) -> tuple[str, str]: """Resolve one profile's busy modes without consulting process env.""" raw_input = str( cfg_get(config, "display", "busy_input_mode", default="") or "" ).strip().lower() input_mode = ( raw_input if raw_input in {"interrupt", "queue", "steer"} else fallback_input ) raw_text = str( cfg_get(config, "display", "busy_text_mode", default="") or "" ).strip().lower() if raw_text in {"interrupt", "queue"}: text_mode = raw_text elif raw_input in {"interrupt", "queue", "steer"}: text_mode = "queue" if input_mode == "queue" else "interrupt" else: text_mode = fallback_text return input_mode, text_mode def _snapshot_profile_busy_modes(self, profile_name: str, config: dict) -> None: """Cache a routed profile's busy policy for this gateway lifetime.""" input_mode, text_mode = self._busy_modes_from_config( config, fallback_input=getattr(self, "_busy_input_mode", "interrupt"), fallback_text=getattr(self, "_busy_text_mode", "interrupt"), ) input_modes = self.__dict__.setdefault("_busy_input_modes_by_profile", {}) text_modes = self.__dict__.setdefault("_busy_text_modes_by_profile", {}) input_modes[profile_name] = input_mode text_modes[profile_name] = text_mode def _busy_profile_name_for_source(self, source: SessionSource) -> Optional[str]: """Return the routed profile whose busy policy applies, if any.""" if not getattr(getattr(self, "config", None), "multiplex_profiles", False): return None name = str(getattr(source, "profile", "") or "").strip() if not name: try: name = str(self._profile_name_for_source(source) or "").strip() except Exception: name = "" return name or None def _effective_busy_input_mode(self, source: SessionSource) -> str: """Resolve busy input mode from the routed profile startup snapshot.""" fallback = getattr(self, "_busy_input_mode", "interrupt") profile_name = self._busy_profile_name_for_source(source) if not profile_name: return fallback modes = getattr(self, "_busy_input_modes_by_profile", None) return modes.get(profile_name, fallback) if isinstance(modes, dict) else fallback def _effective_busy_text_mode(self, source: SessionSource) -> str: """Resolve legacy busy text mode from the routed profile snapshot.""" fallback = getattr(self, "_busy_text_mode", "interrupt") profile_name = self._busy_profile_name_for_source(source) if not profile_name: return fallback modes = getattr(self, "_busy_text_modes_by_profile", None) return modes.get(profile_name, fallback) if isinstance(modes, dict) else fallback @staticmethod def _load_restart_drain_timeout() -> float: """Load graceful gateway restart/stop drain timeout in seconds.""" raw = os.getenv("HERMES_RESTART_DRAIN_TIMEOUT", "").strip() if not raw: cfg = _load_gateway_runtime_config() raw = str(cfg_get(cfg, "agent", "restart_drain_timeout", default="") or "").strip() value = parse_restart_drain_timeout(raw) if raw and value == DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT: try: float(raw) except (TypeError, ValueError): logger.warning( "Invalid restart_drain_timeout '%s', using default %.0fs", raw, DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT, ) return value @staticmethod def _load_restart_after_turn_timeout() -> float: """Load in-band restart wait-for-idle timeout in seconds (#77184).""" env_raw = os.getenv("HERMES_RESTART_AFTER_TURN_TIMEOUT") if env_raw is not None and str(env_raw).strip() != "": raw: object = env_raw else: cfg = _load_gateway_runtime_config() raw = cfg_get(cfg, "agent", "restart_after_turn_timeout", default=None) value = parse_restart_after_turn_timeout(raw) # Warn only when the user supplied a non-empty value that failed to # parse (parser falls back to the default). ``0`` is valid. if raw is not None and str(raw).strip() != "": try: float(raw) except (TypeError, ValueError): logger.warning( "Invalid restart_after_turn_timeout '%s', using default %.0fs", raw, DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT, ) return value @staticmethod def _load_cron_drain_timeout() -> float: """Load the cron-only floor under the stop()/drain wait (#82161).""" env_raw = os.getenv("HERMES_CRON_DRAIN_TIMEOUT") if env_raw is not None and str(env_raw).strip() != "": raw: object = env_raw else: cfg = _load_gateway_runtime_config() raw = cfg_get(cfg, "agent", "cron_drain_timeout", default=None) value = parse_cron_drain_timeout(raw) # Warn only when the user supplied a non-empty value that failed to # parse (parser falls back to the default). ``0`` is valid. if raw is not None and str(raw).strip() != "": try: float(raw) except (TypeError, ValueError): logger.warning( "Invalid cron_drain_timeout '%s', using default %.0fs", raw, DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, ) return value @staticmethod def _load_signal_interrupt_grace_timeout() -> float: """Load the unexpected-signal post-interrupt grace in seconds.""" cfg = _load_gateway_runtime_config() raw = cfg_get( cfg, "gateway", "signal_interrupt_grace_timeout", default=None, ) value = parse_signal_interrupt_grace_timeout(raw) if raw is not None and raw != "": try: float(raw) except (TypeError, ValueError): logger.warning( "Invalid signal_interrupt_grace_timeout '%s', using default %.0fs", raw, DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT, ) return value def _post_interrupt_grace_timeout(self) -> float: """Return the grace before teardown after forcibly interrupting agents.""" if ( getattr(self, "_signal_initiated_shutdown", False) and not getattr(self, "_restart_requested", False) ): return max( 0.0, float( getattr( self, "_signal_interrupt_grace_timeout", DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT, ) ), ) return DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT @staticmethod def _load_background_notifications_mode() -> str: """Load background process notification mode from config or env var. Modes: - ``concise`` — one-line status message on completion (default); failures append a short output tail - ``all`` — running-output updates *and* the final raw-output message - ``result`` — only the final raw-output completion message - ``error`` — only the final raw-output message when exit code is non-zero - ``off`` — no watcher messages at all """ mode = os.getenv("HERMES_BACKGROUND_NOTIFICATIONS", "") if not mode: cfg = _load_gateway_runtime_config() raw = cfg_get(cfg, "display", "background_process_notifications") if raw is False: mode = "off" elif raw not in {None, ""}: mode = str(raw) mode = (mode or "concise").strip().lower() valid = {"concise", "all", "result", "error", "off"} if mode not in valid: logger.warning( "Unknown background_process_notifications '%s', defaulting to 'concise'", mode, ) return "concise" return mode @staticmethod def _load_provider_routing() -> dict: """Load OpenRouter provider routing preferences from config.yaml.""" try: # Canonical gateway loader (fail-open): managed overlay + ${VAR} # expansion now apply to provider_routing too. cfg = _load_gateway_runtime_config() return cfg.get("provider_routing", {}) or {} except Exception: pass return {} @staticmethod def _load_fallback_model() -> list | None: """Load fallback provider chain from config.yaml. Returns the merged effective chain from ``fallback_providers`` plus any legacy ``fallback_model`` entries. ``fallback_providers`` stays first when both keys are present. """ try: # Canonical gateway loader (fail-open): managed overlay + ${VAR} # expansion now apply to the fallback chain too. cfg = _load_gateway_runtime_config() fb = get_fallback_chain(cfg) if fb: return fb except Exception: pass return None def _refresh_fallback_model(self) -> list | None: """Re-read fallback_providers from disk for the next agent create/reuse. Cron already does this per job via ``get_fallback_chain``; the gateway previously froze ``self._fallback_model`` at process start, so a chain configured (or changed) after ``hermes gateway`` was running never reached messaging sessions even though the same process's cron jobs fell back correctly. Fixes #60955. A TRANSIENT read/parse failure (user mid-edit of config.yaml with a non-atomic write) keeps the last known-good chain instead of wiping a cached agent's working fallback for that turn. Only a successful read that genuinely lacks the key clears the chain. """ try: from hermes_cli.config import read_user_config_raw cfg_path = _hermes_home / "config.yaml" if not cfg_path.exists(): self._fallback_model = None return self._fallback_model # Raw primitive (raises on parse failure) is required here: the # canonical fail-open loader would return {} on a torn mid-edit # write and WIPE the last known-good chain. The overlay/expansion # below fixes the managed-scope/${VAR} drift without losing that. cfg = read_user_config_raw(cfg_path) try: from hermes_cli import managed_scope cfg = managed_scope.apply_managed_overlay(cfg) except Exception: pass try: from hermes_cli.config import _expand_env_vars expanded = _expand_env_vars(cfg) if isinstance(expanded, dict): cfg = expanded except Exception: pass except Exception: # Transient failure — keep last known-good chain. logger.debug( "fallback_providers refresh: config.yaml read failed; " "keeping last known-good chain", exc_info=True, ) return self._fallback_model self._fallback_model = get_fallback_chain(cfg) or None return self._fallback_model @staticmethod def _apply_fallback_chain_to_agent(agent: Any, chain: list | None) -> None: """Keep a cached agent's fallback chain aligned with current config. Skips rewrite while a cooldown is holding the agent on an already- activated fallback provider — ``restore_primary_runtime`` owns that turn-scoped lifecycle. When primary is active (or cooldown expired), replace the chain so mid-uptime ``fallback_providers`` edits take effect without requiring a gateway restart (#60955). """ if agent is None: return new_chain = list(chain or []) rate_limited_until = getattr(agent, "_rate_limited_until", 0) or 0 if ( getattr(agent, "_fallback_activated", False) and rate_limited_until > time.monotonic() ): return old_chain = list(getattr(agent, "_fallback_chain", []) or []) agent._fallback_chain = new_chain agent._fallback_model = new_chain[0] if new_chain else None if not getattr(agent, "_fallback_activated", False): agent._fallback_index = 0 # A config edit signals the user changed something — drop the # session-scoped unavailability memo so re-configured entries # (e.g. credentials added mid-uptime for a previously-failing # provider) get retried instead of staying suppressed for the # cached agent's lifetime. Only on actual content change, so # the per-message no-op refresh keeps the memo's rate-limiting # benefit (#60955). if new_chain != old_chain: unavailable = getattr(agent, "_unavailable_fallback_keys", None) if unavailable: unavailable.clear() def _snapshot_running_agents(self) -> Dict[str, Any]: return { session_key: agent for session_key, agent in self._running_agent_items() if agent is not _AGENT_PENDING_SENTINEL } def _get_max_concurrent_sessions(self) -> Optional[int]: """Return the configured active chat session cap, if enabled.""" try: from hermes_cli.active_sessions import resolve_max_concurrent_sessions return resolve_max_concurrent_sessions(getattr(self, "config", None)) except Exception: return None def _active_session_limit_message(self, session_key: str) -> Optional[str]: """Return a user-facing rejection when starting a new session exceeds the cap.""" max_sessions = self._get_max_concurrent_sessions() if max_sessions is None: return None if self._is_session_running(session_key): return None active_count = self._running_agent_count() if active_count < max_sessions: return None from hermes_cli.active_sessions import active_session_limit_message return active_session_limit_message(active_count, max_sessions) def _claim_active_session_slot( self, session_key: str, source: SessionSource, ) -> tuple[Any, Optional[str]]: """Claim a cross-process active-session slot for a new gateway turn.""" if self._is_session_running(session_key): return None, None local_limit_message = self._active_session_limit_message(session_key) if local_limit_message is not None: return None, local_limit_message try: from hermes_cli.active_sessions import try_acquire_active_session platform = source.platform.value if source and source.platform else "gateway" return try_acquire_active_session( session_id=session_key, surface=f"gateway:{platform}", config=getattr(self, "config", None), metadata={ "platform": platform, "chat_id": getattr(source, "chat_id", "") or "", "user_id": getattr(source, "user_id", "") or "", # Writer identity for re-entrancy (#94595): if this # process leaks a lease for this session (exception path # skipped release), the next turn re-acquires its own # entry instead of being fenced out of it forever — # pruning only reclaims entries whose PROCESS died. "live_session_id": str(session_key), }, ) except Exception as exc: logger.warning("Failed to claim active session slot: %s", exc) return None, None @staticmethod def _agent_has_active_subagents(running_agent: Any) -> bool: """Return True when *running_agent* is currently driving subagents via the ``delegate_task`` tool. Background (#30170): ``AIAgent.interrupt()`` cascades through the parent's ``_active_children`` list and calls ``interrupt()`` on every child synchronously, which aborts in-flight subagent work and produces a fallback cascade with no actionable signal. Demoting ``busy_input_mode='interrupt'`` to ``queue`` semantics whenever this helper returns True protects subagent work from conversational follow-ups while leaving the explicit ``/stop`` path (which goes through ``_interrupt_and_clear_session``) untouched. Safe-by-default: returns False on any attribute or lock error so a missing/broken parent never blocks the existing interrupt path. """ if running_agent is None or running_agent is _AGENT_PENDING_SENTINEL: return False children = getattr(running_agent, "_active_children", None) # AIAgent always initialises this as a concrete list (see # agent/agent_init.py). Reject anything that isn't a real # collection — this guards against ``MagicMock()._active_children`` # auto-creating a truthy stub in tests and triggering the demotion # against an agent that doesn't actually have subagents. if not isinstance(children, (list, tuple, set)): return False if not children: return False lock = getattr(running_agent, "_active_children_lock", None) try: if lock is not None: with lock: return bool(children) return bool(children) except Exception: return False async def _session_has_compression_in_flight(self, session_key: str) -> bool: """Return True when a compression lock is held for this session's id. Context compression is interrupt-protected (#23975) but gateway ``interrupt`` busy-input mode can still start a follow-up turn against the pre-rotation parent while compression is mid-flight, producing orphaned compression siblings (#56391). Callers demote interrupt to queue when this returns True. Both blocking sources — the ``session_store`` lock + JSON load, and the SQLite ``get_compression_lock_holder`` SELECT — are offloaded to a worker thread so a large state.db never freezes the event loop (#5). """ session_store = getattr(self, "session_store", None) if not session_key or session_store is None: return False try: session_id = await asyncio.to_thread( self._lookup_session_id_under_store_lock, session_store, session_key ) except (AttributeError, TypeError): return False except Exception: logger.warning( "Compression in-flight check failed while reading session %s; " "treating compression as active to avoid interrupting a possible " "parent-session rotation", session_key, exc_info=True, ) return True if not session_id: return False session_db = getattr(self, "_session_db", None) if session_db is None: return False raw_db = getattr(session_db, "_db", session_db) try: holder = await asyncio.to_thread( raw_db.get_compression_lock_holder, str(session_id) ) # Production returns Optional[str]. Reject non-strings so a # MagicMock auto-attr (or any unexpected truthy) cannot look # like a held lock and skip hygiene (#96953). return isinstance(holder, str) and bool(holder) except (AttributeError, TypeError): return False except Exception: logger.warning( "Compression in-flight check failed while reading lock holder " "for session %s; treating compression as active to avoid " "interrupting a possible parent-session rotation", session_id, exc_info=True, ) return True @staticmethod def _lookup_session_id_under_store_lock(session_store, session_key: str): """Sync helper run in the thread pool: read session_id under the store lock.""" # noqa: SLF001 — intentional private access; runs off the event loop. with session_store._lock: # noqa: SLF001 session_store._ensure_loaded_locked() # noqa: SLF001 entry = session_store._entries.get(session_key) # noqa: SLF001 return getattr(entry, "session_id", None) if entry is not None else None # Hard cap on per-session pending follow-ups for busy_input_mode=queue # (and the draining/steer-fallback/subagent-demotion paths that share # this entry point). Without a cap, a stuck agent + a rapid-fire user # could grow the overflow list unboundedly. 32 turns of queued # follow-ups is far beyond any realistic conversational backlog while # still small enough to never threaten memory. _BUSY_QUEUE_MAX_PENDING = 32 def _queue_or_replace_pending_event(self, session_key: str, event: MessageEvent) -> None: adapter = self._adapter_for_source(event.source) if not adapter: return # #28503 — Previously this called ``merge_pending_message_event`` # with the default ``merge_text=False``, which silently OVERWROTE # the single pending slot when consecutive text messages arrived # in ``busy_input_mode: queue``. Route through the FIFO # infrastructure shared with ``/queue`` so each follow-up gets # its own turn in arrival order. Photo bursts still merge into # the head slot via ``merge_pending_message_event`` (album # semantics); everything else appends to the overflow tail. pending_slot = getattr(adapter, "_pending_messages", None) existing = pending_slot.get(session_key) if isinstance(pending_slot, dict) else None security_metadata_keys = ( "hermes_plugin_id", "hermes_plugin_injection", "gateway_session_key", "gateway_session_id", "gateway_session_strict", ) same_security_context = existing is not None and ( getattr(existing, "internal", False) == getattr(event, "internal", False) and getattr(existing, "allow_gateway_control", True) == getattr(event, "allow_gateway_control", True) and all( (getattr(existing, "metadata", None) or {}).get(key) == (getattr(event, "metadata", None) or {}).get(key) for key in security_metadata_keys ) ) if same_security_context and ( getattr(existing, "message_type", None) == MessageType.PHOTO or event.message_type == MessageType.PHOTO or bool(getattr(existing, "media_urls", None)) or bool(getattr(event, "media_urls", None)) ): # Preserve photo-burst / media-merge semantics for the head slot. merge_pending_message_event( adapter._pending_messages, session_key, event, merge_text=event.message_type == MessageType.TEXT, ) return if self._queue_depth(session_key, adapter=adapter) >= self._BUSY_QUEUE_MAX_PENDING: logger.warning( "Dropping busy-mode follow-up for session %s — pending queue at cap (%d).", session_key, self._BUSY_QUEUE_MAX_PENDING, ) return self._enqueue_fifo(session_key, event, adapter) async def _prepare_busy_steer_text(self, event: MessageEvent) -> str: """Return steerable text for a busy follow-up, transcribing voice first. Fresh and queued voice messages reach the normal inbound STT pipeline, but successful steer messages intentionally bypass that queue. Without preprocessing here, a media-only voice follow-up has an empty text payload and steer mode silently degrades to queue mode. Audio file attachments remain files; only voice-message media follows the automatic STT contract used by ``_prepare_inbound_message_text``. If transcription fails, preserve any caption and let the existing steer fallback handle an otherwise empty event without losing it. Routes through ``_transcribe_and_echo_pending_voice`` — the single out-of-band transcription choke point shared with the interrupt monitor and the pending-drain path — so the STT call is made at most once per platform message (cached on the event) and the transcript echo respects the count-based ledger. If steering later falls back to queue mode, the drain path reuses the cached transcript instead of paying for a second STT call or re-echoing the same line. """ text = (event.text or "").strip() if not self._pending_event_audio_paths(event): return text adapter = self._adapter_for_source(event.source) enriched_text, successful_transcripts = await self._transcribe_and_echo_pending_voice( event, adapter, event.source, text, log_context="Busy-steer", ) if not successful_transcripts: return text return (enriched_text or text).strip() async def _handle_active_session_busy_message(self, event: MessageEvent, session_key: str) -> bool: # --- Authorization gate (#17775) --- # The cold path (_handle_message) checks _is_user_authorized before # creating a session. The busy path must enforce the same check; # otherwise unauthorized users in shared threads (Slack/Telegram/Discord) # can inject messages into an active session they don't own. if not self._is_user_authorized(event.source): logger.warning( "Dropping message from unauthorized user in active session: " "user=%s (%s), platform=%s, session=%s", event.source.user_id, event.source.user_name, event.source.platform.value if event.source.platform else "unknown", session_key, ) return True # handled (silently dropped); do not fall through effective_mode = self._effective_busy_input_mode(event.source) # --- Draining case (gateway restarting/stopping) --- if self._draining: adapter = self._adapter_for_source(event.source) if not adapter: return True reply_anchor = self._reply_anchor_for_event(event) thread_meta = self._thread_metadata_for_source(event.source, reply_anchor) if self._queue_during_drain_enabled(effective_mode): self._queue_or_replace_pending_event(session_key, event) message = f"⏳ Gateway {self._status_action_gerund()} — queued for the next turn after it comes back." else: message = f"⏳ Gateway is {self._status_action_gerund()} and is not accepting another turn right now." await adapter._send_with_retry( chat_id=event.source.chat_id, content=message, reply_to=( reply_anchor if event.source.platform == Platform.TELEGRAM and event.source.chat_type == "dm" and event.source.thread_id else (None if event.source.platform == Platform.TELEGRAM and event.source.thread_id else event.message_id) ), metadata=thread_meta, ) return True # --- Approval response routing (#46866) --- # When the agent is blocked waiting for a dangerous-command approval, # plain-text responses like "yes" or "approve" must be routed to the # approval handler instead of being steered/queued/interrupted. # Otherwise approval via messaging platforms never succeeds — the # reply is queued behind a turn that can't start until the approval # resolves, so the approval times out and auto-denies (a deadlock). # # Slash forms (/approve, /deny) already bypass to the runner at the # base-adapter guard. This handles the bare-word forms (Signal/SMS # users naturally type "yes" rather than "/approve"). Gating on # has_blocking_approval(session_key) is the disambiguator that keeps # a conversational "yes" from triggering a dangerous command when no # approval is actually pending (design intent — see run.py "Pending # exec approvals are handled by /approve and /deny" note). # # We reuse the canonical /approve and /deny handlers rather than # re-deriving the resolution + i18n messaging: they resolve the # waiting thread, resume typing, AND return a localized confirmation # string. The busy-handler path does not auto-send that return, so # we deliver it ourselves (mirroring the draining-case send above). try: from tools.approval import has_blocking_approval if event.allow_gateway_control and has_blocking_approval(session_key): _raw_text = (event.text or "").strip().lower() _approve_words = {"approve", "yes", "ok", "okay", "confirm", "y", "👍"} _deny_words = {"deny", "no", "reject", "cancel", "n", "👎"} _approval_handler = None _normalized_args = "" if _raw_text in _approve_words: _approval_handler = self._handle_approve_command elif _raw_text in _deny_words: _approval_handler = self._handle_deny_command elif _raw_text in {"always", "approve always", "always approve"}: _approval_handler = self._handle_approve_command _normalized_args = "always" elif _raw_text in {"session", "approve session", "session approve"}: _approval_handler = self._handle_approve_command _normalized_args = "session" if _approval_handler is not None: # Synthesize the canonical "/approve [args]" / "/deny" # command text so the slash handlers parse modifiers via # event.get_command_args(). Always use a literal "/" — # MessageEvent.is_command()/get_command_args() only # recognize the "/" prefix, not the per-platform display # prefix ("!" on Slack/Matrix). _verb = "approve" if _approval_handler is self._handle_approve_command else "deny" _synth = f"/{_verb}" if _normalized_args: _synth = f"{_synth} {_normalized_args}" event.text = _synth _reply = await _approval_handler(event) logger.info( "Approval response via plain text: session=%s verb=%s args=%r", session_key, _verb, _normalized_args, ) _adapter = self._adapter_for_source(event.source) if _adapter and _reply: _text, _eph_ttl = _adapter._unwrap_ephemeral(_reply) if _text: _anchor = self._reply_anchor_for_event(event) await _adapter._send_with_retry( chat_id=event.source.chat_id, content=_text, reply_to=_anchor, metadata=self._thread_metadata_for_source(event.source, _anchor), ) return True except Exception: logger.warning( "Plain-text approval routing failed for session %s; " "falling through to busy handling", session_key, exc_info=True, ) # Normal busy case (agent actively running a task) adapter = self._adapter_for_source(event.source) if not adapter: return False # let default path handle it # --- Internal synthetic events must never interrupt/steer --- # Async-delegation completions (delegate_task(background=true)) and # background-process completions (terminal notify_on_complete) re-enter # the originating session as internal MessageEvents. When the session # is busy, treating them like a user TEXT message means interrupt-mode # (the default busy_text_mode) aborts the active turn AND sends a "⚡ # Interrupting current task" ack — exactly the opposite of the design # invariant that a completion surfaces as a NEW turn only when idle and # never splices into a running turn. Plugin events carry untrusted # payload text, so queue those through the gateway FIFO to keep their # security metadata separate from pending user input. if getattr(event, "internal", False) and not event.allow_gateway_control: self._queue_or_replace_pending_event(session_key, event) return True if getattr(event, "internal", False): return False _busy_state = self._peek_session_state(session_key) running_agent = _busy_state.turn.agent if _busy_state else None busy_text_mode = self._effective_busy_text_mode(event.source) if ( event.message_type == MessageType.TEXT and busy_text_mode == "queue" and effective_mode != "steer" ): return False # Steer mode: inject mid-run via running_agent.steer() instead of # queueing + interrupting. If the agent isn't running yet # (sentinel) or lacks steer(), or the payload is empty, fall back # to queue semantics so nothing is lost. # #30170 — Subagent protection. ``AIAgent.interrupt()`` cascades # to every entry in the parent's ``_active_children`` list and # aborts in-flight ``delegate_task`` work. Demote ``interrupt`` # to ``queue`` when the parent is currently driving subagents so # a conversational follow-up doesn't destroy minutes of subagent # work. Explicit ``/stop`` and ``/new`` slash commands go through # ``_interrupt_and_clear_session`` and are unaffected — the # operator still has a way to force-cancel everything. demoted_for_subagents = ( effective_mode == "interrupt" and self._agent_has_active_subagents(running_agent) ) if demoted_for_subagents: logger.info( "Demoting busy_input_mode 'interrupt' to 'queue' for session %s " "because the running agent has active subagents (#30170)", session_key, ) effective_mode = "queue" demoted_for_compression = ( effective_mode == "interrupt" and await self._session_has_compression_in_flight(session_key) ) if demoted_for_compression: logger.info( "Demoting busy_input_mode 'interrupt' to 'queue' for session %s " "because context compression is in flight (#56391)", session_key, ) effective_mode = "queue" steered = False redirected = False if effective_mode == "steer": steer_text = await self._prepare_busy_steer_text(event) # A follow-up qualifies for steering when it is plain text, OR # when every attachment is STT-eligible voice media whose # transcript was just folded into steer_text — otherwise a voice # note in steer mode silently degrades to queue mode (#58780). _steer_media_urls = getattr(event, "media_urls", None) or [] _steer_all_voice = bool(_steer_media_urls) and ( len(self._pending_event_audio_paths(event)) == len(_steer_media_urls) ) can_steer = ( steer_text and ( ( event.message_type == MessageType.TEXT and not event.media_urls and not event.media_types ) or _steer_all_voice ) and running_agent is not None and running_agent is not _AGENT_PENDING_SENTINEL and hasattr(running_agent, "steer") ) if can_steer: try: steered = bool(running_agent.steer(steer_text)) except Exception as exc: logger.warning("Gateway steer failed for session %s: %s", session_key, exc) steered = False if not steered: # Fall back to queue (merge into pending messages, no interrupt) effective_mode = "queue" elif ( effective_mode == "interrupt" and event.message_type == MessageType.TEXT and not event.media_urls and not event.media_types and running_agent is not None and running_agent is not _AGENT_PENDING_SENTINEL and getattr(running_agent, "_supports_active_turn_redirect", False) is True and hasattr(running_agent, "redirect") ): try: redirected = bool(running_agent.redirect((event.text or "").strip())) except Exception as exc: logger.warning("Gateway redirect failed for session %s: %s", session_key, exc) redirected = False # Store the message so it's processed as the next turn after the # current run finishes (or is interrupted). Skip this for a # successful steer — the text already landed inside the run and # must NOT also be replayed as a next-turn user message. # # Route through _queue_or_replace_pending_event (the same FIFO # infrastructure used by busy queue-mode and /queue) rather than a # raw merge_pending_message_event(merge_text=True). The raw merge # newline-joins consecutive TEXT follow-ups into a SINGLE pending # turn, destroying message boundaries — so two separate user # messages sent while the agent was busy (interrupt mode, or a # steer that fell back to queue) arrived as one mashed-together # turn (#43066 sub-bug 2). The FIFO path gives each text its own # turn in arrival order while still preserving photo-burst / album # merge semantics for media. if not steered and not redirected: self._queue_or_replace_pending_event(session_key, event) is_queue_mode = effective_mode == "queue" is_steer_mode = effective_mode == "steer" is_redirect_mode = effective_mode == "interrupt" and redirected # If not in queue/steer mode, interrupt the running agent immediately. # This aborts in-flight tool calls and causes the agent loop to exit # at the next check point. if ( effective_mode == "interrupt" and not redirected and running_agent and running_agent is not _AGENT_PENDING_SENTINEL ): try: _interrupt_text = event.text _media_urls = getattr(event, "media_urls", None) or [] if self._pending_event_audio_paths(event): _interrupt_text, _ = await self._transcribe_and_echo_pending_voice( event, adapter, event.source, event.text or "", log_context="Voice-busy-interrupt", ) elif not _interrupt_text and _media_urls: _interrupt_text = _build_media_placeholder(event) running_agent.interrupt(_interrupt_text) except Exception: pass # don't let interrupt failure block the ack # Check if busy ack is disabled — skip sending but still process the input. # Placed before debounce so we don't stamp a "last ack" timestamp that was # never actually delivered. busy_ack_enabled = os.environ.get("HERMES_GATEWAY_BUSY_ACK_ENABLED", "true").lower() == "true" if not busy_ack_enabled: logger.debug("Busy ack suppressed for session %s", session_key) return True # input still processed, just no ack sent # Debounce before consulting config-heavy display settings. Rapid # follow-ups should be processed but should not trigger another config # read just to discover that no ack will be sent. _BUSY_ACK_COOLDOWN = 30 now = time.time() last_ack = _busy_state.turn.busy_ack_ts if _busy_state else 0 if now - last_ack < _BUSY_ACK_COOLDOWN: return True # interrupt sent (if not queue), ack already delivered recently from gateway.display_config import resolve_display_setting platform_key = _platform_config_key(event.source.platform) # In steer mode the user's text has already been injected into the # active run. Some mobile chat setups want that steering to be silent, # like STT transcript echo suppression: keep the behavior, drop only # the confirmation bubble. if is_steer_mode: steer_ack_env = os.environ.get("HERMES_GATEWAY_BUSY_STEER_ACK_ENABLED") if steer_ack_env is not None: steer_ack_enabled = steer_ack_env.strip().lower() in {"1", "true", "yes", "on"} else: steer_ack_enabled = bool( resolve_display_setting( _load_gateway_config(), platform_key, "busy_steer_ack_enabled", True, ) ) if not steer_ack_enabled: logger.debug("Busy steer ack suppressed for session %s", session_key) return True self._session_state(session_key).turn.busy_ack_ts = now # Build a status-rich acknowledgment. Mobile chat defaults keep this # terse; detailed iteration/tool state is still available in logs and # can be opted in per platform via display.platforms..busy_ack_detail. status_parts = [] busy_ack_detail_enabled = bool( resolve_display_setting( _load_gateway_config(), _platform_config_key(event.source.platform), "busy_ack_detail", True, ) ) if busy_ack_detail_enabled and running_agent and running_agent is not _AGENT_PENDING_SENTINEL: try: summary = running_agent.get_activity_summary() iteration = summary.get("api_call_count", 0) max_iter = summary.get("max_iterations", 0) current_tool = summary.get("current_tool") start_ts = _busy_state.turn.started_ts if _busy_state else 0 if start_ts: elapsed_min = int((now - start_ts) / 60) if elapsed_min > 0: status_parts.append(f"{elapsed_min} min elapsed") if max_iter: status_parts.append(f"iteration {iteration}/{max_iter}") if current_tool: status_parts.append(f"running: {current_tool}") except Exception: pass status_detail = f" ({', '.join(status_parts)})" if status_parts else "" if is_steer_mode: message = ( f"⏩ Steered into current run{status_detail}. " f"Your message arrives after the next tool call." ) elif is_redirect_mode: message = ( f"↪ Redirected current run{status_detail}. " f"I'll adjust using your correction." ) elif is_queue_mode and demoted_for_subagents: # #30170 — explain the demotion so the user knows their # follow-up didn't accidentally kill the subagent and # discovers `/stop` as the explicit escape hatch. message = ( f"⏳ Subagent working{status_detail} — your message is queued for " f"when it finishes (use /stop to cancel everything)." ) elif is_queue_mode and demoted_for_compression: message = ( f"⏳ Compressing context{status_detail} — your message is queued for " f"when it finishes (use /stop to cancel everything)." ) elif is_queue_mode: message = ( f"⏳ Queued for the next turn{status_detail}. " f"I'll respond once the current task finishes." ) else: message = ( f"⚡ Interrupting current task{status_detail}. " f"I'll respond to your message shortly." ) # First-touch onboarding: the very first time a user sends a message # while the agent is busy, append a one-time hint explaining the # queue/interrupt knob. Flag is persisted to config.yaml so it never # fires again on this install. try: from agent.onboarding import ( BUSY_INPUT_FLAG, busy_input_hint_gateway, is_seen, mark_seen, ) _user_cfg = _load_gateway_config() if not is_seen(_user_cfg, BUSY_INPUT_FLAG): if is_steer_mode: _hint_mode = "steer" elif is_queue_mode: _hint_mode = "queue" elif is_redirect_mode: _hint_mode = "redirect" else: _hint_mode = "interrupt" message = ( f"{message}\n\n" f"{busy_input_hint_gateway(_hint_mode)}" ) mark_seen(_hermes_home / "config.yaml", BUSY_INPUT_FLAG) except Exception as _onb_err: logger.debug("Failed to apply busy-input onboarding hint: %s", _onb_err) reply_anchor = self._reply_anchor_for_event(event) thread_meta = self._thread_metadata_for_source(event.source, reply_anchor) try: await adapter._send_with_retry( chat_id=event.source.chat_id, content=message, reply_to=( reply_anchor if event.source.platform == Platform.TELEGRAM and event.source.chat_type == "dm" and event.source.thread_id else (None if event.source.platform == Platform.TELEGRAM and event.source.thread_id else event.message_id) ), metadata=thread_meta, ) except Exception as e: logger.debug("Failed to send busy-ack: %s", e) return True async def _drain_active_agents( self, timeout: float, cron_timeout: Optional[float] = None ) -> tuple[Dict[str, Any], bool]: snapshot = self._snapshot_running_agents() last_active_count = self._running_agent_count() last_cron_count = self._active_cron_job_count() last_api_count = self._active_api_run_count() last_deferred_count = self._active_deferred_agent_worker_count() last_status_at = 0.0 def _maybe_update_status(force: bool = False) -> None: nonlocal last_active_count, last_cron_count, last_api_count nonlocal last_deferred_count, last_status_at now = asyncio.get_running_loop().time() active_count = self._running_agent_count() cron_count = self._active_cron_job_count() api_count = self._active_api_run_count() deferred_count = self._active_deferred_agent_worker_count() if ( force or active_count != last_active_count or cron_count != last_cron_count or api_count != last_api_count or deferred_count != last_deferred_count or (now - last_status_at) >= 1.0 ): self._update_runtime_status("draining") last_active_count = active_count last_cron_count = cron_count last_api_count = api_count last_deferred_count = deferred_count last_status_at = now # Cron jobs run on the scheduler's own thread pool, outside # ``self._running_agents`` — fold their in-flight count into the # same wait/timeout this method already applies to chat sessions, # or a cron job's tool work gets killed with zero warning the # instant it's the only active thing running (#60432). # API-server / desk sessions have the same structural gap (#63529). if ( not self._running_agents and last_cron_count == 0 and last_api_count == 0 and last_deferred_count == 0 ): _maybe_update_status(force=True) return snapshot, False _maybe_update_status(force=True) # Cron work drains on its own deadline. ``timeout`` # (``restart_drain_timeout``) defaults to 0 because interrupting a # chat turn is announced and resumable; a cron run killed mid-flight # is recorded in jobs.json as a permanent failure nobody is waiting # on. Sharing one budget meant the default config could report # ``timed_out=True`` after 0.00s with a cron job in flight and kill # it — the drain never even entered this loop (#82161). loop = asyncio.get_running_loop() started = loop.time() deadline = started + timeout cron_deadline = started + (timeout if cron_timeout is None else cron_timeout) def _still_draining() -> bool: now = loop.time() if ( len(self._running_agents) or self._active_api_run_count() or self._active_deferred_agent_worker_count() ) and now < deadline: return True return bool(self._active_cron_job_count()) and now < cron_deadline # Both budgets at 0 leave this loop unentered, which is the legacy # "interrupt immediately" behaviour — expressed as an expired # deadline rather than a special case, so the timed_out value below # is always computed from real state instead of asserted up front. while _still_draining(): _maybe_update_status() await asyncio.sleep(0.1) timed_out = ( bool(len(self._running_agents)) or bool(self._active_cron_job_count()) or bool(self._active_api_run_count()) or bool(self._active_deferred_agent_worker_count()) ) _maybe_update_status(force=True) return snapshot, timed_out def _interrupt_running_agents(self, reason: str) -> None: for session_key, agent in list(self._running_agents.items()): if agent is _AGENT_PENDING_SENTINEL: continue try: request_hard_interrupt(agent, reason) logger.debug("Interrupted running agent for session %s during shutdown", session_key) except Exception as e: logger.debug("Failed interrupting agent during shutdown: %s", e) # API-server / desk turns are adapter-owned and never enter # _running_agents, so the loop above cannot see them even though # _drain_active_agents() waited for them (#63529). interrupted_api = self._interrupt_api_server_runs(reason) if interrupted_api: logger.debug("Interrupted %d api_server run(s) during shutdown", interrupted_api) interrupted_deferred = self._interrupt_deferred_agent_workers(reason) if interrupted_deferred: logger.debug( "Interrupted %d deferred agent worker(s) during shutdown", interrupted_deferred, ) async def _notify_interrupted_cron_jobs(self, job_ids) -> int: """Tell the owner of each just-interrupted cron job that its run died. The cron worker cannot do this itself. Its thread reaches ``_deliver_result`` asynchronously, and by then ``_bounded_adapter_teardown`` has closed the transport — so the notice never leaves the process, and ``_consume_interrupted_flag`` discards the resulting ``delivery_error`` along with it. The run's only trace is a line in jobs.json nobody reads (#82232). Must therefore be called from the post-interrupt phase, while adapters are still connected — the same window ``_notify_active_sessions_of_shutdown`` relies on for chat sessions, which is blind to cron work because cron runs on the scheduler's own thread pool rather than ``self._running_agents`` (#60432). Best-effort by construction: every failure is swallowed so a wedged adapter can never extend shutdown. Returns the number of notices sent. """ if not job_ids: return 0 try: from cron.jobs import get_job from cron.scheduler import _resolve_delivery_targets except Exception as e: logger.debug("Cron interrupt notification unavailable: %s", e) return 0 action = "restarting" if self._restart_requested else "shutting down" notified: set = set() for job_id in job_ids: try: job = get_job(job_id) if not job: continue # deliver=local jobs — and deliver=origin jobs with no # resolvable origin (#43014) — resolve to zero targets and # must stay silent rather than fall back to a home channel. # Interrupted notices are failure-category engine status, so # they honor the job's failure_deliver override (NS-788). targets = _resolve_delivery_targets(job, for_failure=True) except Exception as e: logger.debug("Cron interrupt targets unresolved for %s: %s", job_id, e) continue if not targets: continue msg = ( f"⚠️ Cron job '{job.get('name') or job_id}' was interrupted — " f"the gateway is {action} and killed the run before it " "finished. No result was produced for this run." ) for target in targets: try: platform = Platform(str(target.get("platform", "")).lower()) except Exception: continue adapter = self.adapters.get(platform) if adapter is None: continue platform_cfg = self.config.platforms.get(platform) if platform_cfg is not None and not platform_cfg.gateway_restart_notification: continue chat_id = str(target.get("chat_id")) thread_id = target.get("thread_id") dedup_key = ( job_id, platform.value, chat_id, str(thread_id) if thread_id else None, ) if dedup_key in notified: continue try: metadata = self._thread_metadata_for_target( platform, chat_id, thread_id, adapter=adapter ) result = await adapter.send(chat_id, msg, metadata=metadata) if result is not None and getattr(result, "success", True) is False: logger.debug( "Cron interrupt notice to %s:%s failed: %s", platform.value, chat_id, getattr(result, "error", "send returned success=False"), ) continue notified.add(dedup_key) except Exception as e: logger.debug( "Cron interrupt notice to %s:%s raised: %s", platform.value, chat_id, e, ) if notified: logger.info( "Shutdown: delivered %d interrupted-cron-job notice(s)", len(notified), ) return len(notified) async def _notify_active_sessions_of_shutdown(self) -> None: """Send shutdown/restart notifications to active chats and home channels. Called at the very start of stop() — adapters are still connected so messages can be delivered. Best-effort: individual send failures are logged and swallowed so they never block the shutdown sequence. """ active = self._snapshot_running_agents() restart_source = self._restart_command_source if self._restart_requested else None action = "restarting" if self._restart_requested else "shutting down" hint = ( "Your current task will be interrupted. " "Send any message after restart and I'll try to resume where you left off." if self._restart_requested else "Your current task will be interrupted." ) msg = f"⚠️ Gateway {action} — {hint}" notified: set[tuple[str, str, Optional[str]]] = set() for session_key in active: source = None try: if getattr(self, "session_store", None) is not None: await self.async_session_store._ensure_loaded() entry = self.session_store._entries.get(session_key) source = getattr(entry, "origin", None) if entry else None except Exception as e: logger.debug( "Failed to load session origin for shutdown notification %s: %s", session_key, e, ) if source is None: source = self._get_cached_session_source(session_key) if source is not None: platform_str = source.platform.value chat_id = str(source.chat_id) thread_id = source.thread_id else: # Fall back to parsing the session key when no persisted # origin is available (legacy sessions/tests). _parsed = _parse_session_key(session_key) if not _parsed: continue platform_str = _parsed["platform"] chat_id = _parsed["chat_id"] thread_id = _parsed.get("thread_id") # Deduplicate only identical delivery targets. Thread/topic-aware # platforms can share a parent chat while still routing to distinct # destinations via metadata. dedup_key = (platform_str, chat_id, str(thread_id) if thread_id else None) if dedup_key in notified: continue try: platform = Platform(platform_str) adapter = self.adapters.get(platform) if not adapter: continue platform_cfg = self.config.platforms.get(platform) if platform_cfg is not None and not platform_cfg.gateway_restart_notification: logger.info( "Shutdown notification suppressed for active session: %s has gateway_restart_notification=false", platform_str, ) continue reply_to_message_id = getattr(source, "message_id", None) if source is not None else None if reply_to_message_id is None and restart_source is not None: try: restart_platform = restart_source.platform.value restart_chat_id = str(restart_source.chat_id) restart_thread_id = str(restart_source.thread_id) if restart_source.thread_id else None if (restart_platform, restart_chat_id, restart_thread_id) == dedup_key: reply_to_message_id = getattr(restart_source, "message_id", None) except Exception: pass metadata = self._thread_metadata_for_target( platform, chat_id, thread_id, chat_type=getattr(source, "chat_type", None) if source is not None else None, reply_to_message_id=reply_to_message_id, adapter=adapter, ) result = await adapter.send(chat_id, msg, metadata=metadata) if result is not None and getattr(result, "success", True) is False: logger.debug( "Failed to send shutdown notification to %s:%s: %s", platform_str, chat_id, getattr(result, "error", "send returned success=False"), ) continue notified.add(dedup_key) logger.info( "Sent shutdown notification to active chat %s:%s", platform_str, chat_id, ) except Exception as e: logger.debug( "Failed to send shutdown notification to %s:%s: %s", platform_str, chat_id, e, ) if self._restart_requested and restart_source is not None: logger.debug("Skipping home-channel shutdown notifications for in-chat restart") return # Suppress ONLY the home-channel broadcast when the drain that is ending # in this shutdown asked us to be quiet (e.g. a NAS auto-update image # migration — drain-gated, then the machine is recreated). On the # always-on Hermes Cloud fleet that broadcast would otherwise fire on # every routine auto-update, spamming home channels with operator- # flavoured "gateway shutting down" pings the user doesn't care about. # The per-active-session interrupt pings above are deliberately NOT # gated: on a drained shutdown they're empty by construction, and in the # force-interrupt (deadline-exceeded) case they carry the genuinely # useful "your task was cut off, message me to resume" hint. The flag is # only honoured for a CURRENT-epoch marker (drain_notification_suppressed # reuses the NS-570 staleness check), so an orphaned marker can never # silence a fresh gateway's legitimate broadcast. try: from gateway.drain_control import drain_notification_suppressed if drain_notification_suppressed(): logger.info( "Home-channel shutdown broadcast suppressed by drain marker " "(suppress_notification=true)" ) return except Exception as e: # Never let the suppression check block the shutdown broadcast — # fail toward the louder, more-visible behaviour. logger.debug("drain_notification_suppressed check failed: %s", e) # Snapshot adapters up front: adapter.send() can hit a fatal error # path that pops the adapter from self.adapters (see _handle_fatal # elsewhere), which would otherwise trigger # ``RuntimeError: dictionary changed size during iteration`` — # observed in a user report during gateway shutdown. for platform, adapter in list(self.adapters.items()): home = self.config.get_home_channel(platform) if not home or not home.chat_id: continue platform_cfg = self.config.platforms.get(platform) if platform_cfg is not None and not platform_cfg.gateway_restart_notification: logger.info( "Shutdown notification suppressed for home channel: %s has gateway_restart_notification=false", platform.value, ) continue dedup_key = (platform.value, str(home.chat_id), str(home.thread_id) if home.thread_id else None) if dedup_key in notified: continue try: metadata = self._thread_metadata_for_target( platform, home.chat_id, home.thread_id, adapter=adapter, ) if metadata: result = await adapter.send(str(home.chat_id), msg, metadata=metadata) else: result = await adapter.send(str(home.chat_id), msg) if result is not None and getattr(result, "success", True) is False: logger.debug( "Failed to send shutdown notification to home channel %s:%s: %s", platform.value, home.chat_id, getattr(result, "error", "send returned success=False"), ) continue notified.add(dedup_key) logger.info( "Sent shutdown notification to home channel %s:%s", platform.value, home.chat_id, ) except Exception as e: logger.debug( "Failed to send shutdown notification to home channel %s:%s: %s", platform.value, home.chat_id, e, ) async def _finalize_shutdown_agents(self, active_agents: Dict[str, Any]) -> None: for agent in active_agents.values(): # Persist any in-flight transcript to the SQLite session store # before teardown (#13121). An agent forcibly interrupted by the # drain-timeout escalation may never reach # ``turn_finalizer.finalize_turn`` (the only place that flushes the # turn to state.db) — e.g. it was blocked in a tool call that did # not abort within the post-interrupt grace window. Its in-flight # tool rounds live only in the in-memory ``_session_messages`` # (refreshed per tool round in ``conversation_loop`` but never # written to SQLite mid-turn), so the immediate pre-restart turn is # silently dropped from ``load_transcript()`` on resume. Flushing # here closes that gap; the resume_pending / fresh-tool-tail # branches in ``_handle_message_with_agent`` already expect a # transcript whose tail may be a pending tool result. The flush is # idempotent (identity-tracked in ``_flush_messages_to_session_db``), # so agents that DID finish gracefully re-flush nothing. try: _flush = getattr(agent, "_flush_messages_to_session_db", None) _session_messages = getattr(agent, "_session_messages", None) if callable(_flush) and isinstance(_session_messages, list) and _session_messages: # Strip private empty-response retry scaffolding from the # tail first, mirroring the graceful ``_persist_session`` # path, so a resumed turn doesn't replay synthetic recovery # nudges. _strip = getattr( agent, "_drop_trailing_empty_response_scaffolding", None ) if callable(_strip): try: _strip(_session_messages) except Exception: pass try: _flush(_session_messages) except Exception as _flush_err: # The in-memory transcript could not be persisted # (e.g. FTS/SQLite index corruption — #72680). A plain # debug log loses the conversation permanently when the # process exits. Dump the live agent history to an # external JSON recovery snapshot so an operator can # salvage it after repairing state.db. The flush is # non-fatal; shutdown must never block on a best-effort # backup. logger.warning( "Shutdown transcript flush failed (%s); preserving " "%d in-memory message(s) to recovery snapshot", _flush_err, len(_session_messages), ) from gateway.shutdown_flush import flush_agent_history_to_file flush_agent_history_to_file( getattr(agent, "session_id", None), _session_messages, ) except Exception as _e: logger.debug("Shutdown transcript flush failed: %s", _e) # Off-loop + bounded: finalize_session fans out to plugin # on_session_finalize hooks that can do arbitrary synchronous # work (e.g. an observability plugin serializing a full-session # trace export). Running it inline on the event loop blocked the # entire shutdown sequence past systemd's TimeoutStopSec on a # multi-day 4.7G session — heartbeats froze and the process was # SIGKILLed mid-export. Same class as the memory-provider hang # below (#53175). await self._finalize_session_off_loop( session_id=getattr(agent, "session_id", None), platform="gateway", reason="shutdown", ) # Off-loop + bounded: a wedged memory provider here used to hang # the whole shutdown so SIGTERM never completed (#53175). await self._cleanup_agent_resources_off_loop( agent, context="shutdown finalize" ) def _should_emit_long_running_notification( self, session_key: Optional[str], agent: Any, executor_task: Optional[Any], ) -> bool: """Only emit the heartbeat while this task still owns the live run. Guards against a stale ``running: delegate_task`` heartbeat outliving the run that started it: stop once the executor finishes, the agent is gone, or the session key has been rebound to a different live agent (e.g. the user sent ``/new`` and a fresh agent took the slot mid-run, #12029). """ if agent is None: return False if executor_task is not None and executor_task.done(): return False if session_key: _hb_state = self._peek_session_state(session_key) if (_hb_state.turn.agent if _hb_state else None) is not agent: return False return True # Upper bound on off-loop agent-resource cleanup invoked from coroutines # running on the gateway's event loop (session-expiry sweep, in-turn # cache-hygiene re-eviction). _cleanup_agent_resources is synchronous and # can block for a long time (agent.close() does subprocess teardown; # shutdown_memory_provider() may do network/SQLite IO via a memory plugin). # Calling it inline wedges the whole loop — the bot goes silent, the # runtime-status updated_at heartbeat freezes, and SIGTERM cannot be # serviced (#53175). Offload to a worker thread under this timeout so the # loop is never blocked; mirrors the /new reset path's fix (#35994). _CLEANUP_TIMEOUT_S = 30.0 def _defer_agent_cleanup_until_future_done( self, future: asyncio.Future, agent: Any, *, context: str, ) -> None: """Clean up ``agent`` only after its executor future has finished. A timed-out executor call keeps running in its worker thread. Closing the agent before that thread exits can tear down clients or providers it is still using. Keep a strong task reference and wait for the real future before invoking the normal bounded, off-loop cleanup path. """ async def _cleanup_when_done() -> None: try: await asyncio.shield(future) except asyncio.CancelledError: # Loop shutdown can cancel this waiter while the executor still # runs. Never turn that cancellation into premature cleanup. return except Exception as exc: logger.debug( "Deferred agent worker%s finished with an error: %s", f" ({context})" if context else "", exc, ) await self._cleanup_agent_resources_off_loop(agent, context=context) self._track_deferred_agent_worker(future, agent) task = asyncio.create_task(_cleanup_when_done()) tasks = getattr(self, "_deferred_agent_cleanup_tasks", None) if tasks is None: tasks = set() self._deferred_agent_cleanup_tasks = tasks tasks.add(task) task.add_done_callback(tasks.discard) # Bounded budget for one finalize_session() dispatch (plugin # on_session_finalize hooks + core Relay conversation close). Generous # enough for a normal trace-export flush, small enough that a wedged # plugin can never eat the systemd stop window. _FINALIZE_TIMEOUT_S = 10.0 async def _finalize_session_off_loop( self, *, session_id: Any, platform: str, reason: str, **extra: Any, ) -> None: """Run hermes_cli.lifecycle.finalize_session off the event loop, bounded. finalize_session() invokes plugin ``on_session_finalize`` hooks synchronously; a hook doing heavy blocking work (observability trace export, network flush) on the event loop freezes heartbeats, adapters, and the shutdown drain itself. Off-loop + ``wait_for`` keeps the loop live; on timeout the worker thread is left to finish (or leak) on its own and the caller proceeds — mirroring ``_cleanup_agent_resources_off_loop`` (#53175). """ def _call() -> None: from hermes_cli.lifecycle import finalize_session finalize_session( session_id=session_id, platform=platform, reason=reason, **extra, ) try: await asyncio.wait_for( self._run_in_executor_with_context(_call), timeout=self._FINALIZE_TIMEOUT_S, ) except asyncio.TimeoutError: logger.warning( "Session finalize hooks (%s, reason=%s) exceeded %ss; " "proceeding without blocking the event loop (the worker " "thread is left to finish on its own).", session_id, reason, self._FINALIZE_TIMEOUT_S, ) except Exception as finalize_exc: logger.debug( "Session finalize hooks (%s, reason=%s) failed: %s", session_id, reason, finalize_exc, ) async def _cleanup_agent_resources_off_loop( self, agent: Any, *, context: str = "" ) -> None: """Run _cleanup_agent_resources in a worker thread with a bounded wait. Safe to await from coroutines on the gateway event loop: a slow or wedged teardown (memory provider IO, subprocess close) can no longer block message processing. On timeout the await is cancelled and the worker thread is left to finish (or leak) on its own — the caller proceeds regardless, exactly as the /new reset path does (#35994). """ if agent is None: return if context.startswith("shutdown") or context == "session expiry": try: agent._end_session_on_close = False except Exception: pass try: await asyncio.wait_for( self._run_in_executor_with_context( self._cleanup_agent_resources, agent ), timeout=self._CLEANUP_TIMEOUT_S, ) except asyncio.TimeoutError: logger.warning( "Agent resource cleanup%s exceeded %ss; proceeding without " "blocking the event loop (the worker thread is left to finish " "on its own). (#53175)", f" ({context})" if context else "", self._CLEANUP_TIMEOUT_S, ) except Exception as cleanup_exc: logger.warning( "Agent resource cleanup%s failed: %s (#53175)", f" ({context})" if context else "", cleanup_exc, ) def _cleanup_agent_resources(self, agent: Any) -> None: """Best-effort cleanup for temporary or cached agent instances.""" if agent is None: return try: if hasattr(agent, "shutdown_memory_provider"): # Drain queued memory writes BEFORE tearing the provider down. # The memory manager persists per-turn sync and end-of-session # extraction on a single serialized background worker. # shutdown_memory_provider() -> shutdown_all() only gives that # worker a ~5s bounded drain and abandons (cancels) anything # still queued past it, so a /reset — or any gateway session # rotation that reaches this cleanup path — could silently drop # writes the session had already handed off. The next session # then loads stale memory (#73297). Give pending work a bounded # head start through the manager's own barrier first, mirroring # the CLI exit path (cli.py). Best-effort: a flush failure must # never block teardown. _mm = getattr(agent, "_memory_manager", None) if _mm is not None and hasattr(_mm, "flush_pending"): try: _mm.flush_pending(timeout=10) except Exception: pass # Pass the agent's own conversation transcript so memory # providers' ``on_session_end`` hooks see the real messages # instead of the empty default (#15165). ``_session_messages`` # is set on ``AIAgent`` (run_agent.py:1518) and refreshed at # the end of every ``run_conversation`` turn via # ``_persist_session``; on an agent built through # ``object.__new__`` (test stubs) the attribute may be # absent, so ``getattr`` with a ``None`` default keeps the # call signature-compatible with the pre-fix behaviour # (``shutdown_memory_provider(messages=None)``). session_messages = getattr(agent, "_session_messages", None) if isinstance(session_messages, list): agent.shutdown_memory_provider(session_messages) else: agent.shutdown_memory_provider() except Exception: pass # Close tool resources (terminal sandboxes, browser daemons, # background processes, httpx clients) to prevent zombie # process accumulation. try: if hasattr(agent, "close"): agent.close() except Exception: pass # Auxiliary async clients (session_search/web/vision/etc.) live in a # process-global cache and are created inside worker threads. Clean up # any entries whose event loop is now dead so their httpx transports do # not accumulate across gateway turns. try: from agent.auxiliary_client import cleanup_stale_async_clients cleanup_stale_async_clients() except Exception: pass _STUCK_LOOP_THRESHOLD = 3 # restarts while active before auto-suspend _STUCK_LOOP_FILE = ".restart_failure_counts" def _increment_restart_failure_counts(self, active_session_keys: set) -> None: """Increment restart-failure counters for sessions active at shutdown. Persists to a JSON file so counters survive across restarts. Sessions NOT in active_session_keys are removed (they completed successfully, so the loop is broken). """ import json path = _hermes_home / self._STUCK_LOOP_FILE try: counts = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} except Exception: counts = {} # Increment active sessions, remove inactive ones (loop broken) new_counts = {} for key in active_session_keys: new_counts[key] = counts.get(key, 0) + 1 # Keep any entries that are still above 0 even if not active now # (they might become active again next restart) try: atomic_json_write(path, new_counts, indent=None) except Exception: pass def _suspend_stuck_loop_sessions(self) -> int: """Suspend sessions that have been active across too many restarts. Returns the number of sessions suspended. Called on gateway startup AFTER suspend_recently_active() to catch the stuck-loop pattern: session loads → agent gets stuck → gateway restarts → repeat. """ import json path = _hermes_home / self._STUCK_LOOP_FILE if not path.exists(): return 0 try: counts = json.loads(path.read_text(encoding="utf-8")) except Exception: return 0 suspended = 0 stuck_keys = [k for k, v in counts.items() if v >= self._STUCK_LOOP_THRESHOLD] for session_key in stuck_keys: try: entry = self.session_store._entries.get(session_key) if entry and not entry.suspended: entry.suspended = True suspended += 1 logger.warning( "Auto-suspended stuck session %s (active across %d " "consecutive restarts — likely a stuck loop)", session_key, counts[session_key], ) except Exception: pass if suspended: try: self.session_store._save() except Exception: pass # Clear the file — counters start fresh after suspension try: path.unlink(missing_ok=True) except Exception: pass return suspended async def _clear_restart_failure_count(self, session_key: str) -> None: """Clear the restart-failure counter for a session that completed OK. Called after a successful agent turn to signal the loop is broken. Offloaded to a thread because the caller (_handle_message_with_agent) runs on the event loop and atomic_json_write calls os.fsync. """ import json path = _hermes_home / self._STUCK_LOOP_FILE if not path.exists(): return try: counts = json.loads(path.read_text(encoding="utf-8")) if session_key in counts: del counts[session_key] if counts: await asyncio.to_thread(atomic_json_write, path, counts, indent=None) else: path.unlink(missing_ok=True) except Exception: pass async def _launch_detached_restart_command(self) -> None: import shutil import subprocess hermes_cmd = _resolve_hermes_bin() if not hermes_cmd: logger.error("Could not locate hermes binary for detached /restart") return if self._detached_restart_helper_started: return self._detached_restart_helper_started = True current_pid = os.getpid() restart_after_s = max(float(getattr(self, "_restart_drain_timeout", 0.0) or 0.0) + 5.0, 5.0) # On Windows there's no bash/setsid chain — spawn a tiny Python # watcher directly via sys.executable instead. The watcher polls # current_pid, waits for our exit, then runs `hermes gateway # restart` with detach flags so the respawn survives the CLI # that triggered the /restart command closing its console. if sys.platform == "win32": import textwrap from hermes_cli._subprocess_compat import ( windows_detach_flags_without_breakaway, windows_detach_popen_kwargs, ) cmd_argv = [*hermes_cmd, "gateway", "restart"] watcher = textwrap.dedent( """ import os, subprocess, sys, time from hermes_cli._subprocess_compat import windows_detach_flags_without_breakaway pid = int(sys.argv[1]) restart_after_s = float(sys.argv[2]) cmd = sys.argv[3:] deadline = time.monotonic() + restart_after_s def _alive(p): # On Windows, os.kill(pid, 0) is NOT a no-op — it maps to # GenerateConsoleCtrlEvent(0, pid) (bpo-14484). Use the # Win32 handle-based existence check instead. if os.name == 'nt': import ctypes k32 = ctypes.windll.kernel32 k32.OpenProcess.restype = ctypes.c_void_p k32.WaitForSingleObject.restype = ctypes.c_uint k32.GetLastError.restype = ctypes.c_uint h = k32.OpenProcess(0x1000 | 0x100000, False, int(p)) if not h: return k32.GetLastError() != 87 try: return k32.WaitForSingleObject(h, 0) == 0x102 finally: k32.CloseHandle(h) try: os.kill(int(p), 0) return True except ProcessLookupError: return False except PermissionError: return True except OSError: return False while time.monotonic() < deadline: if not _alive(pid): break time.sleep(0.2) subprocess.Popen( cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, creationflags=windows_detach_flags_without_breakaway(), ) """ ).strip() from tools.environments.local import build_subprocess_env watcher_env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=True) # This watcher is intentionally outside the running gateway. If it # inherits the gateway marker, `hermes gateway restart` refuses to # run as a self-restart loop guard and the gateway stays stopped. watcher_env.pop("_HERMES_GATEWAY", None) project_root = Path(__file__).resolve().parent.parent # The watcher runs sys.executable (console python) under the # CREATE_NO_WINDOW detach kwargs below: it owns one hidden # console, inherited by the `hermes gateway restart` child, so # nothing flashes. Do NOT swap in GUI-subsystem pythonw.exe — # a console-less watcher forces every console-subsystem # descendant to allocate a visible conhost (#54220/#56747). watcher_python = sys.executable venv_dir = Path(watcher_env.get("VIRTUAL_ENV") or project_root / "venv") site_packages = venv_dir / "Lib" / "site-packages" if site_packages.exists(): watcher_env["VIRTUAL_ENV"] = str(venv_dir) pythonpath = [str(project_root), str(site_packages)] if watcher_env.get("PYTHONPATH"): pythonpath.append(watcher_env["PYTHONPATH"]) watcher_env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(pythonpath)) watcher_argv = [ watcher_python, "-c", watcher, str(current_pid), str(restart_after_s), *cmd_argv, ] # The watcher process must itself break away from any job object the # parent CLI lives in (Electron/Tauri-wrapped Hermes Desktop, Windows # Terminal, schtasks shells); otherwise it is reaped when the CLI # exits and the gateway never respawns. windows_detach_popen_kwargs() # carries CREATE_BREAKAWAY_FROM_JOB, but a restrictive job object # (no JOB_OBJECT_LIMIT_BREAKAWAY_OK) rejects that bit with # ERROR_ACCESS_DENIED, surfaced as OSError. Retry once without the # breakaway bit, preserving argv and the scrubbed watcher_env. # Mirrors the canonical fallback in # hermes_cli/gateway_windows.py::_spawn_detached. try: subprocess.Popen( watcher_argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=watcher_env, **windows_detach_popen_kwargs(), ) except OSError: try: subprocess.Popen( watcher_argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=watcher_env, creationflags=windows_detach_flags_without_breakaway(), ) except OSError as exc: # Both spawn attempts failed (a breakaway-denying job object # is the common cause, but OSError covers others too). # Record a minimal, path-safe diagnostic and return without # crashing the caller: state plainly that no watcher was # started, and log only the interpreter basename and a # numeric error code — never argv, env, watcher source, or # str(exc) (which can carry a full interpreter path for a # FileNotFoundError). winerror = getattr(exc, "winerror", None) error_code = winerror if winerror is not None else exc.errno error_field = "winerror" if winerror is not None else "errno" logger.warning( "Detached restart watcher was not started after the " "no-breakaway retry (%s; %s=%r). The gateway will not " "be respawned by this restart attempt.", os.path.basename(watcher_python), error_field, error_code, ) return cmd = " ".join(shlex.quote(part) for part in hermes_cmd) shell_cmd = ( f"deadline=$(( $(date +%s) + {int(restart_after_s)} )); " f"while kill -0 {current_pid} 2>/dev/null && [ $(date +%s) -lt $deadline ]; do sleep 0.2; done; " f"{cmd} gateway restart" ) # Same marker scrub as the Windows watcher above: this watcher runs # `hermes gateway restart` from outside the gateway, but it inherits # _HERMES_GATEWAY=1 from us, and the CLI's self-restart loop guard # refuses to run when that marker is set — silently (DEVNULL), so the # gateway stops and never comes back. from tools.environments.local import build_subprocess_env watcher_env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=True) watcher_env.pop("_HERMES_GATEWAY", None) setsid_bin = shutil.which("setsid") if setsid_bin: subprocess.Popen( [setsid_bin, "bash", "-lc", shell_cmd], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=watcher_env, start_new_session=True, ) else: subprocess.Popen( ["bash", "-lc", shell_cmd], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=watcher_env, start_new_session=True, ) def _wedged_agent_count(self) -> int: """Count running chat agents already past the inactivity timeout. A turn whose agent has recorded no activity (no API bytes, no tool progress) for longer than ``agent.gateway_timeout`` is wedged — the same threshold at which the turn reaper gives up on it. The restart after-turn wait must not treat such turns as work worth waiting for: a wedged agent pinned ``hermes update`` in "draining" for the full ``restart_after_turn_timeout`` cap because the drain counted it as active while its own inactivity watchdog had already declared it dead (Aug 2026, WhatsApp turn idle 30+ min, drain waited on it anyway). Returns 0 when the inactivity timeout is disabled (``gateway_timeout`` 0/unset ⇒ the operator opted into unbounded turns; the after-turn cap still bounds the wait). Cron/API-server work has no per-turn activity clock and is never counted as wedged. Pending sentinels are brand-new turns, never wedged. Fail-open per agent: an unreadable activity summary means "not wedged". """ timeout = _float_env("HERMES_AGENT_TIMEOUT", 1800) if timeout <= 0: return 0 wedged = 0 for agent in list((getattr(self, "_running_agents", None) or {}).values()): if agent is None or agent is _AGENT_PENDING_SENTINEL: continue summary_fn = getattr(agent, "get_activity_summary", None) if not callable(summary_fn): continue try: summary = summary_fn() if not isinstance(summary, dict): continue idle = float(summary.get("seconds_since_activity", 0.0)) except Exception: continue if idle >= timeout: wedged += 1 return wedged def _awaitable_work_count(self) -> int: """Active work minus wedged turns — what the restart wait waits on.""" return max(0, self._active_work_count() - self._wedged_agent_count()) async def _await_active_work_before_restart(self) -> bool: """Wait for in-flight work to finish before entering ``stop()``. In-band restart used to call ``stop()`` immediately, which folded the requesting turn into the drain wait set and force-interrupted it at ``restart_drain_timeout`` (#77184). Instead we refuse new turns and wait here for active agents/cron/api work to reach zero, then let ``stop()`` run against an idle gateway (drain is instant). Turns already past the inactivity timeout are excluded from the wait (``_wedged_agent_count``): restart is usually the *remedy* for a wedged turn, so deferring it behind one inverts the point of the graceful path. ``stop()``'s drain interrupts them under ``restart_drain_timeout`` instead. Returns True when work drained to zero, False when the safety cap elapsed with work still active — or when only wedged work remains — (caller proceeds to ``stop()``, which may then interrupt remaining runs under ``restart_drain_timeout``). """ active = self._active_work_count() if active <= 0: return True awaitable = self._awaitable_work_count() if awaitable <= 0: logger.warning( "Restart requested with %d active work unit(s), all wedged " "past the inactivity timeout; skipping the after-turn wait " "and proceeding to stop()/drain which will interrupt them", active, ) return False timeout = float(getattr(self, "_restart_after_turn_timeout", 0.0) or 0.0) if timeout <= 0: logger.info( "Restart requested with %d active work unit(s); " "restart_after_turn_timeout=0 — entering stop()/drain immediately", active, ) return False logger.info( "Restart requested with %d active work unit(s); " "deferring stop() until they finish (cap=%.0fs) so in-flight " "turns are not amputated (#77184)", active, timeout, ) try: self._update_runtime_status("draining") except Exception: pass loop = asyncio.get_running_loop() deadline = loop.time() + timeout last_status_at = 0.0 while self._awaitable_work_count() > 0: now = loop.time() if now >= deadline: logger.warning( "Restart after-turn wait timed out after %.0fs with %d " "still active; proceeding to stop()/drain which may " "interrupt remaining work (#77184)", timeout, self._active_work_count(), ) return False if (now - last_status_at) >= 30.0: logger.info( "Restart deferred: waiting on %d active work unit(s) " "(%d wedged and excluded; %.0fs remaining before force drain)", self._awaitable_work_count(), self._wedged_agent_count(), deadline - now, ) try: self._update_runtime_status("draining") except Exception: pass last_status_at = now await asyncio.sleep(0.1) if self._active_work_count() > 0: logger.warning( "Restart deferred wait: %d wedged work unit(s) remain; " "proceeding to stop()/drain which will interrupt them", self._active_work_count(), ) return False logger.info( "Restart deferred wait complete — active work drained; " "proceeding to stop()" ) return True def request_restart(self, *, detached: bool = False, via_service: bool = False) -> bool: if self._restart_task_started: return False self._restart_requested = True self._restart_detached = detached self._restart_via_service = via_service self._restart_task_started = True # Refuse new turns immediately while in-flight work finishes. # Keep ``_running`` True so adapters stay connected and the active # turn can still deliver its final response (#77184). self._draining = True async def _run_restart() -> None: await self._await_active_work_before_restart() # Launch the detached helper only AFTER the after-turn wait. # Its deadline is drain_timeout+5 and covers stop() teardown — # launching earlier would fire `hermes gateway restart` while # the requesting turn was still running. if detached: try: await self._launch_detached_restart_command() except Exception as e: logger.error("Failed to launch detached gateway restart helper: %s", e) await asyncio.sleep(0.05) await self.stop(restart=True, detached_restart=detached, service_restart=via_service) # _run_restart is a short-lived self-terminating task (calls stop() # then returns). Don't add it to _background_tasks — _stop_impl # cancels all entries in that set, which would cancel _run_restart # while it's awaiting _stop_task, propagating CancelledError into # _stop_impl and preventing _shutdown_event.set() / _exit_code = 75. # See #12875. # # We still hold a strong reference in self._restart_task: a bare # asyncio.create_task() keeps only a weak reference, so the event # loop may garbage-collect a still-pending task mid-flight. The # cancel loop in _stop_impl explicitly skips _restart_task for the # same reason it skips _stop_task. self._restart_task = asyncio.create_task(_run_restart()) return True # Drain-timeout reasons set by _stop_impl() when a still-running turn is # force-interrupted; "restart_interrupted" is set by # SessionStore.suspend_recently_active() on crash recovery (no # .clean_shutdown marker). All three mean "the agent was mid-turn and # we killed it" — eligible for startup auto-resume. _AUTO_RESUME_REASONS = frozenset( {"restart_timeout", "shutdown_timeout", "restart_interrupted"} ) async def _run_startup_resume_event( self, adapter: BasePlatformAdapter, event: MessageEvent, session_key: str, ) -> None: """Dispatch one synthetic startup resume and wait for its agent turn. ``BasePlatformAdapter.handle_message()`` returns after it installs the adapter-level guard and spawns the background processing task. Startup restore needs a stronger boundary: inbound messages must stay queued until the resumed agent turn itself has finished, otherwise a user message can race the restore turn immediately after ``handle_message`` returns. """ try: await adapter.handle_message(event) session_tasks = getattr(adapter, "_session_tasks", {}) task = session_tasks.get(session_key) if isinstance(session_tasks, dict) else None if task is not None: await asyncio.shield(task) finally: # _schedule_resume_pending_sessions pre-claims the runner slot # before spawning this task. If adapter.handle_message raises # before _handle_message takes ownership, release that pre-claim; # otherwise the real run's normal cleanup owns the slot. _pre_state = self._peek_session_state(session_key) if (_pre_state.turn.agent if _pre_state else None) is _AGENT_PENDING_SENTINEL: self._release_running_agent_state(session_key) def _queue_startup_restore_event(self, event: MessageEvent) -> None: queue = getattr(self, "_startup_restore_queue", None) if queue is None: queue = [] self._startup_restore_queue = queue queue.append(event) try: source = event.source logger.info( "Queued inbound message during gateway startup restore: platform=%s chat=%s", source.platform.value if source and source.platform else "unknown", source.chat_id if source else "unknown", ) except Exception: pass async def _drain_startup_restore_queue(self) -> int: """Replay inbound messages queued while startup auto-resume ran.""" drained = 0 queue = getattr(self, "_startup_restore_queue", None) if queue is None: return 0 while queue: event = queue.pop(0) source = getattr(event, "source", None) adapter = self._adapter_for_source(source) if adapter is None: logger.debug( "Dropping startup-restore queued message: adapter unavailable for %s", getattr(getattr(source, "platform", None), "value", None), ) continue # Mark this replay so _handle_message does not queue it again while # the restore gate remains closed for any fresh inbound arrivals. try: setattr(event, "_hermes_startup_restore_replay", True) except Exception: pass await adapter.handle_message(event) drained += 1 return drained def _start_startup_warmup(self) -> None: """Kick off the boot turn-machinery warm-up in the background (#99373). Called from ``start()`` right after the startup-restore gate closes, so the warm-up overlaps the (slow, network-bound) platform connects instead of adding boot latency. ``_finish_startup_restore`` awaits it (bounded) before opening the inbound gate. """ timeout = _startup_warmup_timeout_secs() if timeout <= 0: self._startup_warmup_task = None return self._startup_warmup_task = asyncio.ensure_future( self._warm_turn_prerequisites() ) async def _warm_turn_prerequisites(self) -> None: """Initialize turn machinery off-loop before the gate opens (#99373). Runs ``_warm_turn_machinery_sync`` (run_agent import graph, tool schemas + check_fn probe cache, context-file tier) on an executor thread so the event loop — platform heartbeats, connects — stays responsive. Never raises: a failed warm-up degrades to the historical lazy init, it must not block startup. """ try: loop = asyncio.get_running_loop() t0 = time.monotonic() tool_count = await loop.run_in_executor(None, _warm_turn_machinery_sync) logger.info( "Turn machinery warmed in %.1fs (%d tool schema(s) materialized)", time.monotonic() - t0, tool_count, ) except Exception: logger.warning( "Turn-machinery warm-up failed; first inbound turn will " "initialize lazily", exc_info=True, ) async def _await_startup_warmup(self) -> None: """Bounded wait for the boot warm-up before the inbound gate opens. On timeout the gate opens anyway (availability outranks prompt completeness for a WEDGED init — same principle as the bounded restore-drain wait above) and the warm-up continues in the background; a late failure is still logged. """ task = getattr(self, "_startup_warmup_task", None) if task is None or task.done(): return timeout = _startup_warmup_timeout_secs() if timeout <= 0: return done, pending = await asyncio.wait({task}, timeout=timeout) if pending: logger.warning( "Turn-machinery warm-up still running after %.0fs; opening " "inbound gate anyway — the first turn may see lazily " "initialized machinery (#99373). Warm-up continues in the " "background.", timeout, ) task.add_done_callback( lambda t: GatewayRunner._log_late_background_failure( t, "boot turn-machinery warm-up failed after gate release", level=logging.DEBUG, ) ) async def _finish_startup_restore(self) -> None: """Wait (BOUNDED) for startup auto-resume, then release + drain inbound. The wait is bounded by ``_startup_restore_drain_timeout_secs`` so that a single pathologically long boot-resume turn cannot hold the inbound gate shut for every channel. On timeout we release the gate and let the still-running resume turn(s) finish in the background — they are NOT cancelled. This is safe because duplicate-agent protection does not depend on the wait: ``_schedule_resume_pending_sessions`` claims each session's ``_running_agents`` slot SYNCHRONOUSLY before this gate runs, so any inbound message drained while a resume turn is still in flight queues behind that slot instead of spawning a second agent. """ tasks = list(getattr(self, "_startup_restore_tasks", []) or []) if tasks: timeout = _startup_restore_drain_timeout_secs() if timeout > 0: # asyncio.wait (unlike wait_for / gather+timeout) does NOT # cancel the pending tasks on timeout — the slow resume turn # keeps running in the background instead of being killed. done, pending = await asyncio.wait(tasks, timeout=timeout) if pending: logger.warning( "Startup-restore gate released after %.0fs with %d boot " "auto-resume turn(s) still running; draining inbound " "queue now (resume slots already claimed, so no " "duplicate agents). Slow turn(s) continue in the " "background.", timeout, len(pending), ) # These tasks outlive the gate. Their normal done-callback # only discards them from _background_tasks, so a LATER # failure would be silently swallowed. Attach a logging # callback so a background resume turn that fails after the # timeout is still recorded. for task in pending: task.add_done_callback(self._log_background_resume_result) else: # Non-positive timeout => opt out of the bound (historical # "wait forever" behaviour). await asyncio.gather(*tasks, return_exceptions=True) done = set(tasks) for task in done: if task.cancelled(): continue exc = task.exception() if exc is not None: logger.debug( "startup auto-resume task failed", exc_info=(type(exc), exc, exc.__traceback__), ) self._startup_restore_tasks = [] # Warm the turn machinery BEFORE the queue drains: replayed (and # fresh) inbound turns must not build skeleton prompts (#99373). await self._await_startup_warmup() drained = await self._drain_startup_restore_queue() self._startup_restore_in_progress = False if drained: logger.info("Drained %d inbound message(s) queued during startup restore", drained) @staticmethod def _log_background_resume_result(task: "asyncio.Task") -> None: """Done-callback for a boot-resume turn that outlived the startup-restore gate. Logs a late failure that would otherwise be swallowed once the task is discarded from ``_background_tasks``. Cancellation is expected (shutdown) and is not an error.""" GatewayRunner._log_late_background_failure( task, "background startup auto-resume task failed after gate release", level=logging.DEBUG, ) @staticmethod def _log_late_background_failure( task: "asyncio.Task", message: str, *, level: int = logging.WARNING ) -> None: """Shared done-callback body for boot-path tasks that outlive the startup-restore gate: surface a late failure that would otherwise be swallowed once the task is discarded from ``_background_tasks``. Cancellation is expected (shutdown) and is not an error.""" if task.cancelled(): return exc = task.exception() if exc is not None: logger.log( level, message, exc_info=(type(exc), exc, exc.__traceback__), ) async def _await_startup_boot_sends( self, *, planned_restart_notification_pending: bool, ) -> None: """Run boot-path sends without letting them pin the inbound restore gate. ``_send_restart_notification`` and ``_redeliver_pending_obligations`` used to be awaited inline *before* ``_finish_startup_restore`` released the gate. A single Telegram flood-control sleep on either send froze inbound on every platform for the full ``retry_after`` (#91969). This uses the same bounded ``asyncio.wait`` the resume gate already uses: on timeout we return and let the sends finish in the background. Tasks are not cancelled. The ledger claim + ``resume_pending`` clear happen INLINE here, before the send task exists: they are pure DB work (no network, bounded by claimed-row count), and deferring them into the send task left a window where a hung restart notification ahead of the redelivery step let the gate expire with zero rows claimed — the resume scheduler then replayed turns whose answers were already in the ledger, and the background task later redelivered them too (duplicate delivery + re-paid turn). """ claimed = await self._claim_pending_obligations() async def _boot_sends() -> None: await self._send_restart_notification() if planned_restart_notification_pending: try: await self._send_home_channel_startup_notifications( skip_targets=None, ) finally: _clear_planned_restart_notification() await self._redeliver_claimed_obligations(claimed) boot_task = asyncio.create_task(_boot_sends()) timeout = _startup_restore_drain_timeout_secs() if timeout > 0: _done, pending = await asyncio.wait({boot_task}, timeout=timeout) if pending: logger.warning( "Boot-path sends still running after %.0fs; releasing " "inbound gate so other platforms are not frozen. " "Restart notification / obligation redelivery continue " "in the background.", timeout, ) boot_task.add_done_callback(self._log_background_boot_send_result) tasks = getattr(self, "_background_tasks", None) if tasks is None: self._background_tasks = set() tasks = self._background_tasks tasks.add(boot_task) boot_task.add_done_callback(tasks.discard) else: await boot_task @staticmethod def _log_background_boot_send_result(task: "asyncio.Task") -> None: """Done-callback for boot-path sends that outlived the restore gate.""" GatewayRunner._log_late_background_failure( task, "background boot-path send failed after gate release: see traceback" ) async def _clear_resume_pending_for_claimed_obligations( self, claimed: list, *, require_success: bool = False ) -> list: """Clear resume flags and return rows safe to redeliver. Startup recovery preserves its historical best-effort behavior. Runtime reconnect recovery is stricter: if the session-store write fails, the corresponding response must not be sent because the same agent turn could otherwise be resumed immediately afterward. """ sendable = [] for row in claimed: session_key = row.get("session_key") or "" if not session_key: sendable.append(row) continue try: await self.async_session_store.clear_resume_pending(session_key) except Exception: logger.debug( "clear_resume_pending failed for %s", session_key, exc_info=True, ) if not require_success: sendable.append(row) else: sendable.append(row) return sendable async def _claim_pending_obligations(self) -> list: """Claim recoverable delivery-ledger rows and clear their ``resume_pending`` flags. Pure DB work — no network sends. Runs INLINE at startup BEFORE ``_schedule_resume_pending_sessions`` and before the (bounded, abandonable) boot-send task exists. A session with a recoverable obligation already produced its answer — the turn completed and only delivery is owed — so clearing ``resume_pending`` here prevents the resume path from re-running (and re-paying for) a turn whose output we hold, regardless of how long the sends ahead of redelivery take (#91969). Crash-ambiguity contract (see gateway/delivery_ledger.py): rows that were mid-send or previously rejected carry a visible recovered-reply marker so a possible duplicate is labeled, never silent. Returns the claimed rows for redelivery. """ try: from gateway.delivery_ledger import ( ledger_enabled, sweep_recoverable, ) if not await asyncio.to_thread(ledger_enabled): return [] # Only claim rows whose exact transport owner is connected this # boot. A multiplexed gateway can host several bot identities for # one platform; platform-only filtering would spend a disconnected # bot's retry budget merely because another bot is online. _profile_adapters = getattr(self, "_profile_adapters", None) or {} _deliverable_targets = { (getattr(p, "value", str(p)), "default") for p in self.adapters } # Legacy rows predate adapter_profile. They are unambiguous only in # a non-multiplexed gateway; fail closed when multiple bot identities # share the process. if not _profile_adapters: _deliverable_targets.update( (getattr(p, "value", str(p)), None) for p in self.adapters ) for _profile, _adapters in _profile_adapters.items(): _deliverable_targets.update( (getattr(p, "value", str(p)), _profile) for p in _adapters ) _deliverable = {platform for platform, _ in _deliverable_targets} claimed = await asyncio.to_thread( sweep_recoverable, None, deliverable_platforms=_deliverable, deliverable_targets=_deliverable_targets, ) except Exception: logger.debug("delivery ledger sweep failed", exc_info=True) return [] if not claimed: return [] # Clear resume_pending for EVERY claimed row up front, before any # send. Claiming already spent one of the row's redelivery attempts — # the answer is in the ledger, so the resume path must never re-run # these turns (#91969). await self._clear_resume_pending_for_claimed_obligations(claimed) return claimed async def _redeliver_claimed_obligations(self, claimed: list) -> int: """Redeliver final responses for rows already claimed (and resume-cleared) by :meth:`_claim_pending_obligations`. Network half of the split — runs inside the bounded boot-send task, so a flood-limited send can be abandoned by the restore gate without reopening the turn-replay window. Returns redeliveries attempted. """ if not claimed: return 0 try: from gateway.delivery_ledger import ( RECOVERED_MARKER, mark_delivered, mark_failed, release_runtime_claim, ) except Exception: logger.debug("delivery ledger import failed", exc_info=True) return 0 redelivered = 0 for row in claimed: try: platform = Platform(row["platform"]) except Exception: logger.debug( "obligation %s: unknown platform %r", row["obligation_id"], row.get("platform"), ) continue if "profile" in row: adapter = self._authorization_adapter( platform, row.get("profile") ) else: # Startup rows preserve the historical default-adapter route. adapter = self.adapters.get(platform) if adapter is None: # Runtime claims have not reached a transport yet. If the # reconnect vanished before dispatch, release the claim without # spending an attempt so the next reconnect can retry it. if row.get("runtime_recovery"): try: await asyncio.to_thread( release_runtime_claim, row["obligation_id"], "send_path_degraded", ) except Exception: logger.debug( "failed to release undispatched runtime obligation %s", row["obligation_id"], exc_info=True, ) # Startup claims preserve their historical state; attempts cap # + stale cutoff bound later retries. continue content = row["content"] if row.get("needs_marker"): content = row.get("marker", RECOVERED_MARKER) + content metadata = ( {"thread_id": row["thread_id"]} if row.get("thread_id") else None ) try: result = await adapter.send( chat_id=row["chat_id"], content=content, metadata=metadata, ) except Exception as send_err: logger.warning( "obligation %s: redelivery send raised: %s", row["obligation_id"], send_err, ) result = None try: if result is not None and getattr(result, "success", False): await asyncio.to_thread(mark_delivered, row["obligation_id"]) redelivered += 1 logger.info( "Redelivered recovered final response to %s:%s " "(obligation %s, attempt %d)", row["platform"], row["chat_id"], row["obligation_id"], row["attempts"], ) else: await asyncio.to_thread( mark_failed, row["obligation_id"], str(getattr(result, "error", "") or "send failed"), ) except Exception: logger.debug("delivery ledger update failed", exc_info=True) return redelivered async def _redeliver_pending_obligations(self) -> int: """Claim + redeliver in one call — composition of :meth:`_claim_pending_obligations` and :meth:`_redeliver_claimed_obligations`. Kept as the stable public shape (tests and any external callers drive this name); the startup path calls the two halves separately so the DB half can run inline before the abandonable send task. """ return await self._redeliver_claimed_obligations( await self._claim_pending_obligations() ) async def _redeliver_failed_obligations_for_platform( self, platform: Platform, *, profile: Optional[str] = None, ) -> int: """Replay one adapter identity's transient failures after reconnect. The startup sweep cannot claim live-owner rows by design. A platform adapter can reconnect without the gateway process exiting, however, so ``send_path_degraded`` responses otherwise remain failed until the next process restart. Claiming, resume clearing, and sending stay best-effort and reuse the startup redelivery path's attempt and ambiguity contract. """ try: from gateway.delivery_ledger import ( ledger_enabled, release_runtime_claim, sweep_failed_for_runtime, ) if not await asyncio.to_thread(ledger_enabled): return 0 claimed = await asyncio.to_thread( sweep_failed_for_runtime, platform.value, profile=profile, ) except Exception: logger.debug( "runtime delivery ledger sweep failed after %s reconnect", platform.value, exc_info=True, ) return 0 if not claimed: return 0 # Clear before any send so the reconnect path cannot both redeliver an # already-produced answer and schedule the same agent turn for resume. sendable = await self._clear_resume_pending_for_claimed_obligations( claimed, require_success=True ) sendable_ids = {row["obligation_id"] for row in sendable} for row in claimed: if row["obligation_id"] in sendable_ids: continue try: await asyncio.to_thread( release_runtime_claim, row["obligation_id"], "send_path_degraded", ) except Exception: logger.debug( "failed to release runtime delivery claim %s", row["obligation_id"], exc_info=True, ) return await self._redeliver_claimed_obligations(sendable) def _schedule_resume_pending_sessions(self, platform=None) -> int: """Auto-continue fresh restart-interrupted sessions after startup. ``resume_pending`` already preserves the transcript AND the existing ``_is_resume_pending`` branch in ``_handle_message_with_agent`` injects a reason-aware recovery system note on the next turn. This method closes the UX gap by synthesizing that next turn once adapters are back online — the event text is empty so the existing injection path owns the wording and we never double up. Adapters that are not yet ready (adapter missing from ``self.adapters``) are skipped silently; their sessions stay ``resume_pending`` and will auto-resume on the next real user message, or when the platform reconnects — the reconnect watcher calls this again scoped to that ``platform``. ``platform`` (a ``Platform``) restricts the pass to sessions that originated on that platform. The reconnect path passes it so a platform coming back online retries only its own sessions and never re-touches another platform's in-flight recoveries. Sessions whose agent is already running are skipped regardless, so a session scheduled at startup is never resumed a second time. """ window = _auto_continue_freshness_window() try: with self.session_store._lock: # noqa: SLF001 — snapshot under lock self.session_store._ensure_loaded_locked() # noqa: SLF001 candidates = [ entry for entry in self.session_store._entries.values() # noqa: SLF001 if entry.resume_pending and not entry.suspended and entry.origin is not None and entry.resume_reason in self._AUTO_RESUME_REASONS and (platform is None or entry.origin.platform == platform) ] except Exception as exc: logger.warning("Failed to enumerate resume-pending sessions: %s", exc) return 0 # Defense-3 (#30719): break the SIGTERM-respawn loop. Only count this # boot when there are restart-interrupted sessions to resume — a clean # boot must not accrue toward the breaker. If too many such boots have # happened in the configured window, skip auto-resume for THIS boot: # the gateway still comes up and serves real inbound messages, it just # stops replaying the session that keeps killing it. The session stays # resume_pending, so a real user message can still continue it (a human # is now in the loop). Defenses 1-2 cover the cron/CLI/terminal paths; # this catches every other SIGTERM source (e.g. a raw `terminal( # "launchctl kickstart ai.hermes.gateway")`). if candidates: try: from gateway import restart_loop_guard as _rlg _max_restarts, _window, _max_gap = self._restart_loop_guard_config() if _rlg.check_and_record( _max_restarts, _window, max_gap_seconds=_max_gap ): return 0 except Exception as exc: # noqa: BLE001 — breaker must fail OPEN logger.debug("Restart-loop guard check skipped: %s", exc) now = datetime.now() scheduled = 0 for entry in candidates: marker = entry.last_resume_marked_at or entry.updated_at if marker is not None and (now - marker).total_seconds() > window: continue # Already being resumed (e.g. scheduled at startup and still # in-flight) — don't synthesize a second continuation turn. if self._is_session_running(entry.session_key): continue source = entry.origin adapter = self._adapter_for_source(source) if adapter is None: logger.debug( "Skipping auto-resume for %s: adapter not ready for %s", entry.session_key, getattr(source.platform, "value", source.platform), ) continue # Validate the session owner against the current allowlist # before auto-resuming. A session created before # TELEGRAM_ALLOWED_USERS (or equivalent) was configured, or # before the owner was removed from it, must not silently # receive a full agent response on gateway restart just # because it has a resume-pending marker (issue #23778). try: if not self._is_user_authorized(source): logger.warning( "Skipping auto-resume for %s: session owner is no " "longer authorized under the current allowlist", entry.session_key, ) continue except Exception as exc: logger.warning( "Skipping auto-resume for %s: authorization check failed: %s", entry.session_key, exc, ) continue # Claim the session slot *before* spawning the task so that an # inbound message arriving between task creation and the task's # first await (where _process_message_background sets the real # sentinel) sees the slot as occupied and queues behind it # instead of spinning up a duplicate AIAgent (#45456). _resume_state = self._session_state(entry.session_key) _resume_state.turn.agent = _AGENT_PENDING_SENTINEL _resume_state.turn.started_ts = time.time() self._persist_active_agents() # Empty-text internal event — the _is_resume_pending branch in # _handle_message_with_agent prepends the proper reason-aware # system note before the turn runs. event = MessageEvent( text="", message_type=MessageType.TEXT, source=source, internal=True, ) task = asyncio.create_task( self._run_startup_resume_event(adapter, event, entry.session_key) ) self._background_tasks.add(task) task.add_done_callback(self._background_tasks.discard) if getattr(self, "_startup_restore_in_progress", False): tasks = getattr(self, "_startup_restore_tasks", None) if tasks is None: tasks = [] self._startup_restore_tasks = tasks tasks.append(task) scheduled += 1 if scheduled: logger.info( "Scheduled auto-resume for %d restart-interrupted session(s)", scheduled, ) return scheduled def _startup_should_abort(self) -> bool: return ( self._restart_requested or self._draining or self._shutdown_event.is_set() ) async def _abort_startup_if_shutdown_requested( self, adapter: Optional[BasePlatformAdapter] = None, platform: Optional[Platform] = None, ) -> bool: """Clean up and exit startup when restart/shutdown begins mid-startup.""" if not self._startup_should_abort(): return False if adapter is not None and platform is not None: try: await adapter.cancel_background_tasks() except Exception as e: logger.debug("✗ %s background-task cancel error: %s", platform.value, e) await self._safe_adapter_disconnect(adapter, platform) stop_task = self._stop_task current_task = asyncio.current_task() if stop_task is not None and stop_task is not current_task: await stop_task elif not self._shutdown_event.is_set(): await self.stop( restart=self._restart_requested, detached_restart=self._restart_detached, service_restart=self._restart_via_service, ) return True def _start_loop_liveness_guards(self, loop: asyncio.AbstractEventLoop) -> None: """Arm the selector floor and out-of-loop watchdog before adapters. Disabled entirely with ``gateway.loop_watchdog: false`` in config.yaml (no env override — config-only knob, #69089). """ config = getattr(self, "config", None) if config is not None and not getattr(config, "loop_watchdog", True): return if getattr(self, "_loop_floor_timer_handle", None) is None: try: self._loop_floor_timer_handle = _arm_loop_floor_timer(loop) except Exception: logger.debug("Failed to arm gateway loop floor timer", exc_info=True) watchdog = getattr(self, "_loop_liveness_watchdog", None) if watchdog is None or not watchdog.is_alive(): try: # getattr defaults cover the config=None / bare-object test # path; config-loaded values are already validated+clamped # by GatewayConfig.from_dict, so no re-clamping here. interval = getattr( config, "loop_watchdog_probe_interval_s", DEFAULT_LOOP_WATCHDOG_INTERVAL_S, ) timeout = getattr( config, "loop_watchdog_probe_timeout_s", DEFAULT_LOOP_WATCHDOG_TIMEOUT_S, ) strikes = getattr( config, "loop_watchdog_max_strikes", DEFAULT_LOOP_WATCHDOG_MAX_STRIKES, ) self._loop_liveness_watchdog = start_loop_liveness_watchdog( loop, probe_interval=float(interval), probe_timeout=float(timeout), max_strikes=int(strikes), ) except Exception: logger.debug("Failed to start gateway loop liveness watchdog", exc_info=True) def _stop_loop_liveness_guards(self) -> None: """Disarm lifetime liveness guards before shutdown can load the loop.""" watchdog = getattr(self, "_loop_liveness_watchdog", None) self._loop_liveness_watchdog = None if watchdog is not None: try: watchdog.stop() except Exception: logger.debug("Failed to stop gateway loop liveness watchdog", exc_info=True) floor_timer = getattr(self, "_loop_floor_timer_handle", None) self._loop_floor_timer_handle = None if floor_timer is not None: try: floor_timer.cancel() except Exception: logger.debug("Failed to cancel gateway loop floor timer", exc_info=True) # Also disarm the heartbeat writer task itself (ported from #95808): # once shutdown starts loading the loop, a heartbeat that keeps # refreshing the file can make a draining gateway look healthy to # external probes. Cancel is idempotent; the task is also in # _background_tasks so this is belt-and-braces ordering, not new # lifecycle. heartbeat = getattr(self, "_loop_heartbeat_task", None) self._loop_heartbeat_task = None if heartbeat is not None: try: heartbeat.cancel() except Exception: logger.debug("Failed to cancel gateway loop heartbeat task", exc_info=True) async def _consume_clean_shutdown_marker(self, marker_path) -> int: """Discard orphan turn markers before consuming a clean-exit receipt. If either persistence or marker removal fails, startup must fail closed. Continuing with the old receipt would let a later unclean exit masquerade as clean and discard genuinely interrupted turns. """ discarded = await self.async_session_store.discard_active_turn_markers() marker_path.unlink() return discarded async def _recover_unclean_sessions(self) -> tuple[int, int]: """Recover exact active turns, then run the legacy recency fallback.""" exact = 0 fallback = 0 try: agent_timeout = max(1.0, _float_env("HERMES_AGENT_TIMEOUT", 1800)) marker_max_age = max(60 * 60, int(agent_timeout * 2)) exact = await self.async_session_store.recover_interrupted_turns( max_age_seconds=marker_max_age ) except Exception as exc: logger.warning("Exact active-turn recovery on startup failed: %s", exc) try: fallback = await self.async_session_store.suspend_recently_active( max_age_seconds=120 ) except Exception as exc: logger.warning("Legacy session recovery on startup failed: %s", exc) return exact, fallback @staticmethod def _start_hosted_room_worker_sync(): """Start the local Group Chat worker without importing the dashboard.""" import tui_gateway.server # noqa: F401 from tui_gateway import methods_groups service = methods_groups.get_hosted_room_service() if service is None: service = methods_groups.start_hosted_room_service() if service is None: raise RuntimeError("Group Chat worker has no bound session backend") status = service.runtime.status() if not status.get("running") or status.get("stopping"): raise RuntimeError("Group Chat worker did not start") return service async def _ensure_hosted_room_worker(self): return await asyncio.to_thread(self._start_hosted_room_worker_sync) async def _hosted_room_worker_watcher(self, interval: float = 1.0) -> None: """Keep the room worker alive for the messaging gateway lifetime.""" while self._running: await self._ensure_hosted_room_worker() await asyncio.sleep(interval) async def _stop_hosted_room_worker(self, timeout: float = 5.0) -> bool: """Pause room execution durably without interrupting accepted turns.""" from tui_gateway import methods_groups return await asyncio.to_thread( methods_groups.stop_hosted_room_service, timeout=timeout, ) def _start_loop_heartbeat_task(self) -> None: """Start the loop-liveness heartbeat task (#66892), idempotent. An asyncio task so a frozen loop stops refreshing ``state/gateway.heartbeat``. Cancelled with the other background tasks during stop(). Best-effort — a liveness probe must never be able to abort startup. """ try: _existing_hb = getattr(self, "_loop_heartbeat_task", None) if _existing_hb is not None and not _existing_hb.done(): return self._loop_heartbeat_task = asyncio.create_task( loop_heartbeat_forever( interval_s=DEFAULT_HEARTBEAT_INTERVAL_S, start_time=getattr(self, "_gateway_started_at", 0.0), ) ) # PERMANENT for the process lifetime, same as a # _spawn_supervised watcher — tag it so # _scale_to_zero_has_live_background_work() doesn't treat an # armed, otherwise-idle gateway as busy forever. self._loop_heartbeat_task._hermes_supervised_watcher = True # type: ignore[attr-defined] _bg = getattr(self, "_background_tasks", None) if _bg is not None: _bg.add(self._loop_heartbeat_task) self._loop_heartbeat_task.add_done_callback(_bg.discard) except Exception: logger.debug("Failed to start gateway loop heartbeat", exc_info=True) async def start(self) -> bool: """ Start the gateway and all configured platform adapters. Returns True if at least one adapter connected successfully. """ logger.info("Starting Hermes Gateway...") # Enable faulthandler for stack dumps on freezes/crashes (#70344). # Falls back to a log file when sys.stderr is None (Windows VBS / # pythonw / detached service) — otherwise the gateway would die # here and take every adapter offline. See #71671. try: faulthandler.enable() except (RuntimeError, ValueError, OSError): try: _fh_log_dir = getattr(self.config, "log_dir", None) or os.path.join( str(get_hermes_home()), "logs", ) os.makedirs(_fh_log_dir, exist_ok=True) _fh_enable_path = os.path.join(_fh_log_dir, "gateway_faulthandler.log") _fh_enable_file = open(_fh_enable_path, "a", encoding="utf-8") faulthandler.enable(file=_fh_enable_file, all_threads=True) except Exception: logger.debug("faulthandler.enable() unavailable", exc_info=True) # Also dump stacks to a rotating file for off-line analysis when # the gateway is running under a service manager that doesn't # capture stderr. # faulthandler.register() and SIGUSR2 are POSIX-only; skip the # signal-triggered file dump on Windows (faulthandler.enable() # above still covers fatal-error dumps there). _sigusr2 = getattr(signal, "SIGUSR2", None) if _sigusr2 is not None and hasattr(faulthandler, "register"): try: _log_dir = getattr(self.config, "log_dir", None) or os.path.join( str(get_hermes_home()), "logs", ) _faulthandler_path = os.path.join(_log_dir, "gateway_faulthandler.log") os.makedirs(_log_dir, exist_ok=True) _fh = open(_faulthandler_path, "a", encoding="utf-8") faulthandler.register( _sigusr2, file=_fh, all_threads=True, chain=True, ) except Exception: logger.debug("Could not set up faulthandler file logging", exc_info=True) try: self._gateway_loop = asyncio.get_running_loop() except RuntimeError: self._gateway_loop = None if self._gateway_loop is not None: self._start_loop_liveness_guards(self._gateway_loop) # The event loop is confirmed live: the startup-liveness # watchdog's job is done and the loop-liveness watchdog (armed # just above) takes over from here (OOF-298). Disarm even when # the loop guards are config-disabled — the startup watchdog # only covers the pre-loop window, never adapter connects or # steady-state. Deliberately inside the loop-confirmed branch: # if the loop somehow isn't live, startup has NOT reached the # milestone and the watchdog must stay armed. try: from gateway.startup_watchdog import disarm_startup_watchdog disarm_startup_watchdog() except Exception: logger.debug("Startup watchdog disarm failed", exc_info=True) logger.info("Session storage: %s", self.config.sessions_dir) # Sanity-check that systemd's TimeoutStopSec covers our drain # window. When the user upgraded hermes-agent without re-running # ``hermes setup``, their unit file may still encode the old # default — in which case SIGKILL hits mid-drain and looks like # a phantom kill in the journal. Best-effort, never raises. try: from gateway.shutdown_forensics import check_systemd_timing_alignment _alignment = check_systemd_timing_alignment( self._restart_drain_timeout, getattr(self, "_cron_drain_timeout", DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT), ) if _alignment is not None and _alignment.get("mismatch"): logger.warning( "Stale systemd unit detected: %s has TimeoutStopSec=%.0fs but " "drain_timeout=%.0fs cron_drain_timeout=%.0fs (expected >=%.0fs). " "systemd may SIGKILL the gateway mid-drain. Run " "`hermes gateway install --force` to regenerate the unit, or " "shorten agent.restart_drain_timeout / agent.cron_drain_timeout.", _alignment.get("unit", "(unknown)"), _alignment["timeout_stop_sec"], _alignment["drain_timeout"], _alignment.get( "cron_drain_timeout", DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT ), _alignment["expected_min"], ) except Exception as _e: logger.debug("check_systemd_timing_alignment failed: %s", _e) # Log the resolved max_iterations budget so operators can verify the # config.yaml → env bridge did the right thing at a glance (instead # of silently running at a stale .env value for weeks). try: _effective_max_iter = int(os.getenv("HERMES_MAX_ITERATIONS", "500")) logger.info( "Agent budget: max_iterations=%d (agent.max_turns from config.yaml, " "or HERMES_MAX_ITERATIONS from .env, or default 500)", _effective_max_iter, ) except Exception: pass # Redaction status: ON by default (#17691). Surface a prominent # warning if an operator has explicitly opted out so they don't # forget the downgrade is active — the redactor snapshots its # state at import time, so this log line is the source of truth # for this process's lifetime. try: _redact_raw = os.getenv("HERMES_REDACT_SECRETS", "true") _redact_on = _redact_raw.lower() in {"1", "true", "yes", "on"} if _redact_on: logger.info( "Secret redaction: ENABLED (tool output, logs, and chat " "responses are scrubbed before delivery)" ) else: logger.warning( "Secret redaction: DISABLED (HERMES_REDACT_SECRETS=%s). " "API keys and tokens may appear verbatim in chat output, " "session JSONs, and logs. Set security.redact_secrets: true " "in config.yaml to re-enable.", _redact_raw, ) except Exception: pass try: from hermes_cli.profiles import get_active_profile_name _profile = get_active_profile_name() if _profile and _profile != "default": logger.info("Active profile: %s", _profile) except Exception: pass try: from gateway.status import write_runtime_status write_runtime_status( gateway_state="starting", exit_reason=None, clear_profile_platforms=True, ) except Exception: pass try: from hermes_cli.config import load_config from agent.monitoring.gateway_health_export import start_gateway_health_export self._gateway_health_export_runtime = start_gateway_health_export(load_config()) if getattr(self._gateway_health_export_runtime, "enabled", False): logger.info("Gateway health OTLP export: enabled") except Exception: logger.debug("gateway health OTLP export startup failed", exc_info=True) # Log any active supply-chain security advisories. Operators see this # in gateway.log and `hermes status` surfaces it; we do NOT block # startup or surface it inline to user messages, since the gateway # operator is the one who can act on it (uninstall the package, # rotate credentials). See hermes_cli/security_advisories.py. try: from hermes_cli.security_advisories import ( detect_compromised, gateway_log_message, ) _adv_hits = detect_compromised() _adv_msg = gateway_log_message(_adv_hits) if _adv_msg: logger.warning("%s", _adv_msg) logger.warning( "Run `hermes doctor` on the gateway host for full " "remediation steps." ) except Exception: logger.debug( "security advisory check failed at gateway startup", exc_info=True, ) if await self._abort_startup_if_shutdown_requested(): return True # Warn if no user allowlists are configured and open access is not opted in _builtin_allowed_vars = ( "TELEGRAM_ALLOWED_USERS", "DISCORD_ALLOWED_USERS", "WHATSAPP_ALLOWED_USERS", "WHATSAPP_CLOUD_ALLOWED_USERS", "SLACK_ALLOWED_USERS", "SIGNAL_ALLOWED_USERS", "SIGNAL_GROUP_ALLOWED_USERS", "TELEGRAM_GROUP_ALLOWED_USERS", "TELEGRAM_GROUP_ALLOWED_CHATS", "EMAIL_ALLOWED_USERS", "SMS_ALLOWED_USERS", "MATTERMOST_ALLOWED_USERS", "MATRIX_ALLOWED_USERS", "DINGTALK_ALLOWED_USERS", "FEISHU_ALLOWED_USERS", "WECOM_ALLOWED_USERS", "WECOM_CALLBACK_ALLOWED_USERS", "WEIXIN_ALLOWED_USERS", "BLUEBUBBLES_ALLOWED_USERS", "QQ_ALLOWED_USERS", "YUANBAO_ALLOWED_USERS", "GATEWAY_ALLOWED_USERS", ) _builtin_allow_all_vars = ( "TELEGRAM_ALLOW_ALL_USERS", "DISCORD_ALLOW_ALL_USERS", "WHATSAPP_ALLOW_ALL_USERS", "WHATSAPP_CLOUD_ALLOW_ALL_USERS", "SLACK_ALLOW_ALL_USERS", "SIGNAL_ALLOW_ALL_USERS", "EMAIL_ALLOW_ALL_USERS", "SMS_ALLOW_ALL_USERS", "MATTERMOST_ALLOW_ALL_USERS", "MATRIX_ALLOW_ALL_USERS", "DINGTALK_ALLOW_ALL_USERS", "FEISHU_ALLOW_ALL_USERS", "WECOM_ALLOW_ALL_USERS", "WECOM_CALLBACK_ALLOW_ALL_USERS", "WEIXIN_ALLOW_ALL_USERS", "BLUEBUBBLES_ALLOW_ALL_USERS", "QQ_ALLOW_ALL_USERS", "YUANBAO_ALLOW_ALL_USERS", ) # Also pick up plugin-registered platforms — each entry can declare # its own allowed_users_env / allow_all_env, so the warning stays # accurate as plugins like IRC come online. _plugin_allowed_vars: tuple = () _plugin_allow_all_vars: tuple = () try: from gateway.platform_registry import platform_registry _plugin_allowed_vars = tuple( e.allowed_users_env for e in platform_registry.plugin_entries() if e.allowed_users_env ) _plugin_allow_all_vars = tuple( e.allow_all_env for e in platform_registry.plugin_entries() if e.allow_all_env ) except Exception: pass _any_allowlist = any( os.getenv(v) for v in _builtin_allowed_vars + _plugin_allowed_vars ) _allow_all = os.getenv("GATEWAY_ALLOW_ALL_USERS", "").lower() in {"true", "1", "yes"} or any( os.getenv(v, "").lower() in {"true", "1", "yes"} for v in _builtin_allow_all_vars + _plugin_allow_all_vars ) if not _any_allowlist and not _allow_all: logger.warning( "No env user allowlists configured. Messaging platforms default to " "pairing/allowlist policies and will deny unknown senders unless you " "configure platform allowlists (e.g., TELEGRAM_ALLOWED_USERS=your_id) " "or explicitly opt in with GATEWAY_ALLOW_ALL_USERS=true plus " "dm_policy/group_policy: open on the platform." ) reason = _own_policy_open_startup_violation(self.config) if reason: platform_value = reason.split(":", 1)[0] allow_all_env = None for platform, open_env in _OWN_POLICY_OPEN_ENV.items(): if platform.value == platform_value: allow_all_env = open_env[2] break logger.error( "Refusing to start: %s has dm_policy/group_policy set to 'open' " "but neither GATEWAY_ALLOW_ALL_USERS nor %s is enabled.", platform_value, allow_all_env or "a platform allow-all flag", ) try: from gateway.status import write_runtime_status write_runtime_status(gateway_state="startup_failed", exit_reason=reason) except Exception: pass self._request_clean_exit(reason) return True # Discover Python plugins before shell hooks so plugin block # decisions take precedence in tie cases. The CLI startup path # does this via an explicit call in hermes_cli/main.py; the # gateway lazily imports run_agent inside per-request handlers, # so the discover_plugins() side-effect in model_tools.py is NOT # guaranteed to have run by the time we reach this point. try: from hermes_cli.plugins import discover_plugins discover_plugins() except Exception: logger.warning( "plugin discovery failed at gateway startup", exc_info=True, ) # Register the generic relay adapter when a connector relay URL is # configured (GATEWAY_RELAY_URL / gateway.relay_url). No URL -> no-op, so # direct/single-tenant deployments are unaffected. When configured, the # adapter dials the connector over a WebSocket, negotiates its capability # descriptor at handshake, and bridges inbound/outbound like any platform. try: from gateway.relay import ( register_relay_adapter, relay_url, self_provision_relay, send_relay_policy, ) # Boot-time relay self-provision: resolve the agent's NAS token -> # POST /relay/provision -> set GATEWAY_RELAY_* in os.environ BEFORE # registration reads them. No-op when relay is unconfigured, a secret # is already pinned, or no NAS token resolves (self-hosted, unenrolled). # Never raises. self_provision_relay() if register_relay_adapter(): logger.info("relay adapter registered (connector at %s)", relay_url()) # Declare this gateway's relevance policy (mention-gating / # free-response / allow-bots) to the connector so the SAME # behavior governs relay delivery (Phase 6 Unit ζ). Runs after # the secret is resolved; never raises, never blocks boot. send_relay_policy() except Exception: logger.warning( "relay adapter registration failed at gateway startup", exc_info=True, ) # Register declarative shell hooks from cli-config.yaml. Gateway # has no TTY, so consent has to come from one of the three opt-in # channels (--accept-hooks on launch, HERMES_ACCEPT_HOOKS env var, # or hooks_auto_accept: true in config.yaml). We pass # accept_hooks=False here and let register_from_config resolve # the effective value from env + config itself — the CLI-side # registration already honored --accept-hooks, and re-reading # hooks_auto_accept here would just duplicate that lookup. # Failures are logged but must never block gateway startup. try: from hermes_cli.config import load_config from agent.shell_hooks import register_from_config _hooks_cfg = load_config() register_from_config(_hooks_cfg, accept_hooks=False) from agent.outbound_webhooks import ( register_from_config as register_outbound_webhooks, ) register_outbound_webhooks(_hooks_cfg) except Exception: logger.debug( "shell-hook registration failed at gateway startup", exc_info=True, ) # Discover and load event hooks self.hooks.discover_and_load() # Recover background processes from checkpoint (crash recovery) try: from tools.process_registry import process_registry recovered = process_registry.recover_from_checkpoint() if recovered: logger.info("Recovered %s background process(es) from previous run", recovered) except Exception as e: logger.warning("Process checkpoint recovery: %s", e) # Recover sessions that were active when the gateway last exited. # Exact durable turn markers cover long-running work; the 120-second # recency heuristic remains as an upgrade fallback for turns started by # older Hermes versions that did not write exact markers. # # SKIP suspension after a clean (graceful) shutdown — the previous # process already drained active agents, so sessions aren't stuck. # This prevents unwanted auto-resets after `hermes update`, # `hermes gateway restart`, or `/restart`. _clean_marker = _hermes_home / ".clean_shutdown" if _clean_marker.exists(): logger.info("Previous gateway exited cleanly — skipping session suspension") try: discarded = await self._consume_clean_shutdown_marker(_clean_marker) except Exception as exc: logger.error( "Clean-start marker cleanup failed; refusing startup so the " "clean-exit receipt cannot mask a later unclean exit: %s", exc, ) raise RuntimeError("clean-start recovery cleanup failed") from exc if discarded: logger.info( "Discarded %d orphan active-turn marker(s) after clean shutdown", discarded, ) else: exact, fallback = await self._recover_unclean_sessions() recovered = exact + fallback if recovered: logger.info( "Marked %d in-flight session(s) as resumable from previous run " "(%d exact, %d legacy)", recovered, exact, fallback, ) # Stuck-loop detection (#7536): if a session has been active across # 3+ consecutive restarts, it's probably stuck in a loop (the same # history keeps causing the agent to hang). Auto-suspend it so the # user gets a clean slate on the next message. try: stuck = self._suspend_stuck_loop_sessions() if stuck: logger.warning("Auto-suspended %d stuck-loop session(s)", stuck) except Exception as e: logger.debug("Stuck-loop detection failed: %s", e) # Serialize startup restore against inbound dispatch. Platform # adapters can begin receiving messages as soon as they connect, but # restart-interrupted sessions are not auto-resumed until all startup # wiring below completes. Queue inbound messages until the resume # pass runs and every synthetic resume turn has finished. self._startup_restore_in_progress = True self._startup_restore_queue = [] self._startup_restore_tasks = [] # Fresh-boot readiness (#99373): with no resume_pending sessions the # gate above opens almost immediately, while the agent-side turn # machinery (run_agent import graph, tool schemas, check_fn probes, # context tier) is still cold — a message in that window was served # with a skeleton system prompt. Start warming NOW so the work # overlaps the network-bound platform connects below; # _finish_startup_restore awaits it (bounded) before opening the gate. self._start_startup_warmup() connected_count = 0 enabled_platform_count = 0 startup_nonretryable_errors: list[str] = [] startup_retryable_errors: list[str] = [] _multiplex_on = bool(getattr(self.config, "multiplex_profiles", False)) _multiplex_skipped_platforms: list[Platform] = [] # Initialize and connect each configured platform. # # Parallel startup connect (#83791): the original code ran a serial for-loop, # so every platform's connect() (with its own timeout) had to finish before # the next began. A single slow/failing platform (e.g. Telegram behind a dead # proxy) therefore delayed every other platform's connect by a full timeout # window, cascading one platform's failure onto WeChat/QQ/etc. We now launch # all platform connects concurrently and let each resolve on its own timeline; # per-platform timeouts and error handling are unchanged. # The serial pre-filter (cheap checks, adapter creation, handler wiring) stays # sequential -- only the (slow) connect() calls run in parallel. _pending_connects = [] # (platform, platform_config, adapter) for platform, platform_config in self.config.platforms.items(): if await self._abort_startup_if_shutdown_requested(): return True if not platform_config.enabled: continue # Under multiplexing, a platform may be enabled on the default # profile's config.yaml while its bot token lives only in a # secondary profile's .env. Starting that primary adapter with an # empty token fails immediately and queues an infinite reconnect # loop that can never heal (#64674). Secondary profiles still # start their own adapters under _profile_runtime_scope with the # real token -- skip the empty primary instead of failing loudly. if _multiplex_on and not _platform_has_bot_credential(platform, platform_config): logger.info( "Skipping %s on default profile: no bot credential in this " "profile's secrets. Secondary multiplexed profiles that " "provide the token will still connect.", platform.value, ) _multiplex_skipped_platforms.append(platform) continue enabled_platform_count += 1 adapter = self._create_adapter(platform, platform_config) if not adapter: # Distinguish between missing builtin deps and missing plugin _pval = platform.value _builtin_names = {m.value for m in Platform.__members__.values()} if _pval not in _builtin_names: logger.warning( "No adapter for '%s' -- is the plugin installed? " "(platform is enabled in config.yaml but no plugin registered it)", _pval, ) else: logger.warning("No adapter available for %s", _pval) continue # Set up message + fatal error handlers. Under multiplexing the # default profile needs the same whole-handler runtime scope as a # secondary profile: authorization and prompt rendering both run # before the narrower agent-turn scope is installed. adapter.set_message_handler(self._primary_message_handler()) adapter.set_fatal_error_handler(self._handle_adapter_fatal_error) adapter.set_session_store(self.session_store) adapter.set_busy_session_handler(self._handle_active_session_busy_message) _set_reaction = getattr(adapter, "set_reaction_handler", None) if callable(_set_reaction): _set_reaction(self._handle_reaction_event) adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id) adapter.set_authorization_check(self._make_adapter_auth_check(adapter.platform)) adapter.set_platform_event_handler(self._primary_platform_event_handler()) adapter._busy_text_mode = self._busy_text_mode _pending_connects.append((platform, platform_config, adapter)) if await self._abort_startup_if_shutdown_requested(): return True async def _connect_one_startup(p, p_cfg, adp): """Connect a single platform; never let one block the others (#83791).""" if await self._abort_startup_if_shutdown_requested(adp, p): return (p, adp, p_cfg, "aborted", None) logger.info("Connecting to %s...", p.value) self._update_platform_runtime_status( p.value, platform_state="connecting", error_code=None, error_message=None, ) try: ok = await self._connect_initial_adapter_with_timeout(adp, p) except Exception as _exc: # noqa: BLE001 - surfaced below as a retryable error return (p, adp, p_cfg, "exception", _exc) return (p, adp, p_cfg, "ok" if ok else "failed", None) if _pending_connects: # Abort-aware concurrent wait (parity with the serial loop's # between-platforms abort check): a restart/shutdown requested # while connects are in flight must cancel the still-pending # connects — no later platform may finish connecting — clean up # the ones that already completed, and abort startup. _task_map: dict = {} for (p, c, a) in _pending_connects: _t = asyncio.ensure_future(_connect_one_startup(p, c, a)) _task_map[_t] = (p, c, a) _pending_tasks = set(_task_map) _abort_mid_connect = False while _pending_tasks: _done, _pending_tasks = await asyncio.wait( _pending_tasks, timeout=0.05 ) if _pending_tasks and self._startup_should_abort(): _abort_mid_connect = True break if _abort_mid_connect: # Cancel and fully settle the in-flight connects FIRST, so a # completed adapter's disconnect cannot unblock a sibling's # connect() before the sibling is cancelled. for _t in _pending_tasks: _t.cancel() await asyncio.gather(*_pending_tasks, return_exceptions=True) for _t in _pending_tasks: _p, _c, _a = _task_map[_t] try: await _a.cancel_background_tasks() except Exception as e: logger.debug( "✗ %s background-task cancel error: %s", _p.value, e ) await self._safe_adapter_disconnect(_a, _p) # Tear down adapters whose connect already succeeded — they # were never registered, so stop() won't reach them. for _t, (_p, _c, _a) in _task_map.items(): if _t in _pending_tasks or _t.cancelled(): continue _res = _t.exception() is None and _t.result() or None if _res and _res[3] == "ok": try: await _a.cancel_background_tasks() except Exception as e: logger.debug( "✗ %s background-task cancel error: %s", _p.value, e, ) await self._safe_adapter_disconnect(_a, _p) await self._abort_startup_if_shutdown_requested() return True _raw = [ _t.exception() or _t.result() for _t in _task_map ] else: _raw = [] # Aggregate results single-threaded so shared state (self.adapters, # self._failed_platforms, the error lists, connected_count) is mutated # exactly as the original serial loop did -- only the connect() wall-clock # overlap changed. for _item in _raw: if isinstance(_item, Exception): # Unexpected escape from _connect_one_startup (shouldn't happen); # log and skip rather than aborting the whole startup. logger.error("Unexpected startup connect error: %s", _item) continue platform, adapter, platform_config, outcome, exc = _item if outcome == "aborted": continue if outcome == "exception": logger.error("\u2717 %s error: %s", platform.value, exc) # Same defensive cleanup path for exceptions -- an adapter that # raised mid-connect may still have a live aiohttp.ClientSession or # child subprocess. await self._safe_adapter_disconnect(adapter, platform) self._update_platform_runtime_status( platform.value, platform_state="retrying", error_code=None, error_message=str(exc), ) startup_retryable_errors.append(f"{platform.value}: {exc}") # Unexpected exceptions are typically transient -- queue for retry self._failed_platforms[platform] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() + 30, "queued_at": time.monotonic(), "credential_claim": self._adapter_credential_claim(platform, adapter), "listener_claim": self._adapter_listener_claim(platform, adapter), } continue if outcome == "ok": self.adapters[platform] = adapter self._sync_voice_mode_state_to_adapter(adapter) # Wire voice input callback at connect time so voice # transcription is forwarded without requiring /voice join. self._bind_voice_input_callback(adapter) connected_count += 1 _degraded = adapter.send_path_degraded self._update_platform_runtime_status( platform.value, platform_state="retrying" if _degraded else "connected", error_code=None, error_message=adapter.DEGRADED_STATUS_MESSAGE if _degraded else None, ) logger.info("\u2713 %s connected%s", platform.value, " (degraded)" if _degraded else "") else: # outcome == "failed" logger.warning("\u2717 %s failed to connect", platform.value) # Defensive cleanup: a failed connect() may have allocated resources # (aiohttp.ClientSession, poll tasks, bridge subprocesses) before # giving up. Without this call, those resources are orphaned and # Python logs "Unclosed client session" at process exit. await self._safe_adapter_disconnect(adapter, platform) if adapter.has_fatal_error: # A live foreign holder of this bot token / identity is # a single-writer ownership conflict, not a transient # blip — even though ``_acquire_platform_lock`` emits it # retryable so a MID-RUN reconnect can recover (#54167). # At startup route it as non-retryable: with nothing # connected the gateway exits 78 instead of sitting alive # and deaf in the retry queue forever (#83183). _retryable = adapter.fatal_error_retryable and not ( is_global_startup_conflict(adapter.fatal_error_code) ) self._update_platform_runtime_status( platform.value, platform_state="retrying" if _retryable else "fatal", error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message, ) target = ( startup_retryable_errors if _retryable else startup_nonretryable_errors ) target.append(f"{platform.value}: {adapter.fatal_error_message}") # Queue for reconnection if the error is retryable if _retryable: self._failed_platforms[platform] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() + 30, "credential_claim": self._adapter_credential_claim(platform, adapter), "listener_claim": self._adapter_listener_claim(platform, adapter), } else: self._update_platform_runtime_status( platform.value, platform_state="retrying", error_code=None, error_message="failed to connect", ) startup_retryable_errors.append(f"{platform.value}: failed to connect") # No fatal error info means likely a transient issue -- queue for retry self._failed_platforms[platform] = { "config": platform_config, "attempts": 1, "next_retry": time.monotonic() + 30, "queued_at": time.monotonic(), "credential_claim": self._adapter_credential_claim(platform, adapter), "listener_claim": self._adapter_listener_claim(platform, adapter), } if await self._abort_startup_if_shutdown_requested(): return True # Multi-profile multiplexing: bring up adapters for every OTHER profile # this gateway serves. Each profile's adapters connect under that # profile's home + credential scope and stamp their inbound events with # the profile so the agent turn resolves correctly. No-op when off. try: _secondary_connected = await self._start_secondary_profile_adapters() connected_count += _secondary_connected except MultiplexConfigError as e: # Invalid multiplexer config — abort startup cleanly so the operator # fixes config.yaml rather than running a half-wired gateway. reason = str(e) logger.error("Gateway multiplexer config error: %s", reason) try: from gateway.status import write_runtime_status write_runtime_status(gateway_state="startup_failed", exit_reason=reason) except Exception: pass self._exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE self._request_clean_exit(reason) self._startup_restore_in_progress = False return True except Exception as e: logger.error("Secondary-profile adapter startup failed: %s", e, exc_info=True) finally: # Startup authority is one phase, not a persistent runner mode. # From this point onward every adapter retry is non-evicting. self._platform_lock_takeover_on_start = False # A platform we skipped on the primary for a missing credential was # supposed to be picked up by a secondary profile that owns the token. # If none did, the platform is enabled in config.yaml yet silently # unserved — surface it loudly so the operator sees a config problem # instead of a quiet dead channel (#64674 follow-up). for _skipped in _multiplex_skipped_platforms: _served_by_secondary = any( _skipped in _profile_map for _profile_map in self._profile_adapters.values() ) if not _served_by_secondary: logger.warning( "%s is enabled but no profile (default or secondary) " "provided a bot credential for it — the platform is not " "being served. Add its token to the profile that should " "own it, or disable the platform.", _skipped.value, ) if connected_count == 0: if startup_nonretryable_errors and not startup_retryable_errors: reason = "; ".join(startup_nonretryable_errors) logger.error("Gateway hit a non-retryable startup conflict: %s", reason) try: from gateway.status import write_runtime_status write_runtime_status(gateway_state="startup_failed", exit_reason=reason) except Exception: pass self._exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE self._request_clean_exit(reason) self._startup_restore_in_progress = False return True if startup_nonretryable_errors: # Mixed failure mode (NS-609): some platforms are fatally # misconfigured (e.g. WhatsApp enabled but never paired) while # others hit merely transient errors (e.g. Telegram TimedOut # during polling startup). Exiting with # GATEWAY_FATAL_CONFIG_EXIT_CODE here is wrong in both # supervision worlds: under supervisors that honor the # exit-78 contract (systemd RestartPreventExitStatus, s6 # finish→125 since #51228) the gateway goes PERMANENTLY down # over a network blip; under anything else it crash-loops. # Either way the retryable platforms never get their retry. # Log the fatal side loudly, then fall through to the # degraded/retry path below: the reconnect watcher recovers # the retryable platforms; the non-retryable ones remain # fatal-parked and visible in runtime status. logger.error( "%d platform(s) fatally misconfigured and parked: %s. " "Staying alive so retryable platforms can recover.", len(startup_nonretryable_errors), "; ".join(startup_nonretryable_errors), ) if enabled_platform_count > 0: if startup_retryable_errors: # All enabled platforms hit retryable failures (network # blip, bridge not paired, npm install timeout, etc.). # Keep the gateway alive so: # • cron jobs still run # • the reconnect watcher gets a chance to recover the # failing platforms once the underlying problem is # fixed (e.g. user runs `hermes whatsapp`, fixes # proxy, etc.) # Exiting here used to convert a single misconfigured # platform into an infinite systemd restart loop. reason = "; ".join(startup_retryable_errors) logger.warning( "Gateway started with no connected platforms — " "%d platform(s) queued for retry: %s", len(self._failed_platforms), reason, ) try: from gateway.status import write_runtime_status write_runtime_status( gateway_state="degraded", exit_reason=None, ) except Exception: pass # Fall through to the normal "running" state — reconnect # watcher takes it from here. # All enabled platforms had no adapter (missing library or credentials). # In fleet deployments the same config.yaml is shared across nodes that # may only have credentials for a subset of platforms. Rather than # failing hard, degrade gracefully and allow cron jobs to run (#5196). logger.warning( "No adapter could be created for any of the %d configured platform(s). " "Check that required dependencies are installed and credentials are set. " "Gateway will continue for cron job execution.", enabled_platform_count, ) else: logger.warning("No messaging platforms enabled.") logger.info("Gateway will continue running for cron job execution.") # Update delivery router with adapters if await self._abort_startup_if_shutdown_requested(): return True self.delivery_router.adapters = self.adapters self._wire_teams_pipeline_runtime() self._running = True self._install_plugin_message_injector() self._update_runtime_status("running") try: await self._ensure_hosted_room_worker() except Exception: logger.error( "Group Chat worker failed to start; mutating Group Chat commands " "will fail closed until supervision recovers it", exc_info=True, ) self._spawn_supervised( self._hosted_room_worker_watcher, "hosted_room_worker", ) self._start_loop_heartbeat_task() # Emit gateway:startup hook hook_count = len(self.hooks.loaded_hooks) if hook_count: logger.info("%s hook(s) loaded", hook_count) await self.hooks.emit("gateway:startup", { "platforms": [p.value for p in self.adapters.keys()], }) if connected_count > 0: logger.info("Gateway running with %s platform(s)", connected_count) # Build initial channel directory for send_message name resolution try: from gateway.channel_directory import build_channel_directory directory = await build_channel_directory(self.adapters) ch_count = sum(len(chs) for chs in directory.get("platforms", {}).values()) logger.info("Channel directory built: %d target(s)", ch_count) except Exception as e: logger.warning("Channel directory build failed: %s", e) # Check if we're restarting after a /update command. If the update is # still running, keep watching so we notify once it actually finishes. notified = await self._send_update_notification() if not notified and any( path.exists() for path in ( _hermes_home / ".update_pending.json", _hermes_home / ".update_pending.claimed.json", ) ): self._schedule_update_notification_watch() # Give freshly connected platform adapters a brief moment to settle # before sending restart/startup lifecycle messages. In practice this # helps Discord thread deliveries right after reconnect. if connected_count > 0: await asyncio.sleep(1.0) # Notify the chat that initiated /restart that the gateway is back. chat_restart_notification_pending = _restart_notification_pending() planned_restart_notification_pending = _planned_restart_notification_pending() # Capture, before _send_restart_notification() unlinks the marker, # whether this process booted from a chat-originated /restart. Used as # a one-shot signal by the /restart redelivery guard so a missing # dedup marker only suppresses a /restart when we KNOW we just came out # of a restart cycle (see _is_stale_restart_redelivery). if chat_restart_notification_pending: self._booted_from_restart = True # Restart notification, home-channel startup notice, and obligation # redelivery all call adapter.send(). Those sends must not pin the # inbound restore gate — a Telegram flood-control sleep on this path # froze every platform for the full penalty (#91969). Bound them the # same way _finish_startup_restore bounds resume turns. await self._await_startup_boot_sends( planned_restart_notification_pending=planned_restart_notification_pending, ) # Automatically continue fresh sessions that were interrupted by the # previous gateway restart/shutdown. The resume_pending flag is cleared # by the normal successful-turn path, so a failed auto-resume remains # visible for manual recovery on the next user message. # # Delivery-obligation redelivery already ran inside # _await_startup_boot_sends (and clears resume_pending before send): # a session whose final response was generated but never # confirmed-delivered has its answer in the ledger — redelivering it # is strictly cheaper and more correct than re-running the whole turn. self._schedule_resume_pending_sessions() await self._finish_startup_restore() # Surface state.db init failures to the user's messaging platforms # so they know persistence is broken before losing data (#88235). await self._send_session_db_warning_notifications() # Drain any recovered process watchers (from crash recovery checkpoint) try: from tools.process_registry import process_registry # Detach the current batch atomically: reassigning to a fresh list # takes ownership of exactly the watchers present now, so any watcher # appended concurrently during the yield below isn't silently dropped # by a clear() on the shared list. watchers = process_registry.pending_watchers process_registry.pending_watchers = [] # Process in batches of 100 with event-loop yield points to avoid # O(n^2) event-loop blocking when recovering thousands of watchers. for i, watcher in enumerate(watchers): self._spawn_supervised( lambda w=watcher: self._run_process_watcher(w), f"process_watcher:{watcher.get('session_id')}", restart=False, ) logger.info("Resumed watcher for recovered process %s", watcher.get("session_id")) if i % 100 == 99: await asyncio.sleep(0) except Exception as e: logger.error("Recovered watcher setup error: %s", e) # Start background session expiry watcher to finalize expired sessions self._spawn_supervised(self._session_expiry_watcher, "session_expiry_watcher") # Keep the /model picker's remote catalogs (curated manifest, # OpenRouter live list, Nous Portal recommendations) warm on disk so a # delisted or newly-published model reaches the picker within one TTL # window (model_catalog.ttl_minutes, default 20) without waiting for a # cold /model open to trigger the refresh. self._spawn_supervised(self._model_catalog_refresh_watcher, "model_catalog_refresh_watcher") # Stall watchdog: pending inbound + stale agent activity → warn user # to /new (does not kill the turn; see agent.session_stall_timeout). self._spawn_supervised(self._session_stall_watcher, "session_stall_watcher") # Start background kanban notifier — each gateway delivers events for # subscriptions owned by the profiles whose adapters it hosts, even # when another gateway owns the single dispatcher. self._spawn_supervised(self._kanban_notifier_watcher, "kanban_notifier_watcher") # Start background kanban dispatcher — spawns workers for ready # tasks. Gated by `kanban.dispatch_in_gateway` (default True). # When false, users run `hermes kanban daemon` externally or # simply don't use kanban; this loop becomes a no-op. self._spawn_supervised(self._kanban_dispatcher_watcher, "kanban_dispatcher_watcher") # Start background reconnection watcher for platforms that failed at startup if self._failed_platforms: logger.info( "Starting reconnection watcher for %d failed platform(s): %s", len(self._failed_platforms), ", ".join(p.value for p in self._failed_platforms), ) # Track the reconnect watcher task so _ensure_reconnect_watcher_running # can detect if it dies and respawn it (#70344). Spawned via # _spawn_supervised (not a bare asyncio.create_task) so an exception # escaping the watcher's OUTER while-loop -- not just the per-platform # inner try/except -- is caught, logged, and auto-restarted with # backoff instead of silently killing the watcher forever. Without # this, a platform already queued in _failed_platforms when the # watcher dies stays stranded indefinitely: _ensure_reconnect_watcher_running() # only gets called from a NEW fatal-error arrival, so if no other # platform ever fails afterward, nothing ever notices the watcher is # dead (#71758 -- reported as 17.5h of silent downtime for a platform # whose transient upstream outage had long since recovered). # ``on_spawn`` keeps ``_reconnect_watcher_task`` pointed at the CURRENT # live task even when _spawn_supervised's own backoff respawns it — so # _ensure_reconnect_watcher_running never mistakes a superseded handle # for a dead watcher and spawns a duplicate. self._spawn_reconnect_watcher() # Start background handoff watcher — picks up CLI sessions marked # handoff_state='pending' in state.db and re-binds them to the # destination platform's home channel, then forges a synthetic user # turn so the agent kicks off the new chat. self._spawn_supervised(self._handoff_watcher, "handoff_watcher") # Start background async-delegation watcher — drains completion events # from delegate_task(background=true) subagents and injects each # result back into its originating session as a new turn, covering the # idle case where the subagent finishes with no agent turn running. self._spawn_supervised(self._async_delegation_watcher, "async_delegation_watcher") # Start background /loop wakeup watcher — scans persisted loops # (SessionDB loop:* rows) and injects due wakeup prompts into their # originating chats while the session is idle. self._spawn_supervised(self._loop_wakeup_watcher, "loop_wakeup_watcher") # Start the scale-to-zero idle watcher ONLY when this instance is opted # in (the NAS "Labs" HERMES_SCALE_TO_ZERO stamp), messaging is # relay-only/absent, and a wakeUrl is registered (decisions.md D1/D11/ # §3.4(1)). A non-opted instance never starts it, so behaviour is exactly # as today. When armed, the watcher drives the relay dormant on sustained # idle and then suspends the machine itself via the local flaps socket # (Fly Proxy autostop is inbound-only and job-blind, so the gateway owns # the suspend decision; NAS provisions these machines autostop:"off"). try: if self._scale_to_zero_should_arm(): logger.info( "scale-to-zero: armed (idle timeout %.0fs) — watching for idle", self._scale_to_zero_idle_timeout_seconds(), ) self._spawn_supervised(self._scale_to_zero_watcher, "scale_to_zero_watcher") else: # Surface WHY an OPTED-IN instance didn't arm (a non-opted instance # not arming is normal — stay silent there). Without this, a failed # arm is invisible and "why won't it suspend/wake?" needs a box-dive. self._log_scale_to_zero_not_armed_reason() except Exception: # noqa: BLE001 - arming must never block startup logger.debug("scale-to-zero: arm check failed at startup", exc_info=True) # Start background drain-control watcher — reconciles the gateway's # new-turn accept-state with the external ``.drain_request.json`` marker # the dashboard begin/cancel-drain endpoint writes (Phase 2). A marker # left behind by a prior instantiation (durable-volume restart, NS-570) # is ignored via its instantiation epoch; only a current-epoch marker # engages drain on the first tick. self._spawn_supervised(self._drain_control_watcher, "drain_control_watcher") logger.info("Press Ctrl+C to stop") return True _MAX_SUPERVISED_RESTARTS = 5 # A task that ran at least this long before crashing is treated as having # been HEALTHY — its crash is a fresh, isolated failure rather than part of # a rapid crash-loop, so the consecutive-restart counter resets to 0. Only # crashes that happen within this window of a (re)spawn accumulate toward # ``_MAX_SUPERVISED_RESTARTS``. Without this, a long-lived launchd daemon # whose watcher crashes a handful of times over days would hit the cap and # be permanently abandoned (NS: silent loss of platform-reconnect / kanban / # handoff for the rest of the process life). _SUPERVISED_HEALTHY_SECS = 300 @staticmethod def _supervised_backoff(attempt: int) -> float: """Delay before the supervisor's next respawn, in seconds. Capped exponential. A method rather than an inline expression so the schedule has one name, and so a test can collapse it -- the ordering of crash / give-up / slow-tier is what the exhaustion tests assert, and sleeping through the real curve to observe it would make them take minutes. """ return min(60, 2 ** min(attempt, 6)) def _spawn_supervised( self, coro_factory, name, *, restart=True, _attempt=0, on_spawn=None, on_give_up=None, ): """Launch a long-lived background task with task-level supervision. Complements upstream's per-iteration inner-loop try/except (which only guards a single loop-body) by covering what that CANNOT: an exception raised in the watcher's OUTER ``while self._running:`` loop or its pre-try setup region, plus task-level death generally. A bare ``asyncio.create_task`` drops such an exception on the floor — no log, no restart, the watcher silently gone. This retains the handle in ``self._background_tasks``, logs any crash, and restarts with capped exponential backoff up to ``_MAX_SUPERVISED_RESTARTS`` failures in rapid succession (each within ``_SUPERVISED_HEALTHY_SECS`` of its restart). The counter resets after any run that stayed healthy for at least ``_SUPERVISED_HEALTHY_SECS`` — so a long-lived daemon that crashes occasionally over days is never permanently abandoned. Each watcher starts in a fresh ``Context``. These are process-level services, not continuations of whichever message turn happened to spawn them; inheriting a delegated-child marker would make the Kanban dispatcher reject its own writes when ``asyncio.to_thread`` copies the watcher's context. ``on_spawn`` (optional) is invoked with the freshly-created task on every spawn, INCLUDING internal backoff respawns. Callers that also track the live handle elsewhere (e.g. ``self._reconnect_watcher_task`` for ``_ensure_reconnect_watcher_running``) MUST pass it — otherwise the supervisor's own respawn creates a new task without updating that external handle, so ``_ensure_...`` later sees the stale/done handle and spawns a SECOND concurrent watcher (double reconnect attempts). ``on_give_up`` (optional) is invoked with ``name`` when supervision is abandoned — the restart budget is spent and this task will never be respawned by the supervisor again. Supervision being finite is correct; having no owner of the invariant afterwards is not. A task that still has queued work depending on it needs somewhere to hand that fact to, and before this hook existed the only thing standing between budget exhaustion and a permanent silent outage was a *later, unrelated event* happening to call ``_ensure_...`` (#90386). This is the supervisor telling its caller "I am done; the invariant is yours now", which is a thing only the supervisor knows. """ if getattr(self, "_background_tasks", None) is None: self._background_tasks = set() # Monotonic spawn timestamp captured per spawn: the ``_done`` callback # uses it to distinguish a rapid crash-loop from a healthy-run-then-crash. _started = time.monotonic() # Deliberately do NOT pass kwargs to create_task — some test doubles # mock it with a narrow signature. Calling it from a fresh Context has # the same isolation semantics as create_task(..., context=Context()) # while preserving that compatibility. task = Context().run(lambda: asyncio.create_task(coro_factory())) # Mark this as a PERMANENT supervised watcher, not transient background # WORK. The scale-to-zero idle check must ignore these: supervised # watchers (session-expiry, kanban, reconnect, the scale-to-zero watcher # itself, ...) live for the whole process, so counting them as "live # background work" would make the gateway consider itself busy forever # and never go dormant/suspend. Transient tasks added to # _background_tasks elsewhere (startup-resume events etc.) stay counted. task._hermes_supervised_watcher = True # type: ignore[attr-defined] self._background_tasks.add(task) if on_spawn is not None: # Record the live handle NOW so an external tracker (e.g. # _reconnect_watcher_task) always points at the current task, not a # dead one left behind by a prior supervised respawn. try: on_spawn(task) except Exception: # pragma: no cover - defensive; a tracker must never kill the spawn logger.debug("on_spawn callback for %s raised", name, exc_info=True) def _done(t): self._background_tasks.discard(t) if t.cancelled(): return exc = t.exception() if exc is None: # Clean return == deliberate shutdown or a self-disabling watcher # (e.g. a gated no-op that returns synchronously). Respawning here # would busy-spin such a watcher — so NEVER restart on clean exit. return logger.error("Supervised task %s died: %r", name, exc, exc_info=exc) if restart and self._running: ran_for = time.monotonic() - _started if ran_for >= self._SUPERVISED_HEALTHY_SECS: # Ran healthily for a while before crashing — this is a # FRESH failure, not part of a rapid crash-loop. Reset the # consecutive counter so a daemon that crashes a handful of # times over days is never permanently abandoned. effective_attempt = 0 else: effective_attempt = _attempt if effective_attempt >= self._MAX_SUPERVISED_RESTARTS: logger.error( "Supervised task %s died %d times in rapid succession " "(each within %ds of restart) — giving up restarts", name, effective_attempt, self._SUPERVISED_HEALTHY_SECS, ) if on_give_up is not None: try: on_give_up(name) except Exception: # pragma: no cover - defensive logger.debug( "on_give_up callback for %s raised", name, exc_info=True, ) return backoff = self._supervised_backoff(effective_attempt) async def _respawn(): await asyncio.sleep(backoff) if self._running: self._spawn_supervised( coro_factory, name, restart=restart, _attempt=effective_attempt + 1, on_spawn=on_spawn, # Must be threaded through the recursion for the # same reason on_spawn is: the give-up that # matters is the LAST respawn's, and a callback # dropped here would leave the exhaustion branch # with no owner at exactly the moment it needs one. on_give_up=on_give_up, ) # The done callback retains the context in which it was # registered, so isolate the backoff task too; otherwise a # restart could reintroduce the original caller's turn scope. respawn_task = Context().run(lambda: asyncio.create_task(_respawn())) self._background_tasks.add(respawn_task) respawn_task.add_done_callback(self._background_tasks.discard) task.add_done_callback(_done) return task async def _handoff_watcher( self, interval: float = 2.0, drain_timeout: float = 30.0, ) -> None: """Background task that processes pending CLI→gateway session handoffs. Polls ``state.db`` for sessions in ``handoff_state='pending'`` and, for each one: 1. Atomically claims it (pending → running). 2. Resolves the destination platform's configured home channel. 3. Re-binds the gateway's session_key for that home channel to the CLI's existing session_id via ``session_store.switch_session`` so the full role-aware transcript replays on the next agent turn. 4. Forges a synthetic ``MessageEvent`` (``internal=True``) with a handoff-notice text and dispatches through the normal gateway message pipeline so the agent runs and replies on the platform. 5. Marks the row ``completed`` (or ``failed`` with ``handoff_error``). The CLI process is poll-blocked on the row's terminal state and prints the result to the user. """ # Initial delay so the gateway is fully connected to its platforms # before we try to dispatch handoffs through them. await asyncio.sleep(5) # Does this runner's _process_handoff accept the profile argument? # The real one does; test stand-ins bind a one-parameter callable. # Probed once, outside the loop. try: import inspect as _inspect _process_takes_profile = len( _inspect.signature(self._process_handoff).parameters ) >= 2 except Exception: _process_takes_profile = False # In-flight dispatches, keyed by session id. A handoff runs a FULL # agent turn plus delivery, which can take far longer than the CLI's # 60s wait. Processing them inline would make one slow handoff block # every other profile's poll — a legitimate handoff could then time # out purely because another profile was ahead of it in the queue. # Dispatch is therefore fire-and-forget; the poll loop only claims. inflight: Dict[str, "asyncio.Task"] = {} async def _dispatch(row, session_id, session_db, profile_name) -> None: """Run one claimed handoff to a terminal state, off the poll path.""" try: if _process_takes_profile: await self._process_handoff(row, profile_name) else: await self._process_handoff(row) await session_db.complete_handoff(session_id) except asyncio.CancelledError: # Gateway shutting down: leave the row 'running' so the next # start's reclaim marks it failed with a clear reason. raise except Exception as exc: logger.warning( "Handoff for session %s failed: %s", session_id, exc, exc_info=True, ) try: await session_db.fail_handoff(session_id, str(exc)) except Exception: logger.debug("Could not record handoff failure", exc_info=True) finally: inflight.pop(session_id, None) async def _tick(profile_name: Optional[str] = None) -> None: """One poll of the CURRENTLY-SCOPED session store. Deliberately a closure over ``self`` rather than a method: the watcher's unit tests bind ``_handoff_watcher`` onto a minimal ``SimpleNamespace`` stand-in that exposes only ``_session_db``, ``_running`` and ``_process_handoff``. Any ``self.`` call here would raise AttributeError on that stand-in, get swallowed by the loop's ``except Exception``, and silently turn the whole watcher into a no-op — which is exactly what the mutation-survivable assertions caught. Touch only attributes the stand-in already provides. ``profile_name`` is threaded to ``_process_handoff`` so delivery uses that profile's OWN adapter/home channel; see the docstring there. ``None`` means the root/default profile. """ session_db = getattr(self, "_session_db", None) if session_db is None: return pending = await session_db.list_pending_handoffs() for row in pending: session_id = row.get("id") if not session_id or session_id in inflight: continue if not await session_db.claim_handoff(session_id): # Another tick or another gateway already claimed it. continue # Positional, not keyword: the watcher's existing unit tests # bind a stand-in ``_process_handoff(row)`` with no second # parameter, and a keyword call would TypeError into the # failure branch — turning a passing suite into a silent # no-op watcher. Arity is probed above. # # INVARIANT (do not weaken): this task is created inside # ``_profile_runtime_scope(profile_home)`` but typically RUNS # after the scope exits. It still sees the profile's home and # secret scope only because ``set_hermes_home_override`` and # ``set_secret_scope`` are ContextVar-based — ensure_future # copies the current Context into the Task. If either seam is # ever migrated to a thread-local or module global, secondary- # profile handoffs silently regress to primary-config delivery # (the exact bug fixed in #91217) while still recording # handoff_state='completed'. inflight[session_id] = asyncio.ensure_future( _dispatch(row, session_id, session_db, profile_name) ) # A row still in 'running' at startup belongs to a gateway that died # mid-dispatch. It can never reach a terminal state on its own, and # request_handoff refuses new requests while it sits there — so the # session would be permanently unable to hand off again. Reclaim once, # per store, before the first poll. for _pname, _phome in _handoff_watch_scopes(self): try: if _phome is None: await _reclaim_stale(self) else: async with _async_profile_runtime_scope(_phome): await _reclaim_stale(self) except Exception: logger.debug("Stale-handoff reclaim failed", exc_info=True) try: while self._running: try: for profile_name, profile_home in _handoff_watch_scopes(self): if profile_home is None: await _tick(profile_name) else: async with _async_profile_runtime_scope(profile_home): await _tick(profile_name) except asyncio.CancelledError: raise except Exception as exc: logger.debug("Handoff watcher tick error: %s", exc, exc_info=True) await asyncio.sleep(interval) finally: # Drain in-flight dispatches before returning. Cancelling them # outright would strand their rows in 'running'; giving them a # bounded grace period lets an almost-done handoff record its own # terminal state. Whatever is still running after that is # cancelled and reclaimed at the next startup. pending_tasks = [t for t in inflight.values() if not t.done()] if pending_tasks: try: await asyncio.wait(pending_tasks, timeout=drain_timeout) except Exception: logger.debug("Handoff drain raised", exc_info=True) for task in pending_tasks: if not task.done(): task.cancel() async def _process_handoff( self, row: Dict[str, Any], profile_name: Optional[str] = None, ) -> None: """Execute one handoff row. Raises on failure (caller marks failed). ``profile_name`` is the profile whose store queued this handoff (``None`` = root/default). On a multiplexed gateway it is load-bearing for THREE things that otherwise silently resolve to the primary profile and deliver through the wrong bot: - ``self.adapters`` only ever holds the DEFAULT profile's adapters; secondary profiles live in ``self._profile_adapters[name]``. - ``self.config`` is the primary's config, so ``get_home_channel()`` returns the primary's chat — a medicina handoff would be delivered by the default bot, to the default's home channel. - the session key must be namespaced ``agent::...`` to match the key that profile's own adapter uses for organic inbound messages; otherwise the handoff binds a key nobody reads. Passing the name explicitly beats re-deriving it from the active contextvar: the caller already knows which store the row came from. """ from gateway.config import Platform from gateway.session import SessionSource, build_session_key from gateway.platforms.base import MessageEvent cli_session_id = row["id"] platform_name = (row.get("handoff_platform") or "").strip().lower() if not platform_name: raise RuntimeError("handoff_platform is empty") # Resolve platform enum try: platform = Platform(platform_name) except (ValueError, KeyError): raise RuntimeError(f"unknown platform '{platform_name}'") # Resolve the config + adapter map for the profile that queued this # handoff. On a single-profile gateway (or a default-profile handoff) # both fall back to self.config/self.adapters, so behaviour is # byte-identical to before. On a multiplexed gateway a secondary # profile MUST use its own map — self.adapters holds only the primary's # adapters, and self.config only the primary's home channel. handoff_config = self.config handoff_adapters = self.adapters if profile_name and profile_name != "default": secondary = (self._profile_adapters or {}).get(profile_name) if not secondary: raise RuntimeError( f"profile '{profile_name}' has no live adapters in this gateway" ) handoff_adapters = secondary # The watcher already entered _profile_runtime_scope for this # profile, so a fresh load resolves that profile's config.yaml # and .env (home channel, tokens) rather than the primary's. # Fail closed on a load error: self.config is the primary's, so # falling back would deliver through the right bot to the # WRONG chat and report completed. A failed row the CLI can # retry beats a wrong delivery. try: handoff_config = load_gateway_config() except Exception as exc: logger.error( "Handoff: could not load config for profile %s; " "failing the handoff instead of delivering via the " "primary's config", profile_name, exc_info=True, ) raise RuntimeError( f"could not load config for profile '{profile_name}': {exc}" ) from exc # Adapter must be live. A relay-fronted gateway registers ONE adapter # under Platform.RELAY that fronts N logical platforms — so a literal # adapters.get(discord) misses even though "discord" is deliverable. # resolve_delivery_transport is the shared alias-aware resolver (native # adapter wins; relay eligible only when its authenticated transport # advertises it fronts the logical platform). transport = resolve_delivery_transport(platform, handoff_config, handoff_adapters) if not transport: raise RuntimeError( f"platform '{platform_name}' is not active in this gateway" ) adapter = transport.adapter # Home channel must be configured home = handoff_config.get_home_channel(platform) if not home or not home.chat_id: raise RuntimeError( f"no home channel configured for {platform_name}; " f"run /sethome on the desired chat first" ) cli_title = row.get("title") or cli_session_id[:8] # Try to create a fresh thread on the destination so the handoff # has its own scrollback. Adapter returns None if threading isn't # supported (Matrix/WhatsApp/Signal/SMS) or if creation failed # (no permission, topics-mode off, parent is a DM, etc.). When # None we fall through to using the home channel directly — the # synthetic turn still lands; just without thread isolation. thread_name = f"Hermes — {cli_title}" try: new_thread_id = await adapter.create_handoff_thread( str(home.chat_id), thread_name, ) except Exception as exc: logger.debug( "Handoff: create_handoff_thread raised on %s: %s", platform_name, exc, exc_info=True, ) new_thread_id = None # Use the new thread if the adapter created one; otherwise fall # back to whatever thread (if any) the home channel was configured # with. effective_thread_id = new_thread_id or ( str(home.thread_id) if home.thread_id else None ) # Determine chat_type/user_id for the destination source. # # Telegram private-chat DM topics are represented differently from # group/forum threads by the inbound adapter. A handoff-created topic # in a positive Telegram chat_id must therefore use the same DM-topic # source shape as the user's next real message; otherwise the synthetic # handoff turn binds a generic `thread` session key while real replies # arrive on a `dm` session key. home_chat_id = str(home.chat_id) is_telegram_private_chat = ( platform == Platform.TELEGRAM and looks_like_telegram_private_chat_id(home_chat_id) ) if new_thread_id and not is_telegram_private_chat: dest_chat_type = "thread" dest_user_id = "system:handoff" else: # No thread — assume DM-style for the home channel. For Telegram # private-chat topics, use the real user id (same as chat_id) so # topic-mode checks and binding persistence see the same identity as # subsequent inbound user messages. dest_chat_type = "dm" dest_user_id = home_chat_id if is_telegram_private_chat else "system:handoff" # Discord thread destinations must key on the thread's OWN id, not the # parent channel's, because the Discord adapter builds organic in-thread # messages with ``chat_id == thread id`` — so ``build_session_key`` # yields ``…:thread:{thread}:{thread}``. If the handoff keys on the # parent channel (``…:thread:{parent}:{thread}``) the next real user # reply in the thread resolves to a DIFFERENT session_key and spawns a # fresh session instead of continuing the handed-off one. # # This is Discord-specific: Slack and Telegram adapters key organic # thread messages with ``chat_id == parent_channel`` and the thread #/topic id only in ``thread_id``, so for those platforms the parent # channel is correct (and the deeper chat_type normalization — handoff # uses "thread" but Slack organic uses "group" — is a separate issue). if platform == Platform.DISCORD and dest_chat_type == "thread" and effective_thread_id: dest_chat_id = str(effective_thread_id) else: dest_chat_id = home_chat_id dest_source = SessionSource( platform=platform, chat_id=dest_chat_id, chat_name=home.name, chat_type=dest_chat_type, user_id=dest_user_id, user_name="Handoff", thread_id=effective_thread_id, profile=profile_name, ) # Compute the gateway's session_key for that destination using the # same rules its adapters use, so switch_session targets the right # entry. For thread destinations build_session_key keys without # user_id (thread_sessions_per_user defaults to False) — so the # next real user message in the thread shares this same session. platform_cfg = handoff_config.platforms.get(platform) extra = platform_cfg.extra if platform_cfg else {} # Namespace the key to the profile that queued this handoff. Without # it, a multiplexed gateway builds ``agent:main:...`` here while the # profile's own adapter routes real inbound messages on # ``agent::...`` — the handoff would bind a key nobody reads. # # ``profile_name`` comes straight from the watcher, which knows which # store the row was claimed from; prefer it over re-deriving from the # ambient contextvar. The resolver stays as the fallback for the # root/default case (and returns None when multiplexing is off, which # reproduces the previous key byte-for-byte). # # The isinstance check is load-bearing, not defensive noise: a # duck-typed/Mock session store returns a truthy MagicMock from the # resolver, which would be interpolated straight into the key as # ``agent::``. Same pitfall guarded in # ``BasePlatformAdapter._session_key_profile``. handoff_profile = profile_name if (profile_name and profile_name != "default") else None if handoff_profile is None: try: store = getattr(self.async_session_store, "_store", self.async_session_store) resolver = getattr(store, "_resolve_profile_for_key", None) if callable(resolver): resolved = resolver(dest_source) if isinstance(resolved, str) and resolved.strip(): handoff_profile = resolved except Exception: logger.debug("Handoff: could not resolve profile namespace", exc_info=True) session_key = build_session_key( dest_source, group_sessions_per_user=extra.get("group_sessions_per_user", True), thread_sessions_per_user=extra.get("thread_sessions_per_user", False), profile=handoff_profile, ) # Make sure there's an entry in the session_store for this key. If # the home channel has never been used, get_or_create_session # creates one; switch_session then re-points it. await self.async_session_store.get_or_create_session(dest_source) # Re-bind the destination key to the CLI session_id. switch_session # ends the prior session in SQLite and reopens the CLI session under # the new key. The CLI's transcript becomes the active one for the # gateway from this moment on. switched = await self.async_session_store.switch_session(session_key, cli_session_id) if switched is None: raise RuntimeError( f"could not switch session key {session_key} → {cli_session_id}" ) # Evict any cached AIAgent for this session_key so the next dispatch # rebuilds it against the CLI session_id (mirrors /resume / /branch). self._evict_cached_agent(session_key) # Cancel any in-flight running-agent state for the destination key # so the synthetic turn isn't queued behind a stale running flag. self._release_running_agent_state(session_key) synthetic_text = ( f"[Session was just handed off from CLI (\"{cli_title}\") to this " f"channel. The full prior conversation history is loaded above. " f"Briefly confirm you're working here and summarize what we were " f"working on, so the user can continue from this device.]" ) synthetic_event = MessageEvent( text=synthetic_text, source=dest_source, internal=True, ) logger.info( "Handoff: dispatching synthetic turn for CLI session %s → %s " "(home=%s, thread=%s, session_key=%s)", cli_session_id, platform_name, home.chat_id, effective_thread_id, session_key, ) # Dispatch through the runner directly. Going through # adapter.handle_message would spawn a background task and we'd # lose synchronous error visibility; calling _handle_message inline # keeps the success/failure path observable for the watcher. response_text = await self._handle_message(synthetic_event) if not response_text: # Streaming may have already delivered the response inline. # Either way, agent ran without raising — count as success. return # Send the agent's reply to the destination. Route to the new # thread if we created one; otherwise the configured home channel # (which may itself carry a thread_id). Send through the resolved # transport (not adapter.send directly) so a relay-fronted logical # platform is stamped on the outbound frame (send_for_platform). send_metadata: Dict[str, Any] = {} if effective_thread_id: send_metadata["thread_id"] = effective_thread_id try: result = await transport.send( platform, str(home.chat_id), response_text, send_metadata or None, ) except Exception as exc: raise RuntimeError(f"adapter.send failed: {exc}") from exc if not getattr(result, "success", True): err = getattr(result, "error", "send returned success=False") raise RuntimeError(f"adapter.send failed: {err}") async def _session_expiry_watcher(self, interval: int = 300): """Background task that finalizes expired sessions. Runs every ``interval`` seconds (default 5 min). For each session whose reset policy has expired, invokes ``on_session_finalize`` hooks, cleans up the cached AIAgent's tool resources, evicts the cache entry so it can be garbage-collected, and marks the session so it won't be finalized again. """ await asyncio.sleep(60) # initial delay — let the gateway fully start _finalize_failures: dict[str, int] = {} # session_id -> consecutive failure count _MAX_FINALIZE_RETRIES = 3 while self._running: try: await self.async_session_store._ensure_loaded() # Collect expired sessions first, then log a single summary. _expired_entries = [] for key, entry in list(self.session_store._entries.items()): if entry.expiry_finalized: continue if not await self.async_session_store._is_session_expired(entry): continue _expired_entries.append((key, entry)) if _expired_entries: # Extract platform names from session keys for a compact summary. # Keys look like "agent:main:telegram:dm:12345" — platform is field [2]. _platforms: dict[str, int] = {} for _k, _e in _expired_entries: _parts = _k.split(":") _plat = _parts[2] if len(_parts) > 2 else "unknown" _platforms[_plat] = _platforms.get(_plat, 0) + 1 _plat_summary = ", ".join( f"{p}:{c}" for p, c in sorted(_platforms.items()) ) logger.info( "Session expiry: %d sessions to finalize (%s)", len(_expired_entries), _plat_summary, ) for key, entry in _expired_entries: try: try: _parts = key.split(":") _platform = _parts[2] if len(_parts) > 2 else "" # Off-loop + bounded: plugin finalize hooks can # block arbitrarily (see _finalize_session_off_loop) # and this watcher runs on the gateway event loop. await self._finalize_session_off_loop( session_id=entry.session_id, platform=_platform, reason="session_expired", ) except Exception: pass # Shut down memory provider and close tool resources # on the cached agent. Idle agents live in # _agent_cache (not _running_agents), so look there. _cached_agent = None _cache_lock = getattr(self, "_agent_cache_lock", None) if _cache_lock is not None: with _cache_lock: _cached = self._agent_cache.get(key) _cached_agent = _cached[0] if isinstance(_cached, tuple) else _cached if _cached else None # Fall back to _running_agents in case the agent is # still mid-turn when the expiry fires. if _cached_agent is None: _exp_state = self._peek_session_state(key) _cached_agent = _exp_state.turn.agent if _exp_state else None if _cached_agent and _cached_agent is not _AGENT_PENDING_SENTINEL: await self._cleanup_agent_resources_off_loop( _cached_agent, context="session expiry" ) # Drop the cache entry so the AIAgent (and its LLM # clients, tool schemas, memory provider refs) can # be garbage-collected. Otherwise the cache grows # unbounded across the gateway's lifetime. self._evict_cached_agent(key) # Permanently finalizing this session — one funnel # call drops every conversation-scoped dict AND the # boundary security state (approvals, update # prompts, slash-confirm) so the dicts don't grow # unbounded across the gateway's lifetime. (Idle # agent-cache eviction must NOT do this: the # session is still alive and a resumed turn rebuilds # its agent from these overrides. Only true session # finalization, /new, and /reset clear them.) See # _CONVERSATION_SCOPED_STATE. self._clear_conversation_scope( key, reason="expiry_finalized" ) # Persist the finalized flag to sessions.json AND # state.db (single write-path, #9006) — also drops # the persisted /model override, since finalization # is a conversation boundary. await self.async_session_store.set_expiry_finalized(entry) logger.debug( "Session expiry finalized for %s", entry.session_id, ) _finalize_failures.pop(entry.session_id, None) except Exception as e: failures = _finalize_failures.get(entry.session_id, 0) + 1 _finalize_failures[entry.session_id] = failures if failures >= _MAX_FINALIZE_RETRIES: logger.warning( "Session finalize gave up after %d attempts for %s: %s. " "Marking as finalized to prevent infinite retry loop.", failures, entry.session_id, e, ) await self.async_session_store.set_expiry_finalized( entry, clear_model_override=False ) _finalize_failures.pop(entry.session_id, None) else: logger.debug( "Session finalize failed (%d/%d) for %s: %s", failures, _MAX_FINALIZE_RETRIES, entry.session_id, e, ) if _expired_entries: _done = sum( 1 for _, e in _expired_entries if e.expiry_finalized ) _failed = len(_expired_entries) - _done if _failed: logger.info( "Session expiry done: %d finalized, %d pending retry", _done, _failed, ) else: logger.info( "Session expiry done: %d finalized", _done, ) # Sweep agents that have been idle beyond the TTL regardless # of session reset policy. This catches sessions with very # long / "never" reset windows, whose cached AIAgents would # otherwise pin memory for the gateway's entire lifetime. try: _idle_evicted = self._sweep_idle_cached_agents() if _idle_evicted: logger.info( "Agent cache idle sweep: evicted %d agent(s)", _idle_evicted, ) except Exception as _e: logger.debug("Idle agent sweep failed: %s", _e) # Neither the LRU cap nor the idle TTL is aware of how much # memory a cached transcript costs, so a busy gateway keeps # every warm session's tool output resident until RSS hits the # cgroup limit (#80764). Shed LRU transcripts once the heap is # over budget; they reload from the persisted session on the # next turn. try: self._sweep_agent_cache_under_pressure() except Exception as _e: logger.debug("Agent cache pressure sweep failed: %s", _e) # Periodically prune stale SessionStore entries. The # in-memory dict (and sessions.json) would otherwise grow # unbounded in gateways serving many rotating chats / # threads / users over long time windows. Pruning is # invisible to users — a resumed session just gets a # fresh session_id, exactly as if the reset policy fired. _last_prune_ts = getattr(self, "_last_session_store_prune_ts", 0.0) _prune_interval = 3600.0 # once per hour if time.time() - _last_prune_ts > _prune_interval: try: _max_age = int( getattr(self.config, "session_store_max_age_days", 0) or 0 ) if _max_age > 0: _pruned = await self.async_session_store.prune_old_entries(_max_age) if _pruned: logger.info( "SessionStore prune: dropped %d stale entries", _pruned, ) except Exception as _e: logger.debug("SessionStore prune failed: %s", _e) self._last_session_store_prune_ts = time.time() except Exception as e: logger.debug("Session expiry watcher error: %s", e) # Sleep in small increments so we can stop quickly for _ in range(interval): if not self._running: break await asyncio.sleep(1) def _session_stall_timeout_seconds(self) -> float: """Return configured stall timeout (seconds); 0 disables the watchdog.""" return _float_env("HERMES_SESSION_STALL_TIMEOUT", 300) def _iter_gateway_adapters(self): """Yield every live platform adapter (default + multiplex profiles).""" seen: set[int] = set() for adapter in list(getattr(self, "adapters", {}).values()): if adapter is None: continue aid = id(adapter) if aid in seen: continue seen.add(aid) yield adapter for amap in list(getattr(self, "_profile_adapters", {}).values()): for adapter in list(amap.values()): if adapter is None: continue aid = id(adapter) if aid in seen: continue seen.add(aid) yield adapter def _session_activity_for_stall(self, session_key: str) -> Optional[dict]: """Return the shared activity snapshot for stall progress (#72039). Single progress source: ``AIAgent.get_activity_summary()`` / ``agent.session_activity``. No turn-start or pending-inbound clocks. """ agent = (getattr(self, "_running_agents", None) or {}).get(session_key) if agent is None or agent is _AGENT_PENDING_SENTINEL: return None if not hasattr(agent, "get_activity_summary"): return None try: summary = agent.get_activity_summary() except Exception: return None return summary if isinstance(summary, dict) else None async def _check_session_stalls(self, timeout_seconds: float) -> int: """Scan pending inbound sessions and notify once per stall episode. Returns the number of notifications sent this pass (for tests). """ from gateway.session_stall import ( format_session_stall_notification, resolve_session_idle_seconds_from_activity, should_clear_session_stall_notification, should_emit_session_stall_notification, ) notified_map = getattr(self, "_session_stall_notified", None) if notified_map is None: notified_map = {} self._session_stall_notified = notified_map sent = 0 now = time.time() candidates: Dict[str, tuple[Any, Any]] = {} for adapter in self._iter_gateway_adapters(): pending_slot = getattr(adapter, "_pending_messages", None) or {} for session_key, event in list(pending_slot.items()): if session_key and session_key not in candidates and event is not None: candidates[session_key] = (adapter, event) for session_key, overflow in list( (getattr(self, "_queued_events", None) or {}).items() ): if not session_key or session_key in candidates or not overflow: continue event = overflow[0] source = getattr(event, "source", None) adapter = ( self._adapter_for_source(source) if source is not None else None ) if adapter is None: continue candidates[session_key] = (adapter, event) for session_key, (adapter, pending_event) in list(candidates.items()): has_pending = pending_event is not None activity = ( self._session_activity_for_stall(session_key) if has_pending else None ) idle_seconds = ( resolve_session_idle_seconds_from_activity(activity, now=now) if has_pending else None ) already = bool(notified_map.get(session_key)) if should_clear_session_stall_notification( timeout_seconds=timeout_seconds, idle_seconds=idle_seconds, has_pending_inbound=has_pending, ): notified_map.pop(session_key, None) already = False if not should_emit_session_stall_notification( timeout_seconds=timeout_seconds, idle_seconds=idle_seconds, has_pending_inbound=has_pending, already_notified=already, ): continue if idle_seconds is None: continue mins = max(1, int(idle_seconds // 60)) activity = activity or {} logger.warning( "Session stall detected: session=%s idle=%.0fs " "(timeout=%.0fs, ~%d min); pending inbound present " "| last_activity=%s | provenance=%s " "(agent.session_stall_timeout)", session_key, idle_seconds, timeout_seconds, mins, activity.get("last_activity_desc") or activity.get("last_activity_description") or "unknown", activity.get("provenance") or activity.get("last_activity_provenance") or "unknown", ) source = getattr(pending_event, "source", None) chat_id = getattr(source, "chat_id", None) if source is not None else None if not chat_id: logger.warning( "Session stall notify skipped (no chat_id): session=%s", session_key, ) # Cannot deliver; latch to avoid log spam every tick. notified_map[session_key] = True continue # #76354 review S2: re-read pending state + activity timestamp # IMMEDIATELY before delivery. The snapshot above ages while # earlier candidates in this pass await their sends; an agent # that made progress (or drained its queue) in that window must # not receive a false stall notice. Abort and leave the latch # un-set so the next tick re-evaluates from scratch. still_pending = ( (getattr(adapter, "_pending_messages", None) or {}).get( session_key ) is not None or bool( (getattr(self, "_queued_events", None) or {}).get( session_key ) ) ) fresh_idle = resolve_session_idle_seconds_from_activity( self._session_activity_for_stall(session_key), now=time.time(), ) if not still_pending or ( fresh_idle is not None and fresh_idle < timeout_seconds ): logger.info( "Session stall notify aborted (no longer stale): " "session=%s pending=%s fresh_idle=%s", session_key, still_pending, fresh_idle, ) # Re-arm: drop any stale latch so a FUTURE genuine stall # episode notifies again. notified_map.pop(session_key, None) continue try: metadata = ( self._thread_metadata_for_source(source) if source is not None and hasattr(self, "_thread_metadata_for_source") else None ) # Round-2 #2: bound the send. A wedged adapter transport # (network hang, dead websocket) must not block the whole # watcher pass — sibling candidates in this loop would never # be evaluated and the watcher itself would stop ticking. try: result = await asyncio.wait_for( adapter.send( str(chat_id), format_session_stall_notification(idle_seconds), metadata=metadata, ), timeout=_STALL_NOTIFY_SEND_TIMEOUT_SECONDS, ) except asyncio.TimeoutError: logger.warning( "Session stall notify send timed out after %.0fs " "for %s; will retry next tick", _STALL_NOTIFY_SEND_TIMEOUT_SECONDS, session_key, ) continue # do not latch; retry next tick # Adapters often return SendResult(success=False) instead of raising. if result is not None and getattr(result, "success", True) is False: logger.warning( "Session stall notify failed for %s: %s", session_key, getattr(result, "error", "send returned success=False"), ) continue # do not latch; retry next tick sent += 1 notified_map[session_key] = True except Exception as exc: logger.warning( "Session stall notify failed for %s: %s", session_key, exc, ) # Do not latch — retry next watcher tick until delivery or episode clear. # Drop latches for sessions that no longer appear in any pending map. for key in list(notified_map.keys()): if key not in candidates: notified_map.pop(key, None) return sent async def _model_catalog_refresh_watcher(self) -> None: """Refresh the /model picker's remote catalogs every TTL window. The picker itself only refreshes on a cold or stale open, so a gateway that nobody opens ``/model`` in keeps serving whatever was cached. This loop calls ``model_catalog.refresh_catalogs()`` (manifest + OpenRouter live filter + Nous Portal recommendations) off-thread on the configured cadence (``model_catalog.ttl_minutes``, default 20) so the on-disk caches every surface reads are never older than one window. """ from hermes_cli.model_catalog import refresh_catalogs, refresh_interval_seconds await asyncio.sleep(30) # let startup settle while self._running: try: await asyncio.to_thread(refresh_catalogs) except Exception as exc: logger.debug("Model catalog refresh failed: %s", exc) try: interval = refresh_interval_seconds() except Exception: interval = 1200.0 deadline = time.monotonic() + interval while self._running and time.monotonic() < deadline: await asyncio.sleep(min(30.0, max(0.0, deadline - time.monotonic()))) async def _session_stall_watcher(self, interval: float = 30.0): """Periodic pending-inbound + stale-activity stall watchdog (#72016). Progress comes only from ``get_activity_summary()`` (#72039). Pending inbound is a notify policy gate, not a progress clock. Notify-only: does not kill the turn (contrast ``gateway_timeout`` / ``shutdown_watchdog``). """ # Short initial delay so startup reconnect noise does not false-fire. await asyncio.sleep(min(30.0, max(1.0, float(interval)))) while self._running: try: timeout = self._session_stall_timeout_seconds() if timeout > 0: await self._check_session_stalls(timeout) except Exception as exc: logger.debug("Session stall watcher error: %s", exc) # Interruptible sleep steps = max(1, int(float(interval))) for _ in range(steps): if not self._running: break await asyncio.sleep(1) def _active_profile_name(self) -> str: """Return the profile name this gateway represents.""" try: from hermes_cli.profiles import get_active_profile_name return get_active_profile_name() or "default" except Exception: return "default" # ── Kanban board watchers ─────────────────────────────────────────── # The kanban notifier/dispatcher watcher loops + their helpers live in # GatewayKanbanWatchersMixin (gateway/kanban_watchers.py). They use only # self state, so inheriting the mixin keeps every self._kanban_* call site # working unchanged while lifting ~1,000 LOC out of this file. #: Interval of the slow respawn tier that takes over once the reconnect #: watcher has exhausted its supervised restart budget. Long on purpose: #: the budget is spent precisely when the watcher is crashing on contact, #: so the useful cadence is "check back later", not "try again now". A #: tight loop here would be worse than the outage it is healing. _RECONNECT_WATCHER_SLOW_RETRY_SECS = 300 #: How many slow-tier respawns to attempt while work is still queued. #: Bounded, not infinite: if half an hour of five-minute retries cannot #: keep a watcher alive, the fault is not transient and a louder failure #: is more useful than a quieter one that never stops. _MAX_SLOW_WATCHER_RESPAWNS = 6 def _on_reconnect_watcher_gave_up(self, name: str = "") -> None: """Own the reconnect invariant once supervision has abandoned it. The invariant this closes: **while the gateway is running and ``_failed_platforms`` is non-empty, either a reconnect watcher is live or a bounded respawn is scheduled.** Before this, the only thing that noticed a dead watcher was a *later fatal error from some other platform* reaching ``_queue_retryable_fatal_platform``. That is event-coupled recovery: it needs an event that, by construction, may never come. #81036 moved queue publication ahead of disconnect and drops the failed adapter from the live map, so once the watcher's budget is spent there may be no adapter left that can emit the event recovery was waiting on. The platform stays queued, nothing retries it, and the stranded check in ``_handle_adapter_fatal_error_detached`` treats a queued platform as safe — so the process is never restarted either. Deliberately NOT done here: requesting a supervisor/process restart when the slow tier is also exhausted. That is a policy decision about blast radius (a gateway serving healthy platforms would be taken down to heal a sick one) and it belongs to a maintainer, not to this patch. What happens instead is a single loud error naming the still-queued platforms, which is the state an operator or an external supervisor can act on. """ if not getattr(self, "_running", False): return if not getattr(self, "_failed_platforms", None): # No queued work depends on the watcher. Letting it stay dead is # correct -- the enqueue path spawns a fresh one the moment a # platform is queued again. logger.warning( "Reconnect watcher supervision exhausted with an empty retry " "queue — leaving it down until a platform is queued." ) return self._schedule_slow_reconnect_watcher_respawn(attempt=0) def _schedule_slow_reconnect_watcher_respawn(self, *, attempt: int) -> None: """Bounded slow-tier respawn of the reconnect watcher.""" if attempt >= self._MAX_SLOW_WATCHER_RESPAWNS: logger.error( "Reconnect watcher could not be kept alive after %d slow " "respawns; %d platform(s) remain queued and unattended: %s. " "Manual intervention or a gateway restart is required.", attempt, len(self._failed_platforms), ", ".join(str(p) for p in self._failed_platforms), ) return async def _slow_respawn() -> None: await asyncio.sleep(self._RECONNECT_WATCHER_SLOW_RETRY_SECS) if not getattr(self, "_running", False): return if not getattr(self, "_failed_platforms", None): # The queue drained while we waited -- something else healed # it. Nothing to own any more. return task = getattr(self, "_reconnect_watcher_task", None) if task is not None and not task.done(): return # a watcher came back on its own; stand down logger.warning( "Reconnect watcher still down with %d platform(s) queued — " "slow respawn %d/%d", len(self._failed_platforms), attempt + 1, self._MAX_SLOW_WATCHER_RESPAWNS, ) self._spawn_reconnect_watcher( on_give_up=lambda _name: self._schedule_slow_reconnect_watcher_respawn( attempt=attempt + 1 ) ) respawn_task = asyncio.create_task(_slow_respawn()) if getattr(self, "_background_tasks", None) is None: self._background_tasks = set() self._background_tasks.add(respawn_task) respawn_task.add_done_callback(self._background_tasks.discard) def _spawn_reconnect_watcher(self, *, on_give_up=None): """Single place that knows how to launch the reconnect watcher. Three call sites used to repeat this triple (factory, name, on_spawn), and the ``on_spawn`` half of it is load-bearing: without it the supervisor's own respawn leaves ``_reconnect_watcher_task`` pointing at a dead handle and ``_ensure_...`` spawns a second concurrent watcher. """ self._reconnect_watcher_task = self._spawn_supervised( self._platform_reconnect_watcher, "platform_reconnect_watcher", on_spawn=lambda t: setattr(self, "_reconnect_watcher_task", t), on_give_up=on_give_up or self._on_reconnect_watcher_gave_up, ) return self._reconnect_watcher_task def _ensure_reconnect_watcher_running(self) -> None: """Ensure the platform reconnect watcher background task is alive. If the tracked reconnect watcher task has died (e.g. from exhausting its restart budget, or a terminal exception that _spawn_supervised could not recover), respawns it so platforms queued for reconnection are not permanently stranded. Called from _queue_retryable_fatal_platform on BOTH paths (#70344, #90386): after a new enqueue, and after a re-fatal for a platform that is already queued -- the latter being the only case in which the watcher can have been retrying long enough to exhaust its supervised restart budget. """ if not getattr(self, "_running", False): return task = getattr(self, "_reconnect_watcher_task", None) if task is not None and not task.done(): return # already alive logger.warning( "Reconnect watcher task is dead (done=%s) — respawning", task.done() if task is not None else "N/A", ) self._spawn_reconnect_watcher() async def _platform_reconnect_watcher(self) -> None: """Background task that periodically retries connecting failed platforms. Uses exponential backoff: 30s → 60s → 120s → 240s → 300s (cap). Retryable failures (network/DNS blips) keep retrying at the backoff cap indefinitely — they self-heal once connectivity returns, so a transient outage never requires manual intervention. Non-retryable failures (bad auth, etc.) drop out of the queue immediately. The circuit breaker (``_pause_failed_platform`` / ``/platform pause``) remains available for manual operator control via ``/platform list`` and ``/platform resume ``, but is no longer triggered automatically — auto-pausing a recovered platform was the cause of bots silently staying dead after a transient DNS failure. """ await asyncio.sleep(10) # initial delay — let startup finish while self._running: if not self._failed_platforms: # Nothing to reconnect — sleep and check again for _ in range(30): if not self._running: return if self._failed_platforms: break await asyncio.sleep(1) continue now = time.monotonic() for platform in list(self._failed_platforms.keys()): if not self._running: return info = self._failed_platforms.get(platform) if info is None: # Removed concurrently (e.g. a manual /platform resume, # or a reconnect that succeeded via a different path) # between the snapshot above and this lookup. Not an # error -- just nothing to do for it this pass. continue # Skip paused platforms entirely — they need explicit # /platform resume to come back. if info.get("paused"): continue # Long-lived retry-loop escalation (OOF-156): once a platform # has been continuously queued past the attention threshold, # flag it NEEDS_ATTENTION in runtime status so owners and # fleet monitoring see "this is not a blip" — a dead token, # revoked intent, or crash-looping sidecar otherwise presents # as ordinary "retrying" forever. Retries continue unchanged: # this is a signal, NOT a circuit breaker (auto-pause was # deliberately removed — see this docstring's history). if not info.get("attention_flagged") and _reconnect_needs_attention(info, now): info["attention_flagged"] = True queued_for = now - info.get("queued_at", now) retrying_since_iso = ( datetime.now(timezone.utc) - timedelta(seconds=queued_for) ).isoformat() logger.warning( "%s has been failing/reconnecting continuously for " "%.1f hours (%d attempts) — flagging NEEDS_ATTENTION. " "Retries continue, but this usually means a permanent " "problem (revoked credentials, missing intents, broken " "sidecar). Check `hermes status` / `/platform list`.", platform.value, queued_for / 3600.0, info.get("attempts", 0), ) self._update_platform_runtime_status( platform.value, platform_state="retrying", needs_attention=True, retrying_since=retrying_since_iso, ) if now < info["next_retry"]: continue # not time yet platform_config = info["config"] attempt = info["attempts"] + 1 # Empty-token primary configs can never reconnect; drop them so # multiplex setups where a secondary profile owns the bot do # not spin forever (#64674). if not _platform_has_bot_credential(platform, platform_config): logger.warning( "Reconnect %s: no bot credential on queued config, " "removing from retry queue", platform.value, ) del self._failed_platforms[platform] continue logger.info( "Reconnecting %s (attempt %d)...", platform.value, attempt, ) adapter = None try: adapter = self._create_adapter(platform, platform_config) if not adapter: logger.warning( "Reconnect %s: adapter creation returned None, removing from retry queue", platform.value, ) del self._failed_platforms[platform] continue adapter.set_message_handler(self._primary_message_handler()) adapter.set_fatal_error_handler(self._handle_adapter_fatal_error) adapter.set_session_store(self.session_store) adapter.set_busy_session_handler(self._handle_active_session_busy_message) _set_reaction = getattr(adapter, "set_reaction_handler", None) if callable(_set_reaction): _set_reaction(self._handle_reaction_event) adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id) adapter.set_authorization_check(self._make_adapter_auth_check(adapter.platform)) adapter.set_platform_event_handler(self._primary_platform_event_handler()) adapter._busy_text_mode = self._busy_text_mode # Reconnect after an outage: preserve the platform's # server-side update queue so messages sent while the bot # was offline are delivered rather than dropped (#46621). success = await self._connect_adapter_with_timeout( adapter, platform, is_reconnect=True ) if success: self.adapters[platform] = adapter self._sync_voice_mode_state_to_adapter(adapter) # Wire voice input callback on reconnect as well (#60623). self._bind_voice_input_callback(adapter) self.delivery_router.adapters = self.adapters del self._failed_platforms[platform] # connect() returning True does not mean the adapter's # receive path is confirmed -- Telegram's degraded # reconnect returns True so the gateway stays up while # its own ladder retries. Stamping "connected" here # would undo the adapter's accurate status (#101391). _degraded = adapter.send_path_degraded self._update_platform_runtime_status( platform.value, platform_state="retrying" if _degraded else "connected", error_code=None, error_message=adapter.DEGRADED_STATUS_MESSAGE if _degraded else None, needs_attention=False, retrying_since=None, ) if _degraded: logger.info("⚠ %s reconnected in degraded mode (receive path not yet confirmed)", platform.value) else: logger.info("✓ %s reconnected successfully", platform.value) # Final responses rejected while this adapter was down # are still owned by this live process, so startup # recovery cannot claim them. Replay the explicitly # transient subset now that the platform is usable. try: await self._redeliver_failed_obligations_for_platform( platform ) except Exception: logger.debug( "failed-obligation redelivery after %s reconnect failed", platform.value, exc_info=True, ) # Rebuild channel directory with the new adapter try: from gateway.channel_directory import build_channel_directory await build_channel_directory(self.adapters) except Exception: pass # A platform that was offline at gateway startup never # got its restart-interrupted sessions auto-resumed — # the startup pass skips sessions whose adapter isn't # connected yet. Now that it's back, retry the # auto-resume scoped to this platform so recovery # doesn't silently wait for a manual user message. try: self._schedule_resume_pending_sessions(platform=platform) except Exception: logger.debug( "resume-pending reschedule after %s reconnect failed", platform.value, exc_info=True, ) # Check if the failure is non-retryable elif adapter.has_fatal_error and not adapter.fatal_error_retryable: self._update_platform_runtime_status( platform.value, platform_state="fatal", error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message, ) logger.warning( "Reconnect %s: non-retryable error (%s), removing from retry queue", platform.value, adapter.fatal_error_message, ) # The adapter is about to be dropped from the queue # without ever being installed on self.adapters, so # nothing else will call disconnect() on it. We must # dispose it here, otherwise the resource owners it # constructed in __init__ (ResponseStore for # APIServerAdapter, etc.) leak 2 fds each. The # gateway hits the 2560-fd limit after ~12h of # failed reconnects at the 300s backoff cap (#37011). await _dispose_unused_adapter(adapter) del self._failed_platforms[platform] else: self._update_platform_runtime_status( platform.value, platform_state="retrying", error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message or "failed to reconnect", ) backoff = _reconnect_backoff(attempt) info["attempts"] = attempt info["next_retry"] = time.monotonic() + backoff logger.info( "Reconnect %s failed, next retry in %ds", platform.value, backoff, ) # Same fd-leak concern as the non-retryable branch # above: the adapter failed to connect and is being # thrown away. Without an explicit dispose call, the # resources it opened in __init__ stay open until # the next GC pass — and aiohttp/SQLite handles # don't get GC'd promptly, so 2 fds/retry leak at # 300s backoff cap = ~12 fds/hour (#37011). await _dispose_unused_adapter(adapter) # Retryable failures (network/DNS blips) keep retrying # at the backoff cap indefinitely — they self-heal once # connectivity returns. We do NOT auto-pause them: a # transient outage must never require manual `/platform # resume` to recover. Non-retryable failures (bad auth, # etc.) already drop out of the queue via the # `not fatal_error_retryable` branch above, so anything # reaching here is by definition retryable. except Exception as e: if adapter is not None: # An exception escaping the connect call path # (DNS timeout, aiohttp server.start() crash, etc.) # leaves the adapter in the same unowned state as # the two branches above. Dispose so __init__ # resources don't accumulate while the watcher # keeps retrying. await _dispose_unused_adapter(adapter) self._update_platform_runtime_status( platform.value, platform_state="retrying", error_code=None, error_message=str(e), ) backoff = _reconnect_backoff(attempt) info["attempts"] = attempt info["next_retry"] = time.monotonic() + backoff logger.warning( "Reconnect %s error: %s, next retry in %ds", platform.value, e, backoff, ) # A raised exception during reconnect (connect timeout, DNS # resolution failure, etc.) is inherently transient — keep # retrying at the backoff cap rather than auto-pausing. # Check every 10 seconds for platforms that need reconnection for _ in range(10): if not self._running: return await asyncio.sleep(1) async def _cancel_secondary_profile_reconnect_tasks(self) -> None: """Cancel profile-scoped reconnects before tearing down their registry. A reconnect can be waiting in adapter setup while shutdown begins. It must not republish an adapter after the secondary registry is drained. Waiting is bounded by the same adapter-cleanup budget; if a task does not finish in time, the stopped runner state still prevents it from installing an adapter when it eventually resumes. """ pending = self._profile_failed_platforms if not isinstance(pending, dict): return current = asyncio.current_task() tasks: list[asyncio.Task] = [] for profile_pending in pending.values(): if not isinstance(profile_pending, dict): continue for task in profile_pending.values(): if isinstance(task, asyncio.Task) and task is not current and not task.done(): tasks.append(task) for task in tasks: task.cancel() timeout = self._adapter_disconnect_timeout_secs() if tasks and timeout > 0: _done, unfinished = await asyncio.wait(tasks, timeout=timeout) if unfinished: logger.warning( "Timed out waiting for %d secondary profile reconnect task(s) during shutdown", len(unfinished), ) pending.clear() def _start_systemd_watchdog(self) -> bool: """Start sd_notify only after a configured gateway is truly running.""" if not self._running or self.config.systemd_watchdog_seconds <= 0: return False if self._systemd_watchdog is not None: return True from gateway.systemd_notify import SystemdWatchdog watchdog = SystemdWatchdog(config_enabled=True) if not watchdog.start(): return False self._systemd_watchdog = watchdog watchdog.ready("Hermes Gateway running") return True async def _stop_systemd_watchdog(self) -> None: """Stop heartbeats before any potentially long shutdown drain.""" watchdog = self._systemd_watchdog if watchdog is None: return self._systemd_watchdog = None await watchdog.stop() async def stop( self, *, restart: bool = False, detached_restart: bool = False, service_restart: bool = False, ) -> None: """Stop the gateway and disconnect all adapters.""" # getattr-guard: shutdown-path tests build bare runners via # object.__new__ that lack the liveness-guard machinery. _stop_guards = getattr(self, "_stop_loop_liveness_guards", None) if callable(_stop_guards): _stop_guards() if restart: self._restart_requested = True self._restart_detached = detached_restart self._restart_via_service = service_restart if self._stop_task is not None: await self._stop_task return async def _stop_impl() -> None: def _kill_tool_subprocesses(phase: str) -> list: """Kill tool subprocesses + tear down terminal envs + browsers. Returns the cron job IDs this phase marked interrupted, so the caller can notify their owners while adapters are still up (#82232). Empty list when no cron work was in flight. Called twice in the shutdown path: once eagerly after a drain timeout forces agent interrupt (so we reclaim bash/ sleep children before systemd TimeoutStopSec escalates to SIGKILL on the cgroup — #8202), and once as a final catch-all at the end of _stop_impl() for the graceful path or anything respawned mid-teardown. All steps are best-effort; exceptions are swallowed so one subsystem's failure doesn't block the rest. """ try: from tools.process_registry import process_registry _killed = process_registry.kill_all() if _killed: logger.info( "Shutdown (%s): killed %d tool subprocess(es)", phase, _killed, ) except Exception as _e: logger.debug("process_registry.kill_all (%s) error: %s", phase, _e) _marked_cron_jobs: list = [] try: # Any cron job still dispatched at this instant just had # its tool subprocess killed above (kill_all() has no # per-job-ID targeting — it's a global sweep). Its agent # thread is still alive in this process and may go on to # produce a plausible-looking final response from the # now-truncated tool output; mark the run interrupted so # the scheduler can never report that as success (#60432). # No-op when no cron job is in flight. from cron.scheduler import mark_running_jobs_interrupted _interrupted = _marked_cron_jobs = mark_running_jobs_interrupted( f"Gateway shutdown ({phase}) killed the job's tool " "subprocess before the run finished." ) if _interrupted: logger.warning( "Shutdown (%s): marked %d in-flight cron job(s) interrupted: %s", phase, len(_interrupted), ", ".join(_interrupted), ) except Exception as _e: logger.debug("mark_running_jobs_interrupted (%s) error: %s", phase, _e) try: from tools.async_delegation import interrupt_all as _interrupt_async _async_n = _interrupt_async(reason=f"gateway shutdown ({phase})") if _async_n: logger.info( "Shutdown (%s): interrupted %d background delegation(s)", phase, _async_n, ) except Exception as _e: logger.debug("async interrupt_all (%s) error: %s", phase, _e) try: from tools.terminal_tool import cleanup_all_environments cleanup_all_environments() except Exception as _e: logger.debug("cleanup_all_environments (%s) error: %s", phase, _e) try: from tools.browser_tool import cleanup_all_browsers cleanup_all_browsers() except Exception as _e: logger.debug("cleanup_all_browsers (%s) error: %s", phase, _e) return _marked_cron_jobs # Thread-based shutdown watchdog (#66892): asyncio timeouts cannot # recover a frozen loop. Arm a plain OS thread at the start of # stop(); if teardown never finishes within drain+grace it dumps # faulthandler stacks and os._exit so KeepAlive/systemd can revive. # Skip under pytest so stop()-driving unit tests don't get a # delayed hard-exit in the worker. _watchdog_done = threading.Event() self._shutdown_watchdog_done = _watchdog_done _stop_started_at_box: dict[str, float] = {} def _shutdown_watchdog_snapshot() -> dict: started = _stop_started_at_box.get("t") return { "restart_requested": bool(self._restart_requested), "draining": bool(self._draining), "running": bool(self._running), "active_agents": self._running_agent_count(), "active_cron_jobs": self._active_cron_job_count(), "active_api_runs": self._active_api_run_count(), "active_deferred_agent_workers": getattr( self, "_active_deferred_agent_worker_count", lambda: 0, )(), "restart_drain_timeout": self._restart_drain_timeout, "watchdog_delay_s": resolve_shutdown_watchdog_delay( self._restart_drain_timeout ), "phase_elapsed_s": ( time.monotonic() - started if started is not None else None ), } if not os.environ.get("PYTEST_CURRENT_TEST"): arm_shutdown_watchdog( resolve_shutdown_watchdog_delay(self._restart_drain_timeout), done_event=_watchdog_done, snapshot_fn=_shutdown_watchdog_snapshot, exit_code=1, ) try: await _stop_impl_body( _kill_tool_subprocesses, _stop_started_at_box, ) finally: _watchdog_done.set() async def _stop_impl_body(_kill_tool_subprocesses, _stop_started_at_box) -> None: # Shutdown-path tests and third-party runner doubles may only # implement the older drain-count surface. _deferred_worker_count = getattr( self, "_active_deferred_agent_worker_count", lambda: 0, ) logger.info( "Stopping gateway%s...", " for restart" if self._restart_requested else "", ) _stop_started_at = time.monotonic() _stop_started_at_box["t"] = _stop_started_at def _phase_elapsed() -> float: return time.monotonic() - _stop_started_at self._running = False self._clear_plugin_message_injector() self._draining = True stop_room_worker = getattr(self, "_stop_hosted_room_worker", None) if callable(stop_room_worker): try: stopped = await stop_room_worker(timeout=5.0) if not stopped: logger.warning( "Group Chat worker is still settling durable work; " "the next gateway start will recover it" ) except Exception: logger.warning( "Group Chat worker could not stop cleanly; the next gateway " "start will recover durable work", exc_info=True, ) stop_watchdog = getattr(self, "_stop_systemd_watchdog", None) if callable(stop_watchdog): await stop_watchdog() await self._cancel_secondary_profile_reconnect_tasks() # Notify all chats with active agents BEFORE draining. # Adapters are still connected here, so messages can be sent. await self._notify_active_sessions_of_shutdown() logger.info( "Shutdown phase: notify_active_sessions done at +%.2fs", _phase_elapsed(), ) timeout = self._restart_drain_timeout # Pre-mark sessions as resume_pending BEFORE the drain wait. # If the process is killed by the service manager during the # drain, the durable marker is already written so the next # gateway boot can recover in-flight sessions (#27856). _pre_drain_keys: list[str] = [] for _sk, _agent in list(self._running_agents.items()): if _agent is _AGENT_PENDING_SENTINEL: continue try: await self.async_session_store.mark_resume_pending( _sk, "restart_timeout" if self._restart_requested else "shutdown_timeout", ) _pre_drain_keys.append(_sk) except Exception as _e: logger.debug("pre-drain mark_resume_pending failed for %s: %s", _sk, _e) _cron_at_start = self._active_cron_job_count() _api_at_start = self._active_api_run_count() _deferred_at_start = _deferred_worker_count() # In-flight cron work gets its own floor, clamped to the watchdog # leash we're already running under so the extra wait can never # cost us the post-drain cleanup window (#82161). # getattr-guard: shutdown-path tests drive _stop_impl_body from # bare doubles that aren't GatewayRunner instances, so they don't # pick up the class-level default. _cron_drain_cfg = getattr( self, "_cron_drain_timeout", DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT ) _cron_timeout = resolve_cron_drain_budget( timeout, _cron_drain_cfg, watchdog_delay=resolve_shutdown_watchdog_delay(timeout), elapsed=_phase_elapsed(), ) if _cron_at_start and _cron_timeout > timeout: logger.info( "Shutdown drain: %d in-flight cron job(s) — waiting up to " "%.0fs for them (cron_drain_timeout=%.0fs, " "restart_drain_timeout=%.0fs)", _cron_at_start, _cron_timeout, _cron_drain_cfg, timeout, ) _drain_started_at = time.monotonic() active_agents, timed_out = await self._drain_active_agents( timeout, _cron_timeout ) _drain_elapsed = time.monotonic() - _drain_started_at logger.info( "Shutdown phase: drain done at +%.2fs (drain took %.2fs, " "timed_out=%s, active_at_start=%d, active_now=%d, " "cron_at_start=%d, cron_now=%d, " "api_at_start=%d, api_now=%d, " "deferred_at_start=%d, deferred_now=%d)", _phase_elapsed(), _drain_elapsed, timed_out, len(active_agents), self._running_agent_count(), _cron_at_start, self._active_cron_job_count(), _api_at_start, self._active_api_run_count(), _deferred_at_start, _deferred_worker_count(), ) if not timed_out: # Drain completed gracefully — all running sessions finished. # Clear the pre-drain resume_pending markers so sessions that # completed during the drain window don't carry a stale flag. for _sk in _pre_drain_keys: if _sk not in self._running_agents: try: await self.async_session_store.clear_resume_pending(_sk) except Exception as _e: logger.debug( "clear_resume_pending after drain failed for %s: %s", _sk, _e, ) if timed_out: logger.warning( "Gateway drain timed out after %.1fs with %d active agent(s), " "%d in-flight cron job(s), %d api_server run(s), and " "%d deferred agent worker(s); " "interrupting remaining work.", _drain_elapsed, self._running_agent_count(), self._active_cron_job_count(), self._active_api_run_count(), _deferred_worker_count(), ) # Mark forcibly-interrupted sessions as resume_pending BEFORE # interrupting the agents. This preserves each session's # session_id + transcript so the next message on the same # session_key auto-resumes from the existing conversation # instead of getting routed through suspend_recently_active() # and converted into a fresh session. Terminal escalation # for genuinely stuck sessions still flows through the # existing ``.restart_failure_counts`` stuck-loop counter # (incremented below, threshold 3), which sets # ``suspended=True`` and overrides resume_pending. # # Iterate self._running_agents (current) rather than the # drain-start ``active_agents`` snapshot — the snapshot # may include sessions that finished gracefully during # the drain window, and marking those falsely would give # them a stray restart-interruption system note on their # next turn even though their previous turn completed # cleanly. Skip pending sentinels for the same reason # _interrupt_running_agents() does: their agent hasn't # started yet, there's nothing to interrupt, and the # session shouldn't carry a misleading resume flag. _resume_reason = ( "restart_timeout" if self._restart_requested else "shutdown_timeout" ) for _sk, _agent in list(self._running_agents.items()): if _agent is _AGENT_PENDING_SENTINEL: continue try: await self.async_session_store.mark_resume_pending(_sk, _resume_reason) except Exception as _e: logger.debug( "mark_resume_pending failed for %s: %s", _sk, _e, ) self._interrupt_running_agents( _INTERRUPT_REASON_GATEWAY_RESTART if self._restart_requested else _INTERRUPT_REASON_GATEWAY_SHUTDOWN ) interrupt_grace_timeout = ( GatewayRunner._post_interrupt_grace_timeout(self) ) interrupt_deadline = ( asyncio.get_running_loop().time() + interrupt_grace_timeout ) logger.info( "Shutdown phase: allowing %.1fs for interrupted agents to unwind", interrupt_grace_timeout, ) # Wait on API-server work too. The interrupt is cooperative: # without this the settle window closes the instant # _running_agents is empty, and an API turn that was just asked # to stop gets its tool subprocesses killed below before it can # unwind — the exact amputation this interrupt exists to avoid. while ( self._running_agents or self._active_api_run_count() or _deferred_worker_count() ) and asyncio.get_running_loop().time() < interrupt_deadline: self._update_runtime_status("draining") await asyncio.sleep(0.1) # The interrupt above fires exactly once, but work can # materialize AFTER that one shot: a /v1/runs task admitted # before the drain populates _active_run_agents only once # _create_agent returns, and a _running_agents entry claimed # as _AGENT_PENDING_SENTINEL is promoted to a real agent by # track_agent() on its own schedule. Either way the settle # loop waited on work nothing signaled. If any is still live # at settle-loop exit, re-signal so a late-materializing # agent gets a cooperative interrupt instead of going # straight to the tool-subprocess kill. if ( self._running_agents or self._active_api_run_count() or _deferred_worker_count() ): self._interrupt_running_agents( _INTERRUPT_REASON_GATEWAY_RESTART if self._restart_requested else _INTERRUPT_REASON_GATEWAY_SHUTDOWN ) logger.debug( "Re-signaled interrupt for work still live at settle-window exit" ) # Kill lingering tool subprocesses NOW, before we spend more # budget on adapter disconnect / session DB close. Under # systemd (TimeoutStopSec bounded by drain_timeout+headroom), # deferring this to the end of stop() risks systemd escalating # to SIGKILL on the cgroup first — at which point bash/sleep # children left behind by an interrupted terminal tool get # killed by systemd instead of us (issue #8202). The final # catch-all cleanup below still runs for the graceful path. _interrupted_cron_jobs = _kill_tool_subprocesses("post-interrupt") logger.info( "Shutdown phase: post-interrupt tool kill done at +%.2fs", _phase_elapsed(), ) # Last window where the transport is still up. The cron worker # whose run we just killed will try to deliver its own # "interrupted" notice, but it gets there after the adapter # teardown below and the message is lost (#82232). try: await self._notify_interrupted_cron_jobs(_interrupted_cron_jobs) except Exception as _e: logger.debug("Cron interrupt notification failed: %s", _e) logger.info( "Shutdown phase: cron interrupt notices done at +%.2fs", _phase_elapsed(), ) if self._restart_requested and self._restart_detached: try: await self._launch_detached_restart_command() except Exception as e: logger.error("Failed to launch detached gateway restart: %s", e) await self._finalize_shutdown_agents(active_agents) # Also shut down memory providers on idle cached agents. # _finalize_shutdown_agents only handles agents that were # mid-turn at drain time; the _agent_cache may still hold # idle agents whose MemoryProviders never received # on_session_end(). _cache_lock = getattr(self, "_agent_cache_lock", None) _cache = getattr(self, "_agent_cache", None) if _cache_lock is not None and _cache is not None: with _cache_lock: _idle_agents = list(_cache.values()) _cache.clear() for _entry in _idle_agents: _agent = ( _entry[0] if isinstance(_entry, tuple) else _entry ) # Bounded + off-loop so a wedged memory provider on one # idle agent can't hang shutdown indefinitely — that path # is why SIGTERM failed to kill the process (#53175). await self._cleanup_agent_resources_off_loop( _agent, context="shutdown idle-cache" ) # Completion flush tasks can be sleeping in their fan-in window or # blocked in adapter delivery. Cancel and await them while adapters # are still alive so every watcher receives a retryable result # before platform teardown begins. cancel_completion_batches = getattr( self, "_cancel_process_completion_batch_tasks", None ) if cancel_completion_batches is not None: await cancel_completion_batches() for platform, adapter in list(self.adapters.items()): await self._bounded_adapter_teardown(adapter, platform) # Disconnect secondary-profile adapters (multiplex mode). for _prof, _amap in list(getattr(self, "_profile_adapters", {}).items()): for platform, adapter in list(_amap.items()): await self._bounded_adapter_teardown( adapter, platform, profile=_prof ) _amap.clear() if hasattr(self, "_profile_adapters"): self._profile_adapters.clear() logger.info( "Shutdown phase: all adapters disconnected at +%.2fs", _phase_elapsed(), ) for _task in list(self._background_tasks): if _task is self._stop_task: continue if _task is self._restart_task: # The restart orchestration task is awaiting _stop_task # right now; cancelling it would propagate CancelledError # into this _stop_impl and skip _shutdown_event.set() / # _exit_code = 75 (#12875). It self-terminates anyway. continue _task.cancel() self._background_tasks.clear() self.adapters.clear() for _session_key in list(self._running_agents): self._release_running_agent_state(_session_key) # Flush pending messages to disk before clearing (#72680). # When FTS5 corruption prevents message persistence, the # in-memory pending text is the only surviving copy. Clearing # without flushing causes permanent data loss. try: from gateway.shutdown_flush import flush_pending_to_file flush_pending_to_file(dict(self._pending_messages), reason="shutdown") except Exception: pass # The FIFO tail lives in SessionState.conversation.queued_events, # not in the slot dict above — flush it too or every follow-up # parked in overflow at restart time is lost (#99882). try: from gateway.shutdown_flush import flush_overflow_to_file flush_overflow_to_file( { _k: list(_v) for _k, _v in dict(getattr(self, "_queued_events", None) or {}).items() if _v }, reason="shutdown", ) except Exception: pass # On the real runner these are live SessionState views whose # clear() resets one field per session — never a wholesale dict # swap, so a concurrent writer on another session can't lose its # entry. Test fakes borrowing _stop_impl keep plain dicts. self._running_agents.clear() self._running_agents_ts.clear() if hasattr(self, "_active_session_leases"): self._active_session_leases.clear() self._pending_messages.clear() self._pending_approvals.clear() if hasattr(self, '_busy_ack_ts'): self._busy_ack_ts.clear() self._shutdown_event.set() # Global cleanup: kill any remaining tool subprocesses not tied # to a specific agent (catch-all for zombie prevention). On the # drain-timeout path we already did this earlier after agent # interrupt — this second call catches (a) the graceful path # where drain succeeded without interrupt, and (b) anything # that got respawned between the earlier call and adapter # disconnect (defense in depth; safe to call repeatedly). _kill_tool_subprocesses("final-cleanup") logger.info( "Shutdown phase: final-cleanup tool kill done at +%.2fs", _phase_elapsed(), ) # Reap the process-global auxiliary-client cache once at the very # end of teardown. Per-turn cleanup runs in _cleanup_agent_resources # for each active agent, but clients bound to worker-thread loops # that died with their ThreadPoolExecutor (notably cron ticks) only # get swept here. Without this, long-running gateways accumulate # async httpx transports until they hit EMFILE on macOS's default # RLIMIT_NOFILE=256. See #14210. try: from agent.auxiliary_client import shutdown_cached_clients shutdown_cached_clients() except Exception as _e: logger.debug("shutdown_cached_clients error: %s", _e) # Quiesce the gateway thread pool BEFORE the session databases # are closed. This used to run *after* the close block below, # which left two holes: # # (a) `_executor_closing` was still False during the close, so # any coroutine reaching `_run_in_executor_with_context` # minted a brand-new pool and ran more blocking DB work # against handles that had just been closed; # (b) cancelling `self._background_tasks` above does not stop a # `run_in_executor` future that already started — the task # dies, the worker thread keeps writing. # # Either way a write lands after `SessionDB.close()`, which has # already checkpointed the WAL and let SQLite unlink the sidecar. # The late write silently reopens the handle (#94736) and mints a # fresh WAL generation behind that checkpoint, so teardown # checkpoints the same file a second time from a connection the # shutdown log never accounts for — the close-time page-write # damage in #101093 and the split WAL generation in #101064. # # The wait is bounded and clamped to what is left of the shutdown # watchdog leash (minus a second for the close itself), so a stuck # worker can never cost us the post-close cleanup window (#82161). _exec_quiesce_budget = max( 0.0, min( _EXECUTOR_QUIESCE_TIMEOUT, resolve_shutdown_watchdog_delay(timeout) - _phase_elapsed() - 1.0, ), ) _exec_live = GatewayRunner._shutdown_executor( self, drain_timeout=_exec_quiesce_budget ) if _exec_live: # A live worker can still be mid-write against a SessionDB # handle. Checkpointing/closing it now is exactly the # sequence that produced the wrong-page-number corruption in # #101093, so the close path below is skipped entirely # rather than raced — the handle is left open for SQLite to # recover from its own WAL on the next open, which is a # transient "database is locked" on an immediate --replace # at worst, not a corrupt file. logger.warning( "Shutdown phase: %d executor worker(s) still running after " "a %.2fs quiesce — skipping the SessionDB close/checkpoint " "to avoid racing a live write (#101093); handles are left " "open for SQLite to recover on next open", _exec_live, _exec_quiesce_budget, ) else: logger.info( "Shutdown phase: executor quiesced at +%.2fs", _phase_elapsed(), ) # Close SQLite session DBs so the WAL write lock is released. # Without this, --replace and similar restart flows leave the # old gateway's connection holding the WAL lock until Python # actually exits — causing 'database is locked' errors when # the new gateway tries to open the same file. # ``self`` holds the DB at ``_session_db`` (an AsyncSessionDB facade); # unwrap to the sync handle. ``session_store`` holds it at ``_db``. _self_db = getattr(self, "_session_db", None) _self_db = getattr(_self_db, "_db", _self_db) for _db in (_self_db, getattr(getattr(self, "session_store", None), "_db", None)): if _db is None or not hasattr(_db, "close"): continue try: _db.close() except Exception as _e: logger.debug("SessionDB close error: %s", _e) # A multiplexed session_store caches one SessionDB per profile # path (#88532); reading ``_db`` above only resolved the handle # for the shutdown task's own (root) scope. Sweep the rest so # secondary profiles' WAL locks are released before --replace # brings a new gateway up on the same files. _sweep = getattr( getattr(self, "session_store", None), "close_all_db_handles", None ) if _sweep is not None: try: _sweep() except Exception as _e: logger.debug("SessionDB handle sweep error: %s", _e) # Same sweep for the runner's own per-profile session_search # handles (slash commands resolve them under profile scopes). try: GatewayRunner.close_all_session_db_handles(self) except Exception as _e: logger.debug("Runner SessionDB handle sweep error: %s", _e) # Final sweep: close any shared SessionDB instances still held by # the process-wide registry (in-process tools, cron, mirror, etc. # that opened via get_shared_session_db but weren't released by # the sweeps above). This is the safety net that guarantees no # WAL write lock survives past gateway shutdown (#90837). try: from hermes_state import close_shared_session_dbs closed = close_shared_session_dbs() if closed: logger.debug("Closed %d shared SessionDB instance(s) at shutdown", closed) except Exception as _e: logger.debug("Shared SessionDB close error: %s", _e) logger.info( "Shutdown phase: SessionDB close done at +%.2fs", _phase_elapsed(), ) from gateway.status import remove_pid_file, release_gateway_runtime_lock remove_pid_file() release_gateway_runtime_lock() # Write a clean-shutdown marker so the next startup knows this # wasn't a crash. suspend_recently_active() only needs to run # after unexpected exits. However, if the drain timed out and # agents were force-interrupted, their sessions may be in an # incomplete state (trailing tool response, no final assistant # message). Skip the marker in that case so the next startup # suspends those sessions — giving users a clean slate instead # of resuming a half-finished tool loop. if not timed_out: try: (_hermes_home / ".clean_shutdown").touch() except Exception: pass else: logger.info( "Skipping .clean_shutdown marker — drain timed out with " "interrupted agents; next startup will suspend recently " "active sessions." ) # Track sessions that were active at shutdown for stuck-loop # detection (#7536). On each restart, the counter increments # for sessions that were running. If a session hits the # threshold (3 consecutive restarts while active), the next # startup auto-suspends it — breaking the loop. if active_agents: self._increment_restart_failure_counts(set(active_agents.keys())) if self._restart_requested and self._restart_command_source is None: try: atomic_json_write( _planned_restart_notification_path(), { "requested_at": time.time(), "via_service": bool(self._restart_via_service), "detached": bool(self._restart_detached), }, indent=None, ) except Exception as e: logger.debug("Failed to write planned restart notification marker: %s", e) if self._restart_requested and self._restart_via_service: # The service manager is the sole restart owner. Exit 75 # paired with ``RestartForceExitStatus=75`` asks systemd to # replace this process without a second helper racing the # unit's stop/start job. The generated launchd plist's # unconditional ``KeepAlive`` likewise replaces the process # after this planned exit. self._exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE self._exit_reason = self._exit_reason or "Gateway restart requested" self._draining = False # Persist the terminal gateway_state. The default is "stopped", # but when this teardown was triggered by an UNEXPECTED external # signal (container/s6 SIGTERM on `docker restart` or image # upgrade, OOM-killer, bare `kill`) we instead persist "running" # to preserve the operator's run-intent across the restart. # # On Docker (s6-overlay), container_boot.py reads gateway_state # on the next boot and only auto-starts gateways whose last # state was "running" (_AUTOSTART_STATES). Persisting "stopped" # — or leaving the mid-shutdown "draining" marker in place — for # a routine `docker compose up --force-recreate` permanently # suppresses auto-start, so the messaging channels silently stay # dark until the operator manually restarts (issue #42675). # # An operator-initiated stop (`hermes gateway stop`, # systemd/launchd ExecStop, the s6 stop path, Ctrl+C) writes a # planned-stop marker BEFORE signalling, so it is classified as # a planned stop (not signal-initiated) and correctly persists # "stopped" — respecting the explicit intent. A restart also # persists "stopped" here; the restarting process brings the # gateway back up itself. if getattr(self, "_signal_initiated_shutdown", False) and not self._restart_requested: logger.info( "Gateway stopped by an unexpected signal — persisting " "gateway_state=running so container_boot auto-starts on " "the next boot (issue #42675)" ) self._update_runtime_status("running", self._exit_reason) else: self._update_runtime_status("stopped", self._exit_reason) _shutdown_gateway_health_export(self) logger.info("Gateway stopped (total teardown %.2fs)", _phase_elapsed()) self._stop_task = asyncio.create_task(_stop_impl()) await self._stop_task async def wait_for_shutdown(self) -> None: """Wait for shutdown signal.""" await self._shutdown_event.wait() async def _start_secondary_profile_adapters(self) -> int: """Bring up adapters for every non-active profile this gateway serves. Returns the number of secondary adapters that connected. No-op (returns 0) unless ``gateway.multiplex_profiles`` is on. Each profile's adapters are created and connected under that profile's HERMES_HOME + secret scope (``_profile_runtime_scope``), stored in ``self._profile_adapters[profile]``, and given a message handler that stamps ``source.profile`` before delegating to the shared ``_handle_message`` — so the agent turn resolves that profile's config, skills, and credentials. Same-platform credential collisions (two profiles polling the same bot token) are detected and refused here, the only point that sees every profile's resolved credentials together. """ if not getattr(self.config, "multiplex_profiles", False): return 0 try: from hermes_cli.profiles import get_active_profile_name except Exception: return 0 active = get_active_profile_name() or "default" connected = 0 # Resource claim -> profile that owns it. Credential claims prevent two # profiles polling the same account; listener claims prevent sidecars # with distinct credentials from binding the same endpoint. claimed: Dict[tuple, str] = {} for _plat, _ad in self.adapters.items(): fp = self._adapter_credential_fingerprint(_ad) if fp is not None: claimed[(_plat, fp)] = active listener_claim = self._adapter_listener_claim(_plat, _ad) if listener_claim is not None: claimed[listener_claim] = active # A retryable primary still owns its configured credential and listener. # Reserve both while it is queued so a secondary cannot take the endpoint # before the reconnect watcher retries the primary adapter. for retry_info in getattr(self, "_failed_platforms", {}).values(): for claim_name in ("credential_claim", "listener_claim"): retry_claim = retry_info.get(claim_name) if isinstance(retry_claim, tuple): claimed[retry_claim] = active profile_homes = _multiplex_profile_homes(self.config) for profile_name, profile_home in profile_homes: if profile_name == active: continue # handled by the primary startup loop try: connected += await self._start_one_profile_adapters( profile_name, profile_home, claimed ) except SecondaryPortBindingConfigError as e: logger.warning( "Skipping secondary profile '%s' due to port-binding config error: %s", profile_name, e, ) except MultiplexConfigError: raise except Exception as e: logger.error( "Failed to start adapters for profile '%s': %s", profile_name, e, exc_info=True, ) # Record the authoritative served set in runtime status for `hermes status`. # "Served" means eligible for shared routing, HTTP prefixes, cron, and # profile runtime scope; it is intentionally broader than profiles with a # successfully connected secondary adapter (or any adapter configured). try: from gateway.status import write_runtime_status from gateway.pairing import PairingStore served = [active] + sorted( name for name, _home in profile_homes if name != active ) # Per-profile PairingStores so authz_mixin can route pairing # checks to the right whitelist. The active profile gets a store # at its HERMES_HOME; additional served profiles resolve from # their own profile homes. See gateway.pairing.PairingStore. for name in served: if name and name not in self.pairing_stores: self.pairing_stores[name] = ( self.pairing_store if name == active else PairingStore(profile=name) ) write_runtime_status(served_profiles=served) except Exception: logger.debug("could not record served_profiles", exc_info=True) return connected async def _start_one_profile_adapters( self, profile_name: str, profile_home: "Path", claimed: Dict[tuple, str] ) -> int: """Create+connect one profile's adapters under its runtime scope.""" from gateway.config import load_gateway_config from hermes_cli.env_loader import hydrate_profile_secret_sources # Hydrate external secret sources (1Password/vault/...) off-loop ONCE, # then enter the scope without re-hydrating: the sync hydration is # network-bound and would otherwise stall every other profile's # heartbeat while this one boots (same class as the reconnect path). await asyncio.to_thread(hydrate_profile_secret_sources, profile_home) with _profile_runtime_scope(profile_home, hydrate_secrets=False): profile_runtime_cfg = _load_gateway_runtime_config() from hermes_cli.plugins import discover_plugins discover_plugins() # Register this profile's own declarative shell hooks and # outbound webhooks. The startup-time registration in # start() only ever sees the root/default profile's config # (it runs before any profile scope exists), so without this # a secondary profile's `hooks:` block is silently inert — # its turns run under this profile's own plugin manager # (hermes_cli.plugins.get_plugin_manager keys by resolved # home), which never received the callbacks. try: from hermes_cli.config import load_config as _load_profile_config from agent.shell_hooks import ( register_from_config as _register_shell_hooks, ) from agent.outbound_webhooks import ( register_from_config as _register_outbound_webhooks, ) _profile_hooks_cfg = _load_profile_config() _register_shell_hooks(_profile_hooks_cfg, accept_hooks=False) _register_outbound_webhooks(_profile_hooks_cfg) except Exception: logger.warning( "shell-hook/webhook registration failed for profile '%s'", profile_name, exc_info=True, ) profile_cfg = load_gateway_config() violation = _own_policy_open_startup_violation(profile_cfg) self._snapshot_profile_busy_modes(profile_name, profile_runtime_cfg) if violation: raise MultiplexConfigError( f"Profile '{profile_name}' enables {violation}. " "Enable GATEWAY_ALLOW_ALL_USERS or the platform allow-all flag " "for that profile, or change dm_policy/group_policy away from " "'open'." ) port_binding_platforms = sorted( platform.value for platform, platform_config in profile_cfg.platforms.items() if platform_config.enabled and _platform_binds_port(platform.value, platform_config.extra) ) if port_binding_platforms: joined = ", ".join(port_binding_platforms) raise SecondaryPortBindingConfigError( f"Profile '{profile_name}' enables port-binding platform(s) " f"{joined}, but gateway.multiplex_profiles is on. The default " f"profile owns the single shared HTTP listener and serves every " f"profile through the /p/{profile_name}/ URL prefix. Remove " f"these platform entries from profile '{profile_name}'s config.yaml " f"or configure them only on the default profile." ) profile_map = self._profile_adapters.setdefault(profile_name, {}) connected = 0 for platform, platform_config in profile_cfg.platforms.items(): if not platform_config.enabled: continue # A platform enabled in a secondary profile's config.yaml may # have no credential in that profile's secret scope — the shared # YAML enables it for the default profile only (#84079). Building # an adapter here would treat every credential-less profile as # configured for the platform and one inbound message would fan # out across all of them. Mirror the primary startup loop's # credential gate and skip instead; profiles with their own # credential still connect below. if ( getattr(self.config, "multiplex_profiles", False) and not _platform_has_bot_credential(platform, platform_config) ): logger.info( "[MULTIPLEX] Profile '%s': skipping %s - no bot credential " "in this profile's secrets", profile_name, platform.value, ) continue # Relay and WhatsApp are shared process-level ingress in multiplex # mode: one connection owned by the active profile, with # route-stamped source.profile fanning inbound turns out to # secondary profiles. The WhatsApp bridge is a single authenticated # session tied to one phone number -- a secondary profile has no # credential of its own to bring, so constructing an adapter for it # only yields a connect/retry loop that stalls startup for every # profile queued behind it. if ( getattr(self.config, "multiplex_profiles", False) and platform in (Platform.RELAY, Platform.WHATSAPP) ): continue try: with _profile_runtime_scope(profile_home, hydrate_secrets=False): adapter = self._create_adapter(platform, platform_config) except Exception as e: logger.error( "[MULTIPLEX] Profile '%s': _create_adapter('%s') raised %s", profile_name, platform.value, e, exc_info=True, ) continue if not adapter: logger.warning( "[MULTIPLEX] Profile '%s': skipping platform '%s' - adapter creation returned None", profile_name, platform.value, ) continue # Same-token conflict detection — refuse a duplicate poll. credential_claim = self._adapter_credential_claim(platform, adapter) if credential_claim is not None: owner = claimed.get(credential_claim) if owner is not None: message = ( f"Profile '{owner}' and '{profile_name}' both configure " f"{platform.value} with the same credential. Give each " f"profile its own {platform.value} credential." ) logger.error( "Profile '%s' and '%s' both configure %s with the same " "credential — refusing to start the duplicate (one " "credential cannot be consumed twice). Give each profile " "its own %s credential.", owner, profile_name, platform.value, platform.value, ) self._update_platform_runtime_status( f"{profile_name}:{platform.value}", platform_state="fatal", error_code="duplicate_credential", error_message=message, ) # This adapter has not connected and therefore owns no # resources to clean up. Calling disconnect here can mutate # the shared platform state and, for a same-credential Photon # adapter, shut down the primary profile's live sidecar. continue listener_claim = self._adapter_listener_claim(platform, adapter) if listener_claim is not None: owner = claimed.get(listener_claim) if owner is not None: bind, port = listener_claim[-2:] message = ( f"Profile '{owner}' and '{profile_name}' both configure " f"{platform.value} sidecars on the same listener. Configure " f"a distinct listener for profile '{profile_name}'." ) logger.error( "Profile '%s' and '%s' both configure %s sidecars on " "%s:%s — refusing to start the duplicate listener. " "Set platforms.%s.extra.sidecar_port to a distinct port " "for profile '%s'.", owner, profile_name, platform.value, bind, port, platform.value, profile_name, ) self._update_platform_runtime_status( f"{profile_name}:{platform.value}", platform_state="fatal", error_code="duplicate_listener", error_message=message, ) # Like credential conflicts, this adapter never connected # and owns no resources that should be disconnected. continue self._configure_profile_adapter(adapter, profile_name, platform) try: with _profile_runtime_scope(profile_home, hydrate_secrets=False): success = await self._connect_initial_adapter_with_timeout( adapter, platform ) if success: profile_map[platform] = adapter # Restore persisted /voice state for this bot (#84872) — # primary startup and every reconnect path already do. self._sync_voice_mode_state_to_adapter(adapter) if credential_claim is not None: claimed[credential_claim] = profile_name if listener_claim is not None: claimed[listener_claim] = profile_name connected += 1 logger.info("✓ %s connected (profile: %s)", platform.value, profile_name) else: logger.warning("✗ %s failed to connect (profile: %s)", platform.value, profile_name) await self._safe_adapter_disconnect(adapter, platform) self._schedule_secondary_profile_startup_reconnect( profile_name, platform, adapter ) except Exception as e: logger.error("✗ %s error (profile: %s): %s", platform.value, profile_name, e) await self._safe_adapter_disconnect(adapter, platform) self._schedule_secondary_profile_startup_reconnect( profile_name, platform, adapter ) return connected def _configure_profile_adapter( self, adapter: BasePlatformAdapter, profile_name: str, platform: Platform, ) -> None: """Install the profile-scoped handlers shared by startup and reconnect.""" # Runtime status is process-scoped even while message/config work is # profile-scoped. Preserve both dimensions in the key so dashboard # and NAS health aggregation can see which secondary profile failed. adapter._runtime_status_platform_key = f"{profile_name}:{platform.value}" adapter.set_message_handler(self._make_profile_message_handler(profile_name)) adapter.set_fatal_error_handler( self._make_profile_fatal_error_handler(profile_name, platform) ) adapter.set_session_store(self.session_store) # Declare credential ownership BEFORE any inbound event can be handled. # Adapter-level session keys (text/media batching, _active_sessions, the # busy guard) are derived at ingress, before _make_profile_message_handler # stamps source.profile — without this every secondary bot would key into # the default profile's `agent:main:` lane and share it (see # BasePlatformAdapter._session_key_profile). _set_owner = getattr(adapter, "set_owner_profile", None) if callable(_set_owner): _set_owner(profile_name) adapter.set_busy_session_handler( self._make_profile_busy_session_handler(profile_name) ) _set_reaction = getattr(adapter, "set_reaction_handler", None) if callable(_set_reaction): _set_reaction(self._handle_reaction_event) adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id) adapter.set_authorization_check( self._make_adapter_auth_check(platform, profile_name=profile_name) ) adapter.set_platform_event_handler( self._make_profile_platform_event_handler(profile_name) ) # Voice transcripts from this bot's channels dispatch through THIS # adapter (primary wiring lives at connect time; see #75198). self._bind_voice_input_callback(adapter) text_modes = getattr(self, "_busy_text_modes_by_profile", None) adapter._busy_text_mode = ( text_modes.get(profile_name, self._busy_text_mode) if isinstance(text_modes, dict) else self._busy_text_mode ) # Secondary adapters always carry the profile they serve so prune # paths namespace topic bindings correctly under multiplex (#76423). adapter._hermes_profile_name = profile_name async def _run_secondary_profile_reconnect( self, profile_name: str, platform: Platform ) -> None: """Reconnect a retryable secondary adapter under its own profile scope.""" attempts = 0 current_task = asyncio.current_task() try: while self._running: adapter = None try: from hermes_cli.profiles import get_profile_dir from hermes_cli.env_loader import hydrate_profile_secret_sources from gateway.config import load_gateway_config profile_home = get_profile_dir(profile_name) # Like the #16856 MCP discovery path, hydrate external secret # sources off-loop so they cannot starve platform heartbeats. await asyncio.to_thread( hydrate_profile_secret_sources, profile_home ) with _profile_runtime_scope(profile_home, hydrate_secrets=False): profile_config = load_gateway_config().platforms.get(platform) if profile_config is None or not profile_config.enabled: return # Mirrors the startup credential gate (#84079): a # credential removed from this profile's scope must # not rebuild an adapter that would fan out turns. if not _platform_has_bot_credential(platform, profile_config): logger.info( "Secondary %s reconnect skipped: no bot credential " "(profile: %s)", platform.value, profile_name, ) return adapter = self._create_adapter(platform, profile_config) if adapter is None: logger.warning( "Secondary %s reconnect skipped: adapter unavailable (profile: %s)", platform.value, profile_name, ) return self._configure_profile_adapter( adapter, profile_name, platform ) success = await self._connect_adapter_with_timeout( adapter, platform, is_reconnect=True ) if success and self._running: profile_map = self._profile_adapters.setdefault(profile_name, {}) if platform not in profile_map: profile_map[platform] = adapter self._sync_voice_mode_state_to_adapter(adapter) logger.info( "✓ %s reconnected (profile: %s)", platform.value, profile_name, ) await self._redeliver_failed_obligations_for_platform( platform, profile=profile_name ) return # A newer reconnect already won the slot while this # attempt was awaiting connect; do not replace it. await self._safe_adapter_disconnect(adapter, platform) return # Shutdown can begin while connect() is in flight. Do not # republish a newly connected adapter after the registry has # been drained; release its partial resources instead. if success: await self._safe_adapter_disconnect(adapter, platform) return await self._safe_adapter_disconnect(adapter, platform) if ( getattr(adapter, "has_fatal_error", False) and not getattr(adapter, "fatal_error_retryable", True) ): return except asyncio.CancelledError: if adapter is not None: await self._safe_adapter_disconnect(adapter, platform) raise except Exception: if adapter is not None: await self._safe_adapter_disconnect(adapter, platform) logger.debug( "Secondary %s reconnect attempt failed (profile: %s)", platform.value, profile_name, exc_info=True, ) if not self._running: return attempts += 1 backoff = _reconnect_backoff(attempts) logger.info( "Secondary %s reconnect retry in %ds (profile: %s)", platform.value, backoff, profile_name, ) await asyncio.sleep(backoff) finally: pending = self._profile_failed_platforms if isinstance(pending, dict): profile_pending = pending.get(profile_name) task = profile_pending.get(platform) if isinstance(profile_pending, dict) else None if not isinstance(task, asyncio.Task) or task is current_task: if isinstance(profile_pending, dict): profile_pending.pop(platform, None) if not profile_pending: pending.pop(profile_name, None) def _schedule_secondary_profile_startup_reconnect( self, profile_name: str, platform: Platform, adapter: BasePlatformAdapter ) -> None: """Queue a cold-start reconnect for a secondary adapter. Startup failure branches run BEFORE ``self._running`` flips True (``_start_secondary_profile_adapters()`` is called mid-``start()``, while ``self._running`` is still False), so the regular scheduler's ``not self._running`` guard would silently drop the request and the runner's ``while self._running`` loop would exit immediately. This bridge parks a background task across the remainder of startup and hands off to the regular scheduler once the gateway is live (the scheduler's own ``_profile_failed_platforms`` slot dedupes at handoff); if shutdown begins first, the request is released. Non-retryable failures are dropped here exactly as the regular scheduler would. """ if not getattr(adapter, "fatal_error_retryable", True): return if is_global_startup_conflict(getattr(adapter, "fatal_error_code", None)): # Same startup contract as the primary path: a live foreign holder # of this profile's token/identity is an ownership conflict, not # a transient blip. Park it fatal (like ``duplicate_credential``) # instead of retry-storming the token every backoff (#83183). logger.error( "[MULTIPLEX] Profile '%s': %s credential is held by another " "gateway (%s) — parked, not retried. %s", profile_name, platform.value, adapter.fatal_error_code, adapter.fatal_error_message or "", ) self._update_platform_runtime_status( f"{profile_name}:{platform.value}", platform_state="fatal", error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message, ) return async def _await_running_then_schedule() -> None: if self._running: try: self._schedule_secondary_profile_reconnect( profile_name, platform, adapter ) except Exception: # Same GC-time-exception hazard as the post-poll handoff # below; surface it in gateway.log instead. logger.exception( "secondary-startup-reconnect handoff failed " "(profile=%s platform=%s)", profile_name, platform.value, ) return # Modest poll interval: startup completion has no dedicated event, # and the reconnect runner's own backoff makes sub-100ms precision # irrelevant. Bounded so a wedged startup cannot spin the loop. while not self._running and not self._shutdown_event.is_set(): await asyncio.sleep(0.1) if self._running and not self._shutdown_event.is_set(): try: self._schedule_secondary_profile_reconnect( profile_name, platform, adapter ) except Exception: # The handoff touches live registries; if it raises, the # parked task would otherwise die as an unretrieved-task # exception logged only at GC time. Surface it where # operators look. logger.exception( "secondary-startup-reconnect handoff failed " "(profile=%s platform=%s)", profile_name, platform.value, ) task = asyncio.create_task( _await_running_then_schedule(), name=f"secondary-startup-reconnect:{profile_name}:{platform.value}", ) background_tasks = getattr(self, "_background_tasks", None) if not isinstance(background_tasks, set): background_tasks = set() self._background_tasks = background_tasks background_tasks.add(task) task.add_done_callback(background_tasks.discard) def _schedule_secondary_profile_reconnect( self, profile_name: str, platform: Platform, adapter: BasePlatformAdapter ) -> None: """Schedule one runner-owned reconnect without sharing primary secrets.""" if not self._running or not adapter.fatal_error_retryable: return pending = self._profile_failed_platforms if not isinstance(pending, dict): pending = {} self._profile_failed_platforms = pending profile_pending = pending.setdefault(profile_name, {}) if platform in profile_pending: return task = asyncio.create_task( self._run_secondary_profile_reconnect(profile_name, platform), name=f"secondary-reconnect:{profile_name}:{platform.value}", ) profile_pending[platform] = task background_tasks = getattr(self, "_background_tasks", None) if not isinstance(background_tasks, set): background_tasks = set() self._background_tasks = background_tasks background_tasks.add(task) task.add_done_callback(background_tasks.discard) def _make_profile_fatal_error_handler( self, profile_name: str, platform: Platform ) -> Callable[[BasePlatformAdapter], Awaitable[None]]: """Route a secondary-profile fatal error to that profile's reconnect slot.""" async def _handler(adapter: BasePlatformAdapter) -> None: await self._handle_profile_adapter_fatal_error(profile_name, platform, adapter) return _handler async def _handle_profile_adapter_fatal_error( self, profile_name: str, platform: Platform, adapter: BasePlatformAdapter, ) -> None: """Remove a failed multiplexed adapter without touching the primary slot. Secondary adapters are owned by ``_profile_adapters`` rather than ``self.adapters``. The primary-only fatal handler intentionally ignores them; without this route, a fatal secondary Discord client stayed live forever after its liveness sampler stopped. """ profile_map = getattr(self, "_profile_adapters", {}).get(profile_name) if not isinstance(profile_map, dict) or profile_map.get(platform) is not adapter: logger.debug( "Ignoring stale fatal error from secondary %s adapter (profile: %s)", platform.value, profile_name, ) return profile_map.pop(platform, None) await self._safe_adapter_disconnect(adapter, platform) if not self._running: return self._schedule_secondary_profile_reconnect(profile_name, platform, adapter) logger.error( "Fatal %s adapter error for multiplexed profile %s (%s)", platform.value, profile_name, adapter.fatal_error_code or "unknown", ) # Reconnect is scoped to the profile's own config and secret mapping; # never rebuild a secondary adapter with the default profile's credentials. def _make_profile_message_handler(self, profile_name: str): """Return a message handler that stamps source.profile then delegates. Auth runs inside ``_handle_message`` *before* the agent-turn scope is installed. For secondary profiles under multiplex, wrap the whole handler in ``_profile_runtime_scope`` so allowlists/tokens from that profile's ``.env`` are visible to ``get_secret`` / authz. """ from hermes_cli.profiles import get_profile_dir try: profile_home = get_profile_dir(profile_name) except Exception: profile_home = None async def _handler(event): try: if getattr(event, "source", None) is not None and not event.source.profile: event.source.profile = profile_name except Exception: pass if profile_home is not None: async with _async_profile_runtime_scope(profile_home): return await self._handle_message(event) return await self._handle_message(event) return _handler def _make_profile_busy_session_handler(self, profile_name: str): """Stamp an owning adapter's profile before resolving busy policy.""" async def _handler(event, _session_key): try: if getattr(event, "source", None) is not None and not event.source.profile: event.source.profile = profile_name except Exception: pass routed_session_key = self._session_key_for_source(event.source) return await self._handle_active_session_busy_message( event, routed_session_key ) return _handler def _make_default_profile_message_handler(self): """Scope primary-adapter messages to their routed multiplex profile. Profile routes are normally stamped on ``event.source`` before this handler runs. Resolve the home per event so session lookup and transcript loading use the same profile store as the later agent run and persistence path. Authorization still belongs to the primary transport profile: a shared Discord/Telegram adapter can route the turn to a profile that intentionally has no bot credential or platform allowlist. Preserve that transport home on the live source so the auth gate does not re-check the sender against the routed runtime's unrelated secret scope. Sources that bypass adapter routing are resolved here; genuinely unrouted events retain the gateway's launch/default home. """ default_home = Path(get_hermes_home()) async def _handler(event): source = event.source # In-process only (SessionSource serialization ignores dynamic attrs). # The route selects agent/session state, not which bot admitted the # message. Keep those two trust domains separate. source._authorization_profile_home = default_home if ( not getattr(source, "profile", None) and getattr(source, "profile_route_rejected", False) is not True ): from gateway.profile_routing import ProfileRouteRejected try: source.profile = self._profile_name_for_source(source) except ProfileRouteRejected: # NOT write-only: ``_handle_message``'s ingress gate reads # this exact marker and drops the message fail-closed # ("explicit profile route targets an unserved profile"). # Setting it here also stops that gate from re-running # routing for the same source. source.profile_route_rejected = True profile_home = ( self._resolve_profile_home_for_source(source) if getattr(source, "profile", None) else default_home ) async with _async_profile_runtime_scope(profile_home): return await self._handle_message(event) return _handler def _primary_message_handler(self): """Return the correctly scoped handler for a primary adapter.""" if getattr(self.config, "multiplex_profiles", False): return self._make_default_profile_message_handler() return self._handle_message async def _handle_gateway_platform_event(self, event: dict, source) -> None: """Authorize and publish one normalized adapter event to plugin hooks.""" try: from hermes_cli.lifecycle import has_hook, invoke_hook if not has_hook("gateway_platform_event"): return if not self._is_user_authorized_for_source(source): return invoke_hook("gateway_platform_event", **event) except Exception: # Observer failures must never break the adapter's update loop. logger.debug("gateway_platform_event hook dispatch failed", exc_info=True) def _make_profile_platform_event_handler(self, profile_name: str): """Bind platform-event auth and hook dispatch to one multiplex profile.""" from hermes_cli.profiles import get_profile_dir try: profile_home = get_profile_dir(profile_name) except Exception: profile_home = None async def _handler(event, source): if getattr(source, "profile", None) is None: source.profile = profile_name if profile_home is not None: with _profile_runtime_scope(profile_home): return await self._handle_gateway_platform_event(event, source) return await self._handle_gateway_platform_event(event, source) return _handler def _make_default_profile_platform_event_handler(self): """Scope primary-transport events to their routed multiplex profile.""" default_home = Path(get_hermes_home()) async def _handler(event, source): source._authorization_profile_home = default_home with _profile_runtime_scope(self._resolve_profile_home_for_source(source)): return await self._handle_gateway_platform_event(event, source) return _handler def _is_user_authorized_for_source( self, source: SessionSource, *, allow_adapter_delegation: bool = True, ) -> bool: """Authorize under the live transport's profile, not the routed runtime. A primary adapter may route one chat into another profile's agent/session namespace. That runtime profile need not (and normally should not) copy the shared bot token or allowlist. The primary message/platform-event handlers stamp the transport home as an in-process-only attribute before entering the routed scope; consult it here for the narrow authorization read, then restore the routed scope for the remainder of the turn. """ def _check() -> bool: # Preserve the historical one-argument seam used by plugins/tests; # only pass the keyword for the explicit delegation-disabled path. if allow_adapter_delegation: return self._is_user_authorized(source) return self._is_user_authorized( source, allow_adapter_delegation=False, ) authorization_home = getattr(source, "_authorization_profile_home", None) if authorization_home is not None: with _profile_runtime_scope(Path(authorization_home)): return _check() return _check() def _primary_platform_event_handler(self): if getattr(self.config, "multiplex_profiles", False): return self._make_default_profile_platform_event_handler() return self._handle_gateway_platform_event @staticmethod def _adapter_credential_claim( platform: Platform, adapter: Any ) -> Optional[tuple]: """Return the exclusive credential resource claimed by an adapter.""" fingerprint = GatewayRunner._adapter_credential_fingerprint(adapter) if fingerprint is None: return None return (platform, fingerprint) @staticmethod def _adapter_listener_claim(platform: Platform, adapter: Any) -> Optional[tuple]: """Return the exclusive listener resource claimed by an adapter. Photon sidecars are per-profile processes. Even when two profiles use different project credentials, their sidecars cannot share a bind and port. Represent that endpoint as a claim so multiplex startup rejects the later adapter before either ``connect()`` or ``disconnect()`` can disturb the first profile. """ if getattr(platform, "value", None) != "photon": return None bind = getattr(adapter, "_sidecar_bind", None) port = getattr(adapter, "_sidecar_port", None) if not isinstance(bind, str) or not bind.strip(): return None try: port = int(port) except (TypeError, ValueError): return None return ("listener", "photon", bind.strip().lower(), port) @staticmethod def _adapter_credential_fingerprint(adapter: Any) -> Optional[str]: """Return a stable, log-safe fingerprint of an adapter's credential. Used only to detect two profiles claiming the same platform credential. Returns a salted hash (never the credential itself) of the adapter's primary credential, or None when no credential is discoverable (in which case we don't attempt conflict detection for it). """ token = None for attr in ( "token", "bot_token", "_token", "api_token", "_bot_token", # Photon/Spectrum authenticates with project credentials instead # of a bot token. Including its secret keeps multiplexed profiles # from spawning competing sidecars for the same account and port. "_project_secret", # Feishu/Lark authenticates with an app_id/app_secret pair rather # than a single token (one active WebSocket connection per app). # app_id is stable, log-safe, and already used as the adapter's # _app_lock_identity, so including it lets the multiplex guard # refuse cloned profiles competing for the same Feishu app. "_app_id", # Same class: Teams (client_id/client_secret) and WeCom # (bot_id/secret) authenticate with an app-style id pair too. "_client_id", "_bot_id", ): val = getattr(adapter, attr, None) if isinstance(val, str) and val.strip(): token = val.strip() break # Many adapters (e.g. Discord) store the token on their `config` # sub-object rather than directly on the adapter. Without this lookup # those adapters all return None here, the same-token conflict check # is silently skipped, and every profile's adapter for that platform # starts polling the same bot token — producing a per-message race # for which adapter answers. See test_reads_config_token. if not token: cfg = getattr(adapter, "config", None) if cfg is not None: for attr in ("token", "bot_token"): val = getattr(cfg, attr, None) if isinstance(val, str) and val.strip(): token = val.strip() break if not token: config = getattr(adapter, "config", None) val = getattr(config, "token", None) if isinstance(val, str) and val.strip(): token = val.strip() if not token: return None import hashlib return hashlib.sha256(("hermes-mux:" + token).encode("utf-8")).hexdigest()[:16] def _create_adapter( self, platform: Platform, config: Any, ) -> Optional[BasePlatformAdapter]: """Create an adapter and bind it to this gateway runner. Every lifecycle path — primary/secondary startup and reconnect — goes through this method. Keep runner binding here so adapters can resolve inbound profile routes before handlers or ``connect()`` run. """ adapter = self._instantiate_adapter(platform, config) if adapter is not None: adapter.gateway_runner = self return adapter def _instantiate_adapter( self, platform: Platform, config: Any, ) -> Optional[BasePlatformAdapter]: """Instantiate the appropriate adapter for a platform. Checks the platform_registry first (plugin adapters), then falls through to the built-in if/elif chain for core platforms. """ if hasattr(config, "extra") and isinstance(config.extra, dict): config.extra.setdefault( "group_sessions_per_user", self.config.group_sessions_per_user, ) config.extra.setdefault( "thread_sessions_per_user", getattr(self.config, "thread_sessions_per_user", False), ) # ── Plugin-registered platforms (checked first) ─────────────────── try: from gateway.platform_registry import platform_registry if platform_registry.is_registered(platform.value): adapter = platform_registry.create_adapter(platform.value, config) if adapter is not None: return adapter # Registered but failed to instantiate — don't silently fall # through to built-ins (there are none for plugin platforms). logger.error( "Platform '%s' is registered but adapter creation failed " "(check dependencies and config)", platform.value, ) return None except Exception as e: logger.debug("Platform registry lookup for '%s' failed: %s", platform.value, e) # Fall through to built-in adapters below if platform == Platform.WHATSAPP_CLOUD: from gateway.platforms.whatsapp_cloud import ( WhatsAppCloudAdapter, check_whatsapp_cloud_requirements, ) if not check_whatsapp_cloud_requirements(): logger.warning( "WhatsApp Cloud: aiohttp/httpx missing — reinstall hermes-agent" ) return None return WhatsAppCloudAdapter(config) elif platform == Platform.SIGNAL: from gateway.platforms.signal import ( SignalAdapter, check_signal_requirements, validate_signal_config, ) if not check_signal_requirements(): logger.warning("Signal: runtime requirements not met") return None if not validate_signal_config(config): logger.warning("Signal: SIGNAL_HTTP_URL or SIGNAL_ACCOUNT not configured") return None return SignalAdapter(config) elif platform == Platform.WEIXIN: from gateway.platforms.weixin import WeixinAdapter, check_weixin_requirements if not check_weixin_requirements(): logger.warning("Weixin: aiohttp/cryptography not installed") return None return WeixinAdapter(config) elif platform == Platform.API_SERVER: from gateway.platforms.api_server import APIServerAdapter, check_api_server_requirements if not check_api_server_requirements(): logger.warning("API Server: aiohttp not installed") return None return APIServerAdapter(config) elif platform == Platform.WEBHOOK: from gateway.platforms.webhook import WebhookAdapter, check_webhook_requirements if not check_webhook_requirements(): logger.warning("Webhook: aiohttp not installed") return None return WebhookAdapter(config) elif platform == Platform.MSGRAPH_WEBHOOK: from gateway.platforms.msgraph_webhook import ( MSGraphWebhookAdapter, check_msgraph_webhook_requirements, ) if not check_msgraph_webhook_requirements(): logger.warning("MSGraph webhook: aiohttp not installed") return None return MSGraphWebhookAdapter(config) elif platform == Platform.BLUEBUBBLES: from gateway.platforms.bluebubbles import BlueBubblesAdapter, check_bluebubbles_requirements if not check_bluebubbles_requirements(): logger.warning("BlueBubbles: aiohttp/httpx missing or BLUEBUBBLES_SERVER_URL/BLUEBUBBLES_PASSWORD not configured") return None return BlueBubblesAdapter(config) elif platform == Platform.QQBOT: from gateway.platforms.qqbot import QQAdapter, check_qq_requirements if not check_qq_requirements(): logger.warning("QQBot: aiohttp/httpx missing or QQ_APP_ID/QQ_CLIENT_SECRET not configured") return None return QQAdapter(config) elif platform == Platform.YUANBAO: from gateway.platforms.yuanbao import YuanbaoAdapter, WEBSOCKETS_AVAILABLE if not WEBSOCKETS_AVAILABLE: logger.warning("Yuanbao: websockets not installed. Run: pip install websockets") return None return YuanbaoAdapter(config) return None def _make_adapter_auth_check( self, platform: Platform, profile_name: Optional[str] = None, ) -> Callable[[str, Optional[str], Optional[str]], bool]: """Build a platform-bound auth callback for adapter use. Adapters that fetch external context (e.g. Slack ``conversations.replies``) call this through ``BasePlatformAdapter._is_sender_authorized`` to mark non-allowlisted senders as unverified in LLM context, mitigating indirect prompt injection from third parties in shared threads/channels. The returned callback delegates to :meth:`_is_user_authorized` so the full auth chain — platform allowlists, group allowlists, pairing store, allow-all flags — stays the single source of truth. ``profile_name`` binds the callback to the secondary adapter's own multiplex profile, so its ``SessionSource`` resolves that profile's secret scope instead of falling back to the active profile. For the shared primary adapter under ``multiplex_profiles`` (``profile_name`` is None) the callback mirrors the inbound message path exactly: the chat's ``profile_routes`` match is stamped on the source so the routed profile's pairing store is consulted, while the allowlist/gate reads stay under the transport (launch) home via ``_is_user_authorized_for_source`` — the same split ``_make_default_profile_message_handler`` applies. Without this an inline-button caller approved only in the routed profile's pairing store was denied (#86296), because the adapter's callback source was never route-stamped. """ multiplex = bool(getattr(self.config, "multiplex_profiles", False)) transport_home = ( Path(get_hermes_home()) if multiplex and profile_name is None else None ) def check( user_id: str, chat_type: Optional[str] = None, chat_id: Optional[str] = None, *, is_bot: bool = False, thread_id: Optional[str] = None, ) -> bool: if not user_id: return False source = SessionSource( platform=platform, chat_id=chat_id or "", chat_type=chat_type or "group", user_id=user_id, thread_id=thread_id, is_bot=bool(is_bot), profile=profile_name, ) # Same in-process transport provenance ``build_source`` retains, so # adapter-level policy reads (config.yaml group_allowed_chats, # allow_from) resolve the receiving adapter even once the routed # profile is stamped below. registry = ( (getattr(self, "_profile_adapters", None) or {}).get(profile_name) if profile_name else getattr(self, "adapters", None) ) or {} adapter = registry.get(platform) if adapter is not None: source._transport_adapter_ref = _weakref.ref(adapter) if transport_home is None: return self._is_user_authorized(source) source._authorization_profile_home = transport_home from gateway.profile_routing import ProfileRouteRejected try: source.profile = self._profile_name_for_source(source) except ProfileRouteRejected: # Same fail-closed outcome as the ingress gate in # ``_handle_message`` for a route to an unserved profile. return False return self._is_user_authorized_for_source(source) return check async def _deliver_platform_notice(self, source, content: str) -> None: """Deliver a setup/operational notice using platform-specific privacy rules.""" adapter = self._adapter_for_source(source) if not adapter: return config = getattr(self, "config", None) if ( config and getattr(source, "platform", None) == Platform.SLACK and _is_slack_ignored_channel(config, getattr(source, "chat_id", None)) ): logger.info( "Skipping Slack platform notice for configured ignored channel %s", getattr(source, "chat_id", None), ) return notice_delivery = "public" if config and hasattr(config, "get_notice_delivery"): notice_delivery = config.get_notice_delivery(source.platform) metadata = self._thread_metadata_for_source(source) if notice_delivery == "private" and getattr(source, "user_id", None): try: result = await adapter.send_private_notice( source.chat_id, source.user_id, content, metadata=metadata, ) if getattr(result, "success", False): return except Exception: logger.debug( "[%s] send_private_notice failed, falling back to public", getattr(source, "platform", "?"), exc_info=True, ) await adapter.send(source.chat_id, content, metadata=metadata) async def _resolve_async_delegation_session( self, session_entry: SessionEntry, pinned_session_id: str, ) -> Optional[SessionEntry]: """Resolve an async completion to its verified owning gateway session. A compression rotation ends the physical parent row while continuing the same logical conversation in a child. Follow that lineage, but never let a late completion override an unrelated /new or restored route. Unknown ownership remains fail-closed; the result is still available in the delegation records. """ session_db = cast(Any, self._session_db) if session_db is None: logger.warning( "Async-delegation completion has no session database; " "dropping injection (#55578 fail-closed)." ) return None pinned_row = None try: pinned_row = await session_db.get_session(pinned_session_id) except Exception: logger.debug( "Async-delegation parent lookup failed for %s", pinned_session_id, exc_info=True, ) if pinned_row is None: logger.warning( "Async-delegation completion has unknown spawning session %s; " "dropping injection (#55578 fail-closed).", pinned_session_id, ) return None target_session_id = pinned_session_id follows_compression = False if pinned_row.get("ended_at"): _end_reason = str(pinned_row.get("end_reason") or "") if _end_reason in _USER_BOUNDARY_END_REASONS: logger.warning( "Async-delegation completion pinned to user-closed session %s " "(end_reason=%r); dropping injection instead of resurrecting it " "(#55578 fail-closed).", pinned_session_id, _end_reason, ) return None if _end_reason != "compression": # Idle/timeout/lifecycle end (scale-to-zero norm): the chat # route remains valid and ``session_entry`` IS the routing # key's current session for this same chat, so deliver the # finished work there instead of dropping it. This is the # delivery leg _classify_completion_target promises when it # returns "deliver" for non-boundary ends — without it the # pre-flight verdict and this resolver disagree, and the # durable row is acked at adapter acceptance then silently # dropped here (falsely-acknowledged permanent loss; # staging incident 2026-08-09 defect #2). logger.info( "Async-delegation completion pinned to %s-ended session %s; " "retargeting to the chat's current session %s.", _end_reason or "idle", pinned_session_id, session_entry.session_id, ) return session_entry follows_compression = True try: target_session_id = await session_db.get_compression_tip( pinned_session_id ) except Exception: logger.debug( "Async-delegation compression-tip lookup failed for %s", pinned_session_id, exc_info=True, ) target_session_id = None if not target_session_id or target_session_id == pinned_session_id: logger.warning( "Async-delegation completion pinned to compressed session %s " "without a continuation; dropping injection.", pinned_session_id, ) return None try: tip_row = await session_db.get_session(target_session_id) except Exception: tip_row = None if tip_row is None or tip_row.get("ended_at"): logger.warning( "Async-delegation compression continuation %s is %s; " "dropping injection.", target_session_id, "unknown" if tip_row is None else "ended", ) return None route_owns_lineage = session_entry.session_id in { pinned_session_id, target_session_id, } if not route_owns_lineage: # A long-running delegation may survive multiple compression # rotations. Accept an intermediate stale route only when its # own verified compression tip is the same live target. try: route_row = await session_db.get_session(session_entry.session_id) route_tip = ( await session_db.get_compression_tip(session_entry.session_id) if route_row is not None and route_row.get("ended_at") and route_row.get("end_reason") == "compression" else None ) except Exception: route_tip = None route_owns_lineage = route_tip == target_session_id if not route_owns_lineage: logger.warning( "Async-delegation completion for compression lineage %s -> %s " "does not own current route %s; dropping injection.", pinned_session_id, target_session_id, session_entry.session_id, ) return None if target_session_id == session_entry.session_id: return session_entry prior_session_id = session_entry.session_id if follows_compression: switched = await self.async_session_store.advance_compression_session( session_entry.session_key, prior_session_id, target_session_id, ) else: switched = await self.async_session_store.switch_session( session_entry.session_key, target_session_id, ) if switched is None: logger.warning( "Async-delegation completion could not bind routing key %s to " "owning session %s; dropping injection.", session_entry.session_key, target_session_id, ) return None logger.info( "Pinned async-delegation completion to owning session %s " "(was %s) for routing key %s (#57498)", target_session_id, prior_session_id, session_entry.session_key, ) return switched # ------------------------------------------------------------------ # Mid-run (busy-session) slash command dispatch — "Guard 2". # # Replaces the historical hand-written per-command if-chain: each # command's mid-run behavior is declared on its CommandDef # (busy_policy / busy_handler in hermes_cli/commands.py) and resolved # here through a single handler table. Reply strings are byte-identical # to the old chain. # ------------------------------------------------------------------ # Command-specific mid-run reject texts (busy_policy == "reject" with a # busy_handler naming an entry here). All other rejected commands get # the generic catch-all text in _dispatch_busy_slash_command. _BUSY_REJECT_TEXT: Dict[str, str] = { "model": "Agent is running — wait or /stop first, then switch models.", "codex-runtime": ("Agent is running — wait or /stop first, then " "change runtime."), "moa": "Agent is running — wait or /stop first, then run /moa.", } def _gateway_plain_command_handlers(self): """Return ordinary slash handlers shared by idle and busy dispatch.""" return { "status": self._handle_status_command, "context": self._handle_context_command, "restart": self._handle_restart_command, "approve": self._handle_approve_command, "deny": self._handle_deny_command, "pause": self._handle_pause_command, "agents": self._handle_agents_command, "bg": self._handle_background_command, "btw": self._handle_btw_command, "kanban": self._handle_kanban_command, "subgoal": self._handle_subgoal_command, "heartbeat": self._handle_heartbeat_command, "busy": self._handle_busy_command, "yolo": self._handle_yolo_command, "verbose": self._handle_verbose_command, "footer": self._handle_footer_command, "help": self._handle_help_command, "commands": self._handle_commands_command, "profile": self._handle_profile_command, "update": self._handle_update_command, "version": self._handle_version_command, } async def _dispatch_busy_slash_command( self, event: MessageEvent, cmd_def, quick_key: str, source, ): """Dispatch a recognized slash command while an agent is running. Resolution order: 1. ``busy_handler`` — special mid-run variant (e.g. /goal's control-verb whitelist, /queue's FIFO enqueue, /model's custom reject text). 2. ``busy_policy == "dispatch"`` — the command's normal handler. 3. Catch-all busy-reject text. Rejecting is required rather than falling through to interrupt + discard: commands like /model, /reasoning, /voice, /insights, /title, /resume, /retry, /undo, /compress, /usage, /reload-mcp, /sethome, /reset (all registered as Discord slash commands) would interrupt the agent AND get silently discarded by the slash-command safety net, producing a zero-char response. See #5057, #6252, #10370. """ name = cmd_def.name policy = getattr(cmd_def, "busy_policy", "reject") handler_key = getattr(cmd_def, "busy_handler", None) if handler_key: special = { "start": self._busy_start_command, "stop": self._busy_stop_command, "new": self._busy_new_command, "queue": self._busy_queue_command, "steer": self._busy_steer_command, "egress": self._busy_egress_command, "goal": self._busy_goal_command, "loop": self._busy_loop_command, }.get(handler_key) if special is not None: return await special(event, quick_key, source) reject_text = self._BUSY_REJECT_TEXT.get(handler_key) if reject_text is not None: return reject_text if policy in ("dispatch", "interrupt_then_dispatch"): plain = self._gateway_plain_command_handlers().get(name) if plain is not None: return await plain(event) logger.warning( "busy_policy=%s for /%s has no mid-run handler — " "falling back to busy-reject", policy, name, ) # Catch-all: any other recognized slash command reached the # running-agent guard. Reject gracefully rather than falling # through to interrupt + discard. return ( f"⏳ Agent is running — `/{name}` can't run " f"mid-turn. Wait for the current response or `/stop` first." ) async def _handle_pause_command(self, event: MessageEvent): """`/pause [reason]` engages the global emergency stop; `/pause off` (aliases: resume/stop) lifts it. This is the in-band resume path for messaging-only operators — the estop gate above deliberately lets recognized slash commands through while paused so a user without host-shell access is never locked out. """ from agent import estop args = (event.get_command_args() or "").strip() if args.lower() in {"off", "resume", "stop", "disengage"}: if estop.disengage(): return "▶️ Resumed — new work is accepted again." return "Hermes wasn't paused." state = estop.get_state() if state is not None and not args: reason = state.get("reason") suffix = f" (reason: {reason})" if reason else "" return ( f"⏸️ Hermes is already paused{suffix}. " "Use `/pause off` to resume." ) estop.engage(reason=args or None) suffix = f" (reason: {args})" if args else "" return ( f"⏸️ Paused{suffix}. New cron/kanban/gateway work is on hold; " "in-flight work finishes normally. Use `/pause off` to resume." ) async def _busy_start_command(self, event: MessageEvent, quick_key: str, source): # Telegram sends /start for bot launches/deep-links. Treat it as a # platform ping, not a user command: no help dump, no agent # interrupt, no queued text. logger.info("Ignoring /start platform ping for active session %s", quick_key) return "" async def _busy_egress_command(self, event: MessageEvent, quick_key: str, source): from hermes_cli.proxy_cli import format_status_text return format_status_text() async def _busy_stop_command(self, event: MessageEvent, quick_key: str, source): # /stop must hard-kill the session when an agent is running. # A soft interrupt (agent.interrupt()) doesn't help when the agent # is truly hung — the executor thread is blocked and never checks # _interrupt_requested. Force-clean _running_agents so the session # is unlocked and subsequent messages are processed normally. await self._interrupt_and_clear_session( quick_key, source, interrupt_reason=_INTERRUPT_REASON_STOP, invalidation_reason="stop_command", ) logger.info("STOP for session %s — agent interrupted, session lock released", quick_key) return EphemeralReply(t("gateway.stop.stopped")) async def _busy_new_command(self, event: MessageEvent, quick_key: str, source): # /reset and /new must bypass the running-agent guard so they # actually dispatch as commands instead of being queued as user # text (which would be fed back to the agent with the same # broken history — #2170). Interrupt the agent first, then # clear the adapter's pending queue so the stale "/reset" text # doesn't get re-processed as a user message after the # interrupt completes. # Clear any pending messages so the old text doesn't replay await self._interrupt_and_clear_session( quick_key, source, interrupt_reason=_INTERRUPT_REASON_RESET, invalidation_reason="new_command", ) # Clean up the running agent entry so the reset handler # doesn't think an agent is still active. return await self._handle_reset_command(event) async def _busy_queue_command(self, event: MessageEvent, quick_key: str, source): # /queue — queue without interrupting. # Semantics: each /queue invocation produces its own full agent # turn, processed in FIFO order after the current run (and any # earlier /queue items) finishes. Messages are NOT merged. queued_text = event.get_command_args().strip() # Preserve media/reply payloads: a /queue carrying a photo, # document, or reply context is valid even with no prompt text # (e.g. "/queue" as the caption of an image). Dropping these # fields silently lost the attachment when the queued turn ran. has_media = bool(getattr(event, "media_urls", None)) if not queued_text and not has_media: return "Usage: /queue " adapter = self._adapter_for_source(source) if adapter: queued_event = MessageEvent( text=queued_text, message_type=event.message_type if has_media else MessageType.TEXT, source=event.source, raw_message=event.raw_message, message_id=event.message_id, media_urls=list(getattr(event, "media_urls", []) or []), media_types=list(getattr(event, "media_types", []) or []), media_text_inlined=list(getattr(event, "media_text_inlined", []) or []), reply_to_message_id=event.reply_to_message_id, reply_to_text=event.reply_to_text, reply_to_author_id=event.reply_to_author_id, reply_to_author_name=event.reply_to_author_name, reply_to_is_own_message=event.reply_to_is_own_message, auto_skill=event.auto_skill, channel_prompt=event.channel_prompt, channel_context=event.channel_context, internal=event.internal, timestamp=event.timestamp, ) self._enqueue_fifo(quick_key, queued_event, adapter) depth = self._queue_depth(quick_key, adapter=self._adapter_for_source(source)) if depth <= 1: return "Queued for the next turn." return f"Queued for the next turn. ({depth} queued)" async def _busy_steer_command(self, event: MessageEvent, quick_key: str, source): # /steer — inject mid-run after the next tool call. # Unlike /queue (turn boundary), /steer lands BETWEEN tool-call # iterations inside the same agent run, by appending to the # last tool result's content. No interrupt, no new user turn, # no role-alternation violation. steer_text = event.get_command_args().strip() if not steer_text: return "Usage: /steer " _steer_state = self._peek_session_state(quick_key) running_agent = _steer_state.turn.agent if _steer_state else None if running_agent is _AGENT_PENDING_SENTINEL: # Agent hasn't started yet — queue as turn-boundary fallback. adapter = self._adapter_for_source(source) if adapter: queued_event = MessageEvent( text=steer_text, message_type=MessageType.TEXT, source=event.source, message_id=event.message_id, channel_prompt=event.channel_prompt, channel_context=event.channel_context, ) self._enqueue_fifo(quick_key, queued_event, adapter) return "Agent still starting — /steer queued for the next turn." if running_agent and hasattr(running_agent, "steer"): try: accepted = running_agent.steer(steer_text) except Exception as exc: logger.warning("Steer failed for session %s: %s", quick_key, exc) return f"⚠️ Steer failed: {exc}" if accepted: preview = steer_text[:60] + ("..." if len(steer_text) > 60 else "") return f"⏩ Steer queued — arrives after the next tool call: '{preview}'" return "Steer rejected (empty payload)." # Running agent is missing or lacks steer() — fall back to queue. adapter = self._adapter_for_source(source) if adapter: queued_event = MessageEvent( text=steer_text, message_type=MessageType.TEXT, source=event.source, message_id=event.message_id, channel_prompt=event.channel_prompt, channel_context=event.channel_context, ) self._enqueue_fifo(quick_key, queued_event, adapter) return "No active agent — /steer queued for the next turn." async def _busy_goal_command(self, event: MessageEvent, quick_key: str, source): # /goal is safe mid-run for status/pause/clear/wait (inspection # and control-plane only — doesn't interrupt the running turn). # Setting a new goal text mid-run is rejected with the same # "wait or /stop" message as /model so we don't race a second # continuation prompt against the current turn. _goal_arg = (event.get_command_args() or "").strip().lower() _goal_verb = _goal_arg.split(None, 1)[0] if _goal_arg else "" # Exact-match control verbs (unchanged semantics), plus the # wait/unwait barrier verbs which take a pid argument and the # gate management verb (inspection/mutation of the gate list only — # gates run at turn boundary, so editing them mid-run is safe). _is_control = ( not _goal_arg or _goal_arg in {"status", "pause", "resume", "clear", "stop", "done", "unwait"} or _goal_verb in {"wait", "gate"} ) if _is_control: return await self._handle_goal_command(event) return "Agent is running — use /goal status / pause / clear / wait mid-run, or /stop before setting a new goal." async def _busy_loop_command(self, event: MessageEvent, quick_key: str, source): # /loop mirrors /goal: control verbs are safe mid-run (state # only — read at the next idle boundary); setting a new loop # mid-run is rejected so we don't race the current turn. _loop_arg = (event.get_command_args() or "").strip().lower() if not _loop_arg or _loop_arg in {"status", "pause", "resume", "stop", "clear", "cancel", "help", "--help", "-h"}: return await self._handle_loop_command(event) return "Agent is running — use /loop status / pause / stop mid-run, or /stop before setting a new loop." async def _handle_message(self, event: MessageEvent) -> Optional[str]: """ Handle an incoming message from any platform. This is the core message processing pipeline: 1. Check user authorization 2. Check for commands (/new, /reset, etc.) 3. Check for running agent and interrupt if needed 4. Get or create session 5. Build context for agent 6. Run agent conversation 7. Return response """ source = event.source # 🔴 Cross-session leak guard. This handler runs inside a per-message # asyncio task created via create_task(), which snapshots the spawning # context with copy_context(). If a *concurrent* message had already # bound its session via set_session_vars() when this task was created, # we inherited ITS HERMES_SESSION_* ContextVars. Until we bind our own # (a few steps down, in _set_session_env), any subprocess spawned here # would read the foreign session's identity via the subprocess-env # bridge — the _UNSET-strip guard there can't help because the vars are # set-to-foreign, not _UNSET. Reset to _UNSET now so that window strips # safe (no session) instead of leaking the sibling's. See # gateway/session_context.reset_session_vars + the inheritance test. try: from gateway.session_context import reset_session_vars reset_session_vars() except Exception: logger.debug("reset_session_vars failed at handler entry", exc_info=True) # Most adapters resolve profile routes in build_source(), before they # hand us the event. A few internal/voice paths construct SessionSource # directly, so resolve those here as the shared fail-closed ingress gate # before authorization, hooks, or session side effects. if ( getattr(getattr(self, "config", None), "multiplex_profiles", False) and not getattr(source, "profile", None) and getattr(source, "profile_route_rejected", False) is not True ): from gateway.profile_routing import ProfileRouteRejected try: source.profile = self._profile_name_for_source(source) except ProfileRouteRejected: source.profile_route_rejected = True # SessionSource owns a strict boolean marker. Require the literal value # so duck-typed test/internal sources with dynamic attributes are not # mistaken for an explicit matched-route rejection. if getattr(source, "profile_route_rejected", False) is True: logger.warning( "Dropping inbound message because its explicit profile route " "targets an unserved profile" ) return None # Internal events (e.g. background-process completion notifications) # are system-generated and must skip user authorization. is_internal = bool(getattr(event, "internal", False)) # Ignored-channel guard runs FIRST — before startup-restore queueing, # plugin hooks, auth, and session setup — so a configured ignored # channel can never reach pairing/auth/session state (#51899). # getattr: bare test runners construct GatewayRunner via # object.__new__ without config (see AGENTS.md pitfall on # object.__new__ test pattern). if ( not is_internal and getattr(source, "platform", None) == Platform.SLACK and _is_slack_ignored_channel( getattr(self, "config", None), getattr(source, "chat_id", None) ) ): logger.info( "Dropping Slack message from configured ignored channel %s", getattr(source, "chat_id", None), ) return None if ( getattr(self, "_startup_restore_in_progress", False) and not is_internal and not getattr(event, "_hermes_startup_restore_replay", False) ): self._queue_startup_restore_event(event) return None # scale-to-zero (Phase 0, 0.B/F13): stamp the gateway-scoped last-inbound # clock for real (user-originated) inbound only. Internal/system events # (background-process completions, startup-restore replays) are NOT # traffic — counting them would keep a genuinely idle gateway awake. This # clock is what the idle predicate (gateway/scale_to_zero.is_idle) reads. if not is_internal: self._scale_to_zero_note_real_inbound() # Fire pre_gateway_dispatch plugin hook for user-originated messages. # Plugins receive the MessageEvent and may return a dict influencing flow: # {"action": "skip", "reason": ...} -> drop (no reply, plugin handled) # {"action": "rewrite", "text": ...} -> replace event.text, continue # {"action": "allow"} / None -> normal dispatch # Hook runs BEFORE auth so plugins can handle unauthorized senders # (e.g. customer handover ingest) without triggering the pairing flow. if not is_internal: try: from hermes_cli.lifecycle import invoke_hook as _invoke_hook _hook_results = _invoke_hook( "pre_gateway_dispatch", event=event, gateway=self, # getattr: bare-runner tests build GatewayRunner via # object.__new__ without __init__ (pitfall #17), and the # hook must not fail dispatch over a missing attribute. session_store=getattr(self, "session_store", None), ) except Exception as _hook_exc: logger.warning("pre_gateway_dispatch invocation failed: %s", _hook_exc) _hook_results = [] for _result in _hook_results: if not isinstance(_result, dict): continue _action = _result.get("action") if _action == "skip": logger.info( "pre_gateway_dispatch skip: reason=%s platform=%s chat=%s", _result.get("reason"), source.platform.value if source.platform else "unknown", source.chat_id or "unknown", ) return None if _action == "rewrite": _new_text = _result.get("text") if isinstance(_new_text, str): event = dataclasses.replace(event, text=_new_text) source = event.source break if _action == "allow": break if is_internal: pass elif source.user_id is None: # Messages with no user identity (Telegram service messages, # channel forwards, anonymous admin posts, sender_chat) can't # be paired, but they can still be authorized via a # chat-scoped allowlist (e.g. TELEGRAM_GROUP_ALLOWED_CHATS # authorizes every member of the listed chat regardless of # sender). Defer to _is_user_authorized so that path runs. if not self._is_user_authorized_for_source(source): logger.debug("Ignoring message with no user_id from %s", source.platform.value) return None elif not self._is_user_authorized_for_source(source): logger.warning("Unauthorized user: %s (%s) on %s", source.user_id, source.user_name, source.platform.value) # In DMs: offer pairing code. In groups: silently ignore. if ( source.chat_type == "dm" and self._get_unauthorized_dm_behavior( source.platform, profile=source.profile, ) == "pair" ): platform_name = source.platform.value if source.platform else "unknown" pairing_store = self._pairing_store_for(source) if pairing_store is None: logger.error( "Cannot offer pairing code on %s: no pairing store", platform_name, ) return None # Rate-limit ALL pairing responses (code or rejection) to # prevent spamming the user with repeated messages when # multiple DMs arrive in quick succession. if pairing_store._is_rate_limited(platform_name, source.user_id): return None code = pairing_store.generate_code( platform_name, source.user_id, source.user_name or "" ) if code: adapter = self._adapter_for_source(source) if adapter: store_profile = getattr(pairing_store, "profile", None) profile_arg = ( f"-p {store_profile} " if isinstance(store_profile, str) and store_profile and store_profile != "default" else "" ) await adapter.send( source.chat_id, f"Hi~ I don't recognize you yet!\n\n" f"Here's your pairing code: `{code}`\n\n" f"Ask the bot owner to run:\n" f"`hermes {profile_arg}pairing approve " f"{platform_name} {code}`" ) else: adapter = self._adapter_for_source(source) if adapter: await adapter.send( source.chat_id, "Too many pairing requests right now~ " "Please try again later!" ) # Record rate limit so subsequent messages are silently ignored pairing_store._record_rate_limit(platform_name, source.user_id) return None # Global emergency stop (`hermes pause`): give new turns a brief # paused notice instead of starting an agent run. Internal events # (background-process completions from IN-FLIGHT work) bypass the # gate — pause stops NEW work, it never kills or orphans running # work. Placed after auth so unauthorized senders keep the normal # silent/pairing behavior and can't probe pause state. # # Passthroughs (pause blocks new AGENT turns, not control traffic): # * recognized slash commands — /status, /help, /new, /approve and # friends must keep working while paused, and /pause off is the # in-band resume path for messaging-only users; # * replies owned by IN-FLIGHT work — a pending detached-update # prompt, clarify, slash-confirm, or dangerous-command approval, # plus any message steering a session whose agent is already # running. Swallowing those would stall work the pause promised # not to touch. if not is_internal: try: from agent.estop import paused_reply as _estop_paused_reply _paused_notice = _estop_paused_reply() except ImportError: _paused_notice = None if _paused_notice is not None: _estop_allow = False _estop_cmd = None try: _estop_cmd = event.get_command() except Exception: _estop_cmd = None if _estop_cmd: try: from hermes_cli.commands import ( resolve_command as _resolve_estop_cmd, ) _estop_allow = _resolve_estop_cmd(_estop_cmd) is not None except Exception: _estop_allow = False if not _estop_allow: try: _estop_key = self._session_key_for_source(source) _estop_state = self._peek_session_state(_estop_key) if ( _estop_state is not None and _estop_state.persistent.update_prompt_pending ): _estop_allow = True if not _estop_allow and self._is_session_running(_estop_key): # Steering / interrupting in-flight work (which # also covers pending clarify + tool approvals # held by the running agent). _estop_allow = True if not _estop_allow: from tools import slash_confirm as _estop_confirm_mod if _estop_confirm_mod.get_pending(_estop_key): _estop_allow = True if not _estop_allow: from tools.approval import ( has_blocking_approval as _estop_has_approval, ) if _estop_has_approval(_estop_key): _estop_allow = True except Exception: pass if not _estop_allow: logger.info( "Gateway turn paused by global emergency stop (platform=%s chat=%s)", getattr(getattr(source, "platform", None), "value", "unknown"), getattr(source, "chat_id", None) or "unknown", ) return _paused_notice # Intercept messages that are responses to a pending /update prompt. # The update process (detached) wrote .update_prompt.json; the watcher # forwarded it to the user; now the user's reply goes back via # .update_response so the update process can continue. # # IMPORTANT: recognized slash commands must bypass this interception. # Otherwise control/session commands like /new or /help get silently # consumed as update answers instead of being dispatched normally. _quick_key = self._session_key_for_source(source) allow_gateway_control = event.allow_gateway_control _up_state = self._peek_session_state(_quick_key) if ( allow_gateway_control and _up_state is not None and _up_state.persistent.update_prompt_pending ): raw = (event.text or "").strip() # Accept /approve and /deny as shorthand for yes/no cmd = event.get_command() if cmd in {"approve", "yes"}: response_text = "y" elif cmd in {"deny", "no"}: response_text = "n" else: _recognized_cmd = None if cmd: try: from hermes_cli.commands import resolve_command as _resolve_update_cmd except Exception: _resolve_update_cmd = None if _resolve_update_cmd is not None: try: _cmd_def = _resolve_update_cmd(cmd) _recognized_cmd = _cmd_def.name if _cmd_def else None except Exception: _recognized_cmd = None if _recognized_cmd: response_text = "" else: response_text = raw if response_text: response_path = _hermes_home / ".update_response" prompt_path = _hermes_home / ".update_prompt.json" try: tmp = response_path.with_suffix(".tmp") tmp.write_text(response_text, encoding="utf-8") tmp.replace(response_path) prompt_path.unlink(missing_ok=True) except OSError as e: logger.warning("Failed to write update response: %s", e) return f"✗ Failed to send response to update process: {e}" _up_state.persistent.update_prompt_pending = False label = response_text if len(response_text) <= 20 else response_text[:20] + "…" return f"✓ Sent `{label}` to the update process." # Recognized slash command during a pending update prompt: # unblock the detached update subprocess by writing a blank # response so ``_gateway_prompt`` returns the prompt's default # (typically a safe "n" / skip) and exits cleanly instead of # blocking on stdin until the 30-minute watcher timeout. # The slash command then falls through to normal dispatch. if _recognized_cmd: response_path = _hermes_home / ".update_response" prompt_path = _hermes_home / ".update_prompt.json" try: tmp = response_path.with_suffix(".tmp") tmp.write_text("", encoding="utf-8") tmp.replace(response_path) prompt_path.unlink(missing_ok=True) logger.info( "Recognized /%s during pending update prompt for %s; " "cancelled prompt with default and dispatching command", _recognized_cmd, _quick_key, ) except OSError as e: logger.warning( "Failed to write cancel response for pending update prompt: %s", e, ) _up_state.persistent.update_prompt_pending = False # Intercept messages that are responses to a pending clarify. # Open-ended prompts and "Other" responses are captured as free text; # direct replies to multi-choice prompts are accepted too ("2" maps # to the second option). Slash # commands still bypass this path so /stop and friends keep working. _clarify_mod = None try: from tools import clarify_gateway as _clarify_mod _pending_clarify = _clarify_mod.get_pending_for_session( _quick_key, include_choice_prompts=True, ) except Exception: _pending_clarify = None if ( allow_gateway_control and _pending_clarify is not None and _clarify_mod is not None ): _clarify_has_audio = bool(self._pending_event_audio_paths(event)) _raw_clarify_reply = await self._prepare_clarify_reply_text(event) if _clarify_has_audio and not _raw_clarify_reply: logger.info( "Gateway retained pending clarify after voice transcription " "produced no usable text (session=%s, id=%s)", _quick_key, _pending_clarify.clarify_id, ) return "" # Skip slash commands — the user clearly wanted to issue a # command, not answer the clarify. Leave the clarify pending # so the user can retry; if it times out, the agent unblocks # with an empty response. if _raw_clarify_reply and not _raw_clarify_reply.startswith("/"): _text_outcome = _clarify_mod.attempt_text_response_for_session( _quick_key, _raw_clarify_reply, ) if _text_outcome == _clarify_mod.TEXT_RESOLVED: logger.info( "Gateway intercepted clarify text response (session=%s, id=%s)", _quick_key, _pending_clarify.clarify_id, ) # The clarify callback pauses the platform typing/status # indicator while waiting so Slack users can type their # answer. The active agent resumes as soon as this reply # resolves the wait, so re-enable its indicator here too. # Without this, Slack stays silent until the independent # long-running heartbeat fires (three minutes by default). _clarify_adapter = self._adapter_for_source(source) if _clarify_adapter: try: _clarify_adapter.resume_typing_for_chat(source.chat_id) except Exception: logger.debug( "Failed to resume typing after clarify response", exc_info=True, ) # Acknowledge with empty string so adapters that emit # the agent's response don't double-post. The agent # itself will produce the next user-facing message. return "" if _text_outcome == _clarify_mod.TEXT_REJECTED_SELECTION: # Selection-shaped but invalid (out-of-range number, # unrecognised comma-list). Keep the clarify armed so # the user can retry — do not cancel and do not treat # this as an unrelated follow-up turn. logger.info( "Gateway retained pending clarify after invalid " "selection attempt (session=%s, id=%s)", _quick_key, _pending_clarify.clarify_id, ) return "" if _text_outcome == _clarify_mod.TEXT_REJECTED_PROSE: # Native-choice prompts deliberately reject unmatched # prose so it can continue through normal busy-message # routing. Release this clarify first: redirect() # degrades to steer() while tools are executing, and # that steer cannot drain until the clarify tool returns. _clarify_mod.resolve_gateway_clarify( _pending_clarify.clarify_id, "", ) # Intercept messages that are responses to a pending /reload-mcp # (or future) slash-confirm prompt. Recognized confirm replies are # /approve, /always, /cancel (plus short aliases). Anything else # falls through to normal dispatch — a stale pending confirm does # NOT block other commands. # # Important: if a dangerous-command approval is ALSO pending (agent # blocked inside tools/approval.py), the tool approval takes # precedence — /approve there unblocks the waiting tool thread. # Slash-confirm only catches /approve when no tool approval is live. from tools import slash_confirm as _slash_confirm_mod _pending_confirm = _slash_confirm_mod.get_pending(_quick_key) _tool_approval_live = False try: from tools.approval import has_blocking_approval _tool_approval_live = has_blocking_approval(_quick_key) except Exception: _tool_approval_live = False if allow_gateway_control and _pending_confirm and not _tool_approval_live: _raw_reply = (event.text or "").strip() # Accept bang-prefixed replies (`!always`, `!cancel`) verbatim. # Slack/Matrix instruction text shows the `!` prefix (typed `/` # is blocked in Slack threads), but the adapters only rewrite # `!` — `always`/`cancel` are confirm keywords, # not registered commands, so the `!` survives to here. _norm_reply = _raw_reply.lstrip("!/").lower() _cmd_reply = event.get_command() _confirm_choice = None if _cmd_reply in {"approve", "yes", "ok", "confirm"}: _confirm_choice = "once" elif _cmd_reply in {"always", "remember"}: _confirm_choice = "always" elif _cmd_reply in {"cancel", "no", "deny", "nevermind"}: _confirm_choice = "cancel" elif _norm_reply in {"approve", "approve once", "once"}: _confirm_choice = "once" elif _norm_reply in {"always", "always approve"}: _confirm_choice = "always" elif _norm_reply in {"cancel", "nevermind", "no"}: _confirm_choice = "cancel" if _confirm_choice is not None: _resolved = await _slash_confirm_mod.resolve( _quick_key, _pending_confirm.get("confirm_id"), _confirm_choice, ) return _resolved or "" # Stale pending + unrelated command: drop the pending state so # the confirm doesn't block normal usage indefinitely. The user # clearly moved on. _slash_confirm_mod.clear_if_stale(_quick_key) # PRIORITY handling when an agent is already running for this session. # Default behavior is to interrupt immediately so user text/stop messages # are handled with minimal latency. # # Special case: Telegram/photo bursts often arrive as multiple near- # simultaneous updates. Do NOT interrupt for photo-only follow-ups here; # let the adapter-level batching/queueing logic absorb them. # Staleness eviction: detect leaked locks from hung/crashed handlers. # With inactivity-based timeout, active tasks can run for hours, so # wall-clock age alone isn't sufficient. Evict only when the agent # has been *idle* beyond the inactivity threshold (or when the agent # object has no activity tracker and wall-clock age is extreme). _raw_stale_timeout = _float_env("HERMES_AGENT_TIMEOUT", 1800) _quick_state = self._peek_session_state(_quick_key) _stale_ts = _quick_state.turn.started_ts if _quick_state else 0 if _quick_state is not None and _quick_state.turn.agent is not None and _stale_ts: _stale_age = time.time() - _stale_ts _stale_agent = _quick_state.turn.agent # Never evict the pending sentinel — it was just placed moments # ago during the async setup phase before the real agent is # created. Sentinels have no get_activity_summary(), so the # idle check below would always evaluate to inf >= timeout and # immediately evict them, racing with the setup path. _stale_idle = float("inf") # assume idle if we can't check _stale_detail = "" _activity_summary_valid = False if _stale_agent and hasattr(_stale_agent, "get_activity_summary"): try: _sa = _stale_agent.get_activity_summary() from gateway.session_stall import ( resolve_session_idle_seconds_from_activity, ) _resolved_idle = resolve_session_idle_seconds_from_activity( _sa if isinstance(_sa, dict) else None, now=time.time(), ) if _resolved_idle is not None: _stale_idle = _resolved_idle _activity_summary_valid = True _stale_detail = ( f" | last_activity={_sa.get('last_activity_desc', 'unknown') if isinstance(_sa, dict) else 'unknown'} " f"({_stale_idle:.0f}s ago) " f"| iteration={_sa.get('api_call_count', 0) if isinstance(_sa, dict) else 0}/{_sa.get('max_iterations', 0) if isinstance(_sa, dict) else 0}" ) except Exception: pass # A valid activity clock is authoritative: total age alone never # makes an actively progressing turn stale. The emergency wall TTL # is only a fallback when the agent cannot report usable activity. _wall_ttl = max(_raw_stale_timeout * 10, 7200) if _raw_stale_timeout > 0 else float("inf") _should_evict = ( _stale_agent is not _AGENT_PENDING_SENTINEL and ( ( _activity_summary_valid and _raw_stale_timeout > 0 and _stale_idle >= _raw_stale_timeout ) or ( not _activity_summary_valid and _stale_age > _wall_ttl ) ) ) if _should_evict: logger.warning( "Evicting stale _running_agents entry for %s " "(age: %.0fs, idle: %.0fs, timeout: %.0fs)%s", _quick_key, _stale_age, _stale_idle, _raw_stale_timeout, _stale_detail, ) self._invalidate_session_run_generation( _quick_key, reason="stale_running_agent_eviction", ) self._release_running_agent_state(_quick_key) # #99106: durable-reaped guard. A session whose routing row was # ended in state.db (e.g. ``ws_orphan_reap`` / ``agent_close``) while # the gateway stayed alive keeps its in-memory turn slot alive # (``_is_session_running`` stays True). The priority fast-path would # then queue every next user message into the dead runtime instead of # healing the routing via ``get_or_create_session`` → ``reopen``. # This is the live-gateway variant of #54878 and the #632 detached/ # 405 suppressions in production. Evict the stale slot so the next # message falls through to the cold path and re-attaches or creates a # fresh session; /status then correctly shows 代理运行中: 否 before the # heal and a live turn after. if self._is_session_running(_quick_key): try: _reap_store = getattr(self, "session_store", None) # Use the public, lock-held accessors: peek_session_id resolves # key -> session_id under the store lock, and returns a # non-str on stubbed stores in bare test runners — both the # isinstance() gate and the ``is True`` gate below keep this # guard inert unless a real SessionStore answers. _reap_peek = getattr(_reap_store, "peek_session_id", None) _is_ended = getattr(_reap_store, "_is_session_ended_in_db", None) _reap_sid = _reap_peek(_quick_key) if callable(_reap_peek) else None if ( isinstance(_reap_sid, str) and _reap_sid and callable(_is_ended) and _is_ended(_reap_sid) is True ): logger.warning( "Evicting stale _running_agents entry for %s — " "durable session %s is ended (reaped) in state.db; " "healing routing on next message (#99106)", _quick_key, _reap_sid, ) self._invalidate_session_run_generation( _quick_key, reason="reaped_session_eviction", ) self._release_running_agent_state(_quick_key) except Exception: logger.debug("reaped-session staleness check failed", exc_info=True) if self._is_session_running(_quick_key): # Resolve the command once; every command's mid-run behavior is # declared on its CommandDef (busy_policy / busy_handler in # hermes_cli/commands.py) and dispatched through the single # resolver _dispatch_busy_slash_command below — no per-command # if-chain here. from hermes_cli.commands import resolve_command as _resolve_cmd_inner _evt_cmd = event.get_command() _cmd_def_inner = _resolve_cmd_inner(_evt_cmd) if _evt_cmd else None # /status and /context are intentionally pre-gate so users # always see session state. if _cmd_def_inner and _cmd_def_inner.name == "status": return await self._handle_status_command(event) if _cmd_def_inner and _cmd_def_inner.name == "context": return await self._handle_context_command(event) # Slash command access control on the running-agent fast-path. # Mirrors the cold-path gate further below so non-admin users # can't bypass gating just because an agent happens to be busy. # /status above is intentionally pre-gate so users always see # session state. /help and /whoami fall under the always-allowed # floor inside _check_slash_access. if _evt_cmd and _cmd_def_inner is not None: _denied = self._check_slash_access(source, _cmd_def_inner.name) if _denied is not None: return _denied # Any recognized slash command: dispatch according to its # declared busy_policy (dispatch / interrupt_then_dispatch / # reject). Unrecognized commands and plain text fall through # to the interrupt/queue logic below. if _cmd_def_inner: return await self._dispatch_busy_slash_command( event, _cmd_def_inner, _quick_key, source, ) if event.message_type == MessageType.PHOTO: logger.debug("PRIORITY photo follow-up for session %s — queueing without interrupt", _quick_key) adapter = self._adapter_for_source(source) if adapter: merge_pending_message_event(adapter._pending_messages, _quick_key, event) return None effective_busy_input_mode = self._effective_busy_input_mode(source) _telegram_followup_grace = float( os.getenv("HERMES_TELEGRAM_FOLLOWUP_GRACE_SECONDS", "3.0") ) _grace_state = self._peek_session_state(_quick_key) _started_at = _grace_state.turn.started_ts if _grace_state else 0 if ( source.platform == Platform.TELEGRAM and event.message_type == MessageType.TEXT and _telegram_followup_grace > 0 and _started_at and (time.time() - _started_at) <= _telegram_followup_grace ): logger.debug( "Telegram follow-up arrived %.2fs after run start for %s — queueing without interrupt", time.time() - _started_at, _quick_key, ) adapter = self._adapter_for_source(source) if adapter: if effective_busy_input_mode == "queue": self._enqueue_fifo(_quick_key, event, adapter) else: merge_pending_message_event( adapter._pending_messages, _quick_key, event, merge_text=True, ) return None _ra_state = self._peek_session_state(_quick_key) running_agent = _ra_state.turn.agent if _ra_state else None if running_agent is _AGENT_PENDING_SENTINEL: # Agent is being set up but not ready yet. if event.get_command() == "stop": # Force-clean the sentinel so the session is unlocked. self._release_running_agent_state(_quick_key) logger.info("HARD STOP (pending) for session %s — sentinel cleared", _quick_key) return EphemeralReply("⚡ Force-stopped. The agent was still starting — session unlocked.") # Queue the message so it will be picked up after the # agent starts. adapter = self._adapter_for_source(source) if adapter: merge_pending_message_event( adapter._pending_messages, _quick_key, event, merge_text=True, ) return None if self._draining: queue_during_drain = self._queue_during_drain_enabled( effective_busy_input_mode ) if queue_during_drain: self._queue_or_replace_pending_event(_quick_key, event) return ( f"⏳ Gateway {self._status_action_gerund()} — queued for the next turn after it comes back." if queue_during_drain else f"⏳ Gateway is {self._status_action_gerund()} and is not accepting another turn right now." ) if effective_busy_input_mode == "queue": logger.debug("PRIORITY queue follow-up for session %s", _quick_key) self._queue_or_replace_pending_event(_quick_key, event) return None if effective_busy_input_mode == "steer": # Steer mode: inject text into the running agent mid-run via # agent.steer(). Falls back to queue semantics if the payload # is empty, the agent lacks steer(), or steer() rejects. steer_text = (event.text or "").strip() steered = False if ( event.message_type == MessageType.TEXT and not event.media_urls and not event.media_types and steer_text and hasattr(running_agent, "steer") ): try: steered = bool(running_agent.steer(steer_text)) except Exception as exc: logger.warning("PRIORITY steer failed for session %s: %s", _quick_key, exc) steered = False if steered: logger.debug("PRIORITY steer for session %s", _quick_key) return None logger.debug("PRIORITY steer-fallback-to-queue for session %s", _quick_key) self._queue_or_replace_pending_event(_quick_key, event) return None # #30170 — Subagent protection (PRIORITY path). Same rationale # as ``_handle_active_session_busy_message``: an interrupt # cascades through ``_active_children`` and aborts in-flight # delegate_task work. Demote to queue semantics when the # parent is currently driving subagents so a conversational # follow-up doesn't destroy minutes of subagent progress. # /stop reaches its dedicated handler above, so the operator # still has a clean escape hatch. if self._agent_has_active_subagents(running_agent): logger.info( "PRIORITY interrupt demoted to queue for session %s " "because the running agent has active subagents (#30170)", _quick_key, ) self._queue_or_replace_pending_event(_quick_key, event) return None # #56391 — Compression protection (PRIORITY path). Same # rationale as ``_handle_active_session_busy_message``: context # compression is interrupt-protected (#23975), but an interrupt # here starts a new turn against the pre-rotation parent # session while the still-running compression later rotates # the id out from under it, forking orphaned compression # siblings. Demote to queue semantics so the follow-up waits # for the in-flight compression + rotation to land. if await self._session_has_compression_in_flight(_quick_key): logger.info( "PRIORITY interrupt demoted to queue for session %s " "because context compression is in flight (#56391)", _quick_key, ) self._queue_or_replace_pending_event(_quick_key, event) return None # Text-only corrections redirect the live turn (preserving # displayed context) when the runtime supports it; media/voice and # older runtimes fall back to the proven interrupt path below. if ( event.message_type == MessageType.TEXT and not event.media_urls and not event.media_types and getattr(running_agent, "_supports_active_turn_redirect", False) is True and hasattr(running_agent, "redirect") ): try: if running_agent.redirect((event.text or "").strip()): logger.debug("PRIORITY redirect for session %s", _quick_key) return None except Exception as exc: logger.warning( "PRIORITY redirect failed for session %s: %s", _quick_key, exc, ) logger.debug("PRIORITY interrupt for session %s", _quick_key) _interrupt_text = event.text _media_urls = getattr(event, "media_urls", None) or [] if self._pending_event_audio_paths(event): _interrupt_text, _ = await self._transcribe_and_echo_pending_voice( event, self._adapter_for_source(source), source, event.text or "", log_context="Voice-priority-interrupt", ) elif not _interrupt_text and _media_urls: _interrupt_text = _build_media_placeholder(event) running_agent.interrupt(_interrupt_text) # NOTE: self._pending_messages was write-only (never consumed). # The actual interrupt message is delivered via adapter._pending_messages # which is read by _run_agent. Removed to prevent unbounded growth. return None # Check for commands command = event.get_command() from hermes_cli.commands import ( GATEWAY_KNOWN_COMMANDS, is_gateway_known_command, resolve_command as _resolve_cmd, ) # Resolve aliases to canonical name so dispatch and hook names # don't depend on the exact alias the user typed. _cmd_def = _resolve_cmd(command) if command else None canonical = _cmd_def.name if _cmd_def else command # Expand alias quick commands before built-in dispatch so targets like # /model openai/gpt-5.5 --provider openrouter reach the /model handler. # Preserve built-in precedence; aliases only need early handling when # the typed command is not already known. if command and _cmd_def is None: if isinstance(self.config, dict): quick_commands = self.config.get("quick_commands", {}) or {} else: quick_commands = getattr(self.config, "quick_commands", {}) or {} if isinstance(quick_commands, dict) and command in quick_commands: qcmd = quick_commands[command] if qcmd.get("type") == "alias": target = (qcmd.get("target") or "").strip() if target: target = target if target.startswith("/") else f"/{target}" target_command = target.lstrip("/") user_args = event.get_command_args().strip() event.text = f"{target} {user_args}".strip() command = target_command.split()[0] if target_command else target_command _cmd_def = _resolve_cmd(command) if command else None canonical = _cmd_def.name if _cmd_def else command # Per-platform slash command access control. Only kicks in when the # operator has set ``allow_admin_from`` for the source's scope (DM # vs group). When unset → backward-compat: every allowed user can # run every command. When set → non-admins can run only commands in # ``user_allowed_commands`` (plus the always-allowed floor: /help, # /whoami). Plain chat is unaffected — only slash commands gate. if command and canonical and is_gateway_known_command(canonical): _denied = self._check_slash_access(source, canonical) if _denied is not None: return _denied # pre_command observer hook (#64204): fires for every recognized # slash command BEFORE core handling, mirroring the CLI fire-site in # cli.py process_command. Observer-only in v1 (returns ignored). # # Placement matters: this cold-path dispatch is only reached when NO # agent is running for the session. The running-agent intercept path # above (/stop, /approve, busy_policy dispatch via # _dispatch_busy_slash_command) deliberately does NOT fire this hook — # those are control-plane operations on an in-flight run, and giving # plugins an observation (and eventually veto) point there would let # a slow or hostile plugin interfere with the operator's escape # hatches for a live agent. if command and is_gateway_known_command(canonical): try: from hermes_cli.plugins import fire_pre_command_hook fire_pre_command_hook( surface="gateway", command=str(canonical), alias_used=str(command), args_raw=event.get_command_args().strip(), session_key=_quick_key, platform=source.platform.value if source.platform else "", ) except Exception as _pre_cmd_err: logger.debug( "pre_command hook dispatch failed (non-fatal): %s", _pre_cmd_err, ) # Fire the ``command:`` hook for any recognized slash # command — built-in OR plugin-registered. Handlers can return a # dict with ``{"decision": "deny" | "handled" | "rewrite", ...}`` # to intercept dispatch before core handling runs. This replaces # the previous fire-and-forget emit(): return values are now # honored, but handlers that return nothing behave exactly as # before (telemetry-style hooks keep working). if command and is_gateway_known_command(canonical): raw_args = event.get_command_args().strip() hook_ctx = { "platform": source.platform.value if source.platform else "", "user_id": source.user_id, "command": canonical, "raw_command": command, "args": raw_args, "raw_args": raw_args, } try: hook_results = await self.hooks.emit_collect( f"command:{canonical}", hook_ctx ) except Exception as _hook_err: logger.debug( "command:%s hook dispatch failed (non-fatal): %s", canonical, _hook_err, ) hook_results = [] for hook_result in hook_results: if not isinstance(hook_result, dict): continue decision = str(hook_result.get("decision", "")).strip().lower() if not decision or decision == "allow": continue if decision == "deny": message = hook_result.get("message") if isinstance(message, str) and message: return message return f"Command `/{command}` was blocked by a hook." if decision == "handled": message = hook_result.get("message") return message if isinstance(message, str) and message else None if decision == "rewrite": new_command = str( hook_result.get("command_name", "") ).strip().lstrip("/") if not new_command: continue new_args = str(hook_result.get("raw_args", "")).strip() event.text = f"/{new_command} {new_args}".strip() command = event.get_command() _cmd_def = _resolve_cmd(command) if command else None canonical = _cmd_def.name if _cmd_def else command break plain_handler = self._gateway_plain_command_handlers().get(canonical) if plain_handler is not None: return await plain_handler(event) if canonical == "new": if await asyncio.to_thread(self._is_telegram_topic_root_lobby, source): return self._telegram_topic_root_new_message() async def _do_reset(): return await self._handle_reset_command(event) return await self._maybe_confirm_destructive_slash( event=event, command="new", title="/new", detail=( "This starts a fresh session and discards the current " "conversation history." ), execute=_do_reset, ) if canonical == "topic": return await self._handle_topic_command(event) if canonical == "start": logger.info("Ignoring /start platform ping for session %s", _quick_key) return "" if canonical == "whoami": return await self._handle_whoami_command(event) if canonical == "egress": from hermes_cli.proxy_cli import format_status_text return format_status_text() if canonical == "platform": return await self._handle_platform_command(event) if canonical == "stop": return await self._handle_stop_command(event) if canonical == "reasoning": return await self._handle_reasoning_command(event) if canonical == "memory": return await self._handle_memory_command(event) if canonical == "skills": return await self._handle_skills_command(event) if canonical == "learn": # Open-ended: rewrite the turn to a standards-guided prompt and fall # through to normal agent processing. The live agent gathers the # sources the user described (dirs via read_file, URLs via # web_extract, this conversation, pasted text) and authors the skill # via skill_manage. Mirrors the /blueprint fall-through so role # alternation is preserved. No engine, works on any backend. from agent.learn_prompt import build_learn_prompt _learn_req = event.get_command_args().strip() _ack = ( "Learning a skill from what you described…" if _learn_req else "Learning a skill from this conversation…" ) try: adapter = self._adapter_for_source(source) if adapter: _ack_meta = self._thread_metadata_for_source(source) await adapter.send(str(source.chat_id), _ack, metadata=_ack_meta) except Exception: logger.debug("learn ack send failed", exc_info=True) try: event.text = build_learn_prompt(_learn_req) # fall through to agent processing except Exception: return "Could not start /learn — please try again." if canonical == "plan": # /plan: rewrite the turn to the plan-mode prompt and fall # through to normal agent processing (same fall-through as /learn # so role alternation is preserved). The live agent inspects the # workspace with read-only tools and saves the markdown plan # under .hermes/plans/ via write_file. No engine, works on any # backend. from agent.plan_prompt import build_plan_prompt _plan_task = event.get_command_args().strip() _ack = ( f"Planning: {_plan_task[:80]}{'…' if len(_plan_task) > 80 else ''}" if _plan_task else "Planning from this conversation's context…" ) try: adapter = self._adapter_for_source(source) if adapter: _ack_meta = self._thread_metadata_for_source(source) await adapter.send(str(source.chat_id), _ack, metadata=_ack_meta) except Exception: logger.debug("plan ack send failed", exc_info=True) try: event.text = build_plan_prompt(_plan_task) # fall through to agent processing except Exception: return "Could not start /plan — please try again." if canonical == "init": # /init: rewrite the turn to a guidance-laden prompt and fall # through to normal agent processing (same fall-through as /learn # so role alternation is preserved). The live agent scans the # project with its own read-only tools and writes/updates # AGENTS.md via write_file. No engine, works on any backend. from hermes_cli.init_command import build_init_prompt_for_cwd _init_notes = event.get_command_args().strip() try: _init_prompt = build_init_prompt_for_cwd(extra=_init_notes) except Exception: return "Could not start /init — please try again." _ack = ( "Updating AGENTS.md from a project scan…" if "UPDATE the existing AGENTS.md" in _init_prompt else "Generating AGENTS.md from a project scan…" ) try: adapter = self._adapter_for_source(source) if adapter: _ack_meta = self._thread_metadata_for_source(source) await adapter.send(str(source.chat_id), _ack, metadata=_ack_meta) except Exception: logger.debug("init ack send failed", exc_info=True) event.text = _init_prompt # fall through to agent processing if canonical == "fast": return await self._handle_fast_command(event) if canonical == "approvals": return await self._handle_approvals_command(event) if canonical == "model": return await self._handle_model_command(event) if canonical == "codex-runtime": return await self._handle_codex_runtime_command(event) if canonical == "personality": return await self._handle_personality_command(event) if canonical == "suggestions": return await self._handle_suggestions_command(event) if canonical == "blueprint": _blueprint_result = await self._handle_blueprint_command(event) _blueprint_seed = getattr(_blueprint_result, "agent_seed", None) if _blueprint_seed: # Blueprint matched — rewrite the turn to the seed and fall # through to _handle_message_with_agent so the agent asks the # user for each slot value conversationally and then calls the # cronjob tool (the /steer fall-through pattern). The seed # enters as a normal user turn, preserving role alternation. # Send the "Setting up X…" ack first so the user gets the same # immediate feedback CLI users see, instead of silence until # the agent's first question. _ack = getattr(_blueprint_result, "text", "") or "" if _ack: try: adapter = self._adapter_for_source(source) if adapter: _ack_meta = self._thread_metadata_for_source(source) await adapter.send(str(source.chat_id), _ack, metadata=_ack_meta) except Exception: logger.debug("blueprint ack send failed", exc_info=True) try: event.text = _blueprint_seed except Exception: return getattr(_blueprint_result, "text", "") or None else: return getattr(_blueprint_result, "text", "") or None if canonical == "save": return await self._handle_save_command(event) if canonical == "retry": return await self._handle_retry_command(event) if canonical == "undo": async def _do_undo(): return await self._handle_undo_command(event) _undo_n = 1 _undo_raw = event.get_command_args().strip() if _undo_raw: try: _undo_n = max(1, int(_undo_raw.split()[0])) except (ValueError, IndexError): _undo_n = 1 _undo_detail = ( "This removes the last user/assistant exchange from history." if _undo_n == 1 else f"This removes the last {_undo_n} user turns from history." ) return await self._maybe_confirm_destructive_slash( event=event, command="undo", title="/undo", detail=_undo_detail, execute=_do_undo, ) if canonical == "sethome": return await self._handle_set_home_command(event) if canonical == "compress": return await self._handle_compress_command(event) if canonical == "usage": return await self._handle_usage_command(event) if canonical == "topup": return await self._handle_topup_command(event) if canonical == "insights": return await self._handle_insights_command(event) if canonical == "reload-mcp": return await self._handle_reload_mcp_command(event) if canonical == "reload-skills": return await self._handle_reload_skills_command(event) if canonical == "bundles": return await self._handle_bundles_command(event) if canonical == "debug": return await self._handle_debug_command(event) if canonical == "title": return await self._handle_title_command(event) if canonical == "resume": return await self._handle_resume_command(event) if canonical == "sessions": return await self._handle_sessions_command(event) if canonical == "branch": return await self._handle_branch_command(event) if canonical == "rollback": return await self._handle_rollback_command(event) if canonical == "diff": return await self._handle_diff_command(event) if canonical == "queue": queue_payload = event.get_command_args().strip() if not queue_payload: return "Usage: /queue " try: event.text = queue_payload except Exception: pass if canonical == "steer": # No active agent — /steer has no tool call to inject into. # Strip the prefix so downstream treats it as a normal user # message. If the payload is empty, surface the usage hint. steer_payload = event.get_command_args().strip() if not steer_payload: return "Usage: /steer (no agent is running; sending as a normal message)" try: event.text = steer_payload except Exception: pass # Do NOT return — fall through to _handle_message_with_agent # at the end of this function so the rewritten text is sent # to the agent as a regular user turn. if canonical == "goal": return await self._handle_goal_command(event) if canonical == "loop": return await self._handle_loop_command(event) if canonical == "refine": return await self._handle_refine_command(event) if canonical == "review": return await self._handle_review_command(event) if canonical == "moa": # /moa is one-shot sugar only: run a single prompt through the # default MoA preset, then restore the prior model. To *switch* to a # MoA preset for the session, pick it from the model picker (MoA # presets surface as a virtual "Mixture of Agents" provider). from hermes_cli.moa_config import ( moa_usage, normalize_moa_config, ) from hermes_cli.config import load_config moa_payload = event.get_command_args().strip() if not moa_payload: return moa_usage() try: cfg = load_config() moa_cfg = normalize_moa_config(cfg.get("moa") if isinstance(cfg, dict) else {}) except Exception: moa_cfg = normalize_moa_config({}) preset = moa_cfg["default_preset"] try: event.text = moa_payload _moa_state = self._session_state(_quick_key) event._moa_restore_override = _moa_state.conversation.model_override _moa_state.conversation.model_override = { "provider": "moa", "model": preset, "base_url": "moa://local", "api_key": "moa-virtual-provider", "api_mode": "chat_completions", } self._evict_cached_agent(_quick_key) event._moa_disable_after_turn = True except Exception: return "Failed to prepare MoA turn." if canonical == "voice": return await self._handle_voice_command(event) if self._draining: return f"⏳ Gateway is {self._status_action_gerund()} and is not accepting new work right now." # User-defined quick commands (bypass agent loop, no LLM call) if command: if isinstance(self.config, dict): quick_commands = self.config.get("quick_commands", {}) or {} else: quick_commands = getattr(self.config, "quick_commands", {}) or {} if not isinstance(quick_commands, dict): quick_commands = {} if command in quick_commands: # Quick commands are slash capabilities too — and type:exec # ones run a shell command in the gateway process. The early # gate above only fires for registry-known commands, so quick # commands (never in the registry) would otherwise reach this # dispatch sink unchecked. Apply the same admin/user policy to # the raw typed name here so non-admins can't invoke admin-only # quick commands. (#44727) _denied = self._check_slash_access(source, command) if _denied is not None: return _denied qcmd = quick_commands[command] if qcmd.get("type") == "exec": exec_cmd = qcmd.get("command", "") if exec_cmd: try: # Sanitize env to prevent credential leakage — # quick commands run in the gateway process which # has all API keys in os.environ. from tools.environments.local import build_subprocess_env sanitized_env = build_subprocess_env() proc = await asyncio.create_subprocess_shell( exec_cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=sanitized_env, ) stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30) output = (stdout or stderr).decode().strip() # Redact any remaining sensitive patterns in output if output: from agent.redact import redact_sensitive_text output = redact_sensitive_text(output) return output if output else "Command returned no output." except asyncio.TimeoutError: return "Quick command timed out (30s)." except Exception as e: return f"Quick command error: {e}" else: return f"Quick command '/{command}' has no command defined." elif qcmd.get("type") == "alias": target = (qcmd.get("target") or "").strip() if target: target = target if target.startswith("/") else f"/{target}" target_command = target.lstrip("/") user_args = event.get_command_args().strip() event.text = f"{target} {user_args}".strip() command = target_command.split()[0] if target_command else target_command # Fall through to normal command dispatch below else: return f"Quick command '/{command}' has no target defined." else: return f"Quick command '/{command}' has unsupported type (supported: 'exec', 'alias')." # Plugin-registered slash commands if command: try: from hermes_cli.plugins import get_plugin_command_handler # Normalize underscores to hyphens so Telegram's underscored # autocomplete form matches plugin commands registered with # hyphens. See hermes_cli/commands.py:_build_telegram_menu. plugin_handler = get_plugin_command_handler(command.replace("_", "-")) if plugin_handler: user_args = event.get_command_args().strip() result = plugin_handler(user_args) if asyncio.iscoroutine(result): result = await result return str(result) if result else None except Exception as e: logger.warning("Plugin command dispatch failed: %s", e) # Skill slash commands: /skill-name loads the skill and sends to agent. # resolve_skill_command_key() handles the Telegram underscore/hyphen # round-trip so /claude_code from Telegram autocomplete still resolves # to the claude-code skill. if command: # Skill bundles take precedence over individual skill commands — # / loads multiple skills at once. Mirrors CLI dispatch. _bundle_handled = False try: from agent.skill_bundles import ( build_bundle_invocation_message, resolve_bundle_command_key, ) bundle_key = resolve_bundle_command_key(command) if bundle_key is not None: user_instruction = event.get_command_args().strip() # Pass the platform explicitly: bundle skill loading # bypasses get_skill_commands()' scan-time disabled # filter, and the gateway serves multiple platforms in # one process, so env-var platform resolution can't be # trusted here. Mirrors the stacked-skill gate (#58888). _bundle_plat = source.platform.value if source.platform else None bundle_result = build_bundle_invocation_message( bundle_key, user_instruction, task_id=_quick_key, platform=_bundle_plat, ) if bundle_result: msg, _loaded, missing = bundle_result event.text = msg _bundle_handled = True if missing: logger.info( "Bundle %s skipped missing skills: %s", bundle_key, ", ".join(missing), ) # Fall through to normal message processing with bundle content except Exception as exc: logger.warning("Bundle dispatch failed: %s", exc) if command and not locals().get("_bundle_handled", False): try: from agent.skill_commands import ( get_skill_commands, build_skill_invocation_message, resolve_skill_command_key, ) skill_cmds = get_skill_commands() cmd_key = resolve_skill_command_key(command) if cmd_key is not None: # Check per-platform disabled status before executing. # get_skill_commands() only applies the *global* disabled # list at scan time; per-platform overrides need checking # here because the cache is process-global across platforms. _skill_name = skill_cmds[cmd_key].get("name", "") _plat = source.platform.value if source.platform else None if _plat and _skill_name: from agent.skill_utils import get_disabled_skill_names as _get_plat_disabled if _skill_name in _get_plat_disabled(platform=_plat): return ( f"The **{_skill_name}** skill is disabled for {_plat}.\n" f"Enable it with: `hermes skills config`" ) user_instruction = event.get_command_args().strip() # Stacked slash-skill invocations: `/skill-a /skill-b do # XYZ` loads every leading skill (up to 5), not just the # first. Inspired by Claude Code v2.1.199. Mirrors CLI. try: from agent.skill_commands import ( build_stacked_skill_invocation_message as _build_stacked, split_stacked_skill_commands, ) extra_keys, stacked_instruction = ( split_stacked_skill_commands(user_instruction) ) except Exception: _build_stacked = None extra_keys, stacked_instruction = [], user_instruction if extra_keys and _plat: # split_stacked_skill_commands() only resolves that # each extra token is a KNOWN skill command — like # get_skill_commands() itself, it has no per-platform # view. Re-check every stacked skill (not just the # leading one above) against the same disabled list, # or a skill an operator disabled for this platform # still gets its full content loaded via the stack. from agent.skill_utils import get_disabled_skill_names as _get_plat_disabled _plat_disabled = _get_plat_disabled(platform=_plat) _disabled_extra = [ skill_cmds.get(k, {}).get("name", "") for k in extra_keys if skill_cmds.get(k, {}).get("name", "") in _plat_disabled ] if _disabled_extra: return ( f"The **{', '.join(_disabled_extra)}** skill(s) in this " f"stacked invocation are disabled for {_plat}.\n" f"Enable them with: `hermes skills config`" ) if extra_keys and _build_stacked is not None: stacked_result = _build_stacked( [cmd_key, *extra_keys], stacked_instruction, task_id=_quick_key, ) if stacked_result: msg, _loaded, _missing = stacked_result event.text = msg # Fall through to normal message processing else: return f"Failed to load stacked skills for /{command}." else: msg = build_skill_invocation_message( cmd_key, user_instruction, task_id=_quick_key ) if msg: event.text = msg # Fall through to normal message processing with skill content else: # Not an active skill — check if it's a known-but-disabled or # uninstalled skill and give actionable guidance. _unavail_msg = _check_unavailable_skill(command) if _unavail_msg: return _unavail_msg # Genuinely unrecognized /command: not a built-in, not a # plugin, not a skill, not a known-inactive skill. Warn # the user instead of silently forwarding it to the LLM # as free text (which leads to silent-failure behavior # like the model inventing a delegate_task call). # Normalize to hyphenated form before checking known # built-ins (command may be an alias target set by the # quick-command block above, so _cmd_def can be stale). if command.replace("_", "-") not in GATEWAY_KNOWN_COMMANDS: logger.warning( "Unrecognized slash command /%s from %s — " "replying with unknown-command notice", command, source.platform.value if source.platform else "?", ) return ( f"Unknown command `/{command}`. " f"Type /commands to see what's available, " f"or resend without the leading slash to send " f"as a regular message." ) except Exception as e: logger.debug("Skill command check failed (non-fatal): %s", e) # Pending exec approvals are handled by /approve and /deny commands above. # No bare text matching — "yes" in normal conversation must not trigger # execution of a dangerous command. if not is_internal and await asyncio.to_thread( self._is_telegram_topic_root_lobby, source ): # Debounce the lobby reminder so a user who forgets about # topic mode and fires ten prompts doesn't get ten copies. if self._should_send_telegram_lobby_reminder(source): return self._telegram_topic_root_lobby_message() return None # ── External-drain new-turn gate (Phase 2) ──────────────────── # When NAS has engaged an external drain (.drain_request.json present, # observed by _drain_control_watcher), refuse to START a new turn so # the in-flight set can only fall to zero — eliminating the TOCTOU race # (D4a: stop accepting new turns FIRST, then NAS polls until # active_agents==0). In-flight turns are untouched; this only blocks the # claim of a NEW session slot. Internal/system events (restart-recovery # replays, background-process completions) bypass the gate — they are # not user-initiated new work and must still flow during a drain. # Reversible: once the marker is removed the gate opens again. if self._external_drain_active and not is_internal: logger.info( "Refusing new turn for session %s — external drain active.", _quick_key, ) return ( "⏳ This agent is draining for a maintenance action and isn't " "accepting new turns right now. It'll be back in a moment — " "please resend shortly." ) # ── Claim this session before any await ─────────────────────── # Between here and _run_agent registering the real AIAgent, there # are numerous await points (hooks, vision enrichment, STT, # session hygiene compression). Without this sentinel a second # message arriving during any of those yields would pass the # "already running" guard and spin up a duplicate agent for the # same session — corrupting the transcript. _active_session_lease, _limit_message = self._claim_active_session_slot( _quick_key, source, ) if _limit_message is not None: logger.info( "Rejecting new active session %s: max_concurrent_sessions reached", _quick_key, ) return _limit_message # ── FIFO orphan rescue (#99882) ──────────────────────────────── # If this session went idle with a populated overflow (queued # during a busy window whose post-turn drain never promoted — # e.g. a compression-demoted follow-up after the compression # window ended through an exit that skipped the promotion site), # those events were silently orphaned. We are starting the next # turn for this session NOW: re-stage the orphans in FIFO order # and enqueue the incoming event behind them, so arrival order # (#28503) holds: oldest orphan runs as this turn, the rest drain # in order, the new message last. Skipped for control commands # (/stop etc. own their own semantics) and internal events. try: _orphan_adapter = self._adapter_for_source(source) if ( _orphan_adapter is not None and not bool(getattr(event, "internal", False)) and not event.get_command() ): _rescued = self._rescue_orphaned_overflow( _quick_key, _orphan_adapter ) if _rescued is not None: # The oldest orphan runs as THIS turn. Park the # incoming event behind the rest of the chain: into the # slot when the chain was a single orphan (so the # post-turn drain picks it up), otherwise into overflow # behind the already-staged next orphan (FIFO). self._enqueue_fifo(_quick_key, event, _orphan_adapter) event = _rescued # Same session key by construction; carry the orphan's # own source so reply anchors / thread metadata point # at the message that is actually being answered. _rescued_source = getattr(_rescued, "source", None) if _rescued_source is not None: source = _rescued_source is_internal = bool(getattr(_rescued, "internal", False)) except Exception: logger.debug( "FIFO orphan rescue pre-claim failed for %s", _quick_key, exc_info=True, ) _claim_state = self._session_state(_quick_key) if _active_session_lease is not None: _claim_state.turn.lease = _active_session_lease _claim_state.turn.agent = _AGENT_PENDING_SENTINEL _claim_state.turn.started_ts = time.time() self._persist_active_agents() _run_generation = self._begin_session_run_generation(_quick_key) try: try: _agent_result = await self._handle_message_with_agent( event, source, _quick_key, _run_generation ) except TurnLeaseTimeoutError as exc: # This is a rejected message, not a completed agent turn. Return # before the /goal judge below so it cannot consume the resend # notice and enqueue a synthetic continuation loop. logger.error( "Rejecting turn for routing key %s on session %s after " "turn-lease timeout; transcript load was not started and " "the user must resend", _quick_key, exc.session_id, ) return ( "⏳ Another turn is still running on this session. To " "protect the transcript, this message was not processed. " "Wait for the active turn to finish, then resend it." ) try: await self._run_post_turn_hooks( agent_result=_agent_result, source=source, is_internal=is_internal, event=event, ) except Exception as _goal_exc: logger.debug("post-turn hook failed: %s", _goal_exc) return _agent_result finally: # MoA one-shot restore must run on EVERY exit path, not just # success. The restore data lives on the per-turn event object # (_moa_restore_override), which is discarded once the event goes # out of scope — so if _handle_message_with_agent raises, a restore # in the try block would be skipped and the MoA override would leak # permanently (every later message silently fans out through MoA). # Putting it in finally guarantees the revert on success, exception, # and interrupt alike. self._restore_moa_one_shot(event, _quick_key) self._restore_pending_one_turn_model_override(_quick_key) # Normal completion/exception/interrupt owns and clears this exact # durable marker. SIGKILL/OOM skips finally, leaving the marker for # the next unclean startup's recovery pass. await self._clear_durable_active_turn(event) # Unconditional release covers every exit path. _release_running_agent_state # is idempotent (pop-on-absent is harmless) and, called without a # run_generation guard, always clears the slot regardless of which # generation it holds. This evicts the zombie left when session_reset # bumps the generation (N -> N+1) mid-flight: gen-N's guarded release # inside _run_agent returns False, and the old sentinel-only check here # missed the leftover real agent — locking the session out forever (#28686). self._release_running_agent_state(_quick_key) # Turn lease (#64934): release THIS turn's lease token — keyed by # (routing key, run generation) so this unwind can only ever free # the lease its own turn acquired, never a newer turn's. self._release_turn_lease(_quick_key, _run_generation) def _restore_moa_one_shot(self, event: "MessageEvent", quick_key: str) -> None: """Revert a ``/moa `` one-shot model override after its turn. Called from the ``finally`` of the message-handling path so the revert fires whether the turn succeeded, raised, or was interrupted. A no-op unless ``event._moa_disable_after_turn`` is set. ``_moa_restore_override`` carries the prior per-session override (``None`` means the user had no override, so the MoA override is cleared outright). """ if not getattr(event, "_moa_disable_after_turn", False): return try: _restore = getattr(event, "_moa_restore_override", None) self._session_state(quick_key).conversation.model_override = _restore self._evict_cached_agent(quick_key) except Exception: pass def _restore_pending_one_turn_model_override(self, session_key: str) -> None: """Restore a per-session model override after ``/model --once`` runs.""" if not session_key: return try: _otr_state = self._peek_session_state(session_key) snapshot = _otr_state.conversation.one_turn_restore if _otr_state else None if _otr_state is not None: _otr_state.conversation.one_turn_restore = None if not snapshot: return self._restore_session_model_override(session_key, snapshot) except Exception: logger.debug("Failed to restore one-turn model override", exc_info=True) async def _prepare_inbound_message_text( self, *, event: MessageEvent, source: SessionSource, history: List[Dict[str, Any]], session_key: Optional[str] = None, ) -> Optional[str]: """Prepare inbound event text for the agent. Keep the normal inbound path and the queued follow-up path on the same preprocessing pipeline so sender attribution, image enrichment, STT, document notes, reply context, and @ references all behave the same. Side effect: buffers per-session native image paths when the active model supports native vision AND the user has images attached. The caller consumes and clears that session-scoped buffer at the ``run_conversation`` site to build a multimodal user turn. When the list is empty, the ``_enrich_message_with_vision`` text path has already run and images are represented in-text. """ history = history or [] _pending_stt_prepared = hasattr(event, "_gateway_pending_stt_text") message_text = ( getattr(event, "_gateway_pending_stt_text", None) if _pending_stt_prepared else event.text ) or "" _group_sessions_per_user = getattr(self.config, "group_sessions_per_user", True) _thread_sessions_per_user = getattr(self.config, "thread_sessions_per_user", False) # Prefer the already resolved session key from the caller so this write # key matches the consume key at the run_conversation site. Fall back # to deriving it here for tests and legacy standalone callers. session_key = session_key or self._session_key_for_source(source) # Reset only this session's per-call buffer; other sessions may be # concurrently preparing multimodal turns on the same runner. self._consume_pending_native_image_paths(session_key) _is_shared_multi_user = is_shared_multi_user_session( source, group_sessions_per_user=_group_sessions_per_user, thread_sessions_per_user=_thread_sessions_per_user, ) if _is_shared_multi_user and source.user_name: # source.user_name is the platform display name — attacker- # influenceable on any platform that lets participants set their # own name. Neutralize embedded newlines/control chars before # interpolating it into every message in the shared session, or # a hostile name can masquerade as a fake markdown section # (mirrors the same field's treatment in # build_session_context_prompt via _format_untrusted_prompt_value). _safe_user_name = neutralize_untrusted_inline_text(source.user_name) # On Slack, expose the current author's verifiable user ID next to # the display name (#17916): "mention me again" requests need a # trusted `<@U...>` target for the CURRENT speaker — display names # are ambiguous and historical mentions may point at someone else. # The user_id comes from the Slack event envelope (not # user-editable text), so it does not need neutralization. if source.platform == Platform.SLACK and source.user_id: _safe_user_name = ( f"{_safe_user_name} | Slack user <@{source.user_id}>" ) message_text = f"[{_safe_user_name}] {message_text}" # Prepend channel context from history backfill (if any). This # happens after sender-prefix so the prefix only applies to the # trigger message, not the backfill block. if getattr(event, "channel_context", None): message_text = f"{event.channel_context}\n\n[New message]\n{message_text}" # Declare at outer scope so the audio-file-paths handling block below # remains safe when ``event.media_urls`` is empty (no inner block runs). audio_file_paths: list[str] = [] video_paths: list[str] = [] if event.media_urls: image_paths = [] audio_paths = [] for i, path in enumerate(event.media_urls): mtype = event.media_types[i] if i < len(event.media_types) else "" # Classify images per-attachment: trust this attachment's own # MIME, and only honour the message-level PHOTO type when the # per-attachment MIME is unknown. Otherwise a document (or any # non-image) sent alongside an image in the same message gets # mis-routed here as an image and the provider 400s. if _event_media_is_image(event, i): image_paths.append(path) # MessageType.AUDIO = audio file attachment (e.g. .mp3, .m4a) — never STT. # Mixed DOCUMENT events also preserve audio as a file path instead of # dropping it or treating it as a voice note. if _event_media_is_audio(event, i): if event.message_type in {MessageType.AUDIO, MessageType.DOCUMENT}: audio_file_paths.append(path) elif not _pending_stt_prepared and _event_media_is_stt_input(event, i): audio_paths.append(path) if mtype.startswith("video/") or (not mtype and event.message_type == MessageType.VIDEO): video_paths.append(path) if image_paths: # Decide routing: native (attach pixels) vs text (vision_analyze # pre-run + prepend description). See agent/image_routing.py. # Offload to a worker thread: the decision does blocking network # I/O — a models.dev fetch on cache miss, and the Ollama # ``/api/show`` capability probe for local servers — whose # request timeout would otherwise stall the whole gateway event # loop (every session) while a single image is routed. _img_mode = await asyncio.to_thread( self._decide_image_input_mode, source=source, session_key=session_key, ) if _img_mode == "native": # Defer attachment to the run_conversation call site. self._session_state( session_key ).persistent.native_image_paths = list(image_paths) logger.info( "Image routing: native (model supports vision). %d image(s) will be attached inline.", len(image_paths), ) else: logger.info( "Image routing: text (mode=%s). Pre-analyzing %d image(s) via vision_analyze.", _img_mode, len(image_paths), ) # Vision enrichment runs before AIAgent.run_conversation(), # so bind this session's resolved runtime explicitly rather # than consulting process-global compatibility mirrors. vision_runtime = None try: turn_model, runtime_kwargs = self._resolve_session_agent_runtime( source=source, session_key=session_key, ) vision_runtime = dict(runtime_kwargs or {}) vision_runtime["model"] = turn_model except Exception: logger.debug( "vision enrichment: session runtime resolution failed", exc_info=True, ) from agent.auxiliary_client import scoped_runtime_main with scoped_runtime_main(vision_runtime): message_text = await self._enrich_message_with_vision( message_text, image_paths, ) if audio_paths: message_text, _successful_transcripts = await self._enrich_message_with_transcription( message_text, audio_paths, ) # Echo each successful transcript back to the user immediately # when configured. Lets users verify STT quality in real-time, # while allowing quiet STT for users who only want the agent to # receive the transcription. if _successful_transcripts and self._should_echo_stt_transcripts(): _echo_adapter = self._adapter_for_source(source) _echo_meta = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event)) if _echo_adapter: for _tx in _successful_transcripts: try: await _echo_adapter.send( source.chat_id, f'🎙️ "{_tx}"', metadata=_echo_meta, ) except Exception as _echo_exc: logger.debug( "Transcript echo failed (non-fatal): %s", _echo_exc, ) # NOTE: Previously, when transcription failed (e.g. no STT # provider configured), the gateway also emitted a hardcoded # English notice via `_stt_adapter.send()`. That bypassed the # LLM and produced two replies — one pre-canned English clip # (which TTS then spoke aloud, in the wrong language) and one # correct, localized LLM reply from the enriched message text. # The enrichment step now leaves a single neutral marker in the # prompt, so the LLM produces one coherent reply in the user's # language. The hardcoded send has therefore been removed. if audio_file_paths: from tools.credential_files import to_agent_visible_cache_path as _to_agent_path for _apath in audio_file_paths: _basename = os.path.basename(_apath) _parts = _basename.split("_", 2) _display = _parts[2] if len(_parts) >= 3 else _basename _display = re.sub(r'[^\w.\- ]', '_', _display) _agent_path = _to_agent_path(_apath) _note = ( f"[The user sent an audio file attachment: '{_display}'. " f"It is saved at: {_agent_path}. " f"Its content is not inlined here. If the user's request involves " f"what the audio contains, transcribe or process it yourself — for " f"example by passing the path to a transcription or media tool — " f"instead of asking the user to describe it. Only ask what to do " f"with it if their intent is genuinely unclear.]" ) message_text = f"{_note}\n\n{message_text}" if video_paths: from tools.credential_files import to_agent_visible_cache_path as _to_agent_path for _vpath in video_paths: _basename = os.path.basename(_vpath) _parts = _basename.split("_", 2) _display = _parts[2] if len(_parts) >= 3 else _basename _display = re.sub(r'[^\w.\- ]', '_', _display) _agent_path = _to_agent_path(_vpath) _note = ( f"[The user sent a video attachment: '{_display}'. " f"It is saved at: {_agent_path}. " f"Its content is not inlined here. If the user's request involves " f"what the video contains, inspect or process it yourself — for " f"example by passing the path to a video analysis or media tool — " f"instead of asking the user to describe it. Only ask what to do " f"with it if their intent is genuinely unclear.]" ) message_text = f"{_note}\n\n{message_text}" if event.media_urls: import mimetypes as _mimetypes from tools.credential_files import to_agent_visible_cache_path _TEXT_EXTENSIONS = {".txt", ".md", ".csv", ".log", ".json", ".xml", ".yaml", ".yml", ".toml", ".ini", ".cfg"} for i, path in enumerate(event.media_urls): # Per-attachment document handling. Skip anything already routed # as image / audio / video by the buckets above — only genuine # non-media files get a path-pointing context note. This makes a # document mixed into a PHOTO/VOICE message (whole-message type # != DOCUMENT) still reach the agent as a readable cached file, # instead of being silently dropped because the message-level # type wasn't DOCUMENT. if ( _event_media_is_image(event, i) or _event_media_is_audio(event, i) or _event_media_is_video(event, i) ): continue mtype = event.media_types[i] if i < len(event.media_types) else "" if mtype in {"", "application/octet-stream"}: _ext = os.path.splitext(path)[1].lower() if _ext in _TEXT_EXTENSIONS: mtype = "text/plain" else: guessed, _ = _mimetypes.guess_type(path) if guessed: mtype = guessed else: mtype = "application/octet-stream" # Any accepted file gets a path-pointing context note — we accept # all file types now, so a non-text/non-application MIME (font/*, # model/*, etc.) must still tell the agent the file exists. basename = os.path.basename(path) parts = basename.split("_", 2) display_name = parts[2] if len(parts) >= 3 else basename display_name = re.sub(r'[^\w.\- ]', '_', display_name) # Translate host cache path to in-container path if running under Docker backend. # This ensures the agent receives a path it can open inside its sandbox, as the # cache directories are auto-mounted at /root/.hermes/cache/* by get_cache_directory_mounts(). agent_path = to_agent_visible_cache_path(path) inline_flags = getattr(event, "media_text_inlined", None) or [] inline_flag = inline_flags[i] if i < len(inline_flags) else None context_note = _build_document_context_note( display_name, agent_path, mtype, content_inlined=inline_flag is not False, ) message_text = f"{context_note}\n\n{message_text}" # Discord: surface the triggering message id per-turn on the user # message rather than in the cached system prompt. message_id changes # every turn, so baking it into build_session_context_prompt() would # bust the agent-cache signature and rebuild the AIAgent every message # (destroying prompt caching). The static IDs block points the agent # here; the volatile id rides the per-turn user content. if ( source is not None and getattr(source, "platform", None) == Platform.DISCORD and getattr(event, "message_id", None) ): from gateway.session import _discord_tools_loaded as _disc_tools_loaded if _disc_tools_loaded(): message_text = ( f"[Triggering message id: `{event.message_id}` — use as " f"`message_id` for reply/react/pin via the discord tools.]\n\n" f"{message_text}" ) if getattr(event, "reply_to_text", None) and event.reply_to_message_id: # Always inject the reply-to pointer — even when the quoted text # already appears in history. The prefix isn't deduplication, it's # disambiguation: it tells the agent *which* prior message the user # is referencing. History can contain the same or similar text # multiple times, and without an explicit pointer the agent has to # guess (or answer for both subjects). Token overhead is minimal. reply_snippet = event.reply_to_text[:500] if getattr(event, "reply_to_is_own_message", False): message_text = ( f'[Replying to your previous message: "{reply_snippet}"]\n\n' f"{message_text}" ) else: message_text = f'[Replying to: "{reply_snippet}"]\n\n{message_text}' if "@" in message_text: try: from agent.context_references import preprocess_context_references_async from agent.model_metadata import get_model_context_length_async try: from tools.terminal_scope import terminal_env as _ts_env except ImportError: _msg_cwd = os.environ.get("TERMINAL_CWD", os.path.expanduser("~")) else: _msg_cwd = _ts_env("TERMINAL_CWD", os.path.expanduser("~")) _msg_config_ctx = None _msg_cfg = None _msg_model_cfg = {} _msg_custom_providers = [] try: _msg_cfg = _load_gateway_config() _msg_model_cfg = _msg_cfg.get("model", {}) if isinstance(_msg_model_cfg, dict): _msg_raw_ctx = _msg_model_cfg.get("context_length") if _msg_raw_ctx is not None: _msg_config_ctx = int(_msg_raw_ctx) try: from hermes_cli.config import get_compatible_custom_providers _msg_custom_providers = get_compatible_custom_providers(_msg_cfg) except Exception: _msg_custom_providers = _msg_cfg.get("custom_providers") or [] except Exception: pass # Resolve the session's actual model/provider/base_url the # same way the hygiene compression block does (~11080). # GatewayRunner has no self._model/self._base_url attrs # (that was copy-pasted from HermesCLI, which does carry # self.model/self.base_url), so using them here always raised # AttributeError, silently caught below, meaning this feature # never ran. _msg_model, _msg_runtime = self._resolve_session_agent_runtime( source=source, session_key=session_key, user_config=_msg_cfg, ) _msg_base_url = _msg_runtime.get("base_url") or "" # A global model.context_length belongs to the configured # model, not a session /model or channel override. Prefer a # matching per-custom-provider model limit when available. _msg_configured_model = ( _msg_model_cfg.get("default") or _msg_model_cfg.get("model") if isinstance(_msg_model_cfg, dict) else _msg_model_cfg ) if _msg_model != _msg_configured_model: _msg_config_ctx = None if _msg_config_ctx is not None and isinstance(_msg_model_cfg, dict): try: from hermes_cli.route_identity import should_clear_context_pin_async if await should_clear_context_pin_async( None, # model match already checked above None, _msg_model_cfg.get("base_url"), _msg_base_url, _msg_model_cfg.get("provider"), _msg_runtime.get("provider"), ): _msg_config_ctx = None except Exception: _msg_config_ctx = None if _msg_custom_providers and _msg_base_url: try: from hermes_cli.config import get_custom_provider_context_length _msg_custom_ctx = get_custom_provider_context_length( model=_msg_model, base_url=_msg_base_url, custom_providers=_msg_custom_providers, ) if _msg_custom_ctx: _msg_config_ctx = _msg_custom_ctx except Exception: pass _msg_ctx_len = await get_model_context_length_async( _msg_model, base_url=_msg_base_url, api_key=_msg_runtime.get("api_key") or "", config_context_length=_msg_config_ctx, provider=_msg_runtime.get("provider") or "", custom_providers=_msg_custom_providers, ) _ctx_result = await preprocess_context_references_async( message_text, cwd=_msg_cwd, context_length=_msg_ctx_len, allowed_root=_msg_cwd, ) if _ctx_result.blocked: _adapter = self._adapter_for_source(source) if _adapter: await _adapter.send( source.chat_id, "\n".join(_ctx_result.warnings) or "Context injection refused.", ) return None if _ctx_result.expanded: message_text = _ctx_result.message except Exception as exc: logger.warning("@ context reference expansion failed: %s", exc) logger.debug("@ context reference expansion failure detail", exc_info=True) return message_text async def _prepare_profile_scoped_inbound_message_text( self, *, event: MessageEvent, source: SessionSource, history: List[Dict[str, Any]], session_key: Optional[str] = None, ) -> Optional[str]: """Run inbound preprocessing under the routed profile when multiplexed.""" if getattr(getattr(self, "config", None), "multiplex_profiles", False): async with _async_profile_runtime_scope( self._resolve_profile_home_for_source(source) ): return await self._prepare_inbound_message_text( event=event, source=source, history=history, session_key=session_key, ) return await self._prepare_inbound_message_text( event=event, source=source, history=history, session_key=session_key, ) async def _prepare_clarify_reply_text(self, event) -> str: """Return raw text or successful voice transcripts for a clarify reply.""" if not self._pending_event_audio_paths(event): return (event.text or "").strip() _, successful_transcripts = await self._transcribe_pending_audio_event_once( event, "", ) return "\n\n".join( transcript.strip() for transcript in successful_transcripts if transcript.strip() ) def _consume_pending_native_image_paths(self, session_key: str) -> List[str]: state = self._peek_session_state(session_key) if state is None or not state.persistent.native_image_paths: return [] paths = list(state.persistent.native_image_paths) state.persistent.native_image_paths = [] return paths def _cache_session_source(self, session_key: str, source) -> None: if not session_key or source is None: return cached_sources = getattr(self, "_session_sources", None) if cached_sources is None: cached_sources = OrderedDict() self._session_sources = cached_sources try: cached_sources[session_key] = dataclasses.replace(source) except Exception: logger.debug("Failed to cache live session source for %s", session_key, exc_info=True) return # LRU: mark as most-recently-used and trim to max size. try: cached_sources.move_to_end(session_key) max_size = getattr(self, "_session_sources_max", 512) while len(cached_sources) > max_size: cached_sources.popitem(last=False) except Exception: pass @property def async_session_store(self) -> AsyncSessionStore: """Return the single async facade for this runner's SessionStore.""" facade = getattr(self, "_async_session_store", None) if facade is None or facade._store is not self.session_store: facade = AsyncSessionStore(self.session_store) self._async_session_store = facade return facade async def _mark_durable_active_turn( self, event: "MessageEvent", session_key: str, ) -> bool: """Persist the exact resolved routing key for this running turn.""" try: token = await self.async_session_store.mark_turn_active(session_key) except Exception as exc: logger.warning( "Could not persist active-turn marker for %s: %s", session_key, exc, ) return False if not token: return False # Private event attributes are process-local ownership state. Keep the # token out of public metadata, transcripts, and platform payloads. setattr(event, "_gateway_active_turn_session_key", session_key) setattr(event, "_gateway_active_turn_token", token) return True async def _clear_durable_active_turn(self, event: "MessageEvent") -> bool: """Best-effort CAS clear of the marker owned by *event*.""" session_key = getattr(event, "_gateway_active_turn_session_key", None) token = getattr(event, "_gateway_active_turn_token", None) try: if not session_key or not token: return False last_error: Optional[Exception] = None for attempt in range(1, 4): try: return bool( await self.async_session_store.clear_turn_active( session_key, token ) ) except Exception as exc: last_error = exc if attempt < 3: logger.debug( "Retrying active-turn marker cleanup for %s (%d/3): %s", session_key, attempt, exc, ) # Never let marker cleanup block in-memory agent/lease release. A # stale marker is bounded by the configured agent timeout and the # clean-start orphan-marker discard path. logger.warning( "Could not clear active-turn marker for %s after 3 attempts: %s", session_key, last_error, ) return False finally: for attr in ( "_gateway_active_turn_session_key", "_gateway_active_turn_token", ): try: delattr(event, attr) except AttributeError: pass def _install_plugin_message_injector(self) -> None: """Publish this live gateway's plugin message scheduler.""" from hermes_cli.plugins import get_plugin_manager get_plugin_manager().set_gateway_message_injector( self, self._schedule_plugin_message_injection, ) def _clear_plugin_message_injector(self) -> None: """Remove this runner's scheduler without clobbering a newer owner.""" from hermes_cli.plugins import get_plugin_manager get_plugin_manager().clear_gateway_message_injector(self) def _schedule_plugin_message_injection( self, *, session_key: str, content: str, plugin_id: str, ) -> bool: """Schedule a plugin-triggered turn on the live gateway loop.""" loop = getattr(self, "_gateway_loop", None) if not getattr(self, "_running", False) or loop is None or loop.is_closed(): return False coro = self._dispatch_plugin_message_injection( session_key=session_key, content=content, plugin_id=plugin_id, ) try: current_loop = asyncio.get_running_loop() except RuntimeError: current_loop = None if current_loop is loop: try: future = loop.create_task(coro) except Exception: coro.close() logger.warning( "Plugin message injection scheduling failed", exc_info=True, ) return False self._background_tasks.add(future) future.add_done_callback(self._background_tasks.discard) else: future = safe_schedule_threadsafe( coro, loop, logger=logger, log_message="Plugin message injection scheduling failed", log_level=logging.WARNING, ) if future is None: return False def _log_result(completed) -> None: try: accepted = completed.result() except (asyncio.CancelledError, concurrent.futures.CancelledError): return except Exception: logger.warning( "Plugin message injection failed: plugin=%s session=%s", plugin_id, session_key, exc_info=True, ) return if not accepted: logger.warning( "Plugin message injection was not routed: plugin=%s session=%s", plugin_id, session_key, ) future.add_done_callback(_log_result) return True async def _dispatch_plugin_message_injection( self, *, session_key: str, content: str, plugin_id: str, ) -> bool: """Route a plugin-triggered turn through the session's live adapter.""" if not getattr(self, "_running", False) or getattr(self, "_draining", False): return False entry = await self.async_session_store.lookup_by_session_key(session_key) if entry is None or entry.origin is None: return False if not getattr(self, "_running", False) or getattr(self, "_draining", False): return False source = dataclasses.replace(entry.origin) try: if not self._is_user_authorized( source, allow_adapter_delegation=False, ): logger.warning( "Plugin message injection denied by current gateway authorization: " "plugin=%s session=%s", plugin_id, session_key, ) return False except Exception: logger.warning( "Plugin message injection authorization check failed: " "plugin=%s session=%s", plugin_id, session_key, exc_info=True, ) return False adapter = self._adapter_for_source(source) if adapter is None: return False event = MessageEvent( text=content, message_type=MessageType.TEXT, source=source, internal=True, allow_gateway_control=False, metadata={ "hermes_plugin_id": plugin_id, "hermes_plugin_injection": True, "gateway_session_key": session_key, "gateway_session_id": entry.session_id, "gateway_session_strict": True, }, ) await adapter.handle_message(event) logger.info( "Plugin message injection dispatched: plugin=%s session=%s session_id=%s", plugin_id, session_key, entry.session_id, ) return True def _get_cached_session_source(self, session_key: str): if not session_key: return None cached_sources = getattr(self, "_session_sources", None) if not cached_sources: return None source = cached_sources.get(session_key) if source is not None: try: cached_sources.move_to_end(session_key) except Exception: pass return source async def _handle_message_with_agent(self, event, source, _quick_key: str, run_generation: int): """Inner handler that runs under the _running_agents sentinel guard.""" _msg_start_time = time.time() _platform_name = source.platform.value if hasattr(source.platform, "value") else str(source.platform) _msg_preview = (event.text or "")[:80].replace("\n", " ") _reply_id = getattr(event, "reply_to_message_id", None) _reply_txt = (getattr(event, "reply_to_text", None) or "")[:80].replace("\n", " ") logger.info( "inbound message: platform=%s user=%s chat=%s msg=%r reply_to_id=%s reply_to_text=%r", _platform_name, source.user_name or source.user_id or "unknown", source.chat_id or "unknown", _msg_preview, _reply_id, _reply_txt, ) # Get or create session # Topic-mode DMs: rewrite a stale/foreign thread_id to the user's # last-active topic so a cross-topic Reply or stripped plain reply # doesn't fragment the conversation across sessions. recovered = await asyncio.to_thread(self._recover_telegram_topic_thread_id, source) if recovered is not None: logger.info( "telegram topic recovery: chat=%s user=%s %r -> %s", source.chat_id, source.user_id, source.thread_id, recovered, ) source = dataclasses.replace(source, thread_id=recovered) try: event.source = source except Exception: pass event_metadata = getattr(event, "metadata", None) or {} expected_session_key = str( event_metadata.get("gateway_session_key") or "" ).strip() if expected_session_key: derived_session_key = self._session_key_for_source(source) if derived_session_key != expected_session_key: logger.warning( "Dropping internally routed event after route recovery: " "expected session=%s derived=%s", expected_session_key, derived_session_key, ) return strict_session = bool(event_metadata.get("gateway_session_strict")) pinned_session_id = str(event_metadata.get("gateway_session_id") or "").strip() if strict_session: session_entry = await self.async_session_store.lookup_by_session_key( expected_session_key ) if ( session_entry is None or not pinned_session_id or session_entry.session_id != pinned_session_id ): logger.warning( "Dropping internally routed event: expected session id=%s is no " "longer current for key=%s", pinned_session_id or "missing", expected_session_key or "missing", ) return else: # Internal wakes must observe reset policy without becoming user # activity themselves. Otherwise periodic Kanban/process # notifications keep the stable routing key alive across every # daily/idle boundary. session_entry = await self.async_session_store.get_or_create_session( source, touch_activity=not bool(getattr(event, "internal", False)), ) session_key = session_entry.session_key if not strict_session and pinned_session_id: resolved_entry = await self._resolve_async_delegation_session( session_entry, pinned_session_id, ) if resolved_entry is None: return session_entry = resolved_entry self._cache_session_source(session_key, source) if await asyncio.to_thread(self._is_telegram_topic_lane, source): try: binding = (await self._session_db.get_telegram_topic_binding( chat_id=str(source.chat_id), thread_id=str(source.thread_id), profile_name=self._telegram_topic_profile_name(source), )) if self._session_db else None except Exception: logger.debug("Failed to read Telegram topic binding", exc_info=True) binding = None if binding: bound_session_id = str(binding.get("session_id") or "") # Heal bindings that point at a pre-compression parent: walk # the compression-continuation chain forward to its tip so the # next message resumes the compressed child instead of # reloading the oversized parent transcript (#20470/#29712/ # #33414). Returns the input unchanged when the session isn't # a compression parent, so this is cheap and safe. if bound_session_id and self._session_db is not None: try: canonical_session_id = await self._session_db.get_compression_tip( bound_session_id, ) except Exception: logger.debug( "compression-tip lookup failed for %s", bound_session_id, exc_info=True, ) canonical_session_id = bound_session_id if ( canonical_session_id and canonical_session_id != bound_session_id ): bound_session_id = canonical_session_id if bound_session_id and bound_session_id != session_entry.session_id: # Route the override through SessionStore so the session_key # → session_id mapping is persisted to disk and the previous # lane session is ended cleanly. Mutating session_entry in # place here created a split-brain state where the JSON # index pointed at one id but code downstream used another. switched = await self.async_session_store.switch_session(session_key, bound_session_id) if switched is not None: session_entry = switched # If the stored binding pointed at a parent, rewrite it to the # canonical descendant now that we've followed the chain. if ( bound_session_id and bound_session_id != str(binding.get("session_id") or "") ): await asyncio.to_thread( self._sync_telegram_topic_binding, source, session_entry, reason="compression-tip-walk", ) else: try: await asyncio.to_thread(self._record_telegram_topic_binding, source, session_entry) except Exception: logger.debug("Failed to record Telegram topic binding", exc_info=True) # Capture and immediately consume was_auto_reset so it does not # re-fire on subsequent messages — preventing the cleanup from # wiping model/reasoning overrides set between turns (Closes #48031). _was_auto_reset = getattr(session_entry, "was_auto_reset", False) if _was_auto_reset: # Treat auto-reset as a full conversation boundary — clear every # conversation-scoped per-session dict in one funnel call so the # fresh session does not inherit the previous conversation's # model/reasoning overrides, a queued "/model switched" note, or # a stale resolved-model cache (#48031, #58403). See # _CONVERSATION_SCOPED_STATE. self._clear_conversation_scope(session_key, reason="auto_reset") # Evict the cached agent so the fresh session does not inherit the # previous conversation's context_compressor._previous_summary — # the cache is keyed on the stable session_key, so an auto-reset # otherwise reuses the old agent and leaks prior history into new # compaction summaries. Mirrors /reset and the compression-exhausted # path (#9893). Covers daily/idle/suspended auto-reset. self._evict_cached_agent(session_key) session_entry.was_auto_reset = False # Emit session:start for new or auto-reset sessions _is_new_session = ( session_entry.created_at == session_entry.updated_at or _was_auto_reset or getattr(session_entry, "is_fresh_reset", False) ) # Consume the is_fresh_reset flag immediately so it doesn't leak # onto subsequent messages in the same session (issue #6508). if getattr(session_entry, "is_fresh_reset", False): session_entry.is_fresh_reset = False if _is_new_session: await self.hooks.emit("session:start", { "platform": source.platform.value if source.platform else "", "user_id": source.user_id, "session_id": session_entry.session_id, "session_key": session_key, }) # Build session context context = build_session_context(source, self.config, session_entry) # Set session context variables for tools (task-local, concurrency-safe) _session_env_tokens = self._set_session_env(context) # Read privacy.redact_pii from config (re-read per message) _redact_pii = False persist_user_message = None persist_user_timestamp = None # Synthetic self-injected turns (async-delegation batch completions, # background watch notifications, resume wake-ups) arrive as # MessageEvent(internal=True). Persist their user row typed with # display_kind="internal_notification" so transcripts/UIs can render # them as timeline notices instead of user bubbles (#82888). Role and # content are untouched — display_kind is a DB-only sidecar stripped # from every provider-bound payload (see conversation_loop's # api_msg.pop("display_kind")). persist_user_display_kind = ( "internal_notification" if getattr(event, "internal", False) else None ) try: _pcfg = _load_gateway_config() _redact_pii = bool((_pcfg.get("privacy") or {}).get("redact_pii", False)) except Exception: pass # Build the context prompt to inject. The render is pinned per # session, keyed by a hash of the exact renderer inputs # (_ephemeral_change_key). A key hit reuses the pinned bytes verbatim # so the composed system prompt cannot drift turn-over-turn; a key # miss (thread rename, /sethome, redact_pii flip, ...) re-renders # once — the only legitimate cache busts. context_prompt = self._pinned_session_context_prompt( context, _redact_pii, session_key ) # Per-turn must-deliver notes. These used to be appended to # context_prompt (the ephemeral system prompt), which guaranteed a # turn1→turn2 system-prompt diff and a full agent rebuild. They now # ride the current user message via the api_content sidecar instead # (staged below, consumed in run_sync → build_turn_context). turn_sidecar_notes: List[str] = [] # If the previous session expired and was auto-reset, deliver a notice # so the agent knows this is a fresh conversation (not an intentional /reset). if _was_auto_reset: reset_reason = getattr(session_entry, 'auto_reset_reason', None) or 'idle' if reset_reason == "suspended": context_note = "[System note: The user's previous session was stopped and suspended. This is a fresh conversation with no prior context.]" elif reset_reason == "daily": context_note = "[System note: The user's session was automatically reset by the daily schedule. This is a fresh conversation with no prior context.]" elif reset_reason == "resume_pending_expired": context_note = "[System note: The previous gateway session could not be recovered after a restart (API recovery timed out). This is a fresh conversation — use /resume to restore history if needed.]" else: context_note = "[System note: The user's previous session expired due to inactivity. This is a fresh conversation with no prior context.]" # Slack/Discord channels/threads are long-lived: point the agent at # the specific prior same-channel session so it recalls that context # via session_search instead of an unrelated recent session. Returns # None (appends nothing) for other platforms or when there's no prior # activity to recall. Deterministic — no extra API/DB calls (#36220). try: continuity_note = build_channel_continuity_note(session_entry, source) except Exception: continuity_note = None if continuity_note: context_note = context_note + "\n\n" + continuity_note turn_sidecar_notes.append(context_note) # Send a user-facing notification explaining the reset, unless: # - notifications are disabled in config # - the platform is excluded (e.g. api_server, webhook) # - the expired session had no activity (nothing was cleared) try: policy = self.session_store.config.get_reset_policy( platform=source.platform, session_type=getattr(source, 'chat_type', 'dm'), ) platform_name = source.platform.value if source.platform else "" had_activity = getattr(session_entry, 'reset_had_activity', False) # Suspended and restart-recovery-expired sessions always notify # regardless of policy.notify — the user had an active session # that was silently replaced, so they need to know they can # /resume it. Idle/daily resets respect the policy flag. should_notify = reset_reason in {"suspended", "resume_pending_expired"} or ( policy.notify and had_activity and platform_name not in policy.notify_exclude_platforms ) if should_notify: adapter = self._adapter_for_source(source) if adapter: if reset_reason == "suspended": reason_text = "previous session was stopped or interrupted" elif reset_reason == "resume_pending_expired": reason_text = "gateway restart recovery timed out" elif reset_reason == "daily": reason_text = f"daily schedule at {policy.at_hour}:00" else: hours = policy.idle_minutes // 60 mins = policy.idle_minutes % 60 duration = f"{hours}h" if not mins else f"{hours}h {mins}m" if hours else f"{mins}m" reason_text = f"inactive for {duration}" notice = ( f"◐ Session automatically reset ({reason_text}). " f"Conversation history cleared.\n" f"Use /resume to browse and restore a previous session.\n" f"Adjust reset timing in config.yaml under session_reset." ) try: session_info = await asyncio.to_thread( self._reset_notice_session_info, source ) if session_info: notice = f"{notice}\n\n{session_info}" except Exception: pass await adapter.send( source.chat_id, notice, metadata=self._thread_metadata_for_source(source), ) except Exception as e: logger.debug("Auto-reset notification failed (non-fatal): %s", e) # was_auto_reset is already consumed in the cleanup block above # (single source of truth); only the reset reason needs clearing here. session_entry.auto_reset_reason = None # Auto-load skill(s) for topic/channel bindings (Telegram DM Topics, # Discord channel_skill_bindings). Supports a single name or ordered list. # Only inject on NEW sessions — ongoing conversations already have the # skill content in their conversation history from the first message. _auto = getattr(event, "auto_skill", None) if _is_new_session and _auto: _skill_names = [_auto] if isinstance(_auto, str) else list(_auto) try: from agent.skill_commands import _load_skill_payload, _build_skill_message _combined_parts: list[str] = [] _loaded_names: list[str] = [] for _sname in _skill_names: _loaded = _load_skill_payload(_sname, task_id=_quick_key) if _loaded: _loaded_skill, _skill_dir, _display_name = _loaded _note = ( f'[IMPORTANT: The "{_display_name}" skill is auto-loaded. ' f"Follow its instructions for this session.]" ) _part = _build_skill_message(_loaded_skill, _skill_dir, _note) if _part: _combined_parts.append(_part) _loaded_names.append(_sname) else: logger.warning("[Gateway] Auto-skill '%s' not found", _sname) if _combined_parts: # Append the user's original text after all skill payloads _combined_parts.append(event.text) event.text = "\n\n".join(_combined_parts) logger.info( "[Gateway] Auto-loaded skill(s) %s for session %s", _loaded_names, session_key, ) except Exception as e: logger.warning("[Gateway] Failed to auto-load skill(s) %s: %s", _skill_names, e) # ── Turn lease (#64934) ──────────────────────────────────────── # Session resolution is FINAL here (get_or_create → async-delegation # pinning → topic tip-walk switch_session are all above). Serialize # the [load history → run → flush] region per resolved SESSION_ID: # when a second routing key is mapped to this same session_id, its # turn waits here for the previous turn's flush instead of loading a # stale history base and interleaving transcript writes. Same-key # messages never reach this point mid-turn (adapter + runner guards # hold them), so the lock is uncontended outside the alias-key route. # Fail-closed on timeout: never enter the transcript region without a # lease. Outer dispatch returns a bounded rejection/resend notice rather # than recreating the exact concurrent-turn corruption this lease exists # to prevent. Released in _handle_message's finally via # _release_turn_lease — granted per (routing key, run generation) so a # stale unwind can't release a newer turn's lease. _lease_registry = getattr(self, "_turn_leases", None) if _lease_registry is not None: try: _lease_token = await _lease_registry.acquire( session_entry.session_id, owner_key=_quick_key, generation=run_generation, timeout=_float_env( "HERMES_TURN_LEASE_TIMEOUT", DEFAULT_LEASE_WAIT ), ) except TurnLeaseTimeoutError: # The broad session-context cleanup finally starts later in this # method. Restore the tokens here before propagating the rejection # to outer dispatch, or this early exit leaks task-local identity. self._clear_session_env(_session_env_tokens) raise if _lease_token is not None: _lease_state = self._session_state(_quick_key).turn _lease_state.lease_token = _lease_token _lease_state.lease_generation = run_generation # A turn only becomes durable recovery work after it owns (or has # explicitly degraded past) the per-session lease. Marking before the # await above would falsely recover an alias-routed message that never # began processing if the gateway died while it was still waiting. await self._mark_durable_active_turn(event, session_entry.session_key) # Load conversation history from transcript. An unreadable canonical # store is not an empty conversation: stop before the agent can invent # continuity from a plausible-looking []. This return happens before # the broad cleanup finally below, so restore task-local context here; # the outer dispatch still clears the durable marker and turn lease. try: history = await self.async_session_store.load_transcript( session_entry.session_id ) except TranscriptReadError: self._clear_session_env(_session_env_tokens) return ( "⚠️ This session's history is temporarily unavailable, so " "this message was not processed. Ask the operator to inspect " "state.db, then resend after it is healthy. Use /reset only " "if you intentionally want to start a new conversation." ) # ----------------------------------------------------------------- # Session hygiene: auto-compress pathologically large transcripts # # Long-lived gateway sessions can accumulate enough history that # every new message rehydrates an oversized transcript, causing # repeated truncation/context failures. Detect this early and # compress proactively — before the agent even starts. (#628) # # Token source priority: # 1. Actual API-reported prompt_tokens from the last turn # (stored in session_entry.last_prompt_tokens) # 2. Rough char-based estimate (str(msg)//4). Overestimates # by 30-50% on code/JSON-heavy sessions, but that just # means hygiene fires a bit early — safe and harmless. # ----------------------------------------------------------------- if history and len(history) >= 4: from agent.model_metadata import ( estimate_messages_tokens_rough, get_model_context_length_async, ) # Read model + compression config from config.yaml. # NOTE: hygiene threshold is intentionally HIGHER than the agent's # own compressor (0.85 vs 0.50). Hygiene is a safety net for # sessions that grew too large between turns — it fires pre-agent # to prevent API failures. The agent's own compressor handles # normal context management during its tool loop with accurate # real token counts. Having hygiene at 0.50 caused premature # compression on every turn in long gateway sessions. _hyg_model = "anthropic/claude-sonnet-4.6" _hyg_threshold_pct = 0.85 _hyg_compression_enabled = True _hyg_hard_msg_limit = 5000 _hyg_timeout_seconds = 30.0 _hyg_total_ceiling_seconds = 600.0 # Max wall-clock the user's TURN is held waiting on hygiene # compression before the gateway stops waiting and proceeds on the # uncompressed transcript (#TKT-0029). The compressor keeps running # detached; its commit is fenced (revoke_commit_admission) so a # stale result can never clobber turns appended after the wait was # abandoned. Capped well below typical transport idle-timeouts # (Telegram ~30s) so the wire never goes silent long enough to sever. _hyg_max_turn_hold_seconds = 10.0 _hyg_failure_cooldown_seconds = 300.0 _hyg_config_context_length = None _hyg_provider = None _hyg_base_url = None _hyg_api_key = None _hyg_configured_model = None _hyg_configured_provider = None _hyg_configured_base_url = None _hyg_data = {} try: _hyg_data = _load_gateway_config() if _hyg_data: # Resolve model name (same logic as run_sync) _model_cfg = _hyg_data.get("model", {}) if isinstance(_model_cfg, str): _hyg_model = _model_cfg elif isinstance(_model_cfg, dict): _hyg_model = _model_cfg.get("default") or _model_cfg.get("model") or _hyg_model # Read explicit context_length override from model config # (same as run_agent.py lines 995-1005) _raw_ctx = _model_cfg.get("context_length") if _raw_ctx is not None: try: _hyg_config_context_length = int(_raw_ctx) except (TypeError, ValueError): pass # Read provider for accurate context detection _hyg_provider = _model_cfg.get("provider") or None _hyg_base_url = _model_cfg.get("base_url") or None # Read compression settings — only use enabled flag. # The threshold is intentionally separate from the agent's # compression.threshold (hygiene runs higher). _comp_cfg = _hyg_data.get("compression", {}) if isinstance(_comp_cfg, dict): _hyg_compression_enabled = str( _comp_cfg.get("enabled", True) ).lower() in {"true", "1", "yes"} _raw_hard_limit = _comp_cfg.get("hygiene_hard_message_limit") if _raw_hard_limit is not None: try: _parsed = int(_raw_hard_limit) if _parsed > 0: _hyg_hard_msg_limit = _parsed except (TypeError, ValueError): pass _raw_timeout = _comp_cfg.get("hygiene_timeout_seconds") if _raw_timeout is not None: try: _parsed = float(_raw_timeout) if _parsed > 0: _hyg_timeout_seconds = _parsed except (TypeError, ValueError): pass _raw_ceiling = _comp_cfg.get("hygiene_total_ceiling_seconds") if _raw_ceiling is not None: try: _parsed = float(_raw_ceiling) if _parsed > 0: _hyg_total_ceiling_seconds = _parsed except (TypeError, ValueError): pass # The ceiling can never be tighter than one idle # window, or the extension loop would be dead code. _hyg_total_ceiling_seconds = max( _hyg_total_ceiling_seconds, _hyg_timeout_seconds, ) _raw_turn_hold = _comp_cfg.get("hygiene_max_turn_hold_seconds") if _raw_turn_hold is not None: try: _parsed = float(_raw_turn_hold) if _parsed > 0: _hyg_max_turn_hold_seconds = _parsed except (TypeError, ValueError): pass _raw_cooldown = _comp_cfg.get("hygiene_failure_cooldown_seconds") if _raw_cooldown is not None: try: _parsed = float(_raw_cooldown) if _parsed >= 0: _hyg_failure_cooldown_seconds = _parsed except (TypeError, ValueError): pass _hyg_configured_model = _hyg_model _hyg_configured_provider = _hyg_provider _hyg_configured_base_url = _hyg_base_url try: _hyg_model, _hyg_runtime = self._resolve_session_agent_runtime( source=source, session_key=session_key, user_config=_hyg_data if isinstance(_hyg_data, dict) else None, ) _hyg_provider = _hyg_runtime.get("provider") or _hyg_provider _hyg_base_url = _hyg_runtime.get("base_url") or _hyg_base_url _hyg_api_key = _hyg_runtime.get("api_key") or _hyg_api_key except Exception: pass if _hyg_config_context_length is not None: try: from hermes_cli.route_identity import should_clear_context_pin_async if await should_clear_context_pin_async( _hyg_configured_model, _hyg_model, _hyg_configured_base_url, _hyg_base_url, _hyg_configured_provider, _hyg_provider, ): _hyg_config_context_length = None except Exception: _hyg_config_context_length = None # Check custom_providers per-model context_length # (same fallback as run_agent.py lines 1171-1189). # Must run after runtime resolution so _hyg_base_url is set. if _hyg_config_context_length is None and _hyg_base_url: try: try: from hermes_cli.config import ( get_compatible_custom_providers as _gw_gcp, get_custom_provider_context_length as _gw_gccl, ) _hyg_custom_providers = _gw_gcp(_hyg_data) except Exception: _hyg_custom_providers = _hyg_data.get("custom_providers") if not isinstance(_hyg_custom_providers, list): _hyg_custom_providers = [] _hyg_custom_ctx = _gw_gccl( model=_hyg_model, base_url=_hyg_base_url, custom_providers=_hyg_custom_providers, ) if _hyg_custom_ctx: _hyg_config_context_length = int(_hyg_custom_ctx) except (TypeError, ValueError): pass except Exception: pass if _hyg_compression_enabled: _hyg_context_length = await get_model_context_length_async( _hyg_model, base_url=_hyg_base_url or "", api_key=_hyg_api_key or "", config_context_length=_hyg_config_context_length, provider=_hyg_provider or "", ) _compress_token_threshold = int( _hyg_context_length * _hyg_threshold_pct ) _warn_token_threshold = int(_hyg_context_length * 0.95) _msg_count = len(history) # Prefer actual API-reported tokens from the last turn # (stored in session entry) over the rough char-based estimate. _stored_tokens = session_entry.last_prompt_tokens if _stored_tokens > 0: _approx_tokens = _stored_tokens _token_source = "actual" else: _approx_tokens = estimate_messages_tokens_rough(history) _token_source = "estimated" # Note: rough estimates overestimate by 30-50% for code/JSON-heavy # sessions, but that just means hygiene fires a bit early — which # is safe and harmless. The 85% threshold already provides ample # headroom (agent's own compressor runs at 50%). A previous 1.4x # multiplier tried to compensate by inflating the threshold, but # 85% * 1.4 = 119% of context — which exceeds the model's limit # and prevented hygiene from ever firing for ~200K models (GLM-5). # Hard safety valve: force compression if message count is # extreme, regardless of token estimates. This breaks the # death spiral where API disconnects prevent token data # collection, which prevents compression, which causes more # disconnects. 5000 messages is far above any normal session # but catches truly runaway growth before it becomes # unrecoverable. Set well clear of legitimate large-context # (1M+) sessions doing thousands of short turns — those # compress on the token threshold, not this count-based floor. # Threshold is configurable via # compression.hygiene_hard_message_limit. # (#2153) _HARD_MSG_LIMIT = _hyg_hard_msg_limit _needs_compress = ( _approx_tokens >= _compress_token_threshold or _msg_count >= _HARD_MSG_LIMIT ) if _needs_compress: # Use the persistent DB-backed cooldown (same as the # in-conversation compression path in context_compressor.py) # so the cooldown survives gateway restarts. The in-memory # dict was reset on every restart, re-triggering the same # failing compression and wedging session storage (#74136). _session_db = getattr(self, "_session_db", None) if _session_db is not None: _session_db = getattr(_session_db, "_db", _session_db) _getter = getattr(_session_db, "get_compression_failure_cooldown", None) if _getter is not None: try: _cooldown_state = _getter(session_entry.session_id) except Exception: _cooldown_state = None if _cooldown_state and _cooldown_state.get("remaining_seconds", 0) > 0: logger.info( "Session hygiene: skipping compression for %s; " "previous failure cooldown active for %.1fs", session_entry.session_id, _cooldown_state["remaining_seconds"], ) _needs_compress = False if _needs_compress and await self._session_has_compression_in_flight( session_key ): # A prior hygiene/agent compression still holds the # durable lock (typically a shielded worker left behind # by /stop or /restart). Starting another attempt waits # up to the 600s ceiling behind a commit that the fence # will refuse, while inbound messages demote to queue # (#96953). logger.info( "Session hygiene: skipping compression for %s; " "another compression is already in flight", session_entry.session_id, ) _needs_compress = False if _needs_compress: logger.info( "Session hygiene: %s messages, ~%s tokens (%s) — auto-compressing " "(threshold: %s%% of %s = %s tokens)", _msg_count, f"{_approx_tokens:,}", _token_source, int(_hyg_threshold_pct * 100), f"{_hyg_context_length:,}", f"{_compress_token_threshold:,}", ) _hyg_meta = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event)) try: from agent.conversation_compression import CompressionCommitFence from run_agent import AIAgent _hyg_model, _hyg_runtime = self._resolve_session_agent_runtime( source=source, session_key=session_key, user_config=_hyg_data if isinstance(_hyg_data, dict) else None, ) _hyg_api_mode = str( _hyg_runtime.get("api_mode") or "" ).lower() if _hyg_api_mode == "codex_app_server": # codex app-server runtime: the model's real # context is the app-server's server-side thread, # not the transcript mirror. The detached-agent # block below could only rewrite the mirror (a # guaranteed no-op for the thread) and its # finally-clause eviction would destroy the live # thread — the next turn then starts blank # (#73503). Route to the live cached agent's # thread/compact/start instead and KEEP it cached. _hyg_codex_auto = "native" _hyg_comp_cfg = ( _hyg_data.get("compression") if isinstance(_hyg_data, dict) else None ) if isinstance(_hyg_comp_cfg, dict): _hyg_codex_auto = str( _hyg_comp_cfg.get( "codex_app_server_auto", "native" ) or "native" ) _hyg_codex_outcome = await run_codex_hygiene_compaction( self, session_key, session_entry.session_id, auto_mode=_hyg_codex_auto, history=history, approx_tokens=_approx_tokens, timeout_seconds=_hyg_total_ceiling_seconds, failure_cooldown_seconds=_hyg_failure_cooldown_seconds, ) logger.info( "Session hygiene (codex app-server): %s " "(session=%s, mode=%s, ~%s tokens)", _hyg_codex_outcome, session_entry.session_id, _hyg_codex_auto, f"{_approx_tokens:,}", ) elif _hyg_runtime.get("api_key"): # Pass the FULL transcript (tool results included). # Filtering to user/assistant-only starved the # compressor: tool results are usually the bulk of # the context, _prune_old_tool_results never saw # them, and short filtered histories tripped the # protect-first/last early-return so nothing was # compressed at all (#3854). The agent loop passes # its full message list to _compress_context — the # gateway now matches. _hyg_msgs = [ m for m in history if m.get("role") in {"user", "assistant", "tool"} ] if len(_hyg_msgs) >= 4: try: _hyg_session_row = await self._session_db.get_session( session_entry.session_id ) except Exception as exc: _hyg_session_row = None logger.warning( "Session hygiene could not restore the system " "prompt for session %s: %s. Preserving an empty " "prompt so the live turn rebuilds it with its " "configured providers.", session_entry.session_id, exc, exc_info=True, ) _hyg_session_db = getattr(self._session_db, "_db", self._session_db) # Hygiene performs the same lossy rewrite as # normal compression. When the operator enabled # compression.checkpoint_required, the memory # provider must be loaded so the required # checkpoint is created before any transcript # mutation; otherwise keep the historical fast # path (no provider init, no best-effort hook) # for hygiene. from hermes_cli.config import load_config as _load_cfg from utils import is_truthy_value as _is_truthy _hyg_checkpoint_required = _is_truthy( ((_load_cfg() or {}).get("compression") or {}).get( "checkpoint_required" ), default=False, ) _hyg_agent = AIAgent( **_hyg_runtime, model=_hyg_model, max_iterations=4, quiet_mode=True, skip_memory=not _hyg_checkpoint_required, enabled_toolsets=["memory"], session_id=session_entry.session_id, session_db=_hyg_session_db, ) _seed_hygiene_system_prompt( _hyg_agent, _hyg_session_row, ) # If compression must rebuild instead of retaining # the cached prompt, make the persisted result # deliberately stale for every real gateway surface. _hyg_agent.platform = _GATEWAY_HYGIENE_PLATFORM _hyg_cleanup_deferred = False try: # Gateway hygiene runs before the user turn # starts and already owns the session binding. # Prefer in-place compaction here: it archives # old rows under the same session id instead of # minting a continuation child that then has to # be published back to SessionStore/topic # bindings. If no SessionDB is available, # compress_context leaves this flag false and # the guard below preserves the transcript. _hyg_agent.compression_in_place = True _bind_hyg_state = getattr( getattr(_hyg_agent, "context_compressor", None), "bind_session_state", None, ) if callable(_bind_hyg_state): _bind_hyg_state( _hyg_session_db, session_entry.session_id, ) # It must never finalize on close() — close() # would end the live gateway session row. _hyg_agent._end_session_on_close = False _hyg_agent._print_fn = lambda *a, **kw: None loop = asyncio.get_running_loop() _hyg_commit_fence = CompressionCommitFence( total_ceiling_seconds=_hyg_total_ceiling_seconds ) # Default executor (NOT self._get_executor): # a fence-cancelled hung summary must never # occupy one of the gateway's agent-work # slots. But it MUST run inside the caller's # contextvars: under multiplex_profiles the # profile secret scope / HERMES_HOME override # live in ContextVars, and a bare # run_in_executor worker starts with an empty # Context — the summary model's # get_secret(_API_KEY) then fails # closed (UnscopedSecretError) and every # hygiene compaction silently degrades to a # lossy truncation (#100849 bundle). _hyg_future = loop.run_in_executor( None, copy_context().run, lambda: _hyg_agent._compress_context( _hyg_msgs, "", approx_tokens=_approx_tokens, commit_fence=_hyg_commit_fence, ), ) try: # Progress-aware wait: the timeout is an # INACTIVITY budget, not a total one. The # compression worker streams its summary # call and ticks the fence per token # (CompressionCommitFence.touch_progress), # so a slow reasoning model that is still # generating keeps extending the deadline; # only a genuinely silent worker times out. # A hard ceiling bounds the total wait so # a degenerate trickle stream can't hold # the turn forever. _hyg_wait_started = time.monotonic() while True: if _hyg_commit_fence.is_cancelled: raise asyncio.TimeoutError # #76354 S3: charge the idle budget # from the LAST PROGRESS event, not # from the start of this wait slice — # otherwise silence can approach 2x # the configured timeout. _hyg_waited = ( time.monotonic() - _hyg_wait_started ) _slice = min( max( _hyg_timeout_seconds - _hyg_commit_fence.seconds_since_progress(), 0.005, ), max( _hyg_total_ceiling_seconds - _hyg_waited, 0.005, ), ) # Bounded turn-hold (#TKT-0029): cap # this slice at the remaining # turn-hold budget so the wait is # re-evaluated against # _hyg_max_turn_hold_seconds at # least that often — otherwise a # continuously-streaming worker # (which keeps the inactivity slice # large) would hold the turn until # the total ceiling before the # budget check ever runs. _turn_hold_remaining = ( _hyg_max_turn_hold_seconds - (time.monotonic() - _hyg_wait_started) ) if _turn_hold_remaining <= 0: # Budget already exhausted — # force an immediate timeout so # the abandonment path below runs. _slice = 0.005 else: _slice = min( _slice, max(_turn_hold_remaining, 0.005), ) # Re-check the fence on a short poll so a # /stop or /restart cancel is not stuck # behind a full idle window (#96953). _idle_left = max( _hyg_timeout_seconds - _hyg_commit_fence.seconds_since_progress(), 0.005, ) _slice = min(_slice, 0.25) try: _compressed, _ = await asyncio.wait_for( asyncio.shield(_hyg_future), timeout=_slice, ) break except asyncio.TimeoutError: if _hyg_commit_fence.is_cancelled: raise _hyg_waited = time.monotonic() - _hyg_wait_started _idle = _hyg_commit_fence.seconds_since_progress() # Bounded turn-hold (#TKT-0029): # never hold the user's TURN # longer than # _hyg_max_turn_hold_seconds, # even if the summary model is # still streaming. Past the # budget we stop waiting and # fall through to the timeout # path below, which revokes # commit admission and proceeds # on the uncompressed # transcript — the wire never # stays silent long enough to # trip a transport idle-timeout. if ( _hyg_waited >= _hyg_max_turn_hold_seconds ): logger.info( "Session hygiene compression for " "session %s exceeded the turn-hold " "budget (%.1fs >= %.1fs) — " "abandoning inline wait, proceeding " "without compression this turn", session_entry.session_id, _hyg_waited, _hyg_max_turn_hold_seconds, ) raise HygieneTurnHoldExceeded( f"turn-hold budget {_hyg_max_turn_hold_seconds:.1f}s " f"elapsed after {_hyg_waited:.1f}s" ) if hygiene_wait_should_extend( idle=_idle, timeout=_hyg_timeout_seconds, waited=_hyg_waited, ceiling=_hyg_total_ceiling_seconds, fence_cancelled=_hyg_commit_fence.is_cancelled, ): if _slice >= _idle_left - 1e-9: logger.info( "Session hygiene compression for " "session %s still streaming after " "%.0fs (last progress %.1fs ago) — " "extending wait (ceiling %.0fs)", session_entry.session_id, _hyg_waited, _idle, _hyg_total_ceiling_seconds, ) continue raise except HygieneTurnHoldExceeded: # Turn-hold expiry is an availability boundary, # not a failure. The compressor is healthy and # still streaming; we simply cannot hold the # current user turn any longer. Share the safe # mechanics (fence, release, defer, proceed # uncompressed) but with distinct provenance, # user message, and NO failure-cooldown # increment. # # #97963: decouple the TURN from the # COMPRESSION. When the worker's commit is # watermark-fenced (it captured the session's # active-row watermark at compression start, # so rows appended after that point — this # released turn included — survive its late # commit verbatim as cloned concurrent tail), # the already-running attempt KEEPS its commit # admission: the user's turn proceeds on the # uncompressed transcript NOW, and the summary # is adopted when the detached worker reaches # its own watermark-fenced commit transaction # (archive_and_compact / the rotation publish # path — the next safe boundary). Before this, # the fence was ALWAYS cancelled here, burning # the full summary attempt — for a thinking # summary model whose reasoning prefix alone # exceeds the 10s hold, that made hygiene # auto-compression fail 100% of the time while # paying the summary model per turn. The turn # itself is still released at the same budget: # only the fate of the detached worker's # RESULT changes. If the commit is NOT # watermark-fenced (no session_db, watermark # capture failed, legacy lock API), a late # commit could clobber newer turns, so cancel # exactly as before — never worse than the # status quo. _hyg_keep_admission = bool( getattr( _hyg_commit_fence, "commit_watermark_fenced", False, ) ) and not _hyg_commit_fence.is_cancelled if _hyg_keep_admission: self._defer_agent_cleanup_until_future_done( _hyg_future, _hyg_agent, context="session hygiene turn-hold", ) _hyg_cleanup_deferred = True # NO retry-after here (#97963 (b)): the # attempt is still running toward a real # commit, and arming the flat 60s # retry-after would ALSO block the # agent-side preflight compressor from a # fresh chance ("Skipping preflight # compression: same-session cooldown # active"). Re-attempt spacing is covered # by the durable compression lock instead: # the next turn's hygiene pre-check skips # while this worker's lease is held # (_session_has_compression_in_flight). # The flat retry-after is recorded by the # done-callback below ONLY if the worker # ends without committing anything. _hyg_deferred_sid = session_entry.session_id _hyg_deferred_key = session_key _hyg_deferred_agent = _hyg_agent def _hyg_adopt_or_space_retry( _fut, _gw=self, _sid=_hyg_deferred_sid, _skey=_hyg_deferred_key, _agent=_hyg_deferred_agent, ): try: _exc = _fut.exception() except ( asyncio.CancelledError, Exception, ): _exc = None _committed = False else: _committed = _exc is None and ( bool( getattr( _agent, "_last_compaction_in_place", False, ) ) or getattr( _agent, "session_id", _sid ) != _sid ) if _committed: logger.info( "Session hygiene compression for " "session %s finished after the " "turn-hold was released — summary " "adopted at the watermark-fenced " "commit boundary (#97963)", _sid, ) try: _reset_hygiene_failure_streak( _gw, _skey ) except Exception as _rs_err: logger.debug( "hygiene streak reset after " "deferred adoption failed: %s", _rs_err, ) else: # Nothing to adopt (summary failed, # fence refused the commit, or the # attempt was superseded). Restore # the pre-#97963 spacing so # sustained traffic does not spawn # and abandon a fresh compressor # every turn. Flat and # non-escalating: the streak must # not advance for a deferral. _record_hygiene_cooldown( _gw, _sid, _HYGIENE_TURNHOLD_RETRY_SECONDS, "hygiene compression deferred: " "turn-hold budget expired and the " "detached attempt did not commit", ) _hyg_future.add_done_callback( _hyg_adopt_or_space_retry ) from agent.session_activity import ( ActivityProvenance, ) _stamp_hygiene_compression_provenance( _hyg_agent, "session hygiene compression turn-hold", ActivityProvenance.AGENT_COMPRESSION_TURNHOLD, "hygiene compression turn-hold " "activity stamp failed", ) logger.info( "Session hygiene compression for session %s " "exceeded turn-hold budget (%.1fs); " "proceeding without compression this turn — " "the watermark-fenced worker keeps its " "commit admission and the summary will be " "adopted when it finishes", session_entry.session_id, time.monotonic() - _hyg_wait_started, ) _turnhold_msg = t( "gateway.compress.turnhold_deferred" ) try: _adapter = self._adapter_for_source(source) if _adapter and source.chat_id: await _adapter.send( source.chat_id, _turnhold_msg, metadata=_hyg_meta, ) except Exception as _werr: logger.warning( "Failed to deliver compression-turnhold " "notice to user: %s", _werr, ) raise _cancelled = None while _cancelled is None: if _hyg_commit_fence.commit_in_flight: _cancelled = False break _cancelled = ( _hyg_commit_fence.try_cancel_before_commit() ) if _cancelled is None: await asyncio.sleep(0.025) if not _cancelled: # NOTE: bounded overshoot by design. # The turn can be held past # _hyg_max_turn_hold_seconds by up to the # commit duration (summary apply + storage # write). Aborting mid-commit would corrupt # the message-store transaction — the # overshoot is the cheaper failure mode. # Do NOT "fix" this into a mid-commit # cancellation. _compressed, _ = await _hyg_future else: _hyg_commit_fence.release_cancelled_compression_lock() self._defer_agent_cleanup_until_future_done( _hyg_future, _hyg_agent, context="session hygiene turn-hold", ) _hyg_cleanup_deferred = True # Short, NON-escalating retry-after. Without # it, every subsequent turn re-spawns a fresh # compressor, holds it for the turn-hold # budget, and cancels it again — a per-turn # summary-model token burn that never commits # under sustained traffic. This is deliberately # NOT _hygiene_cooldown_for_failure: the # compressor is healthy, so the failure streak # must not advance (behavior witness below); # only the flat retry spacing is recorded. _record_hygiene_cooldown( self, session_entry.session_id, _HYGIENE_TURNHOLD_RETRY_SECONDS, "hygiene compression deferred: " "turn-hold budget expired while the " "summary was still streaming", ) from agent.session_activity import ( ActivityProvenance, ) _stamp_hygiene_compression_provenance( _hyg_agent, "session hygiene compression turn-hold", ActivityProvenance.AGENT_COMPRESSION_TURNHOLD, "hygiene compression turn-hold " "activity stamp failed", ) logger.info( "Session hygiene compression for session %s " "exceeded turn-hold budget (%.1fs); " "proceeding without compression this turn", session_entry.session_id, time.monotonic() - _hyg_wait_started, ) _turnhold_msg = t( "gateway.compress.turnhold_deferred" ) try: _adapter = self._adapter_for_source(source) if _adapter and source.chat_id: await _adapter.send( source.chat_id, _turnhold_msg, metadata=_hyg_meta, ) except Exception as _werr: logger.warning( "Failed to deliver compression-turnhold " "notice to user: %s", _werr, ) raise except asyncio.TimeoutError: _hyg_waited = time.monotonic() - _hyg_wait_started _hyg_total_exhausted = ( _hyg_waited >= _hyg_total_ceiling_seconds or _hyg_commit_fence.deadline_exceeded ) if _hyg_total_exhausted: # The worker cooperatively checks this # deadline between digest calls. Keep # its lease until it exits so an # unchanged session cannot overlap a # retry. Inactivity timeouts retain the # established release behavior for a # provider call that may never return. _hyg_commit_fence.retain_compression_lock_until_worker_done() # Capture fence state BEFORE try_cancel # — that call itself sets is_cancelled, # which would mis-label a genuine idle # timeout as a fence cancel (#96953). _hyg_fence_cancelled = ( _hyg_commit_fence.is_cancelled ) _cancelled = None while _cancelled is None: # #76354 F1: a hung commit retains the # fence lock; the lock-free phase # marker keeps this loop from spinning # forever while the commit blocks. if _hyg_commit_fence.commit_in_flight: _cancelled = False break _cancelled = ( _hyg_commit_fence.try_cancel_before_commit() ) if _cancelled is None: # Round-2 #5: transient # lock-setup windows ride # write patience for seconds; # 25ms keeps sub-tick latency # without 1kHz spin. await asyncio.sleep(0.025) if not _cancelled: # The worker crossed the commit boundary just # before the timeout. The fence poll waited for # that boundary to finish, so consume the # completed result instead of treating a # successful compaction as a timeout. _compressed, _ = await _hyg_future else: # Release an inactivity-timed-out # worker's holder-qualified lease # promptly. Total-ceiling attempts # retained it above, so this is a # no-op until worker cleanup there. _hyg_commit_fence.release_cancelled_compression_lock() self._defer_agent_cleanup_until_future_done( _hyg_future, _hyg_agent, context="session hygiene timeout", ) _hyg_cleanup_deferred = True _hyg_timeout_error = ( "session hygiene compression " "cancelled at commit fence" if _hyg_fence_cancelled else ( "session hygiene compression " "timed out with no output from " "the summary model" ) ) if _hyg_failure_cooldown_seconds >= 0: _hyg_cooldown = await asyncio.to_thread( _hygiene_cooldown_for_failure, self, session_key, _hyg_failure_cooldown_seconds, ) _timeout_reason = ( _hyg_timeout_error if _hyg_fence_cancelled else ( "session hygiene compression total " "ceiling exhausted" if _hyg_total_exhausted else "session hygiene compression " "timed out with no output from the " "summary model" ) ) _record_hygiene_cooldown( self, session_entry.session_id, _hyg_cooldown, _timeout_reason, ) from agent.session_activity import ( ActivityProvenance, ) _stamp_hygiene_compression_provenance( _hyg_agent, ( "session hygiene compression " "cancelled at commit fence" if _hyg_fence_cancelled else "session hygiene compression timed out" ), ActivityProvenance.AGENT_COMPRESSION_TIMEOUT, "hygiene compression timeout " "activity stamp failed", ) if _hyg_fence_cancelled: logger.warning( "Session hygiene compression for " "session %s was cancelled at the " "commit fence; continuing without " "compression", session_entry.session_id, ) else: _hyg_elapsed = ( time.monotonic() - _hyg_wait_started ) if _hyg_total_exhausted: logger.warning( "Session hygiene compression for session %s " "reached its total ceiling after %.1fs " "(progress observed=%s); continuing without " "compression", session_entry.session_id, _hyg_elapsed, _hyg_commit_fence.progress_observed, ) else: logger.warning( "Session hygiene compression for session %s " "made no progress for %.1fs (total wait " "%.1fs, ceiling %.1fs); continuing without " "compression", session_entry.session_id, _hyg_commit_fence.seconds_since_progress(), _hyg_elapsed, _hyg_total_ceiling_seconds, ) _timeout_msg = ( _hygiene_compression_timeout_message( total_exhausted=_hyg_total_exhausted, elapsed=_hyg_elapsed, idle_timeout=_hyg_timeout_seconds, progress_observed=( _hyg_commit_fence.progress_observed ), ) ) try: _adapter = self._adapter_for_source(source) if _adapter and source.chat_id: await _adapter.send( source.chat_id, _timeout_msg, metadata=_hyg_meta, ) except Exception as _werr: logger.warning( "Failed to deliver compression-timeout " "warning to user: %s", _werr, ) raise except BaseException: # #76354 F2: non-timeout unwind while the # detached hygiene worker may still run — # KeyboardInterrupt, task cancellation, or # any unexpected error. Revoke commit # admission (and release the worker's # durable lease via the holder-qualified # hook) BEFORE the host unwinds so the # worker can never commit later. _hyg_commit_fence.revoke_commit_admission() if not _hyg_cleanup_deferred: self._defer_agent_cleanup_until_future_done( _hyg_future, _hyg_agent, context="session hygiene unwind", ) _hyg_cleanup_deferred = True # #96953: restart drain / task cancel used # to re-raise with no cooldown, so the # next turn immediately re-armed hygiene # and waited up to 600s behind a fence # that would refuse the commit again. if _hyg_failure_cooldown_seconds >= 0: try: _hyg_cooldown = _hygiene_cooldown_for_failure( self, session_key, _hyg_failure_cooldown_seconds, ) _record_hygiene_cooldown( self, session_entry.session_id, _hyg_cooldown, "session hygiene compression " "cancelled at commit fence", ) except Exception as _cd_err: logger.debug( "hygiene unwind cooldown " "record failed: %s", _cd_err, ) raise # _compress_context ends the old session and creates # a new session_id. Write compressed messages into # the NEW session so the old transcript stays intact # and searchable via session_search. _hyg_new_sid = _hyg_agent.session_id _hyg_rotated = _hyg_new_sid != session_entry.session_id _hyg_in_place = bool( getattr(_hyg_agent, "_last_compaction_in_place", False) ) # Anti-growth guard: refuse a compression # that did not shrink the transcript # (observed: 427K -> 598K). Compare # like-for-like rough estimates. _hyg_in_toks = estimate_messages_tokens_rough(history) _hyg_out_toks = estimate_messages_tokens_rough(_compressed) if _hyg_rotated and _hyg_out_toks > _hyg_in_toks: logger.warning( "Gateway hygiene compression for session %s " "would grow transcript (~%s -> ~%s tokens); " "keeping the original transcript unchanged", session_entry.session_id, f"{_hyg_in_toks:,}", f"{_hyg_out_toks:,}", ) _hyg_rotated = False _compressed = history # Only rewrite the transcript when rotation produced # a NEW session id. In-place compaction does NOT # need a rewrite: archive_and_compact() has already # soft-archived the previous active rows and inserted # the compacted messages as the new active set inside # _compress_context(). Calling rewrite_transcript() # after in-place compaction would invoke # replace_messages(active_only=False) which DELETEs # ALL rows — including the archived turns that # archive_and_compact() deliberately preserved # (silent data loss, #61145). # # The danger this guards against (mirrors the # /compress fix #44794/#39704): if _compress_context # returns a summary but neither rotates nor completes # archive_and_compact(), the session_id is unchanged # for a FAILURE reason, and an unconditional # rewrite_transcript() would DELETE the original # messages and replace them with only the compressed # summary (permanent data loss, #21301). # # Write-before-repoint (mirrors manual /compress): # if we repointed session_entry onto the child SID # and rewrite_transcript then failed (lock/ENOSPC), # the live entry would already reference a brand-new # empty session while the turn continues — the # conversation silently vanishes. Persist the child # transcript first; only then rebind the live entry. if _hyg_rotated: if not await self.async_session_store.rewrite_transcript( _hyg_new_sid, _compressed ): logger.error( "Session hygiene: failed to persist " "compressed transcript for rotated " "session %s → %s; keeping the live " "entry on the original session so the " "conversation is not dropped", session_entry.session_id, _hyg_new_sid, ) # Fail closed: treat like no rotation. _hyg_rotated = False _hyg_in_place = False else: session_entry.session_id = _hyg_new_sid # The held turn lease follows the # rotation so an alias key resolving # the fresh child still serializes # against this turn (#64934). self._rebind_turn_lease( _quick_key, run_generation, _hyg_new_sid ) await self.async_session_store._save() await asyncio.to_thread( self._sync_telegram_topic_binding, source, session_entry, reason="hygiene-compression", ) if _hyg_rotated: # Reset stored token count — transcript rewritten session_entry.last_prompt_tokens = 0 history = _compressed _new_count = len(_compressed) _new_tokens = estimate_messages_tokens_rough( _compressed ) elif _hyg_in_place: # archive_and_compact() already persisted the # compacted transcript inside _compress_context. # Reset counts to match the new active set. session_entry.last_prompt_tokens = 0 history = _compressed _new_count = len(_compressed) _new_tokens = estimate_messages_tokens_rough( _compressed ) else: # No rewrite happened — transcript preserved # unchanged, so the post-compression counts equal # the pre-compression ones. _new_count = _msg_count _new_tokens = _approx_tokens logger.warning( "Gateway hygiene compression for session %s " "did not rotate or compact in place " "(no session_db on the hygiene agent) — " "preserving the original transcript instead " "of overwriting it with the summary (#21301).", session_entry.session_id, ) logger.info( "Session hygiene: compressed %s → %s msgs, " "~%s → ~%s tokens", _msg_count, _new_count, f"{_approx_tokens:,}", f"{_new_tokens:,}", ) if _new_tokens >= _warn_token_threshold: logger.warning( "Session hygiene: still ~%s tokens after " "compression", f"{_new_tokens:,}", ) # If summary generation failed, the # compressor aborts entirely and returns # messages unchanged — nothing is dropped. # Surface a visible warning to the gateway # user — agent.log alone is invisible on # TG/Discord/etc. — so they know the chat # is "frozen" at the current size and can # /compress to retry or /reset to start # fresh. _comp = getattr(_hyg_agent, "context_compressor", None) _hyg_aborted = _comp is not None and getattr( _comp, "_last_compress_aborted", False ) # Fence-cancelled _compress_context returns # the original transcript with # _last_compress_aborted still False # (failure_class=commit_fence_cancelled, # chunk_count=0). Treat that no-op as an # abort so hygiene records a cooldown # instead of retrying into the 600s wait # (#96953). A successful rotate/in-place # commit is not an abort even if a later # invalidation flipped the fence. _hyg_fence_cancelled = bool( _hyg_commit_fence.is_cancelled and not _hyg_rotated and not _hyg_in_place ) if _hyg_fence_cancelled: _hyg_aborted = True if not _hyg_aborted: # Recovery decision lives in the # extracted, unit-tested predicate — the # degenerate "did not rotate or compact # in place" path (#21301) sets both flags # False and reuses the pre-compression # counts, so a numbers-only check would # read a no-op as success and clear the # streak on every wedged run (#79624). if hygiene_compaction_recovered( aborted=_hyg_aborted, rotated=_hyg_rotated, in_place=_hyg_in_place, msg_count=_msg_count, new_count=_new_count, approx_tokens=_approx_tokens, new_tokens=_new_tokens, ): await asyncio.to_thread( _reset_hygiene_failure_streak, self, session_key, ) if _hyg_aborted: if _hyg_failure_cooldown_seconds >= 0: _hyg_cooldown = await asyncio.to_thread( _hygiene_cooldown_for_failure, self, session_key, _hyg_failure_cooldown_seconds, ) _record_hygiene_cooldown( self, session_entry.session_id, _hyg_cooldown, ( "session hygiene compression " "cancelled at commit fence" if _hyg_fence_cancelled else getattr( _comp, "_last_summary_error", None ) ), ) from agent.session_activity import ( ActivityProvenance, ) _stamp_hygiene_compression_provenance( _hyg_agent, "session hygiene compression aborted", ActivityProvenance.AGENT_COMPRESSION_COOLDOWN, "hygiene compression abort " "activity stamp failed", ) if not _hyg_fence_cancelled: _err = getattr(_comp, "_last_summary_error", None) or "unknown error" # Force-redact: provider exception text # may contain credentials; this message # reaches gateway users directly. from agent.redact import redact_sensitive_text _err = redact_sensitive_text(_err, force=True) _warn_msg = ( "⚠️ Context compression aborted " f"({_err}). No messages were dropped — " "conversation is unchanged. Run /compress " "to retry, /reset for a clean session, or " "check your auxiliary.compression model " "configuration." ) try: _adapter = self._adapter_for_source(source) if _adapter and source.chat_id: await _adapter.send(source.chat_id, _warn_msg, metadata=_hyg_meta) except Exception as _werr: logger.warning( "Failed to deliver compression-failure warning to user: %s", _werr, ) # Separately: if the user's CONFIGURED aux # model failed and we recovered by falling # back to the main model, tell them — a # misconfigured auxiliary.compression.model # is something only they can fix, and # silent recovery would hide it. elif _comp is not None and getattr(_comp, "_last_aux_model_failure_model", None): _aux_model = getattr(_comp, "_last_aux_model_failure_model", "") _aux_err = getattr(_comp, "_last_aux_model_failure_error", None) or "unknown error" _aux_msg = ( f"ℹ️ Configured compression model `{_aux_model}` " f"failed ({_aux_err}). Recovered using your main " "model — context is intact — but you may want to " "check `auxiliary.compression.model` in config.yaml." ) try: _adapter = self._adapter_for_source(source) if _adapter and source.chat_id: await _adapter.send(source.chat_id, _aux_msg, metadata=_hyg_meta) except Exception as _werr: logger.warning( "Failed to deliver aux-model-fallback notice to user: %s", _werr, ) finally: # Evict the cached agent so the next turn # rebuilds its system prompt from current # SOUL.md, memory, and skills. self._evict_cached_agent(session_key) if not _hyg_cleanup_deferred: await self._cleanup_agent_resources_off_loop( _hyg_agent, context="session hygiene" ) except HygieneTurnHoldExceeded: # Availability boundary, not a failure — already logged # at INFO by the turn-hold handler. Must not hit the # generic "auto-compress failed" warning below: that # log is how thinking-model deployments read as # permanently broken (#97963; surfaced by @686f6c61 # in PR #99657). pass except Exception as e: logger.warning( "Session hygiene auto-compress failed: %s", e ) # First-message onboarding -- only on the very first interaction ever. # Delivered on the current user message (sidecar), NOT the ephemeral # system prompt: present-on-turn-1/absent-on-turn-2 was a guaranteed # system-prompt diff and agent rebuild. if not history and not await self.async_session_store.has_any_sessions(): # Default first-contact note: a brief self-introduction. _intro_note = ( "[System note: This is the user's very first message ever. " "Briefly introduce yourself and mention that /help shows available commands. " "Keep the introduction concise -- one or two sentences max.]" ) # Opt-in structured profile-build path. When enabled (default # "ask") and not yet offered on this install, swap the plain intro # for a consent-gated directive that offers to build a user # profile and persists confirmed facts via memory(target="user"). # The offer fires at most once (onboarding.seen flag); set # onboarding.profile_build: off in config.yaml to disable. try: from agent.onboarding import ( PROFILE_BUILD_FLAG, is_seen, mark_seen, profile_build_directive, profile_build_mode, ) _onb_cfg = _load_gateway_config() if ( profile_build_mode(_onb_cfg) == "ask" and not is_seen(_onb_cfg, PROFILE_BUILD_FLAG) ): turn_sidecar_notes.append(profile_build_directive().strip()) mark_seen(_hermes_home / "config.yaml", PROFILE_BUILD_FLAG) else: turn_sidecar_notes.append(_intro_note) except Exception as _pb_err: logger.debug( "Profile-build onboarding directive failed, using plain intro: %s", _pb_err, ) turn_sidecar_notes.append(_intro_note) # One-time prompt if no home channel is set for this platform # Skip for webhooks - they deliver directly to configured targets (github_comment, etc.) if not history and source.platform and source.platform != Platform.LOCAL and source.platform != Platform.WEBHOOK: platform_name = source.platform.value env_key = _home_target_env_var(platform_name) # Multiplex: home channel may live only in the profile secret # scope / PlatformConfig, not process os.environ. home_env = "" try: from agent.secret_scope import get_secret home_env = (get_secret(env_key) or "").strip() if env_key else "" except Exception: home_env = "" if not home_env: home_env = (os.getenv(env_key) or "").strip() if env_key else "" # Also honor in-memory / yaml home_channel on this platform. try: if not home_env and self.config.get_home_channel(source.platform): home_env = "set" except Exception: pass # Secondary-profile platforms (e.g. Slack on yolo) may only exist # under that profile's loaded config — check after scope install. if not home_env: try: from gateway.config import load_gateway_config as _lgc prof = (getattr(source, "profile", None) or "").strip() if prof and prof != "default": # Already inside profile scope for secondary handlers; # re-read live config for home_channel. _pcfg = _lgc() if _pcfg.get_home_channel(source.platform): home_env = "set" except Exception: pass if not home_env: # Slack dispatches all Hermes commands through a single # parent slash command `/hermes`; bare `/sethome` is not # registered and would fail with "app did not respond". sethome_cmd = ( "/hermes sethome" if source.platform == Platform.SLACK else "/sethome" ) notice = ( f"📬 No home channel is set for {platform_name.title()}. " f"A home channel is where Hermes delivers cron job results " f"and cross-platform messages.\n\n" f"Type {sethome_cmd} to make this chat your home channel, " f"or ignore to skip." ) await self._deliver_platform_notice(source, notice) # ----------------------------------------------------------------- # Voice channel awareness — deliver current voice channel state so # the agent knows who is in the channel and who is speaking, without # needing a separate tool call. Delivered on the current user # message and ONLY when it changed since the previous turn: the # member/speaking serialization differs essentially every turn, and # appending it to the ephemeral system prompt forced a full agent # rebuild + prompt-cache re-key per message. The system prompt # carries a static pointer line instead (gateway/session.py). # ----------------------------------------------------------------- _vc_note = self._voice_channel_sidecar_note(event, source, session_key) if _vc_note: turn_sidecar_notes.append(_vc_note) # ----------------------------------------------------------------- # Auto-analyze images sent by the user # # If the user attached image(s), we run the vision tool eagerly so # the conversation model always receives a text description. The # local file path is also included so the model can re-examine the # image later with a more targeted question via vision_analyze. # # We filter to image paths only (by media_type) so that non-image # attachments (documents, audio, etc.) are not sent to the vision # tool even when they appear in the same message. # ----------------------------------------------------------------- message_text = await self._prepare_profile_scoped_inbound_message_text( event=event, source=source, history=history, session_key=session_key, ) if message_text is None: return # Capture the platform event time as message metadata and keep the # persisted transcript clean (strip any leading timestamp prefix). # This runs regardless of the toggle so storage stays clean and the # send-time is preserved. Only the in-context RENDER (prepending the # human-readable prefix the model sees) is gated behind # gateway.message_timestamps.enabled — default OFF. try: from hermes_time import get_timezone as _get_evt_tz from gateway.message_timestamps import ( coerce_message_timestamp as _coerce_msg_ts, render_user_content_with_timestamp as _render_msg_ts, strip_leading_message_timestamps as _strip_msg_ts, ) _evt_tz = _get_evt_tz() _evt_ts = getattr(event, "timestamp", None) if message_text and isinstance(message_text, str): _clean_message_text, _embedded_ts = _strip_msg_ts( message_text, tz=_evt_tz) persist_user_message = _clean_message_text _event_epoch = _coerce_msg_ts(_evt_ts, tz=_evt_tz) persist_user_timestamp = ( _event_epoch if _event_epoch is not None else _embedded_ts ) if _message_timestamps_enabled(_load_gateway_config()): message_text = _render_msg_ts( _clean_message_text, persist_user_timestamp, tz=_evt_tz, ) else: # Toggle off: model sees the clean message; the timestamp # is still stored as metadata for later opt-in. message_text = _clean_message_text except Exception as _ts_err: logger.debug("Message timestamp injection failed (non-fatal): %s", _ts_err) # Stage the collected must-deliver notes for this turn's agent run # (one-shot; consumed in run_sync). Staged AFTER the message_text # early-out above so an aborted turn cannot leak its notes into the # next turn's user message. if turn_sidecar_notes and session_key: self._set_pending_turn_sidecar_notes(session_key, turn_sidecar_notes) # Bind this gateway run generation to the adapter's active-session # event so deferred post-delivery callbacks can be released by the # same run that registered them. self._bind_adapter_run_generation( self._adapter_for_source(source), session_key, run_generation, ) try: # Emit agent:start hook hook_ctx = { "platform": source.platform.value if source.platform else "", "user_id": source.user_id, "chat_id": source.chat_id or "", "thread_id": str(getattr(source, "thread_id", None)) if getattr(source, "thread_id", None) else "", "chat_type": getattr(source, "chat_type", "") or "", "session_id": session_entry.session_id, "message": message_text[:500], } await self.hooks.emit("agent:start", hook_ctx) # Run the agent. Capture the session id that this run was launched # against so post-run compression publication can be identity-guarded # below; a /new or another lifecycle transition may move # session_entry.session_id while the old run is still unwinding. _run_start_session_id = session_entry.session_id _turn_started_monotonic = time.monotonic() agent_result = await self._run_agent( message=message_text, context_prompt=context_prompt, history=history, source=source, session_id=_run_start_session_id, session_key=session_key, run_generation=run_generation, event_message_id=self._reply_anchor_for_event(event), inbound_message_id=( str(event.message_id) if event.message_id else None ), channel_prompt=event.channel_prompt, moa_config=getattr(event, "_moa_config", None), persist_user_message=persist_user_message, persist_user_timestamp=persist_user_timestamp, persist_user_display_kind=persist_user_display_kind, message_type=event.message_type, ) _turn_seconds = time.monotonic() - _turn_started_monotonic # Stop persistent typing indicator now that the agent is done. # Slack AI status is scoped to a thread/workspace, so preserve the # same routing metadata used by the response delivery path. try: _typing_adapter = self._adapter_for_source(source) _stop_with_metadata = getattr( type(_typing_adapter), "_stop_typing_with_metadata", None ) _stop_typing = getattr(type(_typing_adapter), "stop_typing", None) if _typing_adapter and callable(_stop_with_metadata): await _typing_adapter._stop_typing_with_metadata( source.chat_id, self._thread_metadata_for_source( source, self._reply_anchor_for_event(event) ), ) elif _typing_adapter and callable(_stop_typing): await _typing_adapter.stop_typing(source.chat_id) except Exception: pass if not self._is_session_run_current(_quick_key, run_generation): logger.info( "Discarding stale agent result for %s — generation %d is no longer current", _quick_key or "?", run_generation, ) _stale_adapter = self._adapter_for_source(source) if getattr(type(_stale_adapter), "pop_post_delivery_callback", None) is not None: _stale_adapter.pop_post_delivery_callback( _quick_key, generation=run_generation, ) elif _stale_adapter and hasattr(_stale_adapter, "_post_delivery_callbacks"): _stale_adapter._post_delivery_callbacks.pop(_quick_key, None) return None response = agent_result.get("final_response") or "" # Hidden-reasoning-only retry exhaustion: the loop's sentinel text # ("Codex response remained incomplete after 3 continuation # attempts") doubles as final_response, so it would be delivered # verbatim into the channel — where peer agents can ingest it as a # completed assistant turn (#51628). Blank it here so the normal # empty-response handling (and the suppression below) applies. if _is_gateway_hidden_reasoning_incomplete_turn(agent_result): response = "" try: from gateway.response_filters import is_intentional_silence_agent_result _intentional_silence = is_intentional_silence_agent_result( agent_result, response, ) except Exception: _intentional_silence = False # Convert the agent's internal "(empty)" sentinel into a # user-friendly message. "(empty)" means the model failed to # produce visible content after exhausting all retries (nudge, # prefill, empty-retry, fallback). Sending the raw sentinel # looks like a bug; a short explanation is more helpful. if response == "(empty)" and not _intentional_silence: response = ( "⚠️ The model returned no response after processing tool " "results. This can happen with some models — try again or " "rephrase your question." ) agent_messages = agent_result.get("messages", []) _response_time = time.time() - _msg_start_time _api_calls = agent_result.get("api_calls", 0) _resp_len = len(response) logger.info( "response ready: platform=%s chat=%s time=%.1fs api_calls=%d response=%d chars", _platform_name, source.chat_id or "unknown", _response_time, _api_calls, _resp_len, ) # NOTE: the cross-process cache-coherence re-baseline # (_refresh_agent_cache_message_count) is intentionally deferred # until AFTER this turn's transcript persistence block below — it # must include the first-turn `session_meta` marker row and the # compression session_id swap, both of which happen later. See # the call site after the `update_session(...)` write. # Successful turn — clear any stuck-loop counter for this session. # This ensures the counter only accumulates across CONSECUTIVE # restarts where the session was active (never completed). # # Also clear the resume_pending flag (set by drain-timeout # shutdown) — the turn ran to completion, so recovery # succeeded and subsequent messages should no longer receive # the restart-interruption system note. if session_key and _should_clear_resume_pending_after_turn(agent_result): await self._clear_restart_failure_count(session_key) try: await self.async_session_store.clear_resume_pending(session_key) except Exception as _e: logger.debug( "clear_resume_pending failed for %s: %s", session_key, _e, ) # Normalize empty responses: surface errors, partial failures, and # the case where agent did work but returned no text. Fix for #18765. if not _intentional_silence: response = _normalize_empty_agent_response( agent_result, response, history_len=len(history), ) response = _sanitize_gateway_final_response(source.platform, response) # Ordering contract: the agent thread already updated the contextvar # in conversation_compression.py; propagate to SessionEntry + _save(). # If the agent's session_id changed during compression, update # session_entry so transcript writes below go to the right session. if agent_result.get("session_id") and agent_result["session_id"] != session_entry.session_id: if session_entry.session_id == _run_start_session_id: session_entry.session_id = agent_result["session_id"] # The held turn lease follows the rotation: the transcript # persistence below writes to the NEW id, so the # serialization boundary must move with it or an alias # key resolving the fresh child could interleave (#64934). self._rebind_turn_lease( _quick_key, run_generation, session_entry.session_id ) await self.async_session_store._save() await self.async_session_store._record_gateway_session_peer( session_entry.session_id, session_key, source, ) await asyncio.to_thread( self._sync_telegram_topic_binding, source, session_entry, reason="agent-result-compression", ) else: logger.info( "Skipping agent-result session split sync for %s because " "the session binding moved from %s to %s before " "compression finished", session_key or "?", _run_start_session_id, session_entry.session_id, ) # Prepend reasoning/thinking if display is enabled (per-platform). # Mattermost requires explicit per-platform opt-in because this is # scratch text, not ordinary final-answer content. try: _show_reasoning_effective = _resolve_gateway_display_bool( _load_gateway_config(), _platform_config_key(source.platform), "show_reasoning", default=bool(getattr(self, "_show_reasoning", False)), platform=source.platform, require_platform_override_for={Platform.MATTERMOST}, ) except Exception: _show_reasoning_effective = ( False if source.platform == Platform.MATTERMOST else getattr(self, "_show_reasoning", False) ) if _show_reasoning_effective and response and not _intentional_silence: last_reasoning = agent_result.get("last_reasoning") if last_reasoning: from gateway.stream_consumer import escape_code_fences_for_display # Collapse long reasoning to keep messages readable lines = last_reasoning.strip().splitlines() if len(lines) > 15: display_reasoning = "\n".join(lines[:15]) display_reasoning += f"\n_... ({len(lines) - 15} more lines)_" else: display_reasoning = last_reasoning.strip() # Render style is per-platform: Discord defaults to "-# " # subtext (native small grey metadata text); other # platforms keep the fenced code block. try: from gateway.display_config import resolve_display_setting _reasoning_style = resolve_display_setting( _load_gateway_config(), _platform_config_key(source.platform), "reasoning_style", "code", ) except Exception: _reasoning_style = "code" if _reasoning_style == "subtext": _quoted = "\n".join( f"-# {ln}" if ln else "-#" for ln in display_reasoning.splitlines() ) response = f"-# 💭 Reasoning\n{_quoted}\n\n{response}" elif _reasoning_style == "blockquote": _quoted = "\n".join( f"> {ln}" if ln else ">" for ln in display_reasoning.splitlines() ) response = f"> 💭 **Reasoning:**\n{_quoted}\n\n{response}" else: # Escape ``` inside reasoning so inner fences don't # break the outer code block used to render it. display_reasoning = escape_code_fences_for_display(display_reasoning) response = f"💭 **Reasoning:**\n```\n{display_reasoning}\n```\n\n{response}" # Runtime-metadata footer — only on the FINAL message of the turn. # Off by default (display.runtime_footer.enabled=false). When # streaming already delivered the body, we can't mutate the sent # text, so we fire a separate trailing send below. _footer_line = "" try: from gateway.runtime_footer import build_footer_line as _bfl _footer_line = _bfl( user_config=_load_gateway_config(), platform_key=_platform_config_key(source.platform), model=agent_result.get("model"), context_tokens=agent_result.get("last_prompt_tokens", 0) or 0, context_length=agent_result.get("context_length") or None, cwd=_terminal_scope_cwd(""), turn_seconds=_turn_seconds, ) except Exception as _footer_err: logger.debug("runtime_footer build failed: %s", _footer_err) _footer_line = "" if _footer_line and response and not agent_result.get("already_sent") and not _intentional_silence: response = f"{response}\n\n{_footer_line}" # Emit agent:end hook await self.hooks.emit("agent:end", { **hook_ctx, "response": (response or "")[:500], "model": agent_result.get("model", ""), "provider": agent_result.get("provider", ""), }) # Check for pending process watchers (check_interval on background processes) try: from tools.process_registry import process_registry # Detach the current batch atomically (see crash-recovery drain # above): reassign to a fresh list so a watcher appended by a # concurrent session during the yield isn't dropped by clear(). watchers = process_registry.pending_watchers process_registry.pending_watchers = [] for i, watcher in enumerate(watchers): asyncio.create_task(self._run_process_watcher(watcher)) if i % 100 == 99: await asyncio.sleep(0) except Exception as e: logger.error("Process watcher setup error: %s", e) # Drain watch pattern notifications that arrived during the agent run. # Watch events and completions share the same queue; process # completions are already handled by the per-process watcher task # above, so we only inject watch-type events here. # # Async-delegation completions ALSO ride this shared queue but are # owned by the dedicated _async_delegation_watcher (started at # boot), which covers both the idle and post-turn cases with a # single consumer — so we leave them on the queue here. try: from tools.process_registry import process_registry as _pr await self._drain_watch_notifications(_pr.completion_queue) except Exception as e: logger.debug("Watch queue drain error: %s", e) # NOTE: Dangerous command approvals are now handled inline by the # blocking gateway approval mechanism in tools/approval.py. The agent # thread blocks until the user responds with /approve or /deny, so by # the time we reach here the approval has already been resolved. The # old post-loop pop_pending + approval_hint code was removed in favour # of the blocking approach that mirrors CLI's synchronous input(). # Save the full conversation to the transcript, including tool calls. # This preserves the complete agent loop (tool_calls, tool results, # intermediate reasoning) so sessions can be resumed with full context # and transcripts are useful for debugging and training data. # # IMPORTANT: For context-overflow failures (compression exhausted, # generic 400 on large sessions) we must NOT persist the user's # message — doing so would grow the session further and cause the # same failure on the next attempt, an infinite loop. (#1630, #9893) # # Transient failures (429, timeout, connection error, provider 5xx) # are different: the session is not oversized, and silently dropping # the user message causes severe context loss on retry — the agent # forgets what was just asked. Persist the user turn so the # conversation is preserved. (#7100) agent_failed_early = bool(agent_result.get("failed")) hidden_reasoning_incomplete = _is_gateway_hidden_reasoning_incomplete_turn( agent_result ) _err_str_for_classify = str(agent_result.get("error", "")).lower() # Use specific multi-word phrases (not bare "exceed" or "token") # to avoid false positives on transient errors like "rate limit # exceeded" or "invalid auth token". Matches run_agent.py's # own context-length classifier. is_context_overflow_failure = agent_failed_early and ( bool(agent_result.get("compression_exhausted")) or any(p in _err_str_for_classify for p in ( "context length", "context size", "context window", "maximum context", "token limit", "too many tokens", "reduce the length", "exceeds the limit", "request entity too large", "prompt is too long", "payload too large", "input is too long", )) or ("400" in _err_str_for_classify and len(history) > 50) ) if is_context_overflow_failure: logger.info( "Skipping transcript persistence for context-overflow " "failure in session %s to prevent session growth loop.", session_entry.session_id, ) elif agent_failed_early: logger.info( "Transient agent failure in session %s — persisting user " "message so conversation context is preserved on retry.", session_entry.session_id, ) elif hidden_reasoning_incomplete: logger.warning( "Suppressing hidden-reasoning-only incomplete gateway turn " "for session %s: %s", session_entry.session_id, agent_result.get("error", "processing incomplete"), ) # When compression is exhausted, the session is permanently too # large to process. Auto-reset it so the next message starts # fresh instead of replaying the same oversized context in an # infinite fail loop. (#9893) # # A lock-contended defer is the OPPOSITE case: the session is # temporarily uncompressible only because a concurrent path holds # the compression lock and is actively shrinking it. Never wipe # the session for that — retry-next-message semantics apply # (#69870 lock-skip consumer; salvaged from #49874). if agent_result.get("compression_deferred"): logger.info( "Compression deferred for session %s — the compression " "lock is held by a concurrent compressor. Keeping the " "session intact; the next message retries normally.", session_entry.session_id if session_entry else "?", ) elif agent_result.get("compression_exhausted") and session_entry and session_key: logger.info( "Auto-resetting session %s after compression exhaustion.", session_entry.session_id, ) new_entry = await self.async_session_store.reset_session(session_key) self._evict_cached_agent(session_key) # Conversation boundary: one funnel call clears every # conversation-scoped per-session dict (#58403 and siblings). # See _CONVERSATION_SCOPED_STATE. self._clear_conversation_scope( session_key, reason="compression_exhausted_reset" ) if new_entry is not None: # Drop the stale reference to the bloated compressed child and # re-point the Telegram topic binding at the fresh session. # Compression rotated session_entry.session_id to the oversized # compressed child earlier this turn (the agent-result sync # above), and that _sync also rewrote the (chat_id, thread_id) # -> bloated-child binding. reset_session swaps in a clean, # parentless session, but without re-syncing the binding the # next inbound message in this topic gets switch_session'd back # onto the bloated child by the binding-heal walk, reloads the # oversized transcript, and re-triggers compression exhaustion # forever (#35809 — regression of the #9893/#10063 auto-reset). # No-op on non-topic lanes. session_entry = new_entry await asyncio.to_thread( self._sync_telegram_topic_binding, source, session_entry, reason="compression-exhausted-reset", ) response = (response or "") + ( "\n\n🔄 Session auto-reset — the conversation exceeded the " "maximum context size and could not be compressed further. " "Your next message will start a fresh session." ) ts = time.time() # Unix epoch float — consistent with DB storage # If this is a fresh session (no history), write the full tool # definitions as the first entry so the transcript is self-describing # -- the same list of dicts sent as tools=[...] in the API request. if is_context_overflow_failure: pass # Skip all transcript writes — don't grow a broken session elif not history: tool_defs = agent_result.get("tools", []) await self.async_session_store.append_to_transcript( session_entry.session_id, { "role": "session_meta", "tools": tool_defs or [], "model": _resolve_gateway_model(), "platform": source.platform.value if source.platform else "", "timestamp": ts, } ) # The agent already persisted these messages to SQLite via # _flush_messages_to_session_db(), so skip the DB write here # to prevent the duplicate-write bug (#860 / #42039). This holds # for the codex app-server runtime too: although it early-returns # and bypasses conversation_loop's per-step flushes, it flushes its # own projected assistant/tool messages before returning and # reports agent_persisted=True (see agent/codex_runtime.py). Reading # the flag (default = self._session_db is not None) keeps the # persistence contract explicit and lets any future non-persisting # runtime opt into a gateway-side write by returning False. agent_persisted = agent_result.get("agent_persisted", self._session_db is not None) # Find only the NEW messages from this turn (skip history we loaded). # Use the filtered history length (history_offset) that was actually # passed to the agent, not len(history) which includes session_meta # entries that were stripped before the agent saw them. if is_context_overflow_failure: pass # handled above — skip all transcript writes elif agent_failed_early or hidden_reasoning_incomplete: # Transient failure (429/timeout/5xx): persist only the user # message so the next message can load a transcript that # reflects what was said. Skip the assistant error text since # it's a gateway-generated hint, not model output. Hidden- # reasoning-only incomplete turns follow the same persistence # rule so peer-agent channels don't ingest them as completed # assistant turns. (#7100, #51628) _user_entry = { "role": "user", "content": ( persist_user_message if persist_user_message is not None else message_text ), "timestamp": ( persist_user_timestamp if persist_user_timestamp is not None else ts ), } if persist_user_display_kind: _user_entry["display_kind"] = persist_user_display_kind if event.message_id: _user_entry["message_id"] = str(event.message_id) # Dedupe: skip if this platform message_id is already in the # transcript (prevents duplicate user turns on Telegram retries # after transient failures). #47237 _skip_persist = ( event.message_id and await self.async_session_store.has_platform_message_id( session_entry.session_id, str(event.message_id) ) ) if _skip_persist: logger.info( "Skipping duplicate user turn " "(message_id=%s) in session %s", event.message_id, session_entry.session_id, ) else: await self.async_session_store.append_to_transcript( session_entry.session_id, _user_entry, skip_db=agent_persisted, ) else: history_len = agent_result.get("history_offset", len(history)) new_messages = agent_messages[history_len:] if len(agent_messages) > history_len else [] # If no new messages found (edge case), fall back to simple user/assistant if not new_messages: _user_entry = { "role": "user", "content": ( persist_user_message if persist_user_message is not None else message_text ), "timestamp": ( persist_user_timestamp if persist_user_timestamp is not None else ts ), } if persist_user_display_kind: _user_entry["display_kind"] = persist_user_display_kind if event.message_id: _user_entry["message_id"] = str(event.message_id) await self.async_session_store.append_to_transcript( session_entry.session_id, _user_entry, skip_db=agent_persisted, ) if response: await self.async_session_store.append_to_transcript( session_entry.session_id, {"role": "assistant", "content": response, "timestamp": ts}, skip_db=agent_persisted, ) else: # Attach the inbound platform message_id to the first user # entry written this turn so platform-level quote-resolution # (e.g. Yuanbao QuoteContextMiddleware's transcript fallback) # can find earlier @bot messages by their original message_id. _user_msg_id_attached = False for msg in new_messages: # Skip system messages (they're rebuilt each run) if msg.get("role") == "system": continue # Add timestamp to each message for debugging entry = {**msg, "timestamp": ts} if ( not _user_msg_id_attached and msg.get("role") == "user" and event.message_id and "message_id" not in entry ): entry["message_id"] = str(event.message_id) _user_msg_id_attached = True await self.async_session_store.append_to_transcript( session_entry.session_id, entry, skip_db=agent_persisted, ) # Token counts and model are now persisted by the agent directly. # Keep only last_prompt_tokens here for context-window tracking and # compression decisions. await self.async_session_store.update_session( session_entry.session_key, last_prompt_tokens=agent_result.get("last_prompt_tokens", 0), touch_activity=not bool(getattr(event, "internal", False)), ) # Re-baseline the cached agent's message_count snapshot now that # ALL of this turn's transcript writes are done — the agent's # flushed user/assistant/tool rows AND the first-turn `session_meta` # marker appended above. The cross-process coherence guard (#45966) # snapshots the count at agent-BUILD time (before this turn's own # writes) and never refreshes it on reuse, so without this the # process's own turn grows message_count and the next turn sees a # mismatch and rebuilds the agent — destroying prompt caching. # # This MUST run after the `session_meta` append: that row also # increments message_count, so re-baselining before it (the old # position) left the snapshot one short and the guard mis-fired on # turn 2 of EVERY fresh gateway conversation, rebuilding the cached # agent and busting the prompt cache. Running here also uses the # compaction-updated session_id (the agent_result session_id swap # above), matching this function's documented contract. Refreshing # here makes the guard fire only on a DIFFERENT process's writes. # Fail-safe inside the helper. await self._refresh_agent_cache_message_count( session_key, session_entry.session_id ) # Intentional silence is a delivery decision, not a transcript # mutation. The agent's [SILENT]/NO_REPLY assistant turn above is # still persisted in session history so later turns keep normal # user/assistant alternation; only the outbound chat delivery is # suppressed. if _intentional_silence: logger.info( "Suppressing intentional silence marker for session %s", session_entry.session_id, ) response = "" # Auto voice reply: send TTS audio before the text response _already_sent = bool(agent_result.get("already_sent")) # Skip when streaming TTS already delivered audio for this turn (#60671). _stts_adapter = self._adapter_for_source(source) _streaming_tts_done = ( _stts_adapter is not None and bool(getattr(_stts_adapter, "_streaming_tts_turn_completed", lambda *_a, **_k: False)(session_key, run_generation)) ) if ( not _streaming_tts_done and self._should_send_voice_reply(event, response, agent_messages, already_sent=_already_sent) ): await self._send_voice_reply(event, response) # If streaming already delivered the response, extract and # deliver any MEDIA: files before returning None. Streaming # sends raw text chunks that include MEDIA: tags — the normal # post-processing in _process_message_background is skipped # when already_sent is True, so media files would never be # delivered without this. # # Never skip when the agent failed — the error message is new # content the user hasn't seen (streaming only sent earlier # partial output before the failure). Without this guard, # users see the agent "stop responding without explanation." if agent_result.get("already_sent") and not agent_result.get("failed"): if response: _media_adapter = self._adapter_for_source(source) if _media_adapter: await self._deliver_media_from_response( response, event, _media_adapter, ) # Streaming already delivered the body text, but the footer was # intentionally held back (see the `not already_sent` gate above). # Send it now as a small trailing message so Telegram/Discord/etc. # still surface the runtime metadata on the final reply. if _footer_line: try: _foot_adapter = self._adapter_for_source(source) if _foot_adapter: await _foot_adapter.send( source.chat_id, _footer_line, metadata=self._thread_metadata_for_source(source, self._reply_anchor_for_event(event)), ) except Exception as _e: logger.debug("trailing footer send failed: %s", _e) # This branch returns None so the adapter does not send the # body twice. /loop and /goal hooks in _handle_message read # the return value, so stash the delivered text on the event # or those hooks never run and a /loop tick stays awaiting. try: event._streamed_final_response = str(response or "") except Exception: pass return None return response except Exception as e: # Stop typing indicator on error too, retaining Slack thread/workspace # routing so a failed turn cannot leave its status visible. try: _err_adapter = self._adapter_for_source(source) _stop_with_metadata = getattr( type(_err_adapter), "_stop_typing_with_metadata", None ) _stop_typing = getattr(type(_err_adapter), "stop_typing", None) if _err_adapter and callable(_stop_with_metadata): await _err_adapter._stop_typing_with_metadata( source.chat_id, self._thread_metadata_for_source( source, self._reply_anchor_for_event(event) ), ) elif _err_adapter and callable(_stop_typing): await _err_adapter.stop_typing(source.chat_id) except Exception: pass logger.exception("Agent error in session %s", session_key) # Crash-resilience for failures that happen before AIAgent enters # run_conversation() (for example: provider/httpx client init # failures). In that path the agent cannot persist the current # inbound turn itself, so append the user message here once. If the # agent already reached its early turn-start persistence, the latest # transcript user row will match and we skip the duplicate. try: if 'message_text' in locals() and message_text is not None and session_entry is not None: _already_persisted = False try: _recent_transcript = await self.async_session_store.load_transcript(session_entry.session_id) except Exception: _recent_transcript = [] for _msg in reversed(_recent_transcript[-10:]): if _msg.get("role") == "user": _expected_user_content = ( persist_user_message if persist_user_message is not None else message_text ) _already_persisted = (_msg.get("content") == _expected_user_content) break if not _already_persisted: _user_entry = { "role": "user", "content": ( persist_user_message if persist_user_message is not None else message_text ), "timestamp": ( persist_user_timestamp if persist_user_timestamp is not None else time.time() ), } if 'persist_user_display_kind' in locals() and persist_user_display_kind: _user_entry["display_kind"] = persist_user_display_kind if getattr(event, "message_id", None): _user_entry["message_id"] = str(event.message_id) await self.async_session_store.append_to_transcript( session_entry.session_id, _user_entry, ) except Exception: logger.debug("Failed to persist inbound user message after agent exception", exc_info=True) # Log full details server-side only; never expose raw exception # types or messages to end users (info-leakage risk). status_hint = "" status_code = getattr(e, "status_code", None) _hist_len = len(history) if 'history' in locals() else 0 if status_code == 401: status_hint = " Check your API key or run `claude /login` to refresh OAuth credentials." elif status_code == 402: status_hint = " Your API balance or quota is exhausted. Check your provider dashboard." elif status_code == 429: # Check if this is a plan usage limit (resets on a schedule) vs a transient rate limit _err_body = getattr(e, "response", None) _err_json = {} try: if _err_body is not None: _err_json = _err_body.json().get("error", {}) if not isinstance(_err_json, dict): _err_json = {} except Exception: pass if _err_json.get("type") == "usage_limit_reached": _resets_in = _err_json.get("resets_in_seconds") if _resets_in and _resets_in > 0: import math _hours = math.ceil(_resets_in / 3600) status_hint = f" Your plan's usage limit has been reached. It resets in ~{_hours}h." else: status_hint = " Your plan's usage limit has been reached. Please wait until it resets." else: status_hint = " You are being rate-limited. Please wait a moment and try again." elif status_code == 529: status_hint = " The API is temporarily overloaded. Please try again shortly." elif status_code in {400, 500}: # 400 with a large session is context overflow. # 500 with a large session often means the payload is too large # for the API to process — treat it the same way. if _hist_len > 50: return ( "⚠️ Session too large for the model's context window.\n" "Use /compact to compress the conversation, or " "/reset to start fresh." ) elif status_code == 400: status_hint = " The request was rejected by the API." return ( f"Sorry, I encountered an unexpected error.{status_hint}\n" "Try again or use /reset to start a fresh session." ) finally: # Restore session context variables to their pre-handler state self._clear_session_env(_session_env_tokens) def _reset_notice_session_info(self, source: SessionSource) -> str: """Session-info block for the auto-reset notice, profile-scoped. When multiplexing, resolve model/provider/context inside the profile serving ``source`` — otherwise the banner advertises the base config's model while the session actually runs on the profile's (#59003). Mirrors ``_run_agent``'s gating so single-profile gateways never enter the scope. Call via ``asyncio.to_thread`` from async handlers: under the scope, resolution can do blocking work (credential refresh, context-length HTTP probes) that must not run on the event loop. The scope is entered inside this method, so contextvars behave correctly in the worker thread. """ if getattr(getattr(self, "config", None), "multiplex_profiles", False): with _profile_runtime_scope(self._resolve_profile_home_for_source(source)): return self._format_session_info() return self._format_session_info() def _format_session_info(self) -> str: """Resolve current model config and return a formatted info block. Surfaces model, provider, context length, and endpoint so gateway users can immediately see if context detection went wrong (e.g. local models falling to the 128K default). """ resolved = _resolve_gateway_model_context() model = resolved.model provider = resolved.provider base_url = resolved.base_url context_length = resolved.context_length # Format context source hint if resolved.context_source == "config": ctx_source = "config" elif resolved.context_source == "default": ctx_source = "default — set model.context_length in config to override" else: ctx_source = "detected" # Format context length for display if context_length >= 1_000_000: ctx_display = f"{context_length / 1_000_000:.1f}M" elif context_length >= 1_000: ctx_display = f"{context_length // 1_000}K" else: ctx_display = str(context_length) lines = [ f"◆ Model: `{model}`", f"◆ Provider: {provider or 'openrouter'}", f"◆ Context: {ctx_display} tokens ({ctx_source})", ] # Show endpoint for local/custom setups if base_url and base_url_hostname(base_url) in ("localhost", "127.0.0.1", "0.0.0.0"): lines.append(f"◆ Endpoint: {base_url}") return "\n".join(lines) def _check_slash_access( self, source: SessionSource, canonical_cmd: str ) -> Optional[str]: """Return a denial message if ``source`` cannot run ``canonical_cmd``, else None. Used by both the cold and running-agent dispatch paths in ``_handle_message`` so admin/user gating can't be bypassed by an in-flight agent. Backward-compat semantics live in :func:`gateway.slash_access.policy_for_source` — when the operator hasn't set ``allow_admin_from`` for the scope, the policy returns ``enabled=False`` and this method always returns None. """ from gateway.slash_access import policy_for_source as _policy_for_source if not canonical_cmd: return None policy = _policy_for_source(self.config, source) if not policy.enabled or policy.can_run(source.user_id, canonical_cmd): return None logger.info( "Slash command /%s denied for %s:%s (not admin, not in user_allowed_commands)", canonical_cmd, source.platform.value if source.platform else "?", source.user_id, ) allowed_preview = sorted(policy.user_allowed_commands) if allowed_preview: suffix = ( "You can run: " + ", ".join(f"/{c}" for c in allowed_preview[:12]) + ("…" if len(allowed_preview) > 12 else "") + ". Use /whoami for the full list." ) else: suffix = ( "No slash commands are enabled for non-admins on this " "platform. Ask an admin to add you to allow_admin_from " "or to set user_allowed_commands." ) return f"⛔ /{canonical_cmd} is admin-only here. {suffix}" def _sibling_thread_run_keys(self, source: SessionSource, own_key: str) -> list: """Find running-agent keys for OTHER participants in the same thread. Only applies when the message originates in a thread. In per-user thread mode (``thread_sessions_per_user=True``) each participant gets an isolated session key of the form ``agent:main:{platform}:{chat_type}:{chat_id}:{thread_id}:{user_id}``, so a run started by another user is invisible to the caller's own ``/stop``. This returns the keys of any *actually running* agents (not the pending sentinel, not the caller's own key) whose key shares the caller's ``{chat_id}:{thread_id}`` prefix. Returns an empty list when the source is not in a thread, or when no sibling runs exist — callers must still gate on authorization. """ thread_id = getattr(source, "thread_id", None) chat_id = getattr(source, "chat_id", None) if not thread_id or not chat_id: return [] platform = source.platform.value chat_type = getattr(source, "chat_type", None) or "" # Prefix that every per-user key in this thread shares, up to and # including the thread_id segment. Matching either the exact # shared-thread key or any key with a further (user_id) segment # (prefix + ":") avoids cross-matching an unrelated thread whose id # merely starts with this one. prefix = ":".join( ["agent:main", platform, chat_type, str(chat_id), str(thread_id)] ) matches = [] for key, agent in self._running_agent_items(): if key == own_key: continue if agent is _AGENT_PENDING_SENTINEL or not agent: continue if key == prefix or key.startswith(prefix + ":"): matches.append(key) return matches def _is_stale_restart_redelivery(self, event: MessageEvent) -> bool: """Return True if this /restart is a Telegram re-delivery we already handled. The previous gateway wrote ``.restart_last_processed.json`` with the triggering platform + update_id when it processed the /restart. If we now see a /restart on the same platform with an update_id <= that recorded value, it is a redelivery when this process booted from that restart. Otherwise the marker must still be recent (< 5 minutes). Only applies to Telegram today (the only platform that exposes a numeric cross-session update ordering); other platforms return False. """ if event is None or event.source is None: return False if event.platform_update_id is None: return False if event.source.platform is None: return False # Only Telegram populates platform_update_id currently; be explicit # so future platforms aren't accidentally gated by this check. try: platform_value = event.source.platform.value except Exception: return False if platform_value != "telegram": return False try: marker_path = _hermes_home / ".restart_last_processed.json" if not marker_path.exists(): # Belt-and-suspenders for when the dedup marker goes missing # (manually cleaned up, or the previous cycle's write failed). # Without a marker the update_id comparison below can't run, so # a redelivered /restart would sail through and re-restart the # gateway — an infinite loop (issue #18528). # # Suppress ONLY when we can independently confirm we just came # out of a restart cycle: this process booted from a # chat-originated /restart (_booted_from_restart) AND is still # within a short post-boot window. This never swallows a # genuine first /restart on a fresh boot (no restart marker on # boot → flag stays False). Consume the flag one-shot so a # legitimate /restart sent later in the same session is honored. if ( getattr(self, "_booted_from_restart", False) and time.time() - getattr(self, "_startup_time", 0.0) < 60 ): self._booted_from_restart = False return True return False data = json.loads(marker_path.read_text(encoding="utf-8")) except Exception: return False if data.get("platform") != platform_value: return False recorded_uid = data.get("update_id") if not isinstance(recorded_uid, int): return False if event.platform_update_id > recorded_uid: return False # A service-managed restart can legitimately take longer than the # marker's normal five-minute trust window while adapters, cron, and # in-flight deliveries drain. If this process booted from the recorded # chat restart, the first same-or-older update is still that restart's # redelivery regardless of elapsed wall time. Consume the boot signal # one-shot so a later genuine command is evaluated normally. if getattr(self, "_booted_from_restart", False): self._booted_from_restart = False return True # Staleness guard: ignore markers older than 5 minutes. A legitimately # old marker (e.g. crash recovery where notify never fired) should not # swallow a fresh /restart from the user. requested_at = data.get("requested_at") if isinstance(requested_at, (int, float)): if time.time() - requested_at > 300: return False return True async def _handle_suggestions_command(self, event: MessageEvent) -> str: """Handle /suggestions in the gateway. Delegates to the shared handler so CLI and gateway never drift. The origin is built from the event source so an accepted suggestion's job delivers back to this chat/thread. """ args = (event.get_command_args() or "").strip() source = event.source origin = None try: platform = getattr(source.platform, "value", None) or str(getattr(source, "platform", "") or "") chat_id = getattr(source, "chat_id", None) if platform and chat_id: origin = { "platform": platform, "chat_id": str(chat_id), "chat_name": getattr(source, "chat_name", None), "thread_id": getattr(source, "thread_id", None), } except Exception: origin = None try: from hermes_cli.suggestions_cmd import handle_suggestions_command return handle_suggestions_command(args, origin=origin, surface="gateway") except Exception as e: logger.debug("suggestions command failed: %s", e) return f"Suggestions command failed: {e}" async def _handle_blueprint_command(self, event: MessageEvent): """Handle /blueprint in the gateway. Delegates to the shared handler so CLI, TUI, and gateway never drift. Returns a BlueprintCommandResult: ``text`` is shown to the user, and if ``agent_seed`` is set the dispatch site rewrites ``event.text`` to the seed and falls through to the agent (the ``/steer`` pattern) so the agent gathers the slot values conversationally. Origin is built from the event source so a directly created blueprint job delivers back to this chat. """ args = (event.get_command_args() or "").strip() source = event.source origin = None try: platform = getattr(source.platform, "value", None) or str(getattr(source, "platform", "") or "") chat_id = getattr(source, "chat_id", None) if platform and chat_id: origin = { "platform": platform, "chat_id": str(chat_id), "chat_name": getattr(source, "chat_name", None), "thread_id": getattr(source, "thread_id", None), } except Exception: origin = None try: from hermes_cli.blueprint_cmd import handle_blueprint_command return handle_blueprint_command(args, origin=origin, surface="gateway") except Exception as e: logger.debug("blueprint command failed: %s", e) from hermes_cli.blueprint_cmd import BlueprintCommandResult return BlueprintCommandResult(f"Cron blueprint command failed: {e}") # ──────────────────────────────────────────────────────────────── # /goal — persistent cross-turn goals (Ralph-style loop) # ──────────────────────────────────────────────────────────────── def _goal_max_turns_from_config(self) -> int: """Resolve the configured /goal turn budget for gateway sessions. GatewayRunner.config is a GatewayConfig dataclass, not the full user config mapping. Top-level config blocks such as ``goals`` are therefore only available through hermes_cli.config.load_config(). """ try: goals_cfg = ( (self.config or {}).get("goals", {}) if isinstance(self.config, dict) else getattr(self.config, "goals", {}) or {} ) if not goals_cfg: from hermes_cli.config import load_config goals_cfg = (load_config() or {}).get("goals") or {} return int(goals_cfg.get("max_turns", 20) or 20) except Exception: return 20 async def _warm_goals_session_db(self, label: str) -> None: """Warm the goals SessionDB cache off-loop (best-effort). A cold cache runs the state.db init on the loop thread behind the bootstrap windows. That freezes the loop for the init duration. The executor hop keeps the profile home override alive under multiplex, so the warm cache belongs to the caller's profile. On failure the caller falls back to the bootstrap windows, so a dropped warm-up is a bounded stall, never a crash. """ try: from hermes_cli.goals import _get_session_db as _warm_goals_db await self._run_in_executor_with_context(_warm_goals_db) except Exception as exc: logger.warning("%s: session DB warm-up failed: %s", label, exc) async def _get_goal_manager_for_event(self, event: "MessageEvent"): """Return a GoalManager bound to the session for this gateway event. Returns ``(manager, session_entry)`` or ``(None, None)`` if the goals module can't be loaded. """ try: from hermes_cli.goals import GoalManager except Exception as exc: logger.debug("goal manager unavailable: %s", exc) return None, None # Warm the SessionDB cache off-loop. A cold cache freezes the # loop for the init duration and drops the first write: the # /goal reply claims the goal was set. await self._warm_goals_session_db("goal manager") try: # Session lookups on behalf of an internal event must not advance # the user-activity clock that drives idle/daily reset policy # (same class as the wake fix in _handle_message_with_agent). session_entry = await self.async_session_store.get_or_create_session( event.source, touch_activity=not bool(getattr(event, "internal", False)), ) except Exception as exc: logger.debug("goal manager: session lookup failed: %s", exc) return None, None sid = getattr(session_entry, "session_id", None) or "" if not sid: return None, None max_turns = self._goal_max_turns_from_config() return GoalManager(session_id=sid, default_max_turns=max_turns), session_entry async def _get_heartbeat_manager_for_event(self, event: "MessageEvent"): """Return a HeartbeatManager bound to the session for this event. Returns ``(manager, session_entry)`` or ``(None, None)``. """ try: from hermes_cli.heartbeat import HeartbeatManager except Exception as exc: logger.debug("heartbeat manager unavailable: %s", exc) return None, None # Warm the SessionDB cache off-loop. A cold cache can drop the # first /heartbeat write while the reply claims it was set. await self._warm_goals_session_db("heartbeat manager") try: # Same reset-policy contract as _get_goal_manager_for_event: # internal events look up the session without touching activity. session_entry = await self.async_session_store.get_or_create_session( event.source, touch_activity=not bool(getattr(event, "internal", False)), ) except Exception as exc: logger.debug("heartbeat manager: session lookup failed: %s", exc) return None, None sid = getattr(session_entry, "session_id", None) or "" if not sid: return None, None return HeartbeatManager(session_id=sid), session_entry def _register_heartbeat_watch(self, quick_key: str, source: Any, session_id: str) -> None: """Track a session with an active heartbeat and start the poller. The registry maps ``quick_key`` → ``(source, session_id)`` so the poller can rebuild a MessageEvent and enqueue via the adapter FIFO. In-memory by design: heartbeat STATE survives restarts in SessionDB, but firing resumes when the user touches /heartbeat again in the new gateway process (documented; durable schedules belong to cron). """ watch = getattr(self, "_heartbeat_watch", None) if watch is None: watch = {} self._heartbeat_watch = watch watch[quick_key] = (source, session_id) self._start_heartbeat_poller() def _unregister_heartbeat_watch(self, quick_key: str) -> None: watch = getattr(self, "_heartbeat_watch", None) if watch: watch.pop(quick_key, None) def _start_heartbeat_poller(self) -> None: """Start the single gateway-wide heartbeat poll task (idempotent).""" existing = getattr(self, "_heartbeat_poll_task", None) if existing is not None and not existing.done(): return from hermes_cli.heartbeat import POLL_SECONDS async def _poll_loop(): while True: await asyncio.sleep(POLL_SECONDS) watch = getattr(self, "_heartbeat_watch", None) if not watch: continue # Warm the cache off-loop once per poll. A watch can only # be registered through the warmed /heartbeat command, so # this covers only the degraded path where that warm-up # failed. await self._warm_goals_session_db("heartbeat poll") for quick_key, (source, session_id) in list(watch.items()): try: # Busy sessions coalesce their tick to the next idle poll. if quick_key in self._running_agents: continue from hermes_cli.heartbeat import HeartbeatManager mgr = HeartbeatManager(session_id=session_id) if not mgr.has_heartbeat(): watch.pop(quick_key, None) continue prompt = mgr.due_prompt() if not prompt: continue adapter = self._adapter_for_source(source) if adapter is None: continue hb_event = MessageEvent( text=prompt, message_type=MessageType.TEXT, source=source, message_id=None, channel_prompt=None, ) self._enqueue_fifo(quick_key, hb_event, adapter) except Exception as exc: logger.debug("heartbeat poll for %s failed: %s", quick_key, exc) try: task = asyncio.create_task(_poll_loop()) self._heartbeat_poll_task = task # PERMANENT once started (an infinite while-True loop, no exit # condition) — same as a _spawn_supervised watcher. Tag it so # _scale_to_zero_has_live_background_work() doesn't treat a # gateway with an active heartbeat watch as busy forever. task._hermes_supervised_watcher = True # type: ignore[attr-defined] _bg = getattr(self, "_background_tasks", None) if _bg is not None: _bg.add(task) task.add_done_callback(_bg.discard) except Exception: logger.debug("Failed to start heartbeat poller", exc_info=True) async def _send_goal_status_notice(self, source: Any, message: str) -> None: """Send a /goal judge status line back to the originating chat/thread.""" adapter = self._adapter_for_source(source) if not adapter: logger.debug("goal continuation: no adapter for %s", getattr(source, "platform", None)) return try: metadata = self._thread_metadata_for_source(source) except Exception: metadata = None result = await adapter.send(source.chat_id, message, metadata=metadata) if result is not None and not getattr(result, "success", True): logger.warning( "goal continuation: status send failed: %s", getattr(result, "error", "unknown error"), ) async def _defer_goal_status_notice_after_delivery(self, source: Any, message: str) -> None: """Send a /goal status line after the main response is delivered. The gateway message handler returns the agent response to the platform adapter, which sends it after this method's caller has returned. For a natural Discord/Telegram reading order, goal status belongs after that send. Platform adapters provide a one-shot post-delivery callback for exactly this boundary; when unavailable, fall back to direct awaited delivery rather than silently dropping the notice. """ adapter = self._adapter_for_source(source) if not adapter: logger.debug("goal continuation: no adapter for %s", getattr(source, "platform", None)) return async def _deliver() -> None: try: await self._send_goal_status_notice(source, message) except Exception as exc: logger.warning("goal continuation: status send failed: %s", exc, exc_info=True) try: session_key = self._session_key_for_source(source) except Exception: session_key = None if session_key and hasattr(adapter, "register_post_delivery_callback"): try: generation = None active = getattr(adapter, "_active_sessions", {}).get(session_key) if active is not None: generation = getattr(active, "_hermes_run_generation", None) adapter.register_post_delivery_callback( session_key, _deliver, generation=generation, ) return except Exception as exc: logger.debug("goal continuation: post-delivery callback registration failed: %s", exc) await _deliver() async def _post_turn_goal_continuation( self, *, session_entry: Any, source: Any, final_response: str, ) -> None: """Run the goal judge after a gateway turn and, if still active, enqueue a continuation prompt for the same session. Called from ``_handle_message_with_agent`` at turn boundary, AFTER the response has been delivered. Safe when no goal is set. We use the adapter's pending-message / FIFO machinery so any real user message that arrives simultaneously is handled by the same queue and takes priority naturally. """ try: from hermes_cli.goals import GoalManager except Exception as exc: logger.debug("goal continuation: goals module unavailable: %s", exc) return sid = getattr(session_entry, "session_id", None) or "" if not sid: return max_turns = self._goal_max_turns_from_config() # Warm the SessionDB cache off-loop. A cold cache runs the # state.db init on the loop thread at the turn boundary (the # 2026-08-14 crash-loop seam). A slow init can drop the goal # read and silently end the goal loop. await self._warm_goals_session_db("goal continuation") mgr = GoalManager(session_id=sid, default_max_turns=max_turns) if not mgr.is_active(): return try: from hermes_cli.goals import gather_background_processes as _gather_bg _bg_procs = _gather_bg() except Exception: _bg_procs = None # evaluate_after_turn calls judge_goal() which makes a synchronous # HTTP request to the auxiliary LLM. Running it on the event-loop # thread would block Discord heartbeats for 10-40 s and cause # connection flaps, so we offload it to a thread-pool executor. # _run_in_executor_with_context (not bare run_in_executor): the # profile secret scope and auxiliary runtime context are contextvars, # and a default-executor hop would drop them — aux-client provider # resolution would then read credentials unscoped and fail under # multiplexing (same pattern as compression in slash_commands.py). decision = await self._run_in_executor_with_context( lambda: mgr.evaluate_after_turn( final_response or "", user_initiated=True, background_processes=_bg_procs, ), ) msg = decision.get("message") or "" # Defer the status line until after the adapter has delivered the # agent's visible final response. The judge runs after the response is # produced but before BasePlatformAdapter sends it, so sending here # would show "✓ Goal achieved" before the answer itself. Registering # an awaited post-delivery callback preserves delivery reliability # without reversing the user-visible ordering. if msg and source is not None: await self._defer_goal_status_notice_after_delivery(source, msg) if not decision.get("should_continue"): return prompt = decision.get("continuation_prompt") or "" if not prompt or source is None: return # Enqueue via the adapter's FIFO so a user message already in # flight preempts the continuation naturally. try: adapter = self._adapter_for_source(source) _quick_key = self._session_key_for_source(source) if adapter and _quick_key: cont_event = MessageEvent( text=prompt, message_type=MessageType.TEXT, source=source, message_id=None, channel_prompt=None, ) self._enqueue_fifo(_quick_key, cont_event, adapter) except Exception as exc: logger.debug("goal continuation: enqueue failed: %s", exc) async def _run_post_turn_hooks( self, *, agent_result: Any, source: Any, is_internal: bool, event: Any = None, ) -> None: """Run goal and loop bookkeeping after an agent turn returns.""" final_text = self._final_text_for_post_turn_hooks(agent_result, event) try: session_entry = await self.async_session_store.get_or_create_session( source, touch_activity=not is_internal, ) except Exception as exc: logger.debug("post-turn session resolution failed: %s", exc) return # Empty interrupted/errored responses must not drive /goal, but an # in-flight /loop tick still needs to be released and rescheduled. if final_text.strip(): try: await self._post_turn_goal_continuation( session_entry=session_entry, source=source, final_response=final_text, ) except Exception as exc: logger.debug("goal continuation hook failed: %s", exc) try: await self._post_turn_loop_completion( session_entry=session_entry, source=source, final_response=final_text, ) except Exception as exc: logger.debug("loop completion hook failed: %s", exc) @staticmethod def _final_text_for_post_turn_hooks(agent_result, event=None) -> str: """Text for /goal and /loop after a gateway turn. Streamed turns return None from _handle_message_with_agent (already_sent). The delivered reply is stashed on the event so those hooks still see it. """ text = "" if isinstance(agent_result, dict): text = str(agent_result.get("final_response") or "") elif isinstance(agent_result, str): text = agent_result if text.strip(): return text streamed = getattr(event, "_streamed_final_response", None) if isinstance(streamed, str) and streamed.strip(): return streamed return text async def _post_turn_loop_completion( self, *, session_entry: Any, source: Any, final_response: str, ) -> None: """Complete a /loop wakeup tick after a gateway turn. No-op unless the session has a loop whose tick is in flight (``awaiting_response`` — set when the wakeup was injected). Applies the LOOP_COMPLETE marker / --until judge / caps and schedules the next tick; the idle wakeup watcher fires it when due. """ try: from hermes_cli.loops import LoopManager except Exception as exc: logger.debug("loop completion: loops module unavailable: %s", exc) return sid = getattr(session_entry, "session_id", None) or "" if not sid: return # Warm the SessionDB cache off-loop. A cold cache at the turn # boundary stalls the loop for the init duration and can drop # the tick-completion write (the /goal continuation seam, one # sibling over). await self._warm_goals_session_db("loop completion") mgr = LoopManager(session_id=sid) state = mgr.state if state is None or not state.awaiting_response: return # The --until judge is a sync aux-LLM call — keep it off the event loop. decision = await asyncio.get_running_loop().run_in_executor( None, mgr.complete_tick, final_response or "" ) msg = decision.get("message") or "" if msg and source is not None: await self._defer_goal_status_notice_after_delivery(source, msg) async def _loop_wakeup_watcher(self, interval: float = 15.0) -> None: """Fire due /loop wakeups for idle gateway sessions. The gateway has no per-session scheduler thread, so a coarse ticker scans persisted loops (SessionDB ``loop:*`` rows) and injects the wakeup prompt into each due session's chat via the same synthetic- message path used by watch notifications. Deferrals: - session currently running an agent turn → skip (stays due; the adapter FIFO would race the live turn otherwise) - active non-parked /goal on the session → skip (goal owns the idle boundary) - no routing metadata on the loop → skip with a one-time warning (CLI/TUI loops carry no route and are driven by their own surfaces) """ await asyncio.sleep(5) # let platforms finish connecting warned_no_route: set = set() while self._running: try: from hermes_cli.loops import ( LoopManager, goal_blocks_loop_tick, list_active_loops, ) # Warm the cache off-loop once per scan. The scan reads # every persisted loop, so a cold cache runs the state.db # init on the loop thread before the first read. await self._warm_goals_session_db("loop wakeup") # Every SessionDB call in this scan runs off the loop thread. # fire_tick()/complete_tick() are writes (BEGIN IMMEDIATE) that # take the writer lock; a slow writer elsewhere (FTS merge, WAL # checkpoint, a long flush) holding it while the watcher blocked # the loop on the same lock froze the gateway for 90+ s until # the liveness watchdog force-exited. list_active_loops() reads # via _read_ctx (lock-free under WAL) but still convoys on the # writer lock when WAL is unavailable, so it goes off-loop too. # _run_in_executor_with_context keeps the profile HERMES_HOME # override alive under multiplex, like the warm-up above. active_loops = await self._run_in_executor_with_context(list_active_loops) now = time.time() for sid, state in active_loops: if state.awaiting_response or now < state.next_due_at: continue route = state.route or {} platform_name = route.get("platform", "") chat_id = route.get("chat_id", "") if not platform_name or not chat_id: # CLI / TUI-owned loop — their own schedulers drive it. continue adapter = None for p, a in self.adapters.items(): if p.value == platform_name: adapter = a break if adapter is None: if sid not in warned_no_route: warned_no_route.add(sid) logger.debug( "loop wakeup: no adapter for platform %r (session %s)", platform_name, sid, ) continue # Build the source + session key to check business. evt_stub = { "session_key": "", "platform": platform_name, "chat_id": chat_id, "chat_type": route.get("chat_type", ""), "thread_id": route.get("thread_id", ""), "user_id": route.get("user_id", ""), "user_name": route.get("user_name", ""), } source = self._build_process_event_source(evt_stub) if source is None: continue try: session_key = self._session_key_for_source(source) except Exception: session_key = None if session_key and session_key in self._running_agents: continue # busy — stays due, next scan retries if goal_blocks_loop_tick(sid): continue mgr = LoopManager(session_id=sid) if not mgr.is_due(now): continue wakeup = await self._run_in_executor_with_context(mgr.fire_tick) if not wakeup: continue try: synth_event = MessageEvent( text=wakeup, message_type=MessageType.TEXT, source=source, internal=True, ) logger.info( "loop wakeup #%s — injecting for %s chat=%s thread=%s", mgr.state.ticks_fired if mgr.state else "?", platform_name, source.chat_id, source.thread_id, ) await adapter.handle_message(synth_event) # Slash-command loops dispatch through the command # path and never hit the post-turn completion hook — # complete the tick immediately (caps + scheduling). if wakeup.lstrip().startswith("/"): await self._run_in_executor_with_context(mgr.complete_tick, "") except Exception as exc: logger.warning("loop wakeup injection failed for %s: %s", sid, exc) try: mgr.abandon_tick() except Exception: pass except Exception as exc: logger.debug("loop wakeup watcher error: %s", exc) await asyncio.sleep(interval) @staticmethod def _get_guild_id(event: MessageEvent) -> Optional[int]: """Extract Discord guild_id from the raw message object.""" raw = getattr(event, "raw_message", None) if raw is None: return None # Slash command interaction if hasattr(raw, "guild_id") and raw.guild_id: return int(raw.guild_id) # Regular message if hasattr(raw, "guild") and raw.guild: return raw.guild.id return None async def _handle_voice_channel_join(self, event: MessageEvent) -> str: """Join the user's current Discord voice channel.""" adapter = self._adapter_for_source(event.source) if not hasattr(adapter, "join_voice_channel"): return "Voice channels are not supported on this platform." guild_id = self._get_guild_id(event) if not guild_id: return "This command only works in a Discord server." voice_channel = await adapter.get_user_voice_channel( guild_id, event.source.user_id ) if not voice_channel: return "You need to be in a voice channel first." # Wire callbacks BEFORE join so voice input arriving immediately # after connection is not lost. self._bind_voice_input_callback(adapter) voice_profile = self._adapter_profile_for_source(event.source) if hasattr(adapter, "_on_voice_disconnect"): adapter._on_voice_disconnect = functools.partial( self._handle_voice_timeout_cleanup, adapter=adapter ) # Let the adapter's inactivity timer see the live voice-reply mode so it # doesn't disconnect a deliberately text-only (/voice off) session. if hasattr(adapter, "_voice_mode_getter"): adapter._voice_mode_getter = lambda chat_id: self._voice_mode.get( self._voice_key(Platform.DISCORD, str(chat_id), profile=voice_profile), "off", ) try: success = await adapter.join_voice_channel(voice_channel) except Exception as e: logger.warning("Failed to join voice channel: %s", e) adapter._voice_input_callback = None err_lower = str(e).lower() if "pynacl" in err_lower or "nacl" in err_lower or "davey" in err_lower: return ( "Voice dependencies are missing (PyNaCl / davey). " f"Install with: `{sys.executable} -m pip install PyNaCl`" ) return f"Failed to join voice channel: {e}" if success: adapter._voice_text_channels[guild_id] = int(event.source.chat_id) if hasattr(adapter, "_voice_sources"): adapter._voice_sources[guild_id] = event.source.to_dict() self._voice_mode[self._voice_key_for_source(event.source)] = "all" self._save_voice_modes() self._set_adapter_auto_tts_enabled(adapter, event.source.chat_id, enabled=True) return ( f"Joined voice channel **{voice_channel.name}**.\n" f"I'll speak my replies and listen to you. Use /voice leave to disconnect." ) # Join failed — clear callback adapter._voice_input_callback = None return "Failed to join voice channel. Check bot permissions (Connect + Speak)." async def _handle_voice_channel_leave(self, event: MessageEvent) -> str: """Leave the Discord voice channel.""" adapter = self._adapter_for_source(event.source) guild_id = self._get_guild_id(event) if not guild_id or not hasattr(adapter, "leave_voice_channel"): return "Not in a voice channel." if not hasattr(adapter, "is_in_voice_channel") or not adapter.is_in_voice_channel(guild_id): return "Not in a voice channel." try: await adapter.leave_voice_channel(guild_id) except Exception as e: logger.warning("Error leaving voice channel: %s", e) # Always clean up state even if leave raised an exception self._voice_mode[self._voice_key_for_source(event.source)] = "off" self._save_voice_modes() self._set_adapter_auto_tts_disabled(adapter, event.source.chat_id, disabled=True) if hasattr(adapter, "_voice_input_callback"): adapter._voice_input_callback = None return "Left voice channel." def _handle_voice_timeout_cleanup(self, chat_id: str, *, adapter=None) -> None: """Called by the adapter when a voice channel times out. Cleans up runner-side voice_mode state that the adapter cannot reach. ``adapter`` is the Discord adapter that timed out (bound at join time); under multiplexing that is a specific profile's bot, not necessarily ``self.adapters[DISCORD]``. """ if adapter is None: adapter = self.adapters.get(Platform.DISCORD) profile = getattr(adapter, "_owner_profile", None) self._voice_mode[self._voice_key(Platform.DISCORD, chat_id, profile=profile)] = "off" self._save_voice_modes() self._set_adapter_auto_tts_disabled(adapter, chat_id, disabled=True) def _is_duplicate_voice_transcript(self, guild_id: int, user_id: int, transcript: str) -> bool: """Suppress repeated STT outputs for the same recent utterance. Voice capture can occasionally emit the same utterance twice a few seconds apart, which creates a second queued agent run and overlapping spoken replies. Dedup exact and near-exact repeats per guild/user over a short window while allowing genuinely new turns through. """ from difflib import SequenceMatcher normalized = re.sub(r"\s+", " ", transcript).strip().lower() normalized = re.sub(r"[^\w\s]", "", normalized) if not normalized: return False now = time.monotonic() window_seconds = 12.0 key = (guild_id, user_id) recent_store = getattr(self, "_recent_voice_transcripts", None) if not isinstance(recent_store, dict): recent_store = {} self._recent_voice_transcripts = recent_store recent = [ (ts, txt) for ts, txt in recent_store.get(key, []) if now - ts <= window_seconds ] for _, prior in recent: if prior == normalized: recent_store[key] = recent return True if len(prior) >= 16 and len(normalized) >= 16: if SequenceMatcher(None, prior, normalized).ratio() >= 0.95: recent_store[key] = recent return True recent.append((now, normalized)) recent_store[key] = recent[-5:] return False async def _handle_voice_channel_input( self, guild_id: int, user_id: int, transcript: str, *, adapter=None ): """Handle transcribed voice from a user in a voice channel. Creates a synthetic MessageEvent and processes it through the adapter's full message pipeline (session, typing, agent, TTS reply). ``adapter`` is the Discord adapter that captured the audio (bound via ``_bind_voice_input_callback``); under multiplexing each profile's bot must dispatch through its own adapter, never the default profile's. """ if adapter is None: adapter = self.adapters.get(Platform.DISCORD) if not adapter: return text_ch_id = adapter._voice_text_channels.get(guild_id) if not text_ch_id: return # Build source — reuse the linked text channel's metadata when available # so voice input shares the same session as the bound text conversation. source_data = getattr(adapter, "_voice_sources", {}).get(guild_id) if source_data: source = SessionSource.from_dict(source_data) source.user_id = str(user_id) source.user_name = str(user_id) else: source = SessionSource( platform=Platform.DISCORD, chat_id=str(text_ch_id), user_id=str(user_id), user_name=str(user_id), chat_type="channel", profile=getattr(adapter, "_owner_profile", None), ) # Check authorization before processing voice input if not self._is_user_authorized(source): logger.debug("Unauthorized voice input from user %d, ignoring", user_id) return if self._is_duplicate_voice_transcript(guild_id, user_id, transcript): logger.info( "Suppressing duplicate voice transcript for guild=%s user=%s: %s", guild_id, user_id, transcript[:100], ) return # Show transcript in text channel (after auth, with mention sanitization) try: channel = adapter._client.get_channel(text_ch_id) if channel: safe_text = transcript[:2000].replace("@everyone", "@\u200beveryone").replace("@here", "@\u200bhere") await channel.send(f"**[Voice]** <@{user_id}>: {safe_text}") except Exception: pass # Build a synthetic MessageEvent and feed through the normal pipeline # Use SimpleNamespace as raw_message so _get_guild_id() can extract # guild_id and _send_voice_reply() plays audio in the voice channel. from types import SimpleNamespace # Resolve the bound text channel's channel_prompt so voice input gets # the same per-channel context as typed messages (#50149). channel_prompt: Optional[str] = None resolver = getattr(adapter, "_resolve_channel_prompt", None) if callable(resolver): try: resolved = resolver(str(text_ch_id)) channel_prompt = resolved if isinstance(resolved, str) else None except Exception: channel_prompt = None event = MessageEvent( source=source, text=transcript, message_type=MessageType.VOICE, raw_message=SimpleNamespace(guild_id=guild_id, guild=None), channel_prompt=channel_prompt, ) await adapter.handle_message(event) def _should_send_voice_reply( self, event: MessageEvent, response: str, agent_messages: list, already_sent: bool = False, ) -> bool: """Decide whether the runner should send a TTS voice reply. Returns False when: - voice_mode is off for this chat - response is empty or an error - agent already called text_to_speech tool (dedup) - voice input and base adapter auto-TTS already handled it (skip_double) UNLESS streaming already consumed the response (already_sent=True), in which case the base adapter won't have text for auto-TTS so the runner must handle it. """ if not response or response.startswith("Error:"): return False chat_id = event.source.chat_id voice_key = self._voice_key_for_source(event.source) voice_mode = self._voice_mode.get(voice_key) is_voice_input = (event.message_type == MessageType.VOICE) adapter = self._adapter_for_source(event.source) adapter_auto_tts = False if adapter and hasattr(adapter, "_should_auto_tts_for_chat"): try: adapter_auto_tts = bool(adapter._should_auto_tts_for_chat(chat_id)) except Exception: adapter_auto_tts = False should = ( (voice_mode == "all") or (voice_mode == "voice_only" and is_voice_input) # ``voice.auto_tts`` is synced into the adapter on gateway startup. # It is the fallback only when the chat has no explicit mode; # otherwise the chat-level all/voice_only/off choice takes precedence. or (voice_mode is None and adapter_auto_tts) ) if not should: logger.debug( "Auto voice reply skipped: mode=%s adapter_auto_tts=%s chat=%s platform=%s", voice_mode, adapter_auto_tts, chat_id, event.source.platform.value, ) return False # Dedup: agent already called TTS tool in THIS turn only last_user_idx = None for i, msg in enumerate(reversed(agent_messages)): if msg.get("role") == "user": last_user_idx = len(agent_messages) - 1 - i; break turn_messages = agent_messages[last_user_idx:] if last_user_idx is not None else agent_messages has_agent_tts = any( msg.get("role") == "assistant" and any( (tc.get("function") or {}).get("name") == "text_to_speech" for tc in (msg.get("tool_calls") or []) ) for msg in turn_messages ) if has_agent_tts: return False # Dedup: base adapter auto-TTS already handles voice input # (play_tts plays in VC when connected, so runner can skip). # When streaming already delivered the text (already_sent=True), # the base adapter will receive None and can't run auto-TTS, # so the runner must take over. if is_voice_input and not already_sent: return False return True def _should_echo_stt_transcripts(self) -> bool: """Return whether inbound voice/STT transcripts should be echoed to chat.""" return bool(getattr(self.config, "stt_echo_transcripts", True)) async def _send_voice_reply(self, event: MessageEvent, text: str) -> None: """Generate TTS audio and send as a voice message before the text reply.""" audio_path = None actual_paths: List[str] = [] try: from tools.tts_tool import text_to_speech_tool, _strip_markdown_for_tts tts_text = _strip_markdown_for_tts(text) if not tts_text: return # Platform-aware output path: platforms whose native voice # bubbles require Ogg/Opus (OPUS_VOICE_PLATFORMS — Telegram, # Matrix, Feishu, WhatsApp, Signal) get an explicit .ogg path; # the TTS tool's central container repair guarantees real # Ogg/Opus bytes for every provider. Others keep MP3. audio_path = build_auto_tts_output_path(event.source.platform) result_json = await asyncio.to_thread( text_to_speech_tool, text=tts_text, output_path=audio_path ) try: result = json.loads(result_json) except (json.JSONDecodeError, TypeError): logger.warning("Auto voice reply TTS returned invalid JSON: %s", result_json[:200] if result_json else result_json) return # Final delivery may be one combined file or multiple separately # valid files when combination is unavailable or would exceed a # platform limit. Preserve legacy single-file results. actual_paths = result.get("file_paths") or [ result.get("file_path", audio_path) ] actual_paths = [ str(path) for path in actual_paths if path and os.path.isfile(path) ] if not result.get("success") or not actual_paths: logger.warning("Auto voice reply TTS failed: %s", result.get("error")) return adapter = self._adapter_for_source(event.source) # If connected to a voice channel, play there instead of sending a file guild_id = self._get_guild_id(event) play_in_voice_channel = getattr(adapter, "play_in_voice_channel", None) is_in_voice_channel = getattr(adapter, "is_in_voice_channel", None) send_voice = getattr(adapter, "send_voice", None) in_voice_channel = bool( guild_id and callable(play_in_voice_channel) and callable(is_in_voice_channel) and is_in_voice_channel(guild_id) ) reply_anchor = self._reply_anchor_for_event(event) thread_meta = self._thread_metadata_for_source(event.source, reply_anchor) if not in_voice_channel and callable(send_voice): # Mark the auto voice reply as notify-worthy. Mirrors the # final-text path in gateway/platforms/base.py which sets # ``notify=True`` so platform adapters that gate push # notifications (Telegram "important" mode) deliver the # final voice reply as a normal notification instead of a # silent message. Clone first so we don't mutate metadata # shared with concurrent typing-indicator state. if thread_meta is not None: thread_meta = dict(thread_meta) thread_meta["notify"] = True else: thread_meta = {"notify": True} for actual_path in actual_paths: if in_voice_channel: play_voice = cast(Callable[..., Awaitable[Any]], play_in_voice_channel) await play_voice(guild_id, actual_path) elif callable(send_voice): send_voice_call = cast(Callable[..., Awaitable[Any]], send_voice) send_kwargs: Dict[str, Any] = { "chat_id": event.source.chat_id, "audio_path": actual_path, "reply_to": reply_anchor, "metadata": thread_meta, } await send_voice_call(**send_kwargs) except Exception as e: logger.warning("Auto voice reply failed: %s", e, exc_info=True) finally: for p in ({audio_path, *actual_paths} - {None}): try: os.unlink(p) except OSError: pass async def _deliver_media_from_response( self, response: str, event: MessageEvent, adapter, thread_metadata: Optional[Dict[str, Any]] = None, ) -> None: """Extract explicit MEDIA: tags from a response and deliver them. Called after streaming has already sent the text to the user, so the text itself is already delivered — this only handles file attachments that the normal _process_message_background path would have caught. Unlike the non-streaming path in ``gateway/platforms/base.py`` (which also auto-detects bare local paths via ``extract_local_files``), this post-stream rescan is EXPLICIT-ONLY. The visible reply has already been streamed verbatim, so a bare path string here was either (a) already shown to the user as text, or (b) stale tool/inspected content that was never part of the intended visible reply. Promoting such paths into uploads after the fact sent files the model never asked to deliver (#20834). Only ``MEDIA:`` directives — the explicit attachment contract — trigger post-stream uploads. """ from pathlib import Path from urllib.parse import quote as _quote try: # Capture [[as_document]] before extract_media strips it, so the # dispatch partition below can route image-extension files # through send_document (preserving bytes) instead of # send_multiple_images (Telegram sendPhoto recompresses to ~1280px). force_document_attachments = "[[as_document]]" in response from gateway.platforms.base import BasePlatformAdapter, should_send_media_as_audio media_files, cleaned = adapter.extract_media(response) media_files = BasePlatformAdapter.filter_media_delivery_paths(media_files) # Do NOT deduplicate explicit MEDIA tags against prior turns here # (#73771). This rescan is already EXPLICIT-ONLY (see docstring): # a MEDIA: directive in the final streamed reply is the model # deliberately attaching a file — including a user-requested # resend. Stale auto-appended tags are deduped upstream in # _collect_auto_append_media_tags with history_media_paths. # Mirrors the same filter removal on the non-streaming path in # gateway/platforms/base.py. # Strip image URLs from the cleaned text for parity with the # non-streaming chain, but do NOT run extract_local_files here: # post-stream delivery is explicit-only (#20834). Bare local paths # in an already-streamed reply are text the user has seen (or # stale inspected content), not an attachment request. adapter.extract_images(cleaned) _thread_meta = ( dict(thread_metadata) if thread_metadata is not None else self._thread_metadata_for_source( event.source, self._reply_anchor_for_event(event), ) ) _VIDEO_EXTS = {'.mp4', '.mov', '.avi', '.mkv', '.webm', '.3gp'} _IMAGE_EXTS = {'.jpg', '.jpeg', '.png', '.webp', '.gif'} # Partition out images so they can be sent as a single batch # (e.g. Signal's multi-attachment RPC). When [[as_document]] was # set, image-extension files skip the photo path and route to # send_document below — preserving original bytes. image_paths: list = [] non_image_media: list = [] for media_path, is_voice in media_files: ext = Path(media_path).suffix.lower() if (ext in _IMAGE_EXTS and not is_voice and not force_document_attachments): image_paths.append(media_path) else: non_image_media.append((media_path, is_voice)) if image_paths: try: images = [(f"file://{_quote(p)}", "") for p in image_paths] await adapter.send_multiple_images( chat_id=event.source.chat_id, images=images, metadata=_thread_meta, ) except Exception as e: logger.warning("[%s] Post-stream image batch delivery failed: %s", adapter.name, e) for media_path, is_voice in non_image_media: try: ext = Path(media_path).suffix.lower() if should_send_media_as_audio(event.source.platform, ext, is_voice=is_voice): await adapter.send_voice( chat_id=event.source.chat_id, audio_path=media_path, metadata=_thread_meta, is_voice=is_voice, ) elif ext in _VIDEO_EXTS: await adapter.send_video( chat_id=event.source.chat_id, video_path=media_path, metadata=_thread_meta, ) else: await adapter.send_document( chat_id=event.source.chat_id, file_path=media_path, metadata=_thread_meta, ) except Exception as e: logger.warning("[%s] Post-stream media delivery failed: %s", adapter.name, e) except Exception as e: logger.warning("Post-stream media extraction failed: %s", e) async def _deliver_queued_first_response( self, response: str, source: SessionSource, adapter, metadata: Optional[Dict[str, Any]] = None, event_message_id: Optional[str] = None, text_already_delivered: bool = False, deliver_media: bool = True, stream_consumer=None, ) -> None: """Deliver a queued response using the normal text+attachment split.""" if not text_already_delivered: text_content = _strip_response_attachments_for_direct_send(response, adapter) if text_content: # Reconcile-by-edit first (live finding, 2026-08-16 canary): # when the stream consumer delivered/sealed a message but its # recorded payload didn't confirm the final (post-stream # mutation), plain-sending here creates the duplicate — the # sealed message already carries most of the answer. A sealed # native stream is a regular message; chat.update on it is # live-verified working. Fall back to plain send only when # there is no editable message or the edit fails. _reconciled = False _sc_msg_id = getattr(stream_consumer, "message_id", None) if ( _sc_msg_id and _sc_msg_id != "__no_edit__" and not getattr(stream_consumer, "_turn_split_delivery", False) ): try: _edit_res = await adapter.edit_message( chat_id=source.chat_id, message_id=_sc_msg_id, content=text_content, finalize=True, ) if getattr(_edit_res, "success", False): _reconciled = True logger.info( "Queued-lane final reconciled by editing message %s in place (no duplicate send).", _sc_msg_id, ) except Exception as _qe: logger.debug( "Queued-lane reconcile edit failed (%s); falling back to send.", _qe, ) if not _reconciled: await adapter.send( source.chat_id, text_content, metadata=metadata, ) # Failed turns still deliver their (normalized failure) text above, # but must not upload attachments as if the turn succeeded — mirrors # the ``not agent_result.get("failed")`` guard on the completed-turn # delivery path. if not deliver_media: return synthetic_event = MessageEvent( text="", source=source, message_id=event_message_id, ) await self._deliver_media_from_response( response, synthetic_event, adapter, thread_metadata=metadata, ) async def _run_background_task( self, prompt: str, source: "SessionSource", task_id: str, event_message_id: Optional[str] = None, media_urls: Optional[List[str]] = None, media_types: Optional[List[str]] = None, ) -> None: """Profile-scoping wrapper around the background agent task. When multiplexing is active, resolve the inbound source's profile and run the whole task inside ``_profile_runtime_scope`` so credentials resolve from that profile's secret scope. Mirrors the pattern in ``_run_agent``. """ if not getattr(getattr(self, "config", None), "multiplex_profiles", False): return await self._run_background_task_inner( prompt, source, task_id, event_message_id, media_urls, media_types, ) profile_home = self._resolve_profile_home_for_source(source) with _profile_runtime_scope(profile_home): return await self._run_background_task_inner( prompt, source, task_id, event_message_id, media_urls, media_types, ) def _resolve_enabled_toolsets_for_source( self, user_config: dict, source: "SessionSource", platform_key: str, ) -> list: """Resolve enabled toolsets for an agent run, honoring per-source overrides. Asks the receiving adapter for a ``toolsets_for_source()`` override (e.g. per-route webhook toolsets). When present, the override list is validated through the SAME ``_get_platform_tools`` path as normal platform config — by substituting it as the platform's toolset list — so unknown names and platform-restricted toolsets are dropped rather than trusted. When absent, falls back to standard ``platform_toolsets.`` resolution. """ from hermes_cli.tools_config import _get_platform_tools override = None try: adapter = self._adapter_for_source(source) if adapter is not None: override = adapter.toolsets_for_source(source) except Exception: override = None if override and isinstance(override, list): cfg = dict(user_config) pts = dict(cfg.get("platform_toolsets") or {}) pts[platform_key] = [str(t) for t in override] cfg["platform_toolsets"] = pts return sorted(_get_platform_tools(cfg, platform_key)) return sorted(_get_platform_tools(user_config, platform_key)) async def _run_background_task_inner( self, prompt: str, source: "SessionSource", task_id: str, event_message_id: Optional[str] = None, media_urls: Optional[List[str]] = None, media_types: Optional[List[str]] = None, ) -> None: """Execute a background agent task and deliver the result to the chat.""" from run_agent import AIAgent media_urls = media_urls or [] media_types = media_types or [] adapter = self._adapter_for_source(source) if not adapter: logger.warning("No adapter for platform %s in background task %s", source.platform, task_id) return _thread_metadata = self._thread_metadata_for_source(source, event_message_id) try: user_config = _load_gateway_config() model, runtime_kwargs = self._resolve_session_agent_runtime( source=source, user_config=user_config, ) if not runtime_kwargs.get("api_key"): await adapter.send( source.chat_id, f"❌ Background task {task_id} failed: no provider credentials configured.", metadata=_thread_metadata, ) return platform_key = _platform_config_key(source.platform) enabled_toolsets = self._resolve_enabled_toolsets_for_source( user_config, source, platform_key ) agent_cfg = user_config.get("agent") or {} from agent.skill_utils import parse_config_string_list disabled_toolsets = parse_config_string_list(agent_cfg.get("disabled_toolsets")) or None pr = self._provider_routing max_iterations = _current_max_iterations() reasoning_config = self._resolve_session_reasoning_config( source=source, model=model ) self._reasoning_config = reasoning_config self._service_tier = self._resolve_session_service_tier(source=source) turn_route = self._resolve_turn_agent_config(prompt, model, runtime_kwargs) # Enrich the prompt with image descriptions so the background # agent can see user-attached images (same as the main flow). enriched_prompt = prompt if media_urls: image_paths = [] for i, path in enumerate(media_urls): mtype = media_types[i] if i < len(media_types) else "" if mtype.startswith("image/"): image_paths.append(path) if image_paths: try: enriched_prompt = await self._enrich_message_with_vision( prompt, image_paths, ) except Exception as e: logger.warning("Background task vision enrichment failed: %s", e) def run_sync(): agent = AIAgent( model=turn_route["model"], **turn_route["runtime"], **_checkpoint_agent_kwargs(user_config), max_iterations=max_iterations, quiet_mode=True, verbose_logging=False, enabled_toolsets=enabled_toolsets, disabled_toolsets=disabled_toolsets, reasoning_config=reasoning_config, service_tier=self._service_tier, request_overrides=turn_route.get("request_overrides"), providers_allowed=pr.get("only"), providers_ignored=pr.get("ignore"), providers_order=pr.get("order"), provider_sort=pr.get("sort"), provider_require_parameters=pr.get("require_parameters", False), provider_data_collection=pr.get("data_collection"), session_id=task_id, platform=platform_key, user_id=source.user_id, user_id_alt=source.user_id_alt, user_name=source.user_name, chat_id=source.chat_id, chat_name=source.chat_name, chat_type=source.chat_type, thread_id=source.thread_id, session_db=getattr(self._session_db, "_db", self._session_db), # Reload from disk — do not reuse the startup snapshot (#60955). fallback_model=self._refresh_fallback_model(), ) try: return agent.run_conversation( user_message=enriched_prompt, task_id=task_id, ) finally: self._cleanup_agent_resources(agent) result = await self._run_in_executor_with_context(run_sync) response = result.get("final_response", "") if result else "" if not response and result and result.get("error"): response = f"Error: {result['error']}" # Background tasks start a fresh conversation (no prior history), # so history_offset=0: every message in the run belongs to this # turn. Mirrors the repair on the main turn path. if response: response = repair_explicit_computer_use_media_paths( response, result.get("messages", []), ) # Extract media files from the response if response: media_files, response = adapter.extract_media(response) from gateway.platforms.base import BasePlatformAdapter media_files = BasePlatformAdapter.filter_media_delivery_paths(media_files) images, text_content = adapter.extract_images(response) preview = prompt[:60] + ("..." if len(prompt) > 60 else "") header = f'✅ Background task complete\nPrompt: "{preview}"\n\n' if text_content: await adapter.send( chat_id=source.chat_id, content=header + text_content, metadata=_thread_metadata, ) elif not images and not media_files: await adapter.send( chat_id=source.chat_id, content=header + "(No response generated)", metadata=_thread_metadata, ) # Send extracted images for image_url, alt_text in (images or []): try: await adapter.send_image( chat_id=source.chat_id, image_url=image_url, caption=alt_text, metadata=_thread_metadata, ) except Exception: pass # Send media files, routing each by type so a TTS clip # arrives as a voice bubble / a clip as a video rather than # a generic document. Mirrors the streaming + kanban paths. from gateway.platforms.base import ( should_send_media_as_audio as _should_send_media_as_audio, ) _IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp"} _VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".3gp"} for media_path, _is_voice in (media_files or []): _ext = os.path.splitext(media_path)[1].lower() try: if _should_send_media_as_audio(source.platform, _ext, _is_voice): await adapter.send_voice( chat_id=source.chat_id, audio_path=media_path, metadata=_thread_metadata, is_voice=_is_voice, ) elif _ext in _VIDEO_EXTS: await adapter.send_video( chat_id=source.chat_id, video_path=media_path, metadata=_thread_metadata, ) elif _ext in _IMAGE_EXTS: await adapter.send_image_file( chat_id=source.chat_id, image_path=media_path, metadata=_thread_metadata, ) else: await adapter.send_document( chat_id=source.chat_id, file_path=media_path, metadata=_thread_metadata, ) except Exception: pass else: preview = prompt[:60] + ("..." if len(prompt) > 60 else "") await adapter.send( chat_id=source.chat_id, content=f'✅ Background task complete\nPrompt: "{preview}"\n\n(No response generated)', metadata=_thread_metadata, ) except Exception as e: logger.exception("Background task %s failed", task_id) try: await adapter.send( chat_id=source.chat_id, content=f"❌ Background task {task_id} failed: {e}", metadata=_thread_metadata, ) except Exception: pass async def _get_telegram_topic_capabilities(self, source: SessionSource) -> dict: """Read Telegram private-topic capability flags via Bot API getMe.""" adapter = self._adapter_for_source(source) bot = getattr(adapter, "_bot", None) if bot is None or not hasattr(bot, "get_me"): return {"checked": False} try: me = await bot.get_me() except Exception: logger.debug("Failed to fetch Telegram getMe topic capabilities", exc_info=True) return {"checked": False} def _field(name: str): if hasattr(me, name): return getattr(me, name) api_kwargs = getattr(me, "api_kwargs", None) if isinstance(api_kwargs, dict) and name in api_kwargs: return api_kwargs.get(name) if isinstance(me, dict): return me.get(name) return None return { "checked": True, "has_topics_enabled": _field("has_topics_enabled"), "allows_users_to_create_topics": _field("allows_users_to_create_topics"), } async def _ensure_telegram_system_topic(self, source: SessionSource) -> None: """Create/pin the managed System topic after /topic activation when possible.""" adapter = self._adapter_for_source(source) if adapter is None or not source.chat_id: return thread_id = None create_topic = getattr(adapter, "_create_dm_topic", None) if callable(create_topic): try: thread_id = await create_topic(int(source.chat_id), "System") except Exception: logger.debug("Failed to create Telegram System topic", exc_info=True) if not thread_id: return message_id = None try: send_result = await adapter.send( source.chat_id, "System topic for Hermes commands and status.", metadata={"thread_id": str(thread_id)}, ) message_id = getattr(send_result, "message_id", None) except Exception: logger.debug("Failed to send Telegram System topic intro", exc_info=True) if not message_id: return bot = getattr(adapter, "_bot", None) if bot is None or not hasattr(bot, "pin_chat_message"): return try: await bot.pin_chat_message( chat_id=int(source.chat_id), message_id=int(message_id), disable_notification=True, ) except Exception: logger.debug("Failed to pin Telegram System topic intro", exc_info=True) async def _send_telegram_topic_setup_image(self, source: SessionSource) -> None: """Send the bundled BotFather Threads Settings screenshot when available.""" adapter = self._adapter_for_source(source) if adapter is None or not source.chat_id or not hasattr(adapter, "send_image_file"): return image_path = Path(__file__).resolve().parent / "assets" / "telegram-botfather-threads-settings.jpg" if not image_path.exists(): return try: await adapter.send_image_file( chat_id=source.chat_id, image_path=str(image_path), caption="BotFather → Bot Settings → Threads Settings", metadata={"thread_id": str(source.thread_id)} if source.thread_id else None, ) except Exception: logger.debug("Failed to send Telegram topic setup image", exc_info=True) def _sanitize_telegram_topic_title(self, title: str) -> str: """Return a Bot API-safe forum topic name from a generated session title.""" cleaned = re.sub(r"\s+", " ", str(title or "")).strip() if not cleaned: return "Hermes Chat" # Telegram forum topic names are short (currently 1-128 chars). Keep # extra room for multi-byte titles and avoid trailing ellipsis churn. if len(cleaned) > 120: cleaned = cleaned[:117].rstrip() + "..." return cleaned def _is_discord_auto_thread_lane(self, source: SessionSource) -> bool: """Return True only for Discord threads Hermes just auto-created.""" return ( source.platform == Platform.DISCORD and source.chat_type == "thread" and bool(getattr(source, "auto_thread_created", False)) and bool(source.thread_id) and bool(getattr(source, "auto_thread_initial_name", None)) ) def _is_relay_discord_channel_lane(self, source: SessionSource) -> bool: """Shape-only check: a relay-delivered Discord CHANNEL event whose reply the connector MAY auto-thread (title-turn registration gate). Deliberately does NOT consult the send-result cache: at registration time (before delivery) the feedback can't exist yet. The rename lane polls the cache at fire time instead.""" return ( source.platform == Platform.DISCORD and bool(source.chat_id) and not source.thread_id and source.chat_type in ("group", "channel") and getattr(source, "delivered_via_upstream_relay", False) is True ) def _relay_auto_thread_info( self, source: SessionSource ) -> Optional[Tuple[str, str]]: """(thread_id, initial_name) when the RELAY connector auto-threaded our reply to this source's chat — the title-turn sibling of _is_discord_auto_thread_lane. The marker-based check above only lights up for events ARRIVING IN an auto-created thread (turn 2+). The auto-title fires on the FIRST exchange, whose source is the PARENT channel event — the thread did not exist at ingest, so no markers can be present and the native lane check never matches on the relay title turn (staging repro 2026-07-29: initial titles fine, semantic renames never happened). Preferred path: the connector stamps ``prospective_thread_id`` on the inbound (the anchor message id, which IS the id of the thread it will auto-create). It's deterministic and per-message, so it identifies the EXACT thread even when several auto-threads spawn from one channel — unlike the send-result cache below, which held a single slot per parent chat and so only the FIRST thread in a channel ever renamed (staging repro 2026-08-02: thread A renamed, sibling thread B stuck at raw text). The connector's own created-name guard (prefer_connector_created) enforces no-clobber, so no initial name is needed here. Fallback: the connector reports where the reply actually landed on the send result (contract §SendResult thread_id/auto_thread_name); the relay adapter caches it per chat and this reads it back — kept for older connectors that don't stamp prospective_thread_id. """ if source.platform != Platform.DISCORD or not source.chat_id: return None if not getattr(source, "delivered_via_upstream_relay", False): return None prospective = getattr(source, "prospective_thread_id", None) if prospective: # Deterministic per-thread identity; the empty initial-name marker # signals the caller to rely on the connector-side no-clobber guard. return (str(prospective), "") adapter = self._adapter_for_source(source) info_fn = getattr(adapter, "auto_thread_info_for_chat", None) if not callable(info_fn): return None try: return _as_thread_info(info_fn(str(source.chat_id))) except Exception: return None async def _await_relay_auto_thread_info( self, source: SessionSource ) -> Optional[Tuple[str, str]]: """``_relay_auto_thread_info``, waited out until this turn delivers. The legacy send-result path can only answer once the reply is sent, and the caller asks at title time — one turn early. The adapter answers on the send either way, so the timeout is only a backstop for a turn that never sends at all; the turn's own inactivity limit is exactly how long that turn could still be alive. """ # The connector-stamped prospective id is known at ingest, so most # sessions answer here and never wait at all. known = self._relay_auto_thread_info(source) if known is not None: return known adapter = self._adapter_for_source(source) wait_fn = getattr(adapter, "wait_for_auto_thread_info", None) if not callable(wait_fn) or not source.chat_id: return None # 0 means the operator disabled the turn limit; the backstop still needs one. timeout = _float_env("HERMES_AGENT_TIMEOUT", 1800) or 1800 try: return _as_thread_info(await wait_fn(str(source.chat_id), timeout)) except Exception: return None def _sanitize_discord_thread_title(self, title: str) -> str: """Return a Discord-safe semantic thread title from a session title. Discord thread names are capped at 100 characters measured in UTF-16 code units (emoji count double), so truncate with the UTF-16 helpers rather than Python code-point slices. """ cleaned = re.sub(r"\s+", " ", str(title or "")).strip() if not cleaned: return "Hermes Chat" if utf16_len(cleaned) > 80: cleaned = _prefix_within_utf16_limit(cleaned, 77).rstrip() + "..." return cleaned async def _rename_discord_auto_thread_for_session_title( self, source: SessionSource, session_id: str, title: str, relay_info: Optional[Tuple[str, str]] = None, ) -> None: """Best-effort semantic rename of a newly auto-created Discord thread. ``relay_info`` is the (thread_id, initial_name) pair from the relay connector's send-result feedback — supplied on the title turn, where the source is the parent-channel event and carries no auto-thread markers (see _relay_auto_thread_info). When absent, the native marker-based lane supplies thread identity from the source itself. """ if relay_info is None and not await asyncio.to_thread( self._is_discord_auto_thread_lane, source ): # Relay title turn with no feedback captured at schedule time: the # title comes off the user's opening message, so it beats the # delivery that produces the connector's send-result feedback # (thread_id + initial name) by the whole length of the turn. Wait # on the adapter for that send rather than guessing how long the # turn will take. if not self._is_relay_discord_channel_lane(source): return relay_info = await self._await_relay_auto_thread_info(source) if relay_info is None: # True miss: the connector did not auto-thread this reply # (policy off, DM, already-threaded, or send failed). return adapter = self._adapter_for_source(source) if getattr(self, "adapters", None) else None if adapter is None: return rename_thread = getattr(adapter, "rename_thread", None) if rename_thread is None: return target_thread_id = relay_info[0] if relay_info else str(source.thread_id) # Relay lane (relay_info present): ask the CONNECTOR to enforce the # no-clobber guard from its own created-name memory — the gateway # can't reliably reproduce the thread's initial name byte-for-byte # (normalization drift silently declined every rename before this). # Native-marker lane keeps the legacy string guard. use_connector_guard = relay_info is not None guard_name = ( None if use_connector_guard else getattr(source, "auto_thread_initial_name", None) ) thread_name = self._sanitize_discord_thread_title(title) # Relay lane only: the connector's egress guard resolves the owning # tenant from the outbound metadata's scope_id (guild) / user_id # (author). Those discriminator caches are keyed by the PARENT channel # chat_id (learned at inbound), NOT the thread id. rename_thread # defaults chat_id to the thread id when no parent is given, so the # scope/author lookup misses and the connector declines the op # ("target not routed to an onboarded tenant" — the live failure on # staging 2026-08-01). Pass the parent channel id (the relay source's # chat_id IS the parent channel; the thread came from send-result # feedback) so the discriminators resolve. Native lane needs nothing: # its source IS the thread and it renames via the direct Discord API, # not the relay egress guard. parent_chat_id = ( str(source.chat_id) if use_connector_guard and source.chat_id else None ) logger.info( "discord auto-thread rename: thread=%s lane=%s new_title=%r", target_thread_id, "relay" if use_connector_guard else "native", thread_name, ) rename_kwargs = ( { "prefer_connector_created": True, "parent_chat_id": parent_chat_id, } if use_connector_guard else {"only_if_current_name": guard_name} ) try: renamed = await rename_thread( target_thread_id, thread_name, **rename_kwargs, ) logger.info( "discord auto-thread rename result: thread=%s applied=%s", target_thread_id, bool(renamed), ) except TypeError: logger.warning( "Discord semantic thread rename raised TypeError (adapter=%s)", type(adapter).__name__, exc_info=True, ) except Exception: logger.debug("Failed to rename Discord auto-thread for generated session title", exc_info=True) def _schedule_discord_semantic_thread_rename( self, source: SessionSource, session_id: str, title: str, ) -> None: """Schedule Discord auto-thread rename from the auto-title background thread.""" relay_info = None if not title: return if not self._is_discord_auto_thread_lane(source): # Relay title turn: the source is the PARENT channel event (the # thread didn't exist at ingest, so no auto-thread markers). The # connector's send-result feedback tells us where the reply # landed — but the auto-title thread races the delivery that # produces it, so a cache miss HERE is not a verdict. Schedule # whenever the SHAPE matches; the async rename lane polls the # cache (with a bounded wait) and no-ops on a true miss. relay_info = self._relay_auto_thread_info(source) if relay_info is None and not self._is_relay_discord_channel_lane( source ): return try: loop = asyncio.get_running_loop() except RuntimeError: loop = getattr(self, "_gateway_loop", None) if loop is None or loop.is_closed(): return try: copied_source = dataclasses.replace(source) except Exception: copied_source = source future = safe_schedule_threadsafe( self._rename_discord_auto_thread_for_session_title( copied_source, session_id, title, relay_info=relay_info ), loop, logger=logger, log_message="Discord semantic thread rename failed to schedule", ) if future is None: return def _log_rename_failure(fut) -> None: try: fut.result() except Exception: logger.debug("Discord semantic thread rename failed", exc_info=True) future.add_done_callback(_log_rename_failure) async def _rename_telegram_topic_for_session_title( self, source: SessionSource, session_id: str, title: str, ) -> None: """Best-effort rename of a Telegram DM topic when Hermes auto-titles a session.""" if not await asyncio.to_thread(self._is_telegram_topic_lane, source) or not source.chat_id or not source.thread_id: return # Operator can fully disable per-topic auto-rename via # extra.disable_topic_auto_rename. Useful when topics are managed # by the user (ad-hoc Threaded Mode) and auto-rename would # overwrite their chosen names every time the auto-title fires. if self._telegram_topic_auto_rename_disabled(source): return # Skip rename when the topic is operator-declared via # extra.dm_topics. Those topics have fixed names chosen by the # operator (plus optional skill binding); auto-renaming would # silently mutate operator config. # # Check the class, not the instance — getattr() on MagicMock # auto-creates attributes, so `hasattr(adapter, "_get_dm_topic_info")` # would return True for every test double. adapter = self._adapter_for_source(source) if adapter is not None: get_info = getattr(type(adapter), "_get_dm_topic_info", None) if callable(get_info): try: operator_topic = get_info(adapter, str(source.chat_id), str(source.thread_id)) except Exception: operator_topic = None # Only treat dict-shaped returns as operator-declared; a # bare MagicMock or other sentinel shouldn't count. if isinstance(operator_topic, dict): return session_db = getattr(self, "_session_db", None) if session_db is not None: try: binding = await session_db.get_telegram_topic_binding( chat_id=str(source.chat_id), thread_id=str(source.thread_id), profile_name=self._telegram_topic_profile_name(source), ) if binding and str(binding.get("session_id") or "") != str(session_id): return except Exception: logger.debug("Failed to verify Telegram topic binding before rename", exc_info=True) return if adapter is None: return topic_name = self._sanitize_telegram_topic_title(title) try: rename_topic = getattr(adapter, "rename_dm_topic", None) if rename_topic is not None: await rename_topic( chat_id=str(source.chat_id), thread_id=str(source.thread_id), name=topic_name, ) return bot = getattr(adapter, "_bot", None) edit_forum_topic = getattr(bot, "edit_forum_topic", None) if bot is not None else None if edit_forum_topic is None: edit_forum_topic = getattr(bot, "editForumTopic", None) if bot is not None else None if edit_forum_topic is None: return try: await edit_forum_topic( chat_id=int(source.chat_id), message_thread_id=int(source.thread_id), name=topic_name, ) except (TypeError, ValueError): await edit_forum_topic( chat_id=source.chat_id, message_thread_id=source.thread_id, name=topic_name, ) except Exception: logger.debug("Failed to rename Telegram topic for auto-generated title", exc_info=True) def _telegram_topic_auto_rename_disabled(self, source: SessionSource) -> bool: """Return True when operator disabled per-topic auto-rename for this Telegram chat. Controlled via ``gateway.platforms.telegram.extra.disable_topic_auto_rename``. Default is False (auto-rename enabled, preserves prior behaviour). """ platform_cfg = ( self.config.platforms.get(source.platform) if getattr(self, "config", None) and getattr(self.config, "platforms", None) else None ) if platform_cfg is None: return False extra = getattr(platform_cfg, "extra", None) or {} value = extra.get("disable_topic_auto_rename") if value is None: return False if isinstance(value, bool): return value if isinstance(value, str): return value.strip().lower() in {"1", "true", "yes", "on"} return bool(value) def _schedule_telegram_topic_title_rename( self, source: SessionSource, session_id: str, title: str, ) -> None: """Schedule a topic rename from the auto-title background thread.""" if not title or not self._is_telegram_topic_lane(source): return if self._telegram_topic_auto_rename_disabled(source): return try: loop = asyncio.get_running_loop() except RuntimeError: loop = getattr(self, "_gateway_loop", None) if loop is None or loop.is_closed(): return try: copied_source = dataclasses.replace(source) except Exception: copied_source = source future = safe_schedule_threadsafe( self._rename_telegram_topic_for_session_title(copied_source, session_id, title), loop, logger=logger, log_message="Telegram topic title rename failed to schedule", ) if future is None: return def _log_rename_failure(fut) -> None: try: fut.result() except Exception: logger.debug("Telegram topic title rename failed", exc_info=True) future.add_done_callback(_log_rename_failure) _TELEGRAM_CAPABILITY_HINT_COOLDOWN_S = 300.0 def _should_send_telegram_capability_hint(self, source: SessionSource) -> bool: """Rate-limit the BotFather Threads Settings screenshot. If a user sends /topic repeatedly while Threads Settings are still off, we shouldn't keep re-uploading the screenshot every time. """ if not hasattr(self, "_telegram_capability_hint_ts"): self._telegram_capability_hint_ts = {} key = self._telegram_topic_cooldown_key(source) if not key: return True import time as _time now = _time.monotonic() last = self._telegram_capability_hint_ts.get(key, 0.0) if now - last < self._TELEGRAM_CAPABILITY_HINT_COOLDOWN_S: return False self._telegram_capability_hint_ts[key] = now return True def _telegram_topic_help_text(self) -> str: return ( "/topic — enable multi-session DM mode (one bot, many parallel chats)\n" "\n" "Usage:\n" " /topic Enable topic mode, or show status if already on\n" " /topic help Show this message\n" " /topic off Disable topic mode and clear topic bindings\n" " /topic Inside a topic: restore a previous session by ID\n" "\n" "How it works:\n" "1. Run /topic once in this DM — Hermes checks BotFather Threads\n" " Settings are enabled and flips on multi-session mode.\n" "2. Tap All Messages at the top of the bot and send any message.\n" " Telegram creates a new topic for that message; each topic is\n" " an independent Hermes session (fresh history, fresh context).\n" "3. The root DM becomes a system lobby — send /topic, /status,\n" " /help, /usage there. Normal prompts go in a topic.\n" "4. /new inside a topic resets just that topic's session.\n" "5. /topic inside a topic restores an old session into it." ) async def _disable_telegram_topic_mode_for_chat(self, source: SessionSource) -> str: """Cleanly disable topic mode for a chat via /topic off.""" if not self._session_db: from hermes_state import format_session_db_unavailable return format_session_db_unavailable(prefix=t("gateway.shared.session_db_unavailable_prefix")) chat_id = str(source.chat_id or "") if not chat_id: return "Could not determine chat ID." # No-op if never enabled. try: currently_enabled = await self._session_db.is_telegram_topic_mode_enabled( chat_id=chat_id, user_id=str(source.user_id or ""), profile_name=self._telegram_topic_profile_name(source), ) except Exception: currently_enabled = False if not currently_enabled: return "Multi-session topic mode is not currently enabled for this chat." try: await self._session_db.disable_telegram_topic_mode( chat_id=chat_id, profile_name=self._telegram_topic_profile_name(source), ) except Exception as exc: logger.exception("Failed to disable Telegram topic mode") return f"Failed to disable topic mode: {exc}" # Reset per-profile+chat debounce state so the user doesn't see a # stale cooldown on the next activation (issue #76423). cooldown_key = self._telegram_topic_cooldown_key(source) if cooldown_key: for attr in ("_telegram_lobby_reminder_ts", "_telegram_capability_hint_ts"): store = getattr(self, attr, None) if isinstance(store, dict): store.pop(cooldown_key, None) return ( "Multi-session topic mode is now OFF for this chat.\n\n" "Existing topics in Telegram aren't removed — they'll just stop " "being gated as independent sessions. The root DM works as a " "normal Hermes chat again. Run /topic to re-enable later." ) async def _telegram_topic_root_status_message(self, source: SessionSource) -> str: lines = [ "Telegram multi-session topics are enabled.", "", "To create a new Hermes chat, open All Messages at the top of this " "bot interface and send any message there. Telegram will create a " "new topic for it.", "", ] try: sessions = await self._session_db.list_unlinked_telegram_sessions_for_user( chat_id=str(source.chat_id), user_id=str(source.user_id), profile_name=self._telegram_topic_profile_name(source), limit=10, ) except Exception: logger.debug("Failed to list unlinked Telegram sessions", exc_info=True) sessions = [] if sessions: lines.append("Previous unlinked sessions:") for session in sessions: session_id = str(session.get("id") or "") title = str(session.get("title") or "Untitled session") preview = str(session.get("preview") or "").strip() line = f"- {title} — `{session_id}`" if preview: line += f" — {preview}" lines.append(line) lines.extend([ "", "To restore one:", "1. Create or open a topic. To create a new one, open All Messages and send any message there.", "2. Send /topic inside that topic.", f"Example: Send /topic {sessions[0].get('id')} inside a topic.", ]) else: lines.extend([ "No previous unlinked Telegram sessions found.", "", "To restore a previous session later:", "1. Create or open a topic. To create a new one, open All Messages and send any message there.", "2. Send /topic inside that topic.", ]) return "\n".join(lines) async def _restore_telegram_topic_session(self, event: MessageEvent, raw_session_id: str) -> str: """Restore an existing Telegram-owned Hermes session into this topic.""" source = event.source session_id = await self._session_db.resolve_session_id(raw_session_id.strip()) if not session_id: return f"Session not found: {raw_session_id.strip()}" session = await self._session_db.get_session(session_id) if not session: return f"Session not found: {raw_session_id.strip()}" if str(session.get("source") or "") != "telegram": return "That session is not a Telegram session and cannot be restored into this topic." if str(session.get("user_id") or "") != str(source.user_id): return "That session does not belong to this Telegram user." linked = await self._session_db.is_telegram_session_linked_to_topic(session_id=session_id) topic_profile = self._telegram_topic_profile_name(source) current_binding = await self._session_db.get_telegram_topic_binding( chat_id=str(source.chat_id), thread_id=str(source.thread_id), profile_name=topic_profile, ) if linked: if not current_binding or current_binding.get("session_id") != session_id: return "That session is already linked to another Telegram topic." session_key = self._session_key_for_source(source) try: await self._session_db.bind_telegram_topic( chat_id=str(source.chat_id), thread_id=str(source.thread_id), user_id=str(source.user_id), session_key=session_key, session_id=session_id, managed_mode="restored", profile_name=topic_profile, ) except ValueError as exc: if "already linked" in str(exc): return "That session is already linked to another Telegram topic." raise title = await self._session_db.get_session_title(session_id) or session_id last_assistant = None try: for message in reversed(await self._session_db.get_messages(session_id)): if message.get("role") != "assistant": continue projected = project_compaction_message_for_display(message) if projected is not None and projected.get("content"): last_assistant = str(projected.get("content")) break except Exception: last_assistant = None response = f"Session restored: {title}" if last_assistant: response += f"\n\nLast Hermes message:\n{last_assistant}" return response async def _execute_mcp_reload(self, event: MessageEvent) -> str: """Actually disconnect, reconnect, and notify MCP tool changes. Split out from ``_handle_reload_mcp_command`` so the confirmation wrapper can invoke the same path whether the user confirmed via button, text reply, or has the confirm gate disabled. Under multiplex the reload runs inside the requesting profile's runtime scope (entered here when the caller — e.g. a button-confirm callback — did not), and only that profile's servers are torn down and rediscovered (#95518). """ multiplex = bool(getattr(self.config, "multiplex_profiles", False)) if multiplex and not get_hermes_home_override(): profile_home = self._resolve_profile_home_for_source(event.source) with _profile_runtime_scope(Path(profile_home)): return await self._execute_mcp_reload(event) try: from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools, _servers, _lock from tools.mcp_tool import _server_scope_keys, reprobe_tool_availability from tools.registry import registry reload_scope = registry.current_scope_key() if multiplex else None def _scoped_server_names() -> set: with _lock: return { name for name in _servers if reload_scope is None or _server_scope_keys.get(name) == reload_scope } # Capture old server names before shutdown old_servers = _scoped_server_names() # Read new config before shutting down, so we know what will be added/removed # Shutdown existing connections await self._run_in_executor_with_context( lambda: shutdown_mcp_servers(scope=reload_scope) ) # Explicit reload also re-probes tool availability (check_fn). reprobe_tool_availability() # Reconnect by discovering tools (reads config.yaml fresh) new_tools = await self._run_in_executor_with_context(discover_mcp_tools) # Compute what changed connected_servers = _scoped_server_names() if reload_scope is not None: from tools.mcp_tool import _mcp_tool_server_names with _lock: new_tools = [ n for n in new_tools if _mcp_tool_server_names.get(n) in connected_servers ] added = connected_servers - old_servers removed = old_servers - connected_servers reconnected = connected_servers & old_servers lines = [t("gateway.reload_mcp.header")] if reconnected: lines.append(t("gateway.reload_mcp.reconnected", names=", ".join(sorted(reconnected)))) if added: lines.append(t("gateway.reload_mcp.added", names=", ".join(sorted(added)))) if removed: lines.append(t("gateway.reload_mcp.removed", names=", ".join(sorted(removed)))) if not connected_servers: lines.append(t("gateway.reload_mcp.none_connected")) else: lines.append(t("gateway.reload_mcp.tools_available", tools=len(new_tools), servers=len(connected_servers))) # Refresh cached agents so existing sessions see new MCP tools on # their next turn — without this, the user has to `/new` (which # discards conversation history) to pick up tools from a server # that was just added or reconnected. The user has already # consented to the prompt-cache invalidation via the slash-confirm # gate in _handle_reload_mcp_command before we reach this point. try: from tools.mcp_tool import refresh_agent_mcp_tools _cache = getattr(self, "_agent_cache", None) _cache_lock = getattr(self, "_agent_cache_lock", None) if _cache_lock is not None and _cache: # Multiplex: only this profile's sessions. Rebuilding # another profile's agent inside this scope would hand it # this profile's tool registry. _ns_prefix = ( _session_key_namespace(event.source.profile) + ":" if multiplex else None ) with _cache_lock: for _sess_key, _entry in list(_cache.items()): if _ns_prefix and not str(_sess_key).startswith(_ns_prefix): continue try: _agent = _entry[0] if isinstance(_entry, tuple) else _entry except Exception: continue if _agent is None: continue # Preserve each cached agent's build-time toolset # selection EXACTLY: a gateway session built with a # restricted enabled_toolsets (e.g. ["safe"]) must # NOT silently gain tools after a reload. This is the # opposite of the interactive CLI/TUI /reload-mcp, # which is a single user re-applying their own config # edit; gateway agents are per-session and may be # deliberately locked down. (Contract is asserted by # test_reload_mcp_preserves_per_agent_toolset_overrides.) refresh_agent_mcp_tools(_agent, quiet_mode=True) except Exception as _exc: logger.debug( "Failed to update cached agent tools after MCP reload: %s", _exc, ) # Inject a message at the END of the session history so the # model knows tools changed on its next turn. Appended after # all existing messages to preserve prompt-cache for the prefix. change_parts = [] if added: change_parts.append(f"Added servers: {', '.join(sorted(added))}") if removed: change_parts.append(f"Removed servers: {', '.join(sorted(removed))}") if reconnected: change_parts.append(f"Reconnected servers: {', '.join(sorted(reconnected))}") tool_summary = f"{len(new_tools)} MCP tool(s) now available" if new_tools else "No MCP tools available" change_detail = ". ".join(change_parts) + ". " if change_parts else "" reload_msg = { "role": "user", "content": f"[IMPORTANT: MCP servers have been reloaded. {change_detail}{tool_summary}. The tool list for this conversation has been updated accordingly.]", } try: session_entry = await self.async_session_store.get_or_create_session(event.source) await self.async_session_store.append_to_transcript( session_entry.session_id, reload_msg ) except Exception: pass # Best-effort; don't fail the reload over a transcript write return "\n".join(lines) except Exception as e: logger.warning("MCP reload failed: %s", e) return t("gateway.reload_mcp.failed", error=e) # ------------------------------------------------------------------ # Slash-command confirmation primitive (generic) # ------------------------------------------------------------------ # Used by slash commands that have a non-destructive but expensive # side effect worth an explicit user confirmation (currently only # /reload-mcp, which invalidates the prompt cache). Two delivery # paths: # 1. Button UI — adapters that override ``send_slash_confirm`` # (Telegram, Discord, Slack, Matrix, Feishu) render three # inline buttons. The adapter routes the button click back via # ``tools.slash_confirm.resolve(session_key, confirm_id, choice)``. # 2. Text fallback — adapters that don't override the hook get a # plain text prompt. Users reply with /approve, /always, or # /cancel; the early intercept in ``_handle_message`` matches # those replies against ``tools.slash_confirm.get_pending()``. async def _maybe_confirm_destructive_slash( self, *, event: MessageEvent, command: str, title: str, detail: str, execute, ) -> Union[str, "EphemeralReply", None]: """Gate a destructive session slash command (/new, /reset, /undo). ``execute`` is an async callable ``execute() -> str | EphemeralReply`` that performs the destructive action. If the ``approvals.destructive_slash_confirm`` config gate is off, ``execute`` runs immediately (returning its result). Otherwise this routes through ``_request_slash_confirm`` — native yes/no buttons on Telegram/Discord/Slack, text fallback elsewhere. Three-option resolution: - ``once`` — run ``execute`` and return its result - ``always`` — persist ``approvals.destructive_slash_confirm: false``, then run ``execute`` - ``cancel`` — return a "cancelled" message; do not run ``execute`` """ # Gate check. confirm_required = True try: cfg = self._read_user_config() approvals = cfg.get("approvals") if isinstance(cfg, dict) else None if isinstance(approvals, dict): confirm_required = bool(approvals.get("destructive_slash_confirm", True)) except Exception: pass if not confirm_required: return await execute() session_key = self._session_key_for_source(event.source) async def _on_confirm(choice: str): if choice == "cancel": return f"🟡 /{command} cancelled. Conversation unchanged." persisted = False if choice == "always": try: from cli import save_config_value # save_config_value swallows its own errors and reports the # outcome in the return value, so the try block alone says # nothing about whether the write landed. persisted = bool( save_config_value("approvals.destructive_slash_confirm", False) ) if persisted: logger.info( "User opted out of destructive slash confirm (session=%s)", session_key, ) else: logger.warning( "Could not persist destructive_slash_confirm=false " "(session=%s); config.yaml is not writable", session_key, ) except Exception as exc: logger.warning( "Failed to persist destructive_slash_confirm=false: %s", exc, ) result = await execute() if choice == "always": if persisted: note = ( "\n\nℹ️ Future /clear, /new, /reset, and /undo will run " "without confirmation. Re-enable via " "`approvals.destructive_slash_confirm: true` in config.yaml." ) else: # The user did approve this run, so the action still goes # ahead, but the preference did not stick and the prompt # will be back next time. Say so rather than promising an # opt-out that was never written. note = ( "\n\n⚠️ Could not save that preference (config.yaml is not " "writable), so /clear, /new, /reset, and /undo will ask " "again next time. To silence it permanently, set " "`approvals.destructive_slash_confirm: false` in config.yaml." ) if isinstance(result, str): return result + note # EphemeralReply or other: leave untouched, since the note would # mangle structured replies. return result return result _p = self._typed_command_prefix_for(event.source.platform) prompt_message = ( f"⚠️ **Confirm /{command}**\n\n" f"{detail}\n\n" "Choose:\n" "• **Approve Once** — proceed this time only\n" "• **Always Approve** — proceed and silence this prompt permanently\n" "• **Cancel** — keep current conversation\n\n" f"_Text fallback: reply `{_p}approve`, `{_p}always`, or `{_p}cancel`._" ) return await self._request_slash_confirm( event=event, command=command, title=title, message=prompt_message, handler=_on_confirm, ) async def _request_slash_confirm( self, *, event: MessageEvent, command: str, title: str, message: str, handler, ) -> Optional[str]: """Ask the user to confirm an expensive slash command. ``handler`` is an async callable ``handler(choice: str) -> str`` where ``choice`` is ``"once"``, ``"always"``, or ``"cancel"``. The handler runs on the event loop when the user responds; its return value is sent back as a gateway message. Returns a short acknowledgment string to send immediately (before the user's response). If buttons rendered successfully the ack is ``None`` (buttons are self-explanatory); if we fell back to text the message itself IS the ack. """ from tools import slash_confirm as _slash_confirm_mod source = event.source session_key = self._session_key_for_source(source) # Bare-runner test harnesses (object.__new__(GatewayRunner)) skip # __init__ and don't have the counter attribute — fall back to a # local counter so tests don't AttributeError. Real runs always # have the instance attribute. counter = getattr(self, "_slash_confirm_counter", None) if counter is None: import itertools as _itertools counter = _itertools.count(1) self._slash_confirm_counter = counter confirm_id = f"{next(counter)}" # Register the pending confirm FIRST so a super-fast button click # cannot race the send_slash_confirm return. _slash_confirm_mod.register(session_key, confirm_id, command, handler) adapter = self._adapter_for_source(source) metadata = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event)) used_buttons = False if adapter is not None: try: button_result = await adapter.send_slash_confirm( chat_id=source.chat_id, title=title, message=message, session_key=session_key, confirm_id=confirm_id, metadata=metadata, ) if button_result and getattr(button_result, "success", False): used_buttons = True except Exception as exc: logger.debug( "send_slash_confirm failed for %s on %s: %s", command, source.platform, exc, ) if used_buttons: # Buttons rendered — no redundant text ack. return None # Text fallback — return the prompt message as the direct reply. return message def _read_user_config(self) -> Dict[str, Any]: """Read the user's raw config.yaml (cached) for gate lookups. Used by slash-confirm gates that must reflect on-disk state changes (e.g. a prior "Always Approve" click) without a gateway restart. """ try: from hermes_cli.config import load_config cfg = load_config() return cfg if isinstance(cfg, dict) else {} except Exception: return {} def _thread_metadata_for_source( self, source, reply_to_message_id: Optional[str] = None, ) -> Optional[Dict[str, Any]]: """Build the metadata dict platforms need for thread-aware replies.""" metadata = self._thread_metadata_for_target( getattr(source, "platform", None), getattr(source, "chat_id", None), getattr(source, "thread_id", None), chat_type=getattr(source, "chat_type", None), reply_to_message_id=reply_to_message_id or getattr(source, "message_id", None), ) if getattr(source, "platform", None) == Platform.SLACK: # Per-turn egress identity (R3-5, connector PR gateway-gateway#210). # Slack's chat.startStream requires recipient_user_id (+ # recipient_team_id) when streaming to a channel, and the relay # connector fills those from metadata.user_id / metadata.scope_id. # The relay adapter's _with_scope fallback resolves BOTH from # per-chat caches keyed only by chat_id — mutable state that a # CONCURRENT turn overwrites: two users with overlapping turns in # one channel would open U1's stream with U2 as the recipient. # Stamp the authentic per-turn values from THIS turn's source here, # where they are still turn-scoped; _with_scope only fills keys # that are absent, so the cache degrades to what it should be — a # restart/synthetic-send fallback. team_id = getattr(source, "scope_id", None) user_id = getattr(source, "user_id", None) if team_id or user_id: metadata = dict(metadata or {}) if team_id: metadata["slack_team_id"] = str(team_id) metadata.setdefault("scope_id", str(team_id)) if user_id: metadata.setdefault("user_id", str(user_id)) # Routed profile for shared state.db namespaces (#76423): the Telegram # prune path needs it because under profile_routes the transport # adapter's stamp is not the profile that wrote the binding. profile = str(getattr(source, "profile", None) or "").strip() if profile and metadata is not None: metadata = dict(metadata) metadata["hermes_profile"] = profile return metadata def _thread_metadata_for_target( self, platform: Optional[Platform], chat_id: Optional[str], thread_id: Optional[str], *, chat_type: Optional[str] = None, reply_to_message_id: Optional[str] = None, adapter: Optional[Any] = None, ) -> Optional[Dict[str, Any]]: """Build thread metadata for synthetic sends that only have routing state.""" if thread_id is None: return None metadata: Dict[str, Any] = {"thread_id": thread_id} if self._is_telegram_dm_topic_target( platform, chat_id, thread_id, chat_type=chat_type, adapter=adapter, ): metadata["telegram_dm_topic_reply_fallback"] = True # Telegram DM topic lanes need direct_messages_topic_id in metadata # so synthetic/queued messages (goal continuations, status notices) # route to the correct topic even when reply anchor is unavailable. tid = str(thread_id) if tid and tid not in {"", "1"}: metadata["direct_messages_topic_id"] = tid if reply_to_message_id is not None: metadata["telegram_reply_to_message_id"] = str(reply_to_message_id) if platform == Platform.SLACK and reply_to_message_id is not None: # Slack's reply_in_thread=false path uses message_id to distinguish # real existing threads from synthetic top-level session keys. metadata["message_id"] = str(reply_to_message_id) return metadata @staticmethod def _is_telegram_dm_topic_target( platform: Optional[Platform], chat_id: Optional[str], thread_id: Optional[str], *, chat_type: Optional[str] = None, adapter: Optional[Any] = None, ) -> bool: """Return True when a target is a Telegram private DM topic lane.""" if platform != Platform.TELEGRAM or thread_id is None: return False if chat_type == "dm": return True # Inspect operator-declared DM topics via the adapter's lookup. Resolve # the method on the CLASS, not the instance: getattr() on a MagicMock # auto-creates a callable child for any attribute, so an instance-level # lookup would report a DM topic for every test double. Only a # dict-shaped return counts as an operator-declared topic — a bare # MagicMock or other sentinel must not. Mirrors the guard in # _rename_telegram_topic_for_session_title. if adapter is not None and chat_id: get_dm_topic_info = getattr(type(adapter), "_get_dm_topic_info", None) if callable(get_dm_topic_info): try: topic_info = get_dm_topic_info(adapter, str(chat_id), str(thread_id)) except Exception: logger.debug("Failed to inspect Telegram DM topic metadata", exc_info=True) else: return isinstance(topic_info, dict) return False @staticmethod def _reply_anchor_for_event(event: MessageEvent) -> Optional[str]: """Return the platform-specific reply anchor for GatewayRunner sends.""" return _reply_anchor_for_event(event) # ------------------------------------------------------------------ # /approve & /deny — explicit dangerous-command approval # ------------------------------------------------------------------ _APPROVAL_TIMEOUT_SECONDS = 300 # 5 minutes # Built-in messaging platforms where the ``/update`` command is allowed. # ACP, API server, and webhooks are programmatic interfaces that should # not trigger system updates. Plugin-migrated platforms (discord, # mattermost, teams, irc, line, …) are NOT listed here — they declare # ``allow_update_command=True`` on their ``PlatformEntry`` and are # honored via the registry fallback at ``_handle_update_command`` below. _UPDATE_ALLOWED_PLATFORMS = frozenset({ Platform.TELEGRAM, Platform.SLACK, Platform.WHATSAPP, Platform.SIGNAL, Platform.MATRIX, Platform.EMAIL, Platform.SMS, Platform.DINGTALK, Platform.FEISHU, Platform.WECOM, Platform.WECOM_CALLBACK, Platform.WEIXIN, Platform.BLUEBUBBLES, Platform.QQBOT, Platform.LOCAL, }) def _schedule_update_notification_watch(self) -> None: """Ensure a background task is watching for update completion.""" existing_task = getattr(self, "_update_notification_task", None) if existing_task and not existing_task.done(): return try: self._update_notification_task = asyncio.create_task( self._watch_update_progress() ) except RuntimeError: logger.debug("Skipping update notification watcher: no running event loop") async def _watch_update_progress( self, poll_interval: float = 2.0, stream_interval: float = 4.0, timeout: float = 1800.0, ) -> None: """Watch ``hermes update --gateway``, streaming output + forwarding prompts. Polls ``.update_output.txt`` for new content and sends chunks to the user periodically. Detects ``.update_prompt.json`` (written by the update process when it needs user input) and forwards the prompt to the messenger. The user's next message is intercepted by ``_handle_message`` and written to ``.update_response``. """ pending_path = _hermes_home / ".update_pending.json" claimed_path = _hermes_home / ".update_pending.claimed.json" output_path = _hermes_home / ".update_output.txt" exit_code_path = _hermes_home / ".update_exit_code" prompt_path = _hermes_home / ".update_prompt.json" loop = asyncio.get_running_loop() deadline = loop.time() + timeout # Resolve the adapter and chat_id for sending messages adapter = None chat_id = None session_key = None metadata = None for path in (claimed_path, pending_path): if path.exists(): try: pending = json.loads(path.read_text(encoding="utf-8")) platform_str = pending.get("platform") chat_id = pending.get("chat_id") chat_type = pending.get("chat_type") session_key = pending.get("session_key") thread_id = pending.get("thread_id") message_id = pending.get("message_id") if platform_str and chat_id: platform = Platform(platform_str) adapter = self.adapters.get(platform) metadata = self._thread_metadata_for_target( platform, chat_id, thread_id, chat_type=chat_type, reply_to_message_id=message_id, adapter=adapter, ) # Fallback session key if not stored (old pending files) if not session_key: session_key = f"{platform_str}:{chat_id}" break except Exception: pass if not adapter or not chat_id: logger.warning("Update watcher: cannot resolve adapter/chat_id, falling back to completion-only") # Fall back to completion-only: wait for the exit code and send the # final notification. _send_update_notification re-resolves the # adapter on every call, so when the target platform is still # reconnecting it returns False and keeps the markers. Keep polling # until it actually delivers (returns True) instead of giving up # after the first completion check — otherwise a platform that # reconnects a few seconds after completion never gets notified. while (pending_path.exists() or claimed_path.exists()) and loop.time() < deadline: if exit_code_path.exists() and await self._send_update_notification(): return await asyncio.sleep(poll_interval) if (pending_path.exists() or claimed_path.exists()) and not exit_code_path.exists(): exit_code_path.write_text("124", encoding="utf-8") await self._send_update_notification() return def _strip_ansi(text: str) -> str: from tools.ansi_strip import strip_ansi return strip_ansi(text) def _read_output_since(path: Path, offset: int) -> tuple[str, int]: """Read update output defensively; logs may contain invalid UTF-8.""" try: data = path.read_bytes() except OSError: return "", offset if len(data) <= offset: return "", len(data) return data[offset:].decode("utf-8", errors="replace"), len(data) bytes_sent = 0 last_stream_time = loop.time() buffer = "" async def _flush_buffer() -> None: """Send buffered output to the user.""" nonlocal buffer, last_stream_time if not buffer.strip(): buffer = "" return # Chunk to fit message limits (Telegram: 4096, others: generous) clean = _strip_ansi(buffer).strip() buffer = "" last_stream_time = loop.time() if not clean: return # Split into chunks if too long max_chunk = 3500 chunks = [clean[i:i + max_chunk] for i in range(0, len(clean), max_chunk)] for chunk in chunks: try: await adapter.send( chat_id, f"```\n{chunk}\n```", metadata=_non_conversational_metadata(metadata, platform=platform), ) except Exception as e: logger.debug("Update stream send failed: %s", e) while loop.time() < deadline: # Check for completion if exit_code_path.exists(): # Read any remaining output if output_path.exists(): try: chunk, bytes_sent = _read_output_since(output_path, bytes_sent) if chunk: buffer += chunk except OSError: pass await _flush_buffer() # Send final status try: exit_code_raw = exit_code_path.read_text(encoding="utf-8").strip() or "1" exit_code = int(exit_code_raw) if exit_code == 0: await adapter.send( chat_id, "✅ Hermes update finished.", metadata=_non_conversational_metadata(metadata, platform=platform), ) else: await adapter.send( chat_id, "❌ Hermes update failed (exit code {}).".format(exit_code), metadata=_non_conversational_metadata(metadata, platform=platform), ) logger.info("Update finished (exit=%s), notified %s", exit_code, session_key) except Exception as e: logger.warning("Update final notification failed: %s", e) # Cleanup for p in (pending_path, claimed_path, output_path, exit_code_path, prompt_path): p.unlink(missing_ok=True) (_hermes_home / ".update_response").unlink(missing_ok=True) _up_done = self._peek_session_state(session_key) if _up_done is not None: _up_done.persistent.update_prompt_pending = False return # Check for new output if output_path.exists(): try: chunk, bytes_sent = _read_output_since(output_path, bytes_sent) if chunk: buffer += chunk except OSError: pass # Flush buffer periodically if buffer.strip() and (loop.time() - last_stream_time) >= stream_interval: await _flush_buffer() # Check for prompts — only forward if we haven't already sent # one that's still awaiting a response. Without this guard the # watcher would re-read the same .update_prompt.json every poll # cycle and spam the user with duplicate prompt messages. _up_pending_state = ( self._peek_session_state(session_key) if session_key else None ) if (prompt_path.exists() and session_key and not ( _up_pending_state is not None and _up_pending_state.persistent.update_prompt_pending )): try: prompt_data = json.loads(prompt_path.read_text(encoding="utf-8")) prompt_text = prompt_data.get("prompt", "") default = prompt_data.get("default", "") if prompt_text: # Flush any buffered output first so the user sees # context before the prompt await _flush_buffer() # Try platform-native buttons first (Discord, Telegram) sent_buttons = False if getattr(type(adapter), "send_update_prompt", None) is not None: try: await adapter.send_update_prompt( chat_id=chat_id, prompt=prompt_text, default=default, session_key=session_key, metadata=_non_conversational_metadata(metadata, platform=platform), ) sent_buttons = True except Exception as btn_err: logger.debug("Button-based update prompt failed: %s", btn_err) if not sent_buttons: default_hint = f" (default: {default})" if default else "" _p = getattr(adapter, "typed_command_prefix", "/") await adapter.send( chat_id, f"⚕ **Update needs your input:**\n\n" f"{prompt_text}{default_hint}\n\n" f"Reply `{_p}approve` (yes) or `{_p}deny` (no), " f"or type your answer directly.", metadata=_non_conversational_metadata(metadata, platform=platform), ) # Keep the prompt marker on disk until the user # answers. If the gateway restarts mid-prompt, the # next watcher can recover by re-forwarding it from # disk. Duplicate sends in the same process are # still suppressed by _update_prompt_pending. self._session_state( session_key ).persistent.update_prompt_pending = True # .update_response to continue — it doesn't re-check logger.info("Forwarded update prompt to %s: %s", session_key, prompt_text[:80]) except (json.JSONDecodeError, OSError) as e: logger.debug("Failed to read update prompt: %s", e) await asyncio.sleep(poll_interval) # Timeout if not exit_code_path.exists(): logger.warning("Update watcher timed out after %.0fs", timeout) exit_code_path.write_text("124", encoding="utf-8") await _flush_buffer() try: await adapter.send( chat_id, "❌ Hermes update timed out after 30 minutes.", metadata=_non_conversational_metadata(metadata, platform=platform), ) except Exception: pass for p in (pending_path, claimed_path, output_path, exit_code_path, prompt_path): p.unlink(missing_ok=True) (_hermes_home / ".update_response").unlink(missing_ok=True) _up_timeout_state = self._peek_session_state(session_key) if _up_timeout_state is not None: _up_timeout_state.persistent.update_prompt_pending = False async def _send_update_notification(self) -> bool: """If an update finished, notify the user. Returns False when the update is still running so a caller can retry later. Returns True after a definitive send/skip decision. This is the legacy notification path used when the streaming watcher cannot resolve the adapter (e.g. after a gateway restart where the platform hasn't reconnected yet). """ pending_path = _hermes_home / ".update_pending.json" claimed_path = _hermes_home / ".update_pending.claimed.json" output_path = _hermes_home / ".update_output.txt" exit_code_path = _hermes_home / ".update_exit_code" if not pending_path.exists() and not claimed_path.exists(): return False cleanup = True active_pending_path = claimed_path try: if pending_path.exists(): try: pending_path.replace(claimed_path) except FileNotFoundError: if not claimed_path.exists(): return True elif not claimed_path.exists(): return True pending = json.loads(claimed_path.read_text(encoding="utf-8")) platform_str = pending.get("platform") chat_id = pending.get("chat_id") chat_type = pending.get("chat_type") thread_id = pending.get("thread_id") message_id = pending.get("message_id") if not exit_code_path.exists(): logger.info("Update notification deferred: update still running") cleanup = False active_pending_path = pending_path claimed_path.replace(pending_path) return False exit_code_raw = exit_code_path.read_text(encoding="utf-8").strip() or "1" exit_code = int(exit_code_raw) # Read the captured update output output = "" if output_path.exists(): output = output_path.read_bytes().decode("utf-8", errors="replace") # Resolve adapter platform = Platform(platform_str) adapter = self.adapters.get(platform) if not adapter and chat_id: # The update finished, but the target platform has not # reconnected yet (common right after the restart that # `hermes update` triggers). Treating "adapter missing" as a # definitive skip would delete the markers and silently lose the # completion notification — the user never learns whether the # update succeeded or timed out. Preserve the markers instead so # a later retry (the watcher poll loop, or the next gateway # startup) can deliver the result once the adapter is back. logger.info( "Update notification deferred: %s adapter not connected yet", platform_str, ) cleanup = False active_pending_path = pending_path claimed_path.replace(pending_path) return False if adapter and chat_id: metadata = self._thread_metadata_for_target( platform, chat_id, thread_id, chat_type=chat_type, reply_to_message_id=message_id, adapter=adapter, ) # Strip ANSI escape codes for clean display from tools.ansi_strip import strip_ansi output = strip_ansi(output).strip() if output: if len(output) > 3500: output = "…" + output[-3500:] if exit_code == 0: msg = f"✅ Hermes update finished.\n\n```\n{output}\n```" else: msg = f"❌ Hermes update failed.\n\n```\n{output}\n```" elif exit_code == 0: msg = "✅ Hermes update finished successfully." else: msg = "❌ Hermes update failed. Check the gateway logs or run `hermes update` manually for details." await adapter.send( chat_id, msg, metadata=_non_conversational_metadata(metadata, platform=platform), ) logger.info( "Sent post-update notification to %s:%s (exit=%s)", platform_str, chat_id, exit_code, ) except Exception as e: logger.warning("Post-update notification failed: %s", e) finally: if cleanup: active_pending_path.unlink(missing_ok=True) claimed_path.unlink(missing_ok=True) output_path.unlink(missing_ok=True) exit_code_path.unlink(missing_ok=True) return True async def _send_restart_notification(self) -> Optional[tuple[str, str, Optional[str]]]: """Notify the chat that initiated /restart that the gateway is back.""" notify_path = _hermes_home / ".restart_notify.json" if not notify_path.exists(): return None try: data = json.loads(notify_path.read_text(encoding="utf-8")) platform_str = data.get("platform") chat_id = data.get("chat_id") chat_type = data.get("chat_type") thread_id = data.get("thread_id") message_id = data.get("message_id") if not platform_str or not chat_id: return None platform = Platform(platform_str) transport = resolve_delivery_transport(platform, self.config, self.adapters) if transport is None: logger.debug( "Restart notification skipped: no live transport for %s", platform_str, ) return None platform_cfg = self.config.platforms.get(platform) if platform_cfg is not None and not platform_cfg.gateway_restart_notification: logger.info( "Restart notification suppressed: %s has gateway_restart_notification=false", platform_str, ) return None metadata = self._thread_metadata_for_target( platform, chat_id, thread_id, chat_type=chat_type, reply_to_message_id=message_id, adapter=transport.adapter, ) if data.get("delivered_via_upstream_relay") is True: metadata = dict(metadata or {}) if data.get("user_id"): metadata["user_id"] = str(data["user_id"]) if data.get("scope_id"): metadata["scope_id"] = str(data["scope_id"]) result = await transport.send( platform, str(chat_id), "♻ Gateway restarted successfully. Your session continues.", metadata=_non_conversational_metadata(metadata, platform=platform), ) # adapter.send() catches provider errors (e.g. "Chat not found") # and returns SendResult(success=False) rather than raising, so # we must inspect the result before claiming success — otherwise # the log line is misleading and hides real delivery failures. if result is not None and getattr(result, "success", True) is False: logger.warning( "Restart notification to %s:%s was not delivered: %s", platform_str, chat_id, getattr(result, "error", "send returned success=False"), ) return None logger.info( "Sent restart notification to %s:%s", platform_str, chat_id, ) return str(platform_str), str(chat_id), str(thread_id) if thread_id else None except Exception as e: logger.warning("Restart notification failed: %s", e) return None finally: notify_path.unlink(missing_ok=True) async def _send_home_channel_startup_notifications( self, *, skip_targets: Optional[set[tuple[str, str, Optional[str]]]] = None, ) -> set[tuple[str, str, Optional[str]]]: """Notify configured home channels that the gateway is back online. The notification is best-effort and sent once per connected platform home channel. ``skip_targets`` lets startup avoid duplicate messages when a more specific restart notification is queued for the same chat. """ delivered: set[tuple[str, str, Optional[str]]] = set() skipped = skip_targets or set() message = "♻️ Gateway online — Hermes is back and ready." for platform, platform_cfg in self.config.platforms.items(): home = platform_cfg.home_channel if not home or not home.chat_id: continue transport = resolve_delivery_transport(platform, self.config, self.adapters) if transport is None: continue if not platform_cfg.gateway_restart_notification: logger.info( "Home-channel startup notification suppressed: %s has gateway_restart_notification=false", platform.value, ) continue target = (platform.value, str(home.chat_id), str(home.thread_id) if home.thread_id else None) if target in skipped or target in delivered: continue try: metadata = self._thread_metadata_for_target( platform, home.chat_id, home.thread_id, adapter=transport.adapter, ) if transport.is_relay: metadata = dict(metadata or {}) if home.user_id: metadata["user_id"] = home.user_id if home.scope_id: metadata["scope_id"] = home.scope_id send_metadata = _non_conversational_metadata(metadata, platform=platform) if send_metadata is not None or transport.is_relay: result = await transport.send( platform, str(home.chat_id), message, metadata=send_metadata, ) else: result = await transport.adapter.send(str(home.chat_id), message) if result is not None and getattr(result, "success", True) is False: logger.warning( "Home-channel startup notification failed for %s:%s: %s", platform.value, home.chat_id, getattr(result, "error", "send returned success=False"), ) continue delivered.add(target) logger.info( "Sent home-channel startup notification to %s:%s", platform.value, home.chat_id, ) except Exception as exc: logger.warning( "Home-channel startup notification failed for %s:%s: %s", platform.value, home.chat_id, exc, ) return delivered async def _send_session_db_warning_notifications(self) -> None: """Broadcast a state.db failure warning to all home channels (#88235). When SessionDB init fails at gateway startup, messages may flow but nothing is persisted — /resume, /history, and session_search all silently break. This sends a one-time warning to each connected platform's home channel so the user knows to investigate before losing data. Best-effort: failures are logged, not raised. """ error = getattr(self, "_session_db_init_error", None) if not error: return from hermes_state import ( _default_db_path, classify_persistence_error, format_session_db_unavailable, ) cause = classify_persistence_error(error) hint = format_session_db_unavailable() if cause == "corrupt": # Copy-pasteable, so name the real store (profiles / HERMES_HOME # do not live under ~/.hermes). db_path = _default_db_path() message = ( "⚠️ Session database corruption detected. Messages may not be " "persisted. Recovery options:\n" "1. Run `hermes doctor --fix`\n" "2. Stop the gateway, then recover with:\n" f" hermes sessions recover --source {db_path} " "--inspect-only\n" " (if it reports recoverable) hermes sessions recover " f"--source {db_path} --output recovered-state.db\n" " — recovery snapshots the damaged file first; do NOT run " "`sqlite3 ... \".recover\"` against the live state.db, a " "vulnerable sqlite3 CLI can corrupt it further\n" "3. Restore from a backup in ~/.hermes/backups/\n" "Run `hermes doctor` for sanitized diagnostics." ) else: message = ( f"⚠️ Session database unavailable — messages may not be persisted. " f"{hint}\n" f"Run `hermes doctor` for diagnostics." ) logger.warning( "Broadcasting state.db failure warning to home channels: %s", error ) for platform, platform_cfg in self.config.platforms.items(): home = platform_cfg.home_channel if not home or not home.chat_id: continue transport = resolve_delivery_transport(platform, self.config, self.adapters) if transport is None: continue try: metadata = self._thread_metadata_for_target( platform, home.chat_id, home.thread_id, adapter=transport.adapter, ) if transport.is_relay: metadata = dict(metadata or {}) if home.user_id: metadata["user_id"] = home.user_id if home.scope_id: metadata["scope_id"] = home.scope_id send_metadata = _non_conversational_metadata(metadata, platform=platform) if send_metadata is not None or transport.is_relay: result = await transport.send( platform, str(home.chat_id), message, metadata=send_metadata, ) else: result = await transport.adapter.send(str(home.chat_id), message) if result is not None and getattr(result, "success", True) is False: logger.warning( "state.db warning notification failed for %s:%s: %s", platform.value, home.chat_id, getattr(result, "error", "send returned success=False"), ) except Exception as exc: logger.warning( "state.db warning notification failed for %s:%s: %s", platform.value, home.chat_id, exc, ) def _set_session_env(self, context: SessionContext) -> list: """Set session context variables for the current async task. Uses ``contextvars`` instead of ``os.environ`` so that concurrent gateway messages cannot overwrite each other's session state. Returns a list of reset tokens; pass them to ``_clear_session_env`` in a ``finally`` block. """ from gateway.session_context import set_session_vars # Propagate the adapter's async-delivery capability so async tools # (terminal notify_on_complete / watch_patterns, delegate_task # background=True) know whether this channel can wake a later turn. # Default True keeps CLI / unknown paths working; stateless adapters # (api_server) declare supports_async_delivery=False. Use getattr so # bare runners built via object.__new__ (tests) without self.adapters # don't blow up — they simply default to supported. _adapters = getattr(self, "adapters", None) or {} _adapter = _adapters.get(context.source.platform) _async_delivery = getattr(_adapter, "supports_async_delivery", True) return set_session_vars( platform=context.source.platform.value, chat_id=context.source.chat_id, chat_type=( str(context.source.chat_type) if context.source.chat_type else "" ), chat_name=context.source.chat_name or "", thread_id=str(context.source.thread_id) if context.source.thread_id else "", user_id=str(context.source.user_id) if context.source.user_id else "", user_id_alt=str(context.source.user_id_alt) if context.source.user_id_alt else "", user_name=str(context.source.user_name) if context.source.user_name else "", scope_id=str(getattr(context.source, "scope_id", "") or ""), session_key=context.session_key, message_id=str(context.source.message_id) if context.source.message_id else "", profile=getattr(context.source, "profile", "") or "", async_delivery=_async_delivery, cron_session="", ) def _clear_session_env(self, tokens: list) -> None: """Restore session context variables to their pre-handler values.""" from gateway.session_context import clear_session_vars clear_session_vars(tokens) async def _run_in_executor_with_context(self, func, *args): """Run blocking work in the thread pool while preserving session contextvars.""" loop = asyncio.get_running_loop() ctx = copy_context() return await loop.run_in_executor( self._get_executor(), ctx.run, func, *args, ) def _get_executor(self) -> concurrent.futures.ThreadPoolExecutor: """Return the gateway-owned executor for blocking agent work.""" lock = getattr(self, "_executor_lock", None) if lock is None: lock = threading.Lock() self._executor_lock = lock with lock: if getattr(self, "_executor_closing", False): raise RuntimeError("Gateway is shutting down; executor unavailable") executor = getattr(self, "_executor", None) if executor is None or getattr(executor, "_shutdown", False): executor = concurrent.futures.ThreadPoolExecutor( max_workers=10, thread_name_prefix="hermes-gateway", ) self._executor = executor return executor def _shutdown_executor(self, drain_timeout: float = 0.0) -> int: """Stop the gateway-owned executor without touching the loop default. Returns the number of worker threads still running when this returns. With the default ``drain_timeout`` of 0 this is the historical fire-and-forget teardown; shutdown passes a bounded budget so blocking DB work cannot outlive ``SessionDB.close()`` (see ``_stop_impl``). ``cancel_futures`` only drops work that has not started yet, and a cancelled ``run_in_executor`` awaitable does not stop the thread behind it, so the running futures have to be waited on explicitly. """ lock = getattr(self, "_executor_lock", None) if lock is None: return 0 with lock: self._executor_closing = True executor = getattr(self, "_executor", None) self._executor = None if executor is None: return 0 try: executor.shutdown(wait=False, cancel_futures=True) except TypeError: executor.shutdown(wait=False) # ThreadPoolExecutor.shutdown() has no timeout, so join the worker # threads directly. `_threads` is absent on the doubles some tests # pass in, which just means no wait. workers = list(getattr(executor, "_threads", None) or ()) deadline = time.monotonic() + max(float(drain_timeout or 0.0), 0.0) for worker in workers: remaining = deadline - time.monotonic() if remaining <= 0: break worker.join(remaining) return sum(1 for worker in workers if worker.is_alive()) def _decide_image_input_mode( self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None, user_config: Optional[dict] = None, provider: Optional[str] = None, model: Optional[str] = None, ) -> str: """Resolve image-input routing for the effective model this turn. Returns ``"native"`` (attach pixels on the user turn) or ``"text"`` (pre-analyze with vision_analyze and prepend the description). See agent/image_routing.py for the full decision table. Gateway sessions can have /model overrides that live outside config.yaml. Image preprocessing runs before AIAgent sets the auxiliary_client runtime globals, so resolve the same per-session runtime bundle the upcoming agent turn will use instead of consulting only the persisted default model. """ try: from agent.image_routing import decide_image_input_mode from agent.auxiliary_client import _read_main_model, _read_main_provider from hermes_cli.config import load_config cfg = user_config if isinstance(user_config, dict) else load_config() resolved_provider = (provider or "").strip() resolved_model = (model or "").strip() resolved_requested_provider = "" needs_session_runtime = not resolved_provider or not resolved_model has_session_identity = source is not None or session_key if needs_session_runtime and has_session_identity: try: turn_model, runtime_kwargs = self._resolve_session_agent_runtime( source=source, session_key=session_key, user_config=cfg, ) if not resolved_model and isinstance(turn_model, str): resolved_model = turn_model.strip() runtime_provider = runtime_kwargs.get("provider") if isinstance(runtime_kwargs, dict) else None runtime_requested_provider = ( runtime_kwargs.get("requested_provider") if isinstance(runtime_kwargs, dict) else None ) if not resolved_provider and isinstance(runtime_provider, str): resolved_provider = runtime_provider.strip() if isinstance(runtime_requested_provider, str): resolved_requested_provider = runtime_requested_provider.strip() except Exception as exc: logger.debug( "image_routing: session runtime resolution failed, falling back to config — %s", exc, ) if not resolved_provider: resolved_provider = _read_main_provider() if not resolved_model: resolved_model = _read_main_model() return decide_image_input_mode( resolved_provider, resolved_model, cfg, requested_provider=resolved_requested_provider, ) except Exception as exc: logger.debug("image_routing: decision failed, falling back to text — %s", exc) return "text" async def _enrich_message_with_vision( self, user_text: str, image_paths: List[str], ) -> str: """ Auto-analyze user-attached images with the vision tool and prepend the descriptions to the message text. Each image is analyzed with a general-purpose prompt. The resulting description *and* the local cache path are injected so the model can: 1. Immediately understand what the user sent (no extra tool call). 2. Re-examine the image with vision_analyze if it needs more detail. Args: user_text: The user's original caption / message text. image_paths: List of local file paths to cached images. Returns: The enriched message string with vision descriptions prepended. """ from tools.vision_tools import vision_analyze_tool from agent.memory_manager import sanitize_context analysis_prompt = ( "Concisely describe this image in 2-4 sentences " "(~200 Chinese characters or ~150 English words). " "Cover the main subject, key visible text/data/code, and overall context. " "If it is a chart, diagram, or scientific figure, include the important " "labels, legend, and key values. Skip decorative details." ) enriched_parts = [] for path in image_paths: try: logger.debug("Auto-analyzing user image: %s", path) result_json = await vision_analyze_tool( image_url=path, user_prompt=analysis_prompt, ) result = json.loads(result_json) if result.get("success"): description = result.get("analysis", "") description = sanitize_context(description) enriched_parts.append( f"[The user sent an image~ Here's what I can see:\n{description}]\n" f"[If you need a closer look, use vision_analyze with " f"image_url: {path} ~]" ) else: enriched_parts.append( "[The user sent an image but I couldn't quite see it " "this time (>_<) You can try looking at it yourself " f"with vision_analyze using image_url: {path}]" ) except Exception as e: logger.error("Vision auto-analysis error: %s", e) enriched_parts.append( f"[The user sent an image but something went wrong when I " f"tried to look at it~ You can try examining it yourself " f"with vision_analyze using image_url: {path}]" ) # Combine: vision descriptions first, then the user's original text if enriched_parts: prefix = "\n\n".join(enriched_parts) if user_text: return f"{prefix}\n\n{user_text}" return prefix return user_text async def _enrich_message_with_transcription( self, user_text: str, audio_paths: List[str], ) -> tuple[str, List[str]]: """ Auto-transcribe user voice/audio messages using the configured STT provider and prepend the transcript to the message text. Args: user_text: The user's original caption / message text. audio_paths: List of local file paths to cached audio files. Returns: A tuple of ``(enriched_text, successful_transcripts)``: - ``enriched_text``: the message string with transcription wrappers prepended (same as before). - ``successful_transcripts``: the raw transcript strings for audio clips that were successfully transcribed, in input order. Empty list if every clip failed or STT is disabled. Callers can use this to echo transcripts back to the user before the agent loop. """ seen = set() audio_paths = [p for p in audio_paths if p not in seen and not seen.add(p)] if not getattr(self.config, "stt_enabled", True): notes = [] for path in audio_paths: abs_path = os.path.abspath(path) duration_str = await _probe_audio_duration(abs_path) if duration_str: notes.append( f"[The user sent a voice message: {abs_path} (duration: {duration_str})]" ) else: notes.append(f"[The user sent a voice message: {abs_path}]") if not notes: return user_text, [] prefix = "\n\n".join(notes) _placeholder = "(The user sent a message with no text content)" if user_text and user_text.strip() == _placeholder: return prefix, [] if user_text: return f"{prefix}\n\n{user_text}", [] return prefix, [] try: from tools.transcription_tools import ( transcribe_audio, transcribe_audio_local_fallback, ) except ModuleNotFoundError as e: logger.error("Transcription module unavailable: %s", e) unavailable_note = "[voice message could not be transcribed]" _placeholder = "(The user sent a message with no text content)" if user_text and user_text.strip() == _placeholder: return unavailable_note, [] if user_text: return f"{unavailable_note}\n\n{user_text}", [] return unavailable_note, [] enriched_parts = [] successful_transcripts: List[str] = [] for path in audio_paths: try: logger.debug("Transcribing user voice: %s", path) result = await asyncio.to_thread( transcribe_audio, path, None, "gateway", ) if not result.get("success"): fallback = await asyncio.to_thread( transcribe_audio_local_fallback, path, ) if fallback.get("success"): logger.info( "Configured STT failed for %s; recovered with local STT", path, ) result = fallback if result["success"]: transcript = result["transcript"] # Speech-to-text can return success=True with an empty or # whitespace-only transcript on silence, cut-off, or # inaudible audio. Emitting empty quotes ('""') makes the # agent reply to nothing and can loop, so that case gets a # clear sentinel note instead (#41603). if not (transcript or "").strip(): enriched_parts.append( "[The user sent a voice message but it came through " "empty or inaudible — speech-to-text returned no " "words. Do not guess at the content; ask the user " "to resend or type it out.]" ) continue successful_transcripts.append(transcript) # Pass the transcript through as a plain quoted line. The # earlier wording ("The user sent a voice message~ Here's # what they said: ...") read as a meta-instruction and made # the LLM volunteer commentary about voice mode rather than # reply to the content. enriched_parts.append(f'"{transcript}"') else: error = result.get("error", "unknown error") # All failure branches: a single, minimal, neutral marker. # Do NOT mention "no STT provider configured", "setup # instructions", or the "hermes-agent-setup" skill, and do # NOT claim a direct message was sent — those phrases get # persisted in conversation history and poison every later # turn, so the model keeps volunteering STT-setup advice # even after transcription starts working. The cause is # logged for operator diagnosis but kept out of the # LLM-visible prompt. logger.info("Voice transcription failed for %s: %s", path, error) from tools.credential_files import to_agent_visible_cache_path agent_path = to_agent_visible_cache_path(os.path.abspath(path)) enriched_parts.append( "[voice message could not be transcribed automatically; " f"the audio is available at: {agent_path}]" ) except Exception as e: logger.error("Transcription error: %s", e) from tools.credential_files import to_agent_visible_cache_path agent_path = to_agent_visible_cache_path(os.path.abspath(path)) enriched_parts.append( "[voice message could not be transcribed automatically; " f"the audio is available at: {agent_path}]" ) if enriched_parts: prefix = "\n\n".join(enriched_parts) # Strip the empty-content placeholder from the Discord adapter # when we successfully transcribed the audio — it's redundant. _placeholder = "(The user sent a message with no text content)" if user_text and user_text.strip() == _placeholder: return prefix, successful_transcripts if user_text: return f"{prefix}\n\n{user_text}", successful_transcripts return prefix, successful_transcripts return user_text, successful_transcripts def _pending_event_audio_paths(self, event) -> List[str]: """Return STT-eligible paths from a pending voice message.""" audio_paths: List[str] = [] media_urls = getattr(event, "media_urls", None) or [] for i, path in enumerate(media_urls): if _event_media_is_stt_input(event, i): audio_paths.append(path) return audio_paths async def _transcribe_pending_audio_event_once( self, event, user_text: Optional[str] = None, ) -> tuple[str | None, List[str]]: """Transcribe a pending audio event once and cache the result on the event. Voice follow-ups can be inspected first by the interrupt monitor and later consumed by the pending-drain path. Both need the same transcript, but only one STT call and one transcript echo should happen for the platform message. """ if hasattr(event, "_gateway_pending_stt_text"): cached_text = getattr(event, "_gateway_pending_stt_text") cached_transcripts = getattr(event, "_gateway_pending_stt_transcripts", []) or [] return cached_text, list(cached_transcripts) audio_paths = self._pending_event_audio_paths(event) if not audio_paths: return user_text if user_text is not None else (getattr(event, "text", None) or None), [] text = user_text if user_text is not None else (getattr(event, "text", "") or "") enriched_text, successful_transcripts = await self._enrich_message_with_transcription( text, audio_paths, ) setattr(event, "_gateway_pending_stt_text", enriched_text) setattr(event, "_gateway_pending_stt_transcripts", list(successful_transcripts)) return enriched_text, successful_transcripts async def _echo_pending_stt_transcripts_once( self, event, adapter, source, transcripts: List[str], *, metadata=None, log_context: str = "Transcript", ) -> None: """Echo pending-event STT transcripts to the chat at most once. The already-echoed transcripts are tracked as a COUNT rather than a single boolean. ``merge_pending_message_event`` can append a second voice note to an event whose first transcript was already echoed and invalidates the transcription cache; the re-run transcription then returns the earlier transcripts as a prefix of the new list, so echoing only the unsent tail suppresses the repeat while still surfacing the newly merged note. A count rather than a set of seen values because two separate notes that transcribe identically are two distinct deliveries and both must be echoed. """ if ( not transcripts or not self._should_echo_stt_transcripts() or adapter is None ): return already_echoed = int(getattr(event, "_gateway_pending_stt_echoed", 0) or 0) unsent = transcripts[already_echoed:] setattr(event, "_gateway_pending_stt_echoed", already_echoed + len(unsent)) for tx in unsent: try: await adapter.send( source.chat_id, f'🎙️ "{tx}"', metadata=metadata, ) except Exception as echo_exc: logger.debug("%s echo failed (non-fatal): %s", log_context, echo_exc) async def _transcribe_and_echo_pending_voice( self, event, adapter, source, text: str, *, log_context: str, metadata=_UNSET, ) -> tuple[str, List[str]]: """Transcribe a pending voice event and echo transcripts once. Unified helper for all interrupt/monitor/backup/drain paths that need to transcribe a pending voice event and echo the transcript to chat. Returns ``(enriched_text, transcripts)`` so the caller can feed the enriched text into ``agent.interrupt()`` or the pending-drain flow. If the event has no STT-eligible media, returns ``(text, [])`` unchanged. The caller is responsible for the ``_build_media_placeholder`` fallback when ``text`` is empty and the event has non-audio media. """ if not self._pending_event_audio_paths(event): return text, [] try: enriched_text, transcripts = await self._transcribe_pending_audio_event_once( event, text, ) echo_meta = self._thread_metadata_for_source( source, self._reply_anchor_for_event(event), ) if metadata is _UNSET else metadata await self._echo_pending_stt_transcripts_once( event, adapter, source, transcripts, metadata=echo_meta, log_context=log_context, ) return enriched_text or text, transcripts except Exception as trans_exc: logger.warning("%s transcription failed: %s", log_context, trans_exc) return text, [] def _build_process_event_source(self, evt: dict): """Resolve the canonical source for a synthetic background-process event. Prefer the persisted session-store origin for the event's session key. Falling back to the currently active foreground event is what causes cross-topic bleed, so don't do that. """ from gateway.session import SessionSource session_key = str(evt.get("session_key") or "").strip() derived_platform = "" derived_chat_type = "" derived_chat_id = "" if session_key: try: self.session_store._ensure_loaded() entry = self.session_store._entries.get(session_key) if entry and getattr(entry, "origin", None): return entry.origin except Exception as exc: logger.debug( "Synthetic process-event session-store lookup failed for %s: %s", session_key, exc, ) cached_source = self._get_cached_session_source(session_key) if cached_source is not None: return cached_source _parsed = _parse_session_key(session_key) if _parsed: derived_platform = _parsed["platform"] derived_chat_type = _parsed["chat_type"] derived_chat_id = _parsed["chat_id"] platform_name = str(evt.get("platform") or derived_platform or "").strip().lower() chat_type = str(evt.get("chat_type") or derived_chat_type or "").strip().lower() chat_id = str(evt.get("chat_id") or derived_chat_id or "").strip() if not platform_name or not chat_type or not chat_id: logger.warning( "Synthetic event source unresolvable: " "session_key=%r platform=%r chat_type=%r chat_id=%r " "evt_type=%s", session_key, platform_name, chat_type, chat_id, evt.get("type", "?"), ) return None try: platform = Platform(platform_name) # Reject arbitrary strings that create dynamic pseudo-members. # Built-in platforms are always valid; plugin platforms must be # registered in the platform registry. if platform.value not in _BUILTIN_PLATFORM_VALUES: try: from gateway.platform_registry import platform_registry if not platform_registry.is_registered(platform.value): raise ValueError(platform_name) except Exception: raise ValueError(platform_name) except Exception: logger.warning( "Synthetic process event has invalid platform metadata: %r", platform_name, ) return None scope_id = str(evt.get("scope_id") or "").strip() or None if scope_id is None and chat_type not in ("dm", "thread"): # Reconstructed (non-persisted) source for a scoped chat with no # scope discriminator: on a relay-fronted deployment the # connector's fail-closed tenant guard may decline the reply # unless user_id resolves it (resolveByUser). Don't fail here — # DMs and author-bound scoped chats still route, and native # adapters don't need scope_id — but say so, so a post-restart # egress decline isn't silent. logger.warning( "Synthetic event source for %s chat=%s (%s) reconstructed " "without scope_id; scoped relay egress may be declined by " "the connector's tenant guard (user_id fallback only).", platform_name, chat_id, chat_type, ) return SessionSource( platform=platform, chat_id=chat_id, chat_type=chat_type, thread_id=str(evt.get("thread_id") or "").strip() or None, user_id=str(evt.get("user_id") or "").strip() or None, user_name=str(evt.get("user_name") or "").strip() or None, scope_id=scope_id, ) async def _drain_watch_notifications(self, completion_queue) -> None: """Consume queued watch events and inject them when notifications are enabled. The queue is ALWAYS drained (so watch events don't rot or requeue-spin) but injection is skipped entirely when ``display.background_process_notifications`` is ``off`` (#9290). """ watch_events = _drain_gateway_watch_events(completion_queue) if self._load_background_notifications_mode() == "off": return for evt in watch_events: synth_text = _format_gateway_process_notification(evt) if not synth_text: continue try: await self._inject_watch_notification(synth_text, evt) except Exception as exc: logger.error("Watch notification injection error: %s", exc) async def _inject_watch_notification( self, synth_text: str, evt: dict, ) -> Optional[bool]: """Inject a watch/completion notification as a synthetic message event. Routing must come from the queued event itself, not from whatever foreground message happened to be active when the queue was drained. Returns ``True`` after adapter acceptance, ``False`` after a retryable adapter failure, and ``None`` when the event has no gateway route. This is not a transactional boundary: a process crash after adapter acceptance can still cause durable at-least-once replay. """ source = await asyncio.to_thread(self._build_process_event_source, evt) if not source: # API-server-originated sessions bind a RAW session key (the # X-Hermes-Session-Id value — see _bind_api_server_session), not a # structured ``agent:main:...`` key, so _build_process_event_source # cannot derive routing metadata from it and returns None above. # Recover the raw session id and wake the real session via the API # server's own /v1/chat/completions entry point instead of # dropping the event. raw_sid = str(evt.get("origin_session_id") or "").strip() if not raw_sid: _sk = str(evt.get("session_key") or "").strip() if _sk and _parse_session_key(_sk) is None: raw_sid = _sk if raw_sid: adapter = self.adapters.get(Platform.API_SERVER) from gateway.wake import ( adapter_supports_push, deliver_wake, persist_delegation_delivery, ) if adapter is not None and not adapter_supports_push(adapter): if evt.get("type") == "async_delegation": # #85957: after the parent turn's event.complete the # CLIENT owns the next turn on this stateless surface. # Persist the completion as a durable delivery row — # never self-post it as a new role=user prompt. try: logger.info( "Async delegation completion — persisting " "delivery row for api_server session %s " "(no wake turn)", raw_sid, ) await persist_delegation_delivery( adapter, text=synth_text, session_id=raw_sid, evt=evt, ) return True except Exception as e: logger.warning( "Async delegation delivery persist failed " "for session %s: %s", raw_sid, e, ) return False try: logger.info( "Watch pattern notification — waking api_server " "session %s via self-post", raw_sid, ) await deliver_wake(adapter, text=synth_text, session_id=raw_sid) return True except Exception as e: logger.warning( "Watch notification self-post wake failed for " "session %s: %s", raw_sid, e, ) return False logger.warning( "Dropping watch notification for raw session %s: no " "api_server adapter to self-post through", raw_sid, ) return None logger.warning( "Dropping watch notification with no routing metadata for process %s", evt.get("session_id", "unknown"), ) return None platform_name = source.platform.value if hasattr(source.platform, "value") else str(source.platform) # Alias-aware resolution (relay-plane): a relay-fronted gateway # registers ONE adapter under Platform.RELAY fronting N logical # platforms, so a literal ``p.value == platform_name`` scan misses # "slack" and silently drops the completion as "no gateway route" # (staging incident 2026-08-09, second occurrence). Resolve through # the shared transport resolver — native adapter wins; relay is # eligible only when it advertises fronting the logical platform. adapter = None try: _platform_enum = Platform(platform_name) except (ValueError, KeyError): _platform_enum = None if _platform_enum is not None: try: _transport = resolve_delivery_transport( _platform_enum, self.config, self.adapters, ) except Exception: _transport = None if _transport is not None: adapter = _transport.adapter if adapter is None: # Legacy literal scan — still correct for native adapters, and # keeps minimal runner stubs (tests) and exotic platform strings # working when the resolver can't run. for p, a in self.adapters.items(): if p.value == platform_name: adapter = a break if not adapter: return None from gateway.wake import adapter_supports_push as _wake_push_ok if not _wake_push_ok(adapter): # Non-push adapter (api_server) resolved WITH routing metadata: # its chat_id is the raw session id (see _bind_api_server_session, # which binds chat_id = session_id). handle_message would run the # wake under a build_session_key()-derived key that never matches # the raw X-Hermes-Session-Id session — self-post instead. from gateway.wake import deliver_wake, persist_delegation_delivery raw_sid = str(evt.get("origin_session_id") or "").strip() or str(source.chat_id or "") if evt.get("type") == "async_delegation": # #85957: same client-owns-the-turn rule as the raw-key branch # above — persist the completion as a delivery row, never # self-post it as a new role=user prompt. try: logger.info( "Async delegation completion — persisting delivery " "row for api_server session %s (no wake turn)", raw_sid, ) await persist_delegation_delivery( adapter, text=synth_text, session_id=raw_sid, evt=evt, ) return True except Exception as e: logger.warning( "Async delegation delivery persist failed for " "session %s: %s", raw_sid, e, ) return False try: logger.info( "Watch pattern notification — waking api_server session " "%s via self-post", raw_sid, ) await deliver_wake(adapter, text=synth_text, session_id=raw_sid) return True except Exception as e: logger.warning( "Watch notification self-post wake failed for session " "%s: %s", raw_sid, e, ) return False try: metadata = {} parent_session_id = str(evt.get("parent_session_id") or "").strip() if parent_session_id: metadata["gateway_session_id"] = parent_session_id synth_event = MessageEvent( text=synth_text, message_type=MessageType.TEXT, source=source, internal=True, message_id=str(evt.get("message_id") or "").strip() or None, metadata=metadata, ) logger.info( "Watch pattern notification — injecting for %s chat=%s thread=%s", platform_name, source.chat_id, source.thread_id, ) # Relay-plane egress priming (defect #4, staging 2026-08-09): a # synthetic turn injected right after a restart reaches a relay # adapter whose per-chat routing caches are cold (they warm only # on inbound), so its replies egress without tenant # discriminators and the connector's fail-closed guard declines # them. Prime the caches from this event's session-store origin. _prime = getattr(adapter, "prime_routing_cache", None) if callable(_prime): _prime(synth_event) await adapter.handle_message(synth_event) return True except Exception as e: logger.error("Watch notification injection error: %s", e) return False @staticmethod def _completion_delivery_identity(evt: dict) -> Optional[tuple[str, str, object]]: """Return a producer-stable identity when one is available. Delegation UUIDs identify one producer completion. Process session IDs are normally unique too, but include the persisted spawn epoch so an explicitly reused ID represents a distinct process incarnation. Legacy process events without ``started_at`` are delivered without deduplication rather than risking suppression of a real completion. """ evt_type = str(evt.get("type") or "") if evt_type == "async_delegation": producer_id = str(evt.get("delegation_id") or "") return (evt_type, producer_id, "") if producer_id else None if evt_type == "completion": producer_id = str(evt.get("session_id") or "") started_at = evt.get("started_at") if producer_id and started_at is not None: return (evt_type, producer_id, started_at) return None async def _classify_completion_target(self, parent_session_id: str) -> str: """Classify an async-completion delivery target before adapter acceptance. Returns one of: - ``"deliver"`` — the spawning session is live, or ended by a compression rotation with a verified live continuation. The inner #55578 resolver (:meth:`_resolve_async_delegation_session`) still owns the actual route retarget; this pre-flight only proves the completion is deliverable so the durable ack stays honest. - ``"terminal"`` — the spawning session is gone for good (unknown, or ended at an explicit user boundary such as /new). Delivery can never succeed; the durable row should be terminally dropped rather than falsely acknowledged as delivered or replayed forever as pending. - ``"retry"`` — transient uncertainty (session DB unavailable, lookup error, or a compression rotation caught mid-flight before its continuation exists). The claim should be released so a later consumer can retry; the attempt cap bounds the churn. """ session_db = getattr(self, "_session_db", None) if session_db is None: return "retry" try: parent = await session_db.get_session(parent_session_id) except Exception: logger.debug( "Async-completion pre-flight parent lookup failed for %s", parent_session_id, exc_info=True, ) return "retry" if parent is None: return "terminal" if not parent.get("ended_at"): return "deliver" end_reason = str(parent.get("end_reason") or "") if end_reason != "compression": # An ended parent is only unreachable when the USER closed the # thread of work (explicit boundary: /new -> session_reset / # new_session, user_exit, session_switch). Idle/timeout ends are # the norm on scale-to-zero relay deployments — the platform chat # remains routable, and the #55578 resolver retargets the # completion to the chat's current session. Dropping those loses # finished work (staging incident 2026-08-09: completed # delegation batch never delivered because the parent had # idle-ended). The boundary set is shared with the resolver # (_USER_BOUNDARY_END_REASONS) so this verdict and the pipeline's # routing decision cannot drift apart. if end_reason in _USER_BOUNDARY_END_REASONS: return "terminal" return "deliver" try: tip_session_id = await session_db.get_compression_tip(parent_session_id) if not tip_session_id or tip_session_id == parent_session_id: # Rotation caught mid-flight: parent is compression-ended but # its continuation isn't visible yet. Retry, don't drop. return "retry" tip = await session_db.get_session(tip_session_id) except Exception: logger.debug( "Async-completion pre-flight tip lookup failed for %s", parent_session_id, exc_info=True, ) return "retry" if tip is None or tip.get("ended_at"): return "retry" return "deliver" async def _deliver_completion_notification( self, synth_text: str, evt: dict, ) -> Optional[bool]: """Deliver once per live gateway, or return False for a retry. ``True`` means this caller reached adapter acceptance, ``False`` means injection failed and the claim was released for retry, and ``None`` means either another same-lifecycle caller owns/delivered the producer event or the event has no gateway route. No cross-process exactly-once guarantee is claimed. """ identity = self._completion_delivery_identity(evt) durable_claim_id = "" durable_delegation_id = "" if evt.get("type") == "async_delegation": durable_delegation_id = str(evt.get("delegation_id") or "") if durable_delegation_id: try: from tools.async_delegation import claim_completion_delivery durable_claim_id = f"gateway:{id(self)}:{__import__('uuid').uuid4().hex}" if not claim_completion_delivery( durable_delegation_id, durable_claim_id, ): return None except Exception as exc: logger.warning( "Could not claim durable async completion %s: %s", durable_delegation_id, exc, ) return False parent_session_id = str(evt.get("parent_session_id") or "").strip() if parent_session_id: # Pre-flight (#65838-class): adapter acceptance is NOT proof of # delivery — the inner #55578 resolver can still fail closed # inside the message pipeline AFTER the adapter accepted, which # would falsely acknowledge the durable row as delivered. # Verify the target here, before acceptance, and give drops an # honest durable disposition. verdict = await self._classify_completion_target(parent_session_id) if verdict == "terminal": logger.warning( "Async delegation %s targets permanently-gone session %s; " "terminally dropping delivery (result remains in the " "delegation records).", durable_delegation_id or "", parent_session_id, ) if durable_claim_id: try: from tools.async_delegation import drop_completion_delivery drop_completion_delivery( durable_delegation_id, durable_claim_id, ) except Exception: logger.debug( "Could not drop durable completion claim", exc_info=True, ) return None if verdict == "retry": if durable_claim_id: try: from tools.async_delegation import release_completion_delivery release_completion_delivery( durable_delegation_id, durable_claim_id, ) except Exception: logger.debug( "Could not release durable completion claim", exc_info=True, ) return False elif evt.get("type") == "completion": # Background-process completions carry only session_key (chat/ # thread routing), so after /new the notification from the OLD # session would land in the chat's NEW session. Stamped events # (spawn-time parent_session_id from terminal_tool) get the same # session-boundary pre-flight as async delegations — one policy # owner (_classify_completion_target), never a forked predicate. # Legacy/unstamped events keep today's behavior and deliver. parent_session_id = str(evt.get("parent_session_id") or "").strip() if parent_session_id: verdict = await self._classify_completion_target(parent_session_id) if verdict == "terminal": logger.warning( "Background process %s completion targets " "permanently-gone session %s (user boundary such as " "/new); dropping notification (output remains " "available via process(action='log')).", evt.get("session_id") or "", parent_session_id, ) return None if verdict == "retry": # Transient uncertainty (session DB unavailable or a # compression rotation mid-flight): signal the watcher to # re-poll and try again rather than dropping or # misrouting the result. return False if identity is not None: with self._completion_delivery_lock: if ( identity in self._completion_deliveries_inflight or identity in self._completion_deliveries_delivered ): return None self._completion_deliveries_inflight.add(identity) accepted = False try: injection_result = await self._inject_watch_notification(synth_text, evt) if injection_result is not True: return injection_result accepted = True if identity is not None: with self._completion_delivery_lock: self._completion_deliveries_inflight.discard(identity) self._completion_deliveries_delivered[identity] = None while ( len(self._completion_deliveries_delivered) > self._completion_delivery_retention ): self._completion_deliveries_delivered.popitem(last=False) # If the durable async-delegation producer branch is present, its # SQLite row remains the authoritative replay state. Acknowledge it # after adapter acceptance; this gateway keeps no parallel ledger. if durable_claim_id: try: from tools.async_delegation import complete_completion_delivery complete_completion_delivery( durable_delegation_id, durable_claim_id, ) except Exception as exc: logger.warning( "Could not acknowledge durable async completion %s: %s", durable_delegation_id, exc, ) return True finally: if identity is not None and not accepted: with self._completion_delivery_lock: self._completion_deliveries_inflight.discard(identity) if durable_claim_id and not accepted: try: from tools.async_delegation import release_completion_delivery release_completion_delivery( durable_delegation_id, durable_claim_id, ) except Exception: logger.debug("Could not release durable completion claim", exc_info=True) @staticmethod def _completion_notification_batch_key(evt: dict) -> tuple[str, ...]: """Return a routing-complete key for short-window process fan-in.""" return tuple(str(evt.get(field) or "") for field in ( "session_key", "platform", "chat_type", "chat_id", "thread_id", "user_id", )) @staticmethod def _format_coalesced_process_completions(entries: list[tuple[str, dict, asyncio.Future]]) -> str: """Build one bounded synthetic event from several redacted completions.""" lines = [ f"[IMPORTANT: {len(entries)} background processes completed for this session.", "Treat these results as one completion batch and send at most one " "consolidated user-facing response.", ] shown = entries[:10] for _text, evt, _future in shown: session_id = str(evt.get("session_id") or "unknown") exit_code = evt.get("exit_code") reason = str(evt.get("completion_reason") or "exited") # Completion-event output is normally passed through the terminal # redactor at the producer seam, but that redactor is deliberately # configurable. This synthetic turn is gateway user-facing input, # so keep the unconditional gateway floor here as defence in depth. # Redact before slicing: truncating first can leave a credential # fragment that no longer matches the authoritative patterns. output = _redact_gateway_user_facing_secrets( str(evt.get("output") or "") ).strip() if len(output) > 800: output = f"[… truncated …]\n{output[-800:]}" lines.append( f"\n- {session_id}: exit_code={exit_code}, reason={reason}" ) if output: lines.append(output) omitted = len(entries) - len(shown) if omitted: lines.append( f"\n- … and {omitted} more completion(s); inspect them with " "the process tool if they affect the conclusion." ) lines.append( "If a result does not change the current conclusion, absorb it silently.]" ) return "\n".join(lines) def _record_coalesced_completion_siblings(self, events: list[dict]) -> None: """Extend a successful primary delivery claim to its batched siblings.""" with self._completion_delivery_lock: for evt in events: identity = self._completion_delivery_identity(evt) if identity is None: continue self._completion_deliveries_inflight.discard(identity) self._completion_deliveries_delivered[identity] = None while ( len(self._completion_deliveries_delivered) > self._completion_delivery_retention ): self._completion_deliveries_delivered.popitem(last=False) async def _flush_process_completion_batch(self, key: tuple[str, ...]) -> None: """Deliver one short-window completion batch and resolve its waiters.""" current_task = asyncio.current_task() entries: list[tuple[str, dict, asyncio.Future]] = [] delivered: Optional[bool] = False try: await asyncio.sleep(self._completion_notification_batch_window) entries = self._completion_notification_batches.pop(key, []) # Detach before adapter delivery. A completion that arrives while # this batch is in flight must be able to schedule the next flush. if self._completion_notification_batch_tasks.get(key) is current_task: self._completion_notification_batch_tasks.pop(key, None) if not entries: return if len(entries) == 1: synth_text = entries[0][0] else: synth_text = self._format_coalesced_process_completions(entries) # A duplicate primary can legitimately return None from the # lifecycle dedupe seam. Try the next batch identity so a # fresh sibling is never discarded with that duplicate. delivered = None for _text, candidate_evt, _future in entries: delivered = await self._deliver_completion_notification( synth_text, candidate_evt, ) if delivered is not None: break if delivered is True and len(entries) > 1: self._record_coalesced_completion_siblings( [evt for _text, evt, _future in entries] ) except asyncio.CancelledError: # Shutdown may cancel us either during the fan-in window or while # adapter delivery is blocked. Recover entries that have not yet # detached and resolve every waiter as retryable before adapters # are torn down. delivered = False if not entries: entries = self._completion_notification_batches.pop(key, []) raise except Exception: logger.exception("Coalesced process completion delivery failed") delivered = False finally: # Never strand watcher futures if formatting, delivery, or task # cancellation interrupts a batch. False follows the existing # watcher retry path; None remains the ordinary dedupe result. for _text, _evt, future in entries: if not future.done(): future.set_result(delivered) # Do not remove a newer flush task that reused the same route key. if self._completion_notification_batch_tasks.get(key) is current_task: self._completion_notification_batch_tasks.pop(key, None) async def _cancel_process_completion_batch_tasks(self) -> None: """Settle pending completion batches before adapter teardown.""" self._completion_notification_batches_stopping = True tasks = { task for task in getattr( self, "_completion_notification_batch_flush_tasks", set() ) if not task.done() } for task in tasks: task.cancel() if tasks: await asyncio.gather(*tasks, return_exceptions=True) # Defensive cleanup for an orphaned queue with no live flush task. batches = getattr(self, "_completion_notification_batches", {}) for entries in batches.values(): for _text, _evt, future in entries: if not future.done(): future.set_result(False) batches.clear() getattr(self, "_completion_notification_batch_tasks", {}).clear() getattr(self, "_completion_notification_batch_flush_tasks", set()).clear() async def _enqueue_process_completion_notification( self, synth_text: str, evt: dict, ) -> Optional[bool]: """Fan in concurrent process completions that share one conversation.""" # Some unit tests construct GatewayRunner with object.__new__. Keep the # batching seam lazy so those focused lifecycle tests remain valid. if not hasattr(self, "_completion_notification_batches"): self._completion_notification_batches = {} if not hasattr(self, "_completion_notification_batch_tasks"): self._completion_notification_batch_tasks = {} if not hasattr(self, "_completion_notification_batch_flush_tasks"): self._completion_notification_batch_flush_tasks = set() if not hasattr(self, "_completion_notification_batch_window"): self._completion_notification_batch_window = 0.1 if not hasattr(self, "_completion_notification_batches_stopping"): self._completion_notification_batches_stopping = False if self._completion_notification_batches_stopping: return False key = self._completion_notification_batch_key(evt) future = asyncio.get_running_loop().create_future() self._completion_notification_batches.setdefault(key, []).append( (synth_text, evt, future) ) if key not in self._completion_notification_batch_tasks: task = asyncio.create_task( self._flush_process_completion_batch(key) ) self._completion_notification_batch_tasks[key] = task # Keep the flush alive and include it in the gateway's normal # lifecycle accounting. Focused tests that construct a runner via # object.__new__ lazily receive the same ownership set. if not hasattr(self, "_background_tasks"): self._background_tasks = set() self._background_tasks.add(task) self._completion_notification_batch_flush_tasks.add(task) task.add_done_callback(self._background_tasks.discard) task.add_done_callback( self._completion_notification_batch_flush_tasks.discard ) return await future def _enrich_async_delegation_routing(self, evt: dict) -> None: """Fill platform/chat_id/thread_id/chat_type on an async-delegation event. Async-delegation completion events only carry ``session_key`` (the daemon worker has no access to the per-message routing metadata the terminal background watcher captures at spawn time). Parse the session_key into the routing fields ``_build_process_event_source`` expects. Best-effort: a CLI-origin event (empty session_key) is left as-is and simply won't route on the gateway. """ if evt.get("platform"): return # already enriched parsed = _parse_session_key(evt.get("session_key", "") or "") if not parsed: return evt["platform"] = parsed.get("platform", "") evt["chat_type"] = parsed.get("chat_type", "") evt["chat_id"] = parsed.get("chat_id", "") if parsed.get("thread_id"): evt["thread_id"] = parsed["thread_id"] @staticmethod def _async_delegation_group_key(evt: dict) -> tuple[str, ...]: """Return the same-session routing key for async completion coalescing. Two events coalesce only when every routing dimension matches — the originating session key, the parent session the result re-enters, and the full gateway route. Events for different sessions never coalesce. """ return tuple(str(evt.get(field) or "") for field in ( "session_key", "parent_session_id", "platform", "chat_type", "chat_id", "thread_id", "user_id", )) @staticmethod def _format_coalesced_async_delegations(blocks: list[str]) -> str: """Join per-delegation formatted blocks into one consolidated turn.""" header = ( f"[IMPORTANT: {len(blocks)} background subagent delegations " "completed for this session. Treat these results as one " "completion batch and send at most one consolidated user-facing " "response. If a result does not change the current conclusion, " "absorb it silently.]" ) return "\n\n".join([header, *blocks]) async def _deliver_async_delegation_group( self, group: list[dict], ) -> Optional[bool]: """Deliver a same-session batch of async completions as ONE turn. A single-event group rides the existing per-event path unchanged. For a multi-event group the primary event is delivered through ``_deliver_completion_notification`` (which owns its durable claim, the lifecycle dedupe, and the target preflight), carrying a consolidated text that also contains every sibling result whose durable row THIS runner successfully claimed up front. Only after adapter acceptance are the sibling claims acknowledged — the durable ledger never acks work that was not delivered, and a sibling claimed by another consumer is excluded from the consolidated text entirely so its content cannot be double-delivered. Returns ``True`` after adapter acceptance, ``False`` when the caller should requeue the group for retry, and ``None`` when nothing in the group is deliverable by this runner (siblings that still need a retry are requeued here before returning). """ from tools.process_registry import process_registry as _pr deliverable: list[tuple[dict, str]] = [] for evt in group: synth_text = _format_gateway_process_notification(evt) if not synth_text: continue identity = self._completion_delivery_identity(evt) if identity is not None: with self._completion_delivery_lock: if ( identity in self._completion_deliveries_inflight or identity in self._completion_deliveries_delivered ): continue deliverable.append((evt, synth_text)) if not deliverable: return None if len(deliverable) == 1: evt, synth_text = deliverable[0] return await self._deliver_completion_notification(synth_text, evt) from tools.async_delegation import ( claim_event_delivery, complete_event_delivery, release_event_delivery, ) primary_evt, primary_text = deliverable[0] blocks = [primary_text] siblings: list[tuple[dict, str]] = [] for evt, synth_text in deliverable[1:]: claim_id = claim_event_delivery(evt, f"gateway-batch:{id(self)}") if claim_id is None: # Another consumer owns this row's delivery; keep its result # out of our consolidated text so it is never double-injected. continue siblings.append((evt, claim_id)) blocks.append(synth_text) if not siblings: return await self._deliver_completion_notification( primary_text, primary_evt, ) consolidated = self._format_coalesced_async_delegations(blocks) delivered: Optional[bool] = False try: delivered = await self._deliver_completion_notification( consolidated, primary_evt, ) finally: if delivered is True: for evt, claim_id in siblings: try: complete_event_delivery(evt, claim_id) except Exception: logger.debug( "Could not acknowledge coalesced durable completion", exc_info=True, ) self._record_coalesced_completion_siblings( [evt for evt, _claim_id in siblings] ) else: # Not delivered — release every sibling claim so a retry (or # another consumer) can claim it, honestly leaving the durable # rows pending. for evt, claim_id in siblings: try: release_event_delivery(evt, claim_id) except Exception: logger.debug( "Could not release coalesced durable claim", exc_info=True, ) if delivered is None: # The primary was dropped/owned elsewhere but the siblings # still need delivery — requeue just them for the next tick. for evt, _claim_id in siblings: _pr.completion_queue.put(evt) return delivered async def _async_delegation_watcher(self, interval: float = 2.0) -> None: """Drain async-delegation completions and inject them as new turns. Background subagents (``delegate_task(background=true)``) run on the async-delegation daemon executor — they have no per-process watcher task, so their completion events would only be seen by the post-turn queue drain. This watcher covers the IDLE case: when a background subagent finishes while no agent turn is running, its result still re-enters the originating session promptly. Mirrors the CLI's idle ``process_loop`` drain. Stays silent when the queue has nothing for us; ignores non-async event types (those are handled by ``_run_process_watcher`` / the post-turn drain). """ await asyncio.sleep(3) # let platforms finish connecting from tools.process_registry import process_registry as _pr while self._running: try: # Peek the queue for async-delegation events. We must NOT # consume watch/completion events here (other drains own them), # so requeue anything that isn't ours. requeue = [] async_events = [] while not _pr.completion_queue.empty(): try: evt = _pr.completion_queue.get_nowait() except Exception: break if evt.get("type") == "async_delegation": async_events.append(evt) else: requeue.append(evt) for evt in requeue: _pr.completion_queue.put(evt) # A same-tick drain often carries several completions for the # SAME originating session (a fan-out of background subagents # finishing together). Delivering each one individually floods # the session with N synthetic turns (#70300) — group by full # gateway route + parent session and inject one consolidated # turn per group. Events for different sessions never coalesce. groups: dict[tuple[str, ...], list[dict]] = {} group_order: list[tuple[str, ...]] = [] for evt in async_events: self._enrich_async_delegation_routing(evt) key = self._async_delegation_group_key(evt) if key not in groups: groups[key] = [] group_order.append(key) groups[key].append(evt) for key in group_order: group = groups[key] try: delivered = await self._deliver_async_delegation_group(group) if delivered is False: for evt in group: _pr.completion_queue.put(evt) except Exception as e: for evt in group: _pr.completion_queue.put(evt) logger.error("Async delegation injection error: %s", e) except Exception as e: logger.debug("Async delegation watcher error: %s", e) await asyncio.sleep(interval) async def _run_process_watcher(self, watcher: dict) -> None: """ Periodically check a background process and push updates to the user. Runs as an asyncio task. Stays silent when nothing changed. Auto-removes when the process exits or is killed. Notification mode (from ``display.background_process_notifications``): - ``concise`` — one-line status message on completion (default); failures append a short output tail - ``all`` — running-output updates + final raw-output message - ``result`` — final raw-output completion message only - ``error`` — final raw-output message only when exit code != 0 - ``off`` — no messages at all """ from tools.process_registry import process_registry session_id = watcher["session_id"] interval = watcher["check_interval"] session_key = watcher.get("session_key", "") platform_name = watcher.get("platform", "") chat_id = watcher.get("chat_id", "") thread_id = watcher.get("thread_id", "") user_id = watcher.get("user_id", "") user_name = watcher.get("user_name", "") message_id = str(watcher.get("message_id") or "").strip() or None agent_notify = watcher.get("notify_on_complete", False) notify_mode = self._load_background_notifications_mode() logger.debug("Process watcher started: %s (every %ss, notify=%s, agent_notify=%s)", session_id, interval, notify_mode, agent_notify) if notify_mode == "off" and not agent_notify: # Still wait for the process to exit so we can log it, but don't # push any messages to the user. while True: await asyncio.sleep(interval) session = process_registry.get(session_id) if session is None or session.exited: break logger.debug("Process watcher ended (silent): %s", session_id) return last_output_len = 0 while True: await asyncio.sleep(interval) session = process_registry.get(session_id) if session is None: break current_output_len = len(session.output_buffer) has_new_output = current_output_len > last_output_len last_output_len = current_output_len if session.exited: # --- Agent-triggered completion: inject synthetic message --- # Skip if the agent already consumed the result via wait/log. # poll() is read-only and intentionally does NOT mark consumed # (#10156) — a status check must not suppress this delivery turn. from tools.process_registry import format_process_notification, process_registry as _pr_check if agent_notify and not _pr_check.is_completion_consumed(session_id): from agent.redact import redact_terminal_output from tools.ansi_strip import strip_ansi _command = getattr(session, "command", "") or "" _raw = strip_ansi(session.output_buffer) if session.output_buffer else "" _raw = redact_terminal_output(_raw, _command) _command = _redact_gateway_user_facing_secrets(_command) # Truncate at line boundaries so notifications never start # mid-line (fixes #23284). Keep the last ~2000 chars but # snap to the nearest preceding newline, then prepend a # truncation marker when output was cut. _LIMIT = 2000 if len(_raw) > _LIMIT: _tail = _raw[-_LIMIT:] _nl = _tail.find("\n") _tail = _tail[_nl + 1:] if _nl != -1 else _tail _out = f"[… output truncated — showing last {len(_tail)} chars]\n{_tail}" else: _out = _raw _out = _redact_gateway_user_facing_secrets(_out) completion_evt = { "type": "completion", "session_id": session_id, "session_key": session_key, "platform": platform_name, "chat_type": watcher.get("chat_type", ""), "chat_id": chat_id, "thread_id": thread_id, "user_id": user_id, "user_name": user_name, "message_id": message_id, "started_at": getattr(session, "started_at", None), "command": _command, "exit_code": session.exit_code, "completion_reason": getattr(session, "completion_reason", "exited"), "termination_source": getattr(session, "termination_source", ""), "output": _out, # Spawning conversation's session-db id (stamped at # spawn time in terminal_tool). Lets the delivery # pre-flight drop this completion when the user closed # that session (/new) before the process finished. "parent_session_id": ( watcher.get("parent_session_id") or getattr(session, "parent_session_id", "") or "" ), } synth_text = format_process_notification(completion_evt) if not synth_text: break delivered = await self._enqueue_process_completion_notification( synth_text, completion_evt, ) if delivered is False: # The process remains terminal; retry after failed # adapter injection instead of suppressing the result. continue break # --- Normal text-only notification --- # Skip when the agent already consumed this completion via # wait/log (#65379): process(wait) returned the exit code and # output inline, so the raw "[Background process ... finished # with exit code ...]" message would be a duplicate delivery # of the same completion. The agent_notify branch above # already honors _completion_consumed; without this check its # skip FALLS THROUGH to this block and re-delivers the output # the agent is actively summarizing. poll() is read-only and # intentionally does not mark consumed (#10156), so a status # check never suppresses this message. if _pr_check.is_completion_consumed(session_id): logger.debug( "Process watcher: completion for %s already consumed " "via wait/log — skipping raw notification (#65379)", session_id, ) break # Decide whether to notify based on mode should_notify = ( notify_mode in {"concise", "all", "result"} or (notify_mode == "error" and session.exit_code not in {0, None}) ) if should_notify: new_output = session.output_buffer[-1000:] if session.output_buffer else "" if new_output: from agent.redact import redact_terminal_output new_output = redact_terminal_output( new_output, getattr(session, "command", "") or "" ) # redact_terminal_output() is unforced, so it returns raw # text when security.redact_secrets is off. This send # goes straight to the platform adapter, so it needs the # same unconditional floor as the agent-notify path. new_output = _redact_gateway_user_facing_secrets(new_output) if notify_mode == "concise": _cmd_disp = _redact_gateway_user_facing_secrets( getattr(session, "command", "") or "" ) _started = getattr(session, "started_at", None) _dur = None if isinstance(_started, (int, float)): _dur = max(0.0, time.time() - _started) message_text = _format_concise_process_notification( session_id, _cmd_disp, session.exit_code, new_output, duration_seconds=_dur, ) else: message_text = ( f"[Background process {session_id} finished with exit code {session.exit_code}~ " f"Here's the final output:\n{new_output}]" ) adapter = None for p, a in self.adapters.items(): if p.value == platform_name: adapter = a break if adapter and chat_id: try: send_meta = {"thread_id": thread_id} if thread_id else None await adapter.send( chat_id, message_text, metadata=_non_conversational_metadata(send_meta, platform=platform_name), ) except Exception as e: logger.error("Watcher delivery error: %s", e) break elif has_new_output and notify_mode == "all" and not agent_notify: # New output available -- deliver status update (only in "all" mode) # Skip periodic updates for agent_notify watchers (they only care about completion) new_output = session.output_buffer[-500:] if session.output_buffer else "" if new_output: from agent.redact import redact_terminal_output new_output = redact_terminal_output( new_output, getattr(session, "command", "") or "" ) new_output = _redact_gateway_user_facing_secrets(new_output) message_text = ( f"[Background process {session_id} is still running~ " f"New output:\n{new_output}]" ) adapter = None for p, a in self.adapters.items(): if p.value == platform_name: adapter = a break if adapter and chat_id: try: send_meta = {"thread_id": thread_id} if thread_id else None await adapter.send( chat_id, message_text, metadata=_non_conversational_metadata(send_meta, platform=platform_name), ) except Exception as e: logger.error("Watcher delivery error: %s", e) logger.debug("Process watcher ended: %s", session_id) _MAX_INTERRUPT_DEPTH = 3 # Cap recursive interrupt handling (#816) # Config keys whose values MUST invalidate the gateway's cached agent # when they change. The agent bakes these into its compressor / context # handling at construction time, so a mid-running-gateway config edit # would otherwise be silently ignored until the user triggers a # different cache eviction (model switch, /reset, etc.). # # Each entry is a tuple of (section, key) read from the raw config dict. # Add more here as new baked-at-construction config settings are added. _CACHE_BUSTING_CONFIG_KEYS: tuple = ( ("model", "context_length"), ("model", "max_tokens"), ("compression", "enabled"), ("compression", "progress_notices"), ("compression", "threshold"), ("compression", "model_thresholds"), ("compression", "threshold_tokens"), ("compression", "codex_gpt55_autoraise"), ("compression", "codex_app_server_auto"), ("compression", "codex_responses_native"), ("compression", "codex_responses_compact_threshold"), ("compression", "in_place"), ("compression", "checkpoint_required"), ("compression", "micro_compact"), ("compression", "micro_compact_every_n_turns"), ("compression", "micro_compact_defrag_threshold_tokens"), ("compression", "target_ratio"), ("compression", "tail_mode"), ("compression", "protect_last_n"), ("compression", "proactive_prune_tokens"), ("compression", "proactive_prune_min_result_chars"), ("compression", "proactive_prune_min_reclaim_tokens"), ("compression", "min_tail_user_messages"), ("agent", "disabled_toolsets"), ("memory", "provider"), ("checkpoints", "enabled"), ("checkpoints", "max_snapshots"), ("checkpoints", "max_total_size_mb"), ("checkpoints", "max_file_size_mb"), ) _HONCHO_CACHE_BUSTING_KEYS = ( "honcho.peer_name", "honcho.ai_peer", "honcho.pin_peer_name", "honcho.runtime_peer_prefix", "honcho.user_peer_aliases", ) _HONCHO_CACHE_BUSTING_MEMO: dict[tuple[str, int | None], dict[str, Any]] = {} @classmethod def _empty_honcho_cache_busting_config(cls) -> dict[str, Any]: return {key: None for key in cls._HONCHO_CACHE_BUSTING_KEYS} @classmethod def _extract_honcho_cache_busting_config(cls) -> dict[str, Any]: """Extract Honcho identity keys, memoized by honcho.json mtime.""" try: from plugins.memory.honcho.client import HonchoClientConfig, resolve_config_path path = resolve_config_path() try: mtime_ns = path.stat().st_mtime_ns except OSError: mtime_ns = None memo_key = (str(path), mtime_ns) cached = cls._HONCHO_CACHE_BUSTING_MEMO.get(memo_key) if cached is not None: return dict(cached) hcfg = HonchoClientConfig.from_global_config(config_path=path) aliases = hcfg.user_peer_aliases or {} values = { "honcho.peer_name": hcfg.peer_name, "honcho.ai_peer": hcfg.ai_peer, "honcho.pin_peer_name": bool(hcfg.pin_peer_name), "honcho.runtime_peer_prefix": hcfg.runtime_peer_prefix or "", "honcho.user_peer_aliases": sorted(aliases.items()) if isinstance(aliases, dict) else [], } cls._HONCHO_CACHE_BUSTING_MEMO = {memo_key: values} return dict(values) except Exception: return cls._empty_honcho_cache_busting_config() @classmethod def _extract_cache_busting_config(cls, user_config: dict | None) -> dict: """Pull values that must bust the cached agent. Returns a flat dict keyed by 'section.key'. Missing config keys and non-dict sections yield None values, which still contribute to the signature (so 'absent' vs 'present-and-null' differ). The live tool registry generation is included too. MCP reloads and dynamic MCP tool-list changes mutate the registry without necessarily changing config.yaml. Cached AIAgent instances freeze their tool schemas at construction time, so a registry generation change must rebuild the agent before the next turn. """ out: Dict[str, Any] = {} cfg = user_config if isinstance(user_config, dict) else {} for section, key in cls._CACHE_BUSTING_CONFIG_KEYS: section_val = cfg.get(section) if section == "checkpoints" and isinstance(section_val, bool): # Preserve legacy ``checkpoints: true`` behavior. A live # toggle must still rebuild the cached agent. out[f"{section}.{key}"] = section_val if key == "enabled" else None elif isinstance(section_val, dict): out[f"{section}.{key}"] = section_val.get(key) else: out[f"{section}.{key}"] = None try: from tools.registry import registry out["tools.registry_generation"] = getattr(registry, "_generation", None) except Exception: out["tools.registry_generation"] = None # Honcho identity-mapping keys live in honcho.json, not user_config. # Only read that file when Honcho is the active memory provider. provider = cfg_get(cfg, "memory", "provider") if isinstance(provider, str) and provider.lower() == "honcho": out.update(cls._extract_honcho_cache_busting_config()) else: out.update(cls._empty_honcho_cache_busting_config()) return out @staticmethod def _agent_config_signature( model: str, runtime: dict, enabled_toolsets: list, ephemeral_prompt: str, cache_keys: dict | None = None, user_id: str | None = None, user_id_alt: str | None = None, skip_context_files: bool = False, ) -> str: """Compute a stable string key from agent config values. When this signature changes between messages, the cached AIAgent is discarded and rebuilt. When it stays the same, the cached agent is reused — preserving the frozen system prompt and tool schemas for prompt cache hits. ``cache_keys`` is an optional flat dict of additional config values that should invalidate the cache when they change. Callers pass the output of ``_extract_cache_busting_config(user_config)`` so edits to model.context_length / compression.* in config.yaml are picked up on the next gateway message without a manual restart. ``user_id`` and ``user_id_alt`` are the runtime user identities carried by the current message's gateway source. They participate in the cache key because the Honcho memory provider freezes them into ``HonchoSessionManager`` at first-message init (see ``plugins/memory/honcho/__init__.py::_do_session_init``). Without them in the signature, a shared-thread session_key (one in which ``build_session_key`` intentionally omits the participant ID, e.g. ``thread_sessions_per_user=False``) would reuse the cached AIAgent across distinct users, causing the second user's messages to be attributed to the first user's resolved Honcho peer. This broke #27371's per-user-peer contract in multi-user gateways. Per-user agent rebuilds in shared threads trade prompt-cache warmth for correct memory attribution. """ import hashlib, json as _j # Fingerprint the FULL credential string instead of using a short # prefix. OAuth/JWT-style tokens frequently share a common prefix # (e.g. "eyJhbGci"), which can cause false cache hits across auth # switches if only the first few characters are considered. _api_key = str(runtime.get("api_key", "") or "") _api_key_fingerprint = hashlib.sha256(_api_key.encode()).hexdigest() if _api_key else "" _cache_keys_sorted = sorted((cache_keys or {}).items()) blob = _j.dumps( [ model, _api_key_fingerprint, runtime.get("base_url", ""), runtime.get("provider", ""), runtime.get("requested_provider", ""), runtime.get("api_mode", ""), sorted((runtime.get("capabilities") or {}).items()), sorted(enabled_toolsets) if enabled_toolsets else [], # reasoning_config excluded — it's set per-message on the # cached agent and doesn't affect system prompt or tools. ephemeral_prompt or "", _cache_keys_sorted, str(user_id or ""), str(user_id_alt or ""), # skip_context_files changes the agent's frozen system prompt # (context files in vs out) — a toggled config edit must # rebuild the cached agent, not silently reuse it. bool(skip_context_files), ], sort_keys=True, default=str, ) return hashlib.sha256(blob.encode()).hexdigest()[:16] def _rehydrate_session_model_override(self, session_key: str) -> None: """Lazily restore a persisted /model override after a gateway restart. ``_session_model_overrides`` is in-memory only, so before persistence a restart silently reverted every session to the global default model. The non-secret parts (model/provider/base_url) are written through to the session store when /model runs (and cleared on /new); here we read them back on first use and re-resolve credentials via the normal runtime provider resolution — api_key is never persisted to disk. No-op when an in-memory override already exists (live state wins) or when the store has nothing persisted (e.g. the user ran /new, which clears both the in-memory dict and the persisted field). """ _rehydrate_state = self._peek_session_state(session_key) if ( _rehydrate_state is not None and _rehydrate_state.conversation.model_override is not None ): return store = getattr(self, "session_store", None) if store is None: return try: persisted = store.get_model_override(session_key) except Exception: logger.debug( "Failed to read persisted session model override", exc_info=True ) return if not persisted: return override: Dict[str, Any] = { "model": persisted.get("model"), "provider": persisted.get("provider"), "base_url": persisted.get("base_url"), } provider = persisted.get("provider") if provider: # Re-resolve credentials for the persisted provider. On failure # (e.g. credentials were removed since the switch) keep the # credential-less override — _resolve_session_agent_runtime falls # back to env-based resolution and applies model/provider on top. try: runtime = _resolve_runtime_agent_kwargs_for_provider(provider) override["api_key"] = runtime.get("api_key") override["api_mode"] = runtime.get("api_mode") override["credential_pool"] = runtime.get("credential_pool") override["request_overrides"] = dict( runtime.get("request_overrides") or {} ) override["requested_provider"] = runtime.get("requested_provider") override["capabilities"] = dict(runtime.get("capabilities") or {}) override["max_tokens"] = runtime.get("max_tokens") if not override.get("base_url"): override["base_url"] = runtime.get("base_url") except Exception: logger.debug( "Credential re-resolution failed for persisted override " "(provider=%s); using credential-less override", provider, exc_info=True, ) self._session_state(session_key).conversation.model_override = override logger.info( "Rehydrated persisted /model override for session=%s: model=%s provider=%s", session_key, override.get("model"), provider or "", ) def _apply_session_model_override( self, session_key: str, model: str, runtime_kwargs: dict ) -> tuple: """Apply /model session overrides if present, returning (model, runtime_kwargs). The gateway /model command stores per-session overrides in ``_session_model_overrides``. These must take precedence over config.yaml defaults so the switched model is actually used for subsequent messages. Fields with ``None`` values are skipped so partial overrides don't clobber valid config defaults. """ _apply_state = self._peek_session_state(session_key) override = _apply_state.conversation.model_override if _apply_state else None if not override: return model, runtime_kwargs model = override.get("model", model) for key in ( "provider", "requested_provider", "api_key", "base_url", "api_mode", "credential_pool", "capabilities", "max_tokens", ): val = override.get(key) if val is not None: runtime_kwargs[key] = val # request_overrides reflects the switched-to provider; apply whenever # the override recorded it (even as None) so switching to a provider # without configured overrides clears a stale value left by the # default provider's runtime resolution. if "request_overrides" in override: override_request_overrides = override.get("request_overrides") if isinstance(override_request_overrides, dict) and override_request_overrides: runtime_kwargs["request_overrides"] = dict(override_request_overrides) else: runtime_kwargs["request_overrides"] = override_request_overrides if ( runtime_kwargs.get("api_key") and runtime_kwargs.get("credential_pool") is None and override.get("provider") ): runtime_kwargs["credential_pool"] = _credential_pool_for_provider( override.get("provider") ) return model, runtime_kwargs def _snapshot_session_model_override(self, session_key: str) -> dict: """Capture a gateway session override before a one-turn switch.""" _snap_state = self._peek_session_state(session_key) override = _snap_state.conversation.model_override if _snap_state else None return { "had_override": override is not None, "override": dict(override) if override is not None else None, } def _restore_session_model_override(self, session_key: str, snapshot: dict) -> None: """Restore the session override captured before a one-turn switch.""" if not session_key: return if snapshot.get("had_override"): self._session_state(session_key).conversation.model_override = dict( snapshot.get("override") or {} ) else: _rst_state = self._peek_session_state(session_key) if _rst_state is not None: _rst_state.conversation.model_override = None self._evict_cached_agent(session_key) def _is_intentional_model_switch(self, session_key: str, agent_model: str) -> bool: """Return True if *agent_model* matches an active /model session override.""" _ims_state = self._peek_session_state(session_key) override = _ims_state.conversation.model_override if _ims_state else None return override is not None and override.get("model") == agent_model def _release_running_agent_state( self, session_key: str, *, run_generation: Optional[int] = None, ) -> bool: """Pop ALL per-running-agent state entries for ``session_key``. Replaces ad-hoc ``del self._running_agents[key]`` calls scattered across the gateway. Those sites had drifted: some popped only ``_running_agents``; some also ``_running_agents_ts``; only one path also cleared ``_busy_ack_ts``. Each missed entry was a small, persistent leak — a (str_key → float) tuple per session per gateway lifetime. Use this at every site that ends a running turn, regardless of cause (normal completion, /stop, /reset, /resume, sentinel cleanup, stale-eviction). Per-session state that PERSISTS across turns (``_session_model_overrides``, ``_voice_mode``, ``_pending_approvals``, ``_update_prompt_pending``) is NOT touched here — those have their own lifecycles. When ``run_generation`` is provided, only clear the slot if that generation is still current for the session. This prevents an older async run whose generation was bumped by /stop or /new from clobbering a newer run's state during its own unwind. Returns True when the slot was cleared, False when an ownership guard blocked it. """ if not session_key: return False if run_generation is not None and not self._is_session_run_current( session_key, run_generation ): return False state = self._peek_session_state(session_key) if state is not None: lease = state.turn.lease if lease is not None: try: lease.release() except Exception: logger.debug( "Failed to release active session slot", exc_info=True ) # One structured reset instead of the old drifting pop-list # (agent / started_ts / lease / busy_ack_ts). Turn-lease tokens # are deliberately NOT cleared here — _release_turn_lease owns # them (#64934). state.turn.clear() # Turn boundary: a running-agent slot was just released. Persist the # new (lower) in-flight count so the dashboard readout stays current # between lifecycle transitions. Preserves gateway_state (see # _persist_active_agents). self._persist_active_agents() return True def _release_turn_lease(self, session_key: str, run_generation: int) -> bool: """Release the turn lease acquired by (``session_key``, ``run_generation``). Companion to the acquisition in ``_handle_message_with_agent`` (#64934). The token map is keyed by (routing key, run generation), so this can only ever free the lease its own turn acquired — a stale unwind whose generation was bumped by /stop or /new pops ITS token, and the registry's identity check refuses it if a newer turn already holds the lease. Idempotent and safe for bare test runners built via ``object.__new__`` (getattr defaults). """ if not session_key: return False registry = getattr(self, "_turn_leases", None) state = self._peek_session_state(session_key) if state is None or registry is None: return False turn = state.turn if turn.lease_token is None or turn.lease_generation != run_generation: return False token = turn.lease_token turn.lease_token = None turn.lease_generation = None try: return registry.release(token) except Exception: logger.debug("Failed to release turn lease", exc_info=True) return False def _rebind_turn_lease( self, session_key: str, run_generation: int, new_session_id: str ) -> bool: """Follow a mid-turn session_id rotation with the held turn lease. Compression (session-hygiene pre-compression or the agent's own compressor) can rotate ``session_entry.session_id`` while this turn is in flight. The turn's flush targets the NEW id, so the serialization boundary must follow it — otherwise an alias routing key resolving the new id (topic tip-walk onto the fresh child) could start a concurrent turn the lease never sees (#64934 rotation-alias window). Call at every site that reassigns session_entry.session_id mid-turn. Fail-open no-op when there is no held token. """ if not session_key or not new_session_id: return False registry = getattr(self, "_turn_leases", None) state = self._peek_session_state(session_key) if state is None or registry is None: return False turn = state.turn if turn.lease_token is None or turn.lease_generation != run_generation: return False try: return registry.rebind(turn.lease_token, new_session_id) except Exception: logger.debug("Failed to rebind turn lease", exc_info=True) return False def _clear_conversation_scope(self, session_key: str, *, reason: str) -> None: """Clear ALL conversation-scoped per-session state for ``session_key``. THE single conversation-boundary funnel. Call this — and nothing else — whenever a session_key crosses a conversation boundary: /new, /resume, auto-reset (idle/daily/suspended), expiry finalization, and the compression-exhausted auto-reset. Why a funnel: these boundaries used to each carry a hand-copied pop-list of the per-session dicts, and the lists drifted every time a new dict was added (#48031, #58403, #10702, #35809 were all "boundary X forgot dict Y" bugs — e.g. /new cleared the /model override but not the /model --once restore snapshot). Adding a new conversation-scoped dict now means adding its attribute name to _CONVERSATION_SCOPED_STATE below; every boundary picks it up automatically. Scope rules: - Conversation-scoped (cleared here): model/reasoning overrides, one-turn restore snapshots, pending model notes, last-resolved model cache, queued follow-up events, and the boundary security state (approvals, /yolo, slash-confirm, update prompts). - Turn-scoped (NOT cleared here): _running_agents/_ts, slot leases, turn-lease tokens — owned by _release_running_agent_state and the dispatch finally. - Idle agent-cache eviction is NOT a conversation boundary: the session is still alive and a resumed turn rebuilds from these overrides. Only true boundaries call this. Safe on bare test runners built via ``object.__new__`` (every access is getattr-guarded). """ if not session_key: return # Structural clear: every conversation-scoped field resets in one # call — no per-attribute pop-list to drift. state = self._peek_session_state(session_key) if state is not None: state.conversation.clear() # Legacy plain-dict stores still registered in # _CONVERSATION_SCOPED_STATE (not yet folded into SessionState), # e.g. _pending_model_notes. SessionState-backed names resolve to # MutableMapping views (not dict), so the isinstance(dict) guard # skips them — already handled above. for attr in _CONVERSATION_SCOPED_STATE: store = getattr(self, attr, None) if isinstance(store, dict): store.pop(session_key, None) self._clear_session_boundary_security_state(session_key) logger.debug( "Cleared conversation scope for %s (%s)", session_key, reason ) def _clear_session_boundary_security_state(self, session_key: str) -> None: """Clear per-session control state that must not survive a boundary switch.""" if not session_key: return pending_skills_reload_notes = getattr( self, "_pending_skills_reload_notes", None ) if isinstance(pending_skills_reload_notes, dict): pending_skills_reload_notes.pop(session_key, None) _sec_state = self._peek_session_state(session_key) if _sec_state is not None: _sec_state.persistent.approvals = None _sec_state.persistent.update_prompt_pending = False try: from tools import slash_confirm as _slash_confirm_mod except Exception: _slash_confirm_mod = None if _slash_confirm_mod is not None: try: _slash_confirm_mod.clear(session_key) except Exception as e: logger.debug( "Failed to clear slash-confirm state for session boundary %s: %s", session_key, e, ) try: from tools.approval import clear_session as _clear_approval_session except Exception: return try: _clear_approval_session(session_key) except Exception as e: logger.debug( "Failed to clear approval state for session boundary %s: %s", session_key, e, ) def _begin_session_run_generation(self, session_key: str) -> int: """Claim a fresh run generation token for ``session_key``. Every top-level gateway turn gets a monotonically increasing token. If a later command like /stop or /new invalidates that token while the old worker is still unwinding, the late result can be recognized and dropped instead of bleeding into the fresh session. """ if not session_key: return 0 persistent = self._session_state(session_key).persistent # Monotonic by design (#28686): incremented here, NEVER reset. persistent.run_generation = int(persistent.run_generation) + 1 return persistent.run_generation def _invalidate_session_run_generation(self, session_key: str, *, reason: str = "") -> int: """Invalidate any in-flight run token for ``session_key``.""" generation = self._begin_session_run_generation(session_key) if reason: logger.info( "Invalidated run generation for %s → %d (%s)", session_key, generation, reason, ) return generation def _is_session_run_current(self, session_key: str, generation: int) -> bool: """Return True when ``generation`` is still current for ``session_key``.""" if not session_key: return True state = self._peek_session_state(session_key) current = state.persistent.run_generation if state is not None else 0 return int(current) == int(generation) def _bind_adapter_run_generation( self, adapter: Any, session_key: str, generation: int | None, ) -> None: """Bind a gateway run generation to the adapter's active-session event.""" if not adapter or not session_key or generation is None: return try: interrupt_event = getattr(adapter, "_active_sessions", {}).get(session_key) if interrupt_event is not None: setattr(interrupt_event, "_hermes_run_generation", int(generation)) except Exception: pass async def _interrupt_and_clear_session( self, session_key: str, source: SessionSource, *, interrupt_reason: str, invalidation_reason: str, release_running_state: bool = True, ) -> None: """Interrupt the current run and clear queued session state consistently.""" if not session_key: return _iac_state = self._peek_session_state(session_key) running_agent = _iac_state.turn.agent if _iac_state else None _process_task_id = "" _process_baseline = None if running_agent and running_agent is not _AGENT_PENDING_SENTINEL: request_hard_interrupt(running_agent, interrupt_reason) _process_task_id = getattr( running_agent, "_gateway_turn_process_task_id", "" ) _process_baseline = getattr( running_agent, "_gateway_turn_process_baseline", None ) # Bump the generation *before* scheduling the reap thread and capture # the post-bump value: task_id is session-scoped (task_id == # session_id), so if a replacement turn claims this session and # spawns its own process before the reap thread actually runs, that # claim bumps the generation again. The closure below then sees a # stale generation and skips — the replacement turn's own baseline # covers its own cleanup, so nothing is left permanently unreaped. _generation_at_interrupt = self._invalidate_session_run_generation( session_key, reason=invalidation_reason ) if _process_task_id and _process_baseline is not None: threading.Thread( target=_reap_gateway_turn_processes, args=(_process_task_id, _process_baseline), kwargs={ "source": "gateway_turn_interrupt", "is_still_current": lambda: self._is_session_run_current( session_key, _generation_at_interrupt ), }, name=f"gateway-turn-reaper-{_process_task_id[:12]}", daemon=True, ).start() adapter = self._adapter_for_source(source) interrupt_session_activity = getattr( type(adapter), "interrupt_session_activity", None ) if adapter and callable(interrupt_session_activity): metadata = self._thread_metadata_for_source(source) try: params = inspect.signature(interrupt_session_activity).parameters accepts_metadata = "metadata" in params or any( param.kind is inspect.Parameter.VAR_KEYWORD for param in params.values() ) except (TypeError, ValueError): accepts_metadata = False if accepts_metadata: await adapter.interrupt_session_activity( session_key, source.chat_id, metadata=metadata ) else: await adapter.interrupt_session_activity(session_key, source.chat_id) if adapter and hasattr(adapter, "get_pending_message"): adapter.get_pending_message(session_key) # consume and discard if _iac_state is not None: _iac_state.persistent.pending_command_text = None if release_running_state: self._release_running_agent_state(session_key) # Evict the cached agent: ``_interrupt_requested`` is only # cleared by the turn finalizer, so on a hung or still-draining # run the flag survives the lock release and kills the session's # NEXT message at the top of the tool loop (interrupted=True, # api_calls=0, empty response — silently swallowed, #44212). # Evicting mirrors the /new and /model paths: the next message # rebuilds the agent from session history, while the old agent # object keeps its interrupt flag so a hung drain still dies # when it unblocks. self._evict_cached_agent(session_key) async def _refresh_agent_cache_message_count( self, session_key: str, session_id: Optional[str] ) -> None: """Re-baseline a cached agent's stored message_count after THIS turn. The cross-process coherence guard (#45966) compares the session's on-disk ``message_count`` against the count snapshotted next to the cached agent, and rebuilds the agent on a mismatch. But the snapshot is taken at agent-BUILD time — before this turn writes its own user + assistant (+ tool) rows — and the cache entry is never rewritten on a reuse. So without this re-baseline, THIS process's own turn would grow ``message_count`` and the very next turn would see a mismatch and rebuild the agent — every turn, for every conversation — silently destroying the per-conversation prompt caching the cache exists to protect. Call this once a turn has completed and the agent has flushed its rows to the SessionDB. It snapshots the now-current count (which includes this process's own writes) so the guard only fires when a DIFFERENT process changes the transcript out from under us. The ``_sig`` is left untouched; only the count element is refreshed, and only when the same agent is still cached (no rebuild/eviction raced in between). Fail-safe: any DB error leaves the snapshot as-is, which at worst costs one unnecessary rebuild on the next turn. When the cache entry records a ``session_id`` (4-tuple form, #54947) that differs from the current ``session_id`` — meaning the cache was built for a DIFFERENT conversation under the same ``session_key`` — the snapshot is intentionally left untouched. Overwriting it with the current session's count would corrupt the original conversation's baseline and cause the next switch back to fire the cross-process guard spuriously. Fail-safe: the legacy 3-tuple shape (no ``session_id``) is still re-baselined as before. """ if self._session_db is None or not session_id: return _cache_lock = getattr(self, "_agent_cache_lock", None) _cache = getattr(self, "_agent_cache", None) if not _cache_lock or _cache is None: return try: _sess_row = await self._session_db.get_session(session_id) _live = _sess_row.get("message_count", 0) if _sess_row else None except Exception: return if _live is None: return with _cache_lock: cached = _cache.get(session_key) # Only re-baseline a live 3-tuple entry; skip pending sentinels, # legacy 2-tuples (they intentionally opt out of the guard), and # the case where the entry was evicted/rebuilt mid-turn. if ( isinstance(cached, tuple) and len(cached) > 2 and cached[0] is not _AGENT_PENDING_SENTINEL ): # If the snapshot was taken for a different session_id # (same session_key, different conversation), leave the # snapshot alone — the current session_id's count belongs # to a different DB row (#54947). _snapshot_sid = cached[3] if len(cached) > 3 else None if _snapshot_sid is not None and _snapshot_sid != session_id: return if cached[2] != _live: if _snapshot_sid is None: # Legacy 3-tuple: preserve the original 3-element # shape so existing entries stay compatible with # callers that index ``cached[2]`` directly. _cache[session_key] = (cached[0], cached[1], _live) else: _cache[session_key] = ( cached[0], cached[1], _live, _snapshot_sid, ) def _set_pending_turn_sidecar_notes(self, session_key: str, notes: List[str]) -> None: """Stage per-turn must-deliver notes for the next agent run (one-shot).""" if not session_key or not notes: return self._session_state(session_key).conversation.sidecar_notes = list(notes) def _consume_pending_turn_sidecar_notes(self, session_key: str) -> List[str]: if not session_key: return [] state = self._peek_session_state(session_key) if state is None: return [] staged = state.conversation.sidecar_notes state.conversation.sidecar_notes = [] return list(staged) if isinstance(staged, list) else [] def _voice_channel_sidecar_note(self, event, source: SessionSource, session_key: str) -> Optional[str]: """Return a ``[Voice channel now: ...]`` note when VC state changed. Compares the live Discord voice-channel context against the last value delivered for this session and returns a note only on change (including leaving the channel). Unchanged state returns ``None`` so the per-turn member/speaking serialization cannot churn the prompt. """ if source.platform != Platform.DISCORD: return None adapter = self.adapters.get(Platform.DISCORD) guild_id = self._get_guild_id(event) if not (guild_id and adapter and hasattr(adapter, "get_voice_channel_context")): return None try: vc_now = adapter.get_voice_channel_context(guild_id) or "" except Exception: logger.debug("voice-channel context read failed", exc_info=True) return None vc_prev = None if session_key: _vc_state = self._session_state(session_key) vc_prev = _vc_state.conversation.vc_last _vc_state.conversation.vc_last = vc_now if vc_now == (vc_prev if vc_prev is not None else ""): return None if not vc_now: return "[Voice channel now: not connected to a voice channel]" return f"[Voice channel now: {vc_now}]" def _pinned_session_context_prompt( self, context, redact_pii: bool, session_key: Optional[str] ) -> str: """Return the session-context prompt, pinned per session. Key hit → the pinned bytes are reused VERBATIM (immunizes the composed system prompt against renderer nondeterminism); key miss → re-render ``build_session_context_prompt`` and re-pin (a legitimate cache bust: rename, topic edit, /sethome, redact_pii flip, ...). """ _eph_key = self._ephemeral_change_key(context, redact_pii) _eph_pin = None if session_key: _pin_state = self._peek_session_state(session_key) _eph_pin = _pin_state.conversation.ephemeral_pin if _pin_state else None if _eph_pin is not None and _eph_pin[0] == _eph_key: return _eph_pin[1] text = build_session_context_prompt(context, redact_pii=redact_pii) if session_key: self._session_state(session_key).conversation.ephemeral_pin = ( _eph_key, text, ) return text @staticmethod def _ephemeral_change_key(context, redact_pii: bool) -> str: """Hash the exact inputs ``build_session_context_prompt`` renders. This key decides when the pinned per-session context-prompt bytes are reused verbatim vs re-rendered. The maintained invariant (guarded by the parity test in tests/gateway/test_prompt_tail_freeze.py): any input whose change alters the rendered bytes MUST appear here — omission means a stale pinned prompt (cosmetic staleness); inclusion of an extra field only costs a spurious re-render. """ import hashlib src = context.source platform = src.platform.value if src.platform else "" discord_ids: tuple = () discord_tools = "" if src.platform == Platform.DISCORD: from gateway.session import _discord_tools_loaded discord_tools = "1" if _discord_tools_loaded() else "0" discord_ids = ( str(src.guild_id or ""), str(src.parent_chat_id or ""), str(src.thread_id or ""), str(src.chat_id or ""), # Only PRESENCE is rendered (the id itself is delivered # per-turn in the user message) — keying on the value would # re-render every message for zero byte change. "1" if src.message_id else "0", ) # Slack renders a capability-aware platform note gated on # _slack_tools_loaded() — the gate state must appear in the key # (same parity contract as the Discord gate above) so a config / # MCP-registration flip re-renders once instead of serving a # stale pinned note for the rest of the session. slack_tools = "" if src.platform == Platform.SLACK: from gateway.session import _slack_tools_loaded slack_tools = "1" if _slack_tools_loaded() else "0" try: from hermes_constants import display_hermes_home home_display = str(display_hermes_home()) except Exception: home_display = "" key_tuple = ( platform, str(src.chat_id or ""), str(src.thread_id or ""), str(src.chat_type or ""), str(src.chat_name or ""), str(src.chat_topic or ""), str(src.user_name or ""), str(src.user_id or ""), str(getattr(src, "profile", None) or ""), bool(context.shared_multi_user_session), discord_ids, discord_tools, slack_tools, tuple(p.value for p in context.connected_platforms), tuple( ( p.value, str(getattr(hc, "name", "") or ""), str(getattr(hc, "chat_id", "") or ""), ) for p, hc in context.home_channels.items() ), bool(redact_pii), home_display, ) return hashlib.sha256(repr(key_tuple).encode("utf-8")).hexdigest() def _evict_cached_agent(self, session_key: str) -> None: """Remove a cached agent for a session (called on /new, /model, etc). Pops the entry AND soft-releases the evicted agent's LLM client pool so the httpx connection (sockets + held buffers) is freed promptly rather than waiting on CPython GC — AIAgent holds reference cycles (callbacks, tool state) that delay refcount collection, so a manual release is required to keep gateway RSS flat across many /new, /model, undo and reset operations (#29298, same leak class as #25315). The release is soft (``release_clients()``): it frees the client pool and per-turn child subagents but PRESERVES the session's terminal sandbox, browser daemon, and tracked bg processes (keyed on task_id), because the session may resume with a freshly-built agent. Call sites that want a hard teardown (true conversation boundaries like /new) already call ``_cleanup_agent_resources`` before evicting; ``release_clients`` is idempotent and safe to run again after that (the client is already None). Cleanup runs on a daemon thread so we never block holding ``_agent_cache_lock`` on slow socket teardown — mirrors the cap-enforcer and idle-sweeper paths. """ # Prompt-stability state rides the agent-cache lifecycle: a fresh # agent must re-render its session-context bytes (the pin) and re-see # the current voice-channel state once. _evict_state = self._peek_session_state(session_key) if _evict_state is not None: _evict_state.conversation.ephemeral_pin = None _evict_state.conversation.vc_last = None _lock = getattr(self, "_agent_cache_lock", None) evicted = None if _lock: with _lock: evicted = self._agent_cache.pop(session_key, None) else: _cache = getattr(self, "_agent_cache", None) if _cache is not None: evicted = _cache.pop(session_key, None) agent = evicted[0] if isinstance(evicted, tuple) and evicted else evicted if agent is None or agent is _AGENT_PENDING_SENTINEL: return # Don't tear down an agent that's actively mid-turn — its client, # sandbox and child subagents are in use by the running request. running_ids = { id(a) for _, a in self._running_agent_items() if a is not None and a is not _AGENT_PENDING_SENTINEL } if id(agent) in running_ids: return try: threading.Thread( target=self._release_evicted_agent_soft, args=(agent,), daemon=True, name=f"agent-evict-{str(session_key)[:24]}", ).start() except Exception: # If we can't spawn a thread (interpreter shutdown), release # inline as a best-effort fallback. try: self._release_evicted_agent_soft(agent) except Exception: pass @staticmethod def _init_cached_agent_for_turn(agent: Any, interrupt_depth: int) -> None: """Reset per-turn state on a cached agent before a new turn starts. ``_last_activity_ts``, ``_last_activity_desc``, and ``_last_activity_provenance`` are only reset for fresh external turns (depth 0); they are a semantic triple - description and provenance describe the activity *at* ts, so updating one without the others would make get_activity_summary() misleading. For interrupt-recursive turns all three are preserved so the inactivity watchdog can accumulate stuck-turn idle time and fire the 30-min timeout (#15654). The depth-0 reset is still needed: a session idle for 29 min would otherwise trip the watchdog before the new turn makes its first API call (#9051). """ if interrupt_depth == 0: from agent.session_activity import ActivityProvenance agent._last_activity_ts = time.time() agent._last_activity_desc = "starting new turn (cached)" agent._last_activity_provenance = ActivityProvenance.UNKNOWN # Reset the SessionDB flush cursor so the new turn's messages are # fully persisted - a stale value from the previous turn would # cause `_flush_messages_to_session_db` to skip new rows (#44327). if hasattr(agent, "_last_flushed_db_idx"): agent._last_flushed_db_idx = 0 agent._api_call_count = 0 def _commit_memory_before_soft_evict(self, agent: Any, key: str) -> None: """Fire on_session_end extraction before soft-evicting a live agent. Soft eviction (``_release_evicted_agent_soft``) deliberately keeps the session resumable and does NOT fire ``on_session_end`` — that hook is reserved for the true session boundary, tear-down done by ``_session_expiry_watcher`` when the session finally expires. But the watcher tears down whatever agent it finds in ``_agent_cache`` at expiry time. If cache pressure (the LRU cap) soft-evicts a finalizable session's agent BEFORE it expires, the watcher later finds no cached agent and ``on_session_end`` is silently skipped — memory providers never see the transcript (#11205, LRU-cap variant). We hold the live, fully-scoped agent right now, so commit its end-of-session memory extraction here using the agent's own memory manager (correct per-user/chat scoping, no reconstruction). This uses ``commit_memory_session`` — extraction WITHOUT provider teardown — so the eviction stays soft and a resumed turn keeps working. Only fires for sessions the expiry watcher will eventually finalize (finite reset policy). For ``mode == "none"`` sessions the watcher never runs, so there is no missed-boundary to compensate for and we skip the commit (the agent is simply released). Best-effort: any failure is swallowed so eviction still proceeds. """ if agent is None or not hasattr(agent, "commit_memory_session"): return if getattr(agent, "_memory_manager", None) is None: return # no external memory provider — nothing to commit try: _store = getattr(self, "session_store", None) if _store is None: return _store._ensure_loaded() entry = _store._entries.get(key) if entry is None: return # Only compensate when the watcher would otherwise expect to find # this agent at expiry (finite policy, not yet expired). Expired # sessions are torn down by the watcher directly; mode="none" # sessions are never finalized. if not _store.is_session_finalizable(entry): return if _store._is_session_expired(entry): return messages = getattr(agent, "_session_messages", None) agent.commit_memory_session(messages if isinstance(messages, list) else None) logger.debug( "Committed on_session_end extraction before soft-evicting " "finalizable session=%s (cache pressure, pre-expiry)", key, ) except Exception as _e: logger.debug("Pre-evict memory commit failed for %s: %s", key, _e) def _commit_then_release_soft(self, agent: Any, key: str) -> None: """Commit end-of-session memory (if warranted), then soft-release. Runs on the daemon eviction thread so the memory-provider call and the client teardown never block the caller's held cache lock. Order matters: commit uses the live agent's memory manager before ``release_clients`` drops the message buffer. """ self._commit_memory_before_soft_evict(agent, key) self._release_evicted_agent_soft(agent) def _release_evicted_agent_soft(self, agent: Any) -> None: """Soft cleanup for cache-evicted agents — preserves session tool state. Called from _enforce_agent_cache_cap and _sweep_idle_cached_agents. Distinct from _cleanup_agent_resources (full teardown) because a cache-evicted session may resume at any time — its terminal sandbox, browser daemon, and tracked bg processes must outlive the Python AIAgent instance so the next agent built for the same task_id inherits them. """ if agent is None: return try: if hasattr(agent, "release_clients"): agent.release_clients() else: # Older agent instance (shouldn't happen in practice) — # fall back to the legacy full-close path. self._cleanup_agent_resources(agent) except Exception: pass # Free conversation history memory — can be tens of MB with tool # outputs (file reads, terminal output, search results) on heavy # 100+-tool-call sessions. release_clients() deliberately preserves # session tool state for resume, but the message list is rebuilt from # persisted session JSON on the next turn, so dropping it here is safe. if hasattr(agent, "_session_messages"): agent._session_messages = [] # _db_flush_scan_prefix is a shallow copy of the flushed transcript # (run_agent.py, stamped on every successful flush) — it shares every # message dict, so leaving it pins the multi-MB content strings the # eviction exists to free. Pressure-evictable agents have flushed by # definition, so this attribute is always populated on exactly the # agents the memory valve targets. if hasattr(agent, "_db_flush_scan_prefix"): agent._db_flush_scan_prefix = None def _agent_cache_bounds(self): """Operator-configured agent-cache bounds, resolved once per process. Resolved lazily rather than in ``__init__`` so it also works for the ``__new__``-constructed runners used by tests and by the slash-command mixin. """ bounds = getattr(self, "_agent_cache_bounds_cache", None) if bounds is None: from gateway.agent_cache_pressure import resolve_agent_cache_bounds try: bounds = resolve_agent_cache_bounds(_load_gateway_config()) except Exception as _e: logger.debug("Agent cache bounds config read failed: %s", _e) # Resolve from an empty config rather than bare # AgentCacheBounds(): the dataclass default has # memory_high_mb=None (pressure pass OFF), but an *absent* # config section means "auto" — a transient config read # failure must not permanently disable the OOM valve this # feature exists to provide. bounds = resolve_agent_cache_bounds({}) self._agent_cache_bounds_cache = bounds return bounds def _agent_cache_cap(self) -> int: """Effective LRU cap — the configured override, else the default.""" configured = self._agent_cache_bounds().max_size return configured if configured else _AGENT_CACHE_MAX_SIZE def _agent_cache_idle_ttl(self) -> float: """Effective idle TTL in seconds — configured override, else default.""" configured = self._agent_cache_bounds().idle_ttl_secs return configured if configured else _AGENT_CACHE_IDLE_TTL_SECS def _sweep_agent_cache_under_pressure(self) -> int: """Shed cached transcripts once the gateway's own heap nears its budget. The LRU cap counts entries and the idle sweep counts seconds; neither knows that one cached agent pins a full ``_session_messages`` transcript — tens of MB on a session with 100+ tool calls. A gateway serving many chats therefore holds every warm transcript indefinitely: agents that took a turn within the TTL are never idle-swept, and the sweep additionally defers finalizable sessions until they expire. RSS climbs until the cgroup throttles and SIGTERM can no longer flush inside systemd's stop timeout (#80764). This is the missing valve. Above the configured anonymous-RSS budget it evicts LRU agents through the same soft path the cap enforcer uses, so the transcript is dropped and rebuilt from the persisted session on the next turn. Three things are never touched: agents mid-turn (their clients and sandboxes are in use), the most recently used sessions (whose prompt cache is worth the most), and any session whose live transcript has not finished reaching disk. Returns the number of entries evicted (0 when memory is fine). """ from gateway.agent_cache_pressure import ( plan_pressure_evictions, read_anon_rss_mb, transcript_persistence_caught_up, ) bounds = self._agent_cache_bounds() if not bounds.memory_high_mb: return 0 _cache = getattr(self, "_agent_cache", None) _lock = getattr(self, "_agent_cache_lock", None) if not _cache or _lock is None: # Nothing cached — whatever is using the heap, it isn't us, and # warning about it every tick would point at the wrong subsystem. return 0 rss_mb = read_anon_rss_mb() if rss_mb is None or rss_mb < bounds.memory_high_mb: return 0 running_ids = { id(a) for _, a in self._running_agent_items() if a is not None and a is not _AGENT_PENDING_SENTINEL } def _is_evictable(key: str, agent: Any) -> bool: if agent is None or agent is _AGENT_PENDING_SENTINEL: return False if id(agent) in running_ids: return False return transcript_persistence_caught_up(agent) with _lock: ordered = [ (key, entry[0] if isinstance(entry, tuple) and entry else entry) for key, entry in _cache.items() ] plan = plan_pressure_evictions( ordered, is_evictable=_is_evictable, max_evictions=bounds.max_evictions_per_pass, protect_recent=bounds.protect_recent, ) for key, _ in plan: _cache.pop(key, None) if not plan: _mid_turn = sum(1 for _, a in ordered if a is not None and id(a) in running_ids) _unflushed = sum( 1 for _, a in ordered if a is not None and a is not _AGENT_PENDING_SENTINEL and id(a) not in running_ids and not transcript_persistence_caught_up(a) ) logger.warning( "Agent cache pressure: anon RSS %dMB over budget %dMB but no " "evictable session (%d cached, %d mid-turn, %d blocked on " "un-flushed persistence)%s", rss_mb, bounds.memory_high_mb, len(ordered), _mid_turn, _unflushed, ( " — transcripts are not reaching the session DB " "(session persistence disabled or failing?); the memory " "valve cannot shed sessions until they persist." if _unflushed and not _mid_turn else " — memory will keep climbing until those turns finish." ), ) return 0 evicted_count = len(plan) logger.warning( "Agent cache pressure: anon RSS %dMB over budget %dMB — evicting " "%d LRU session(s): %s", rss_mb, bounds.memory_high_mb, evicted_count, ", ".join(key for key, _ in plan), ) try: threading.Thread( target=self._release_pressure_batch, args=(plan,), daemon=True, name="agent-cache-pressure", ).start() except Exception: self._release_pressure_batch(plan) # NOTE: _release_pressure_batch drains `plan` in place (so the trim # runs with no lingering agent references) — len(plan) is 0 by the # time the daemon thread finishes, hence the pre-captured count. return evicted_count def _release_pressure_batch(self, plan: List[tuple]) -> None: """Release a pressure-evicted batch, then return the heap to the OS. Sequential on one daemon thread rather than a thread per agent: the batch is already capped, and the point of the pass is to reclaim memory, not to race N teardowns. The trailing ``malloc_trim`` is what turns "Python dropped the transcript" into "RSS actually fell" — without it glibc keeps the freed arenas and the cgroup never notices. The plan is drained (``pop`` + ``del``) rather than iterated so that no local reference pins the evicted agents when ``gc.collect`` + ``malloc_trim`` run — otherwise the trim frees almost nothing in this pass, the next tick re-reads a still-high RSS, and the valve over-evicts an extra batch of warm prompt caches every cycle. """ while plan: key, agent = plan.pop(0) # FIFO — evict LRU-first order preserved try: self._commit_then_release_soft(agent, key) except Exception as _e: logger.debug("Pressure release failed for %s: %s", key, _e) del agent try: from hermes_cli.mem_trim import trim_memory trim_memory(force=True, reason="agent_cache_pressure") except Exception: pass def _enforce_agent_cache_cap(self) -> None: """Evict oldest cached agents when cache exceeds the LRU cap. Must be called with _agent_cache_lock held. Resource cleanup (memory provider shutdown, tool resource close) is scheduled on a daemon thread so the caller doesn't block on slow teardown while holding the cache lock. Agents currently in _running_agents are SKIPPED — their clients, terminal sandboxes, background processes, and child subagents are all in active use by the running turn. Evicting them would tear down those resources mid-turn and crash the request. If every candidate in the LRU order is active, we simply leave the cache over the cap; it will be re-checked on the next insert. """ _cache = getattr(self, "_agent_cache", None) if _cache is None: return # OrderedDict.popitem(last=False) pops oldest; plain dict lacks the # arg so skip enforcement if a test fixture swapped the cache type. if not hasattr(_cache, "move_to_end"): return # Snapshot of agent instances that are actively mid-turn. Use id() # so the lookup is O(1) and doesn't depend on AIAgent.__eq__ (which # MagicMock overrides in tests). running_ids = { id(a) for _, a in self._running_agent_items() if a is not None and a is not _AGENT_PENDING_SENTINEL } # Walk LRU → MRU and evict excess-LRU entries that aren't mid-turn. # We only consider entries in the first (size - cap) LRU positions # as eviction candidates. If one of those slots is held by an # active agent, we SKIP it without compensating by evicting a # newer entry — that would penalise a freshly-inserted session # (which has no cache history to retain) while protecting an # already-cached long-running one. The cache may therefore stay # temporarily over cap; it will re-check on the next insert, # after active turns have finished. cap = self._agent_cache_cap() excess = max(0, len(_cache) - cap) evict_plan: List[tuple] = [] # [(key, agent), ...] if excess > 0: ordered_keys = list(_cache.keys()) for key in ordered_keys[:excess]: entry = _cache.get(key) agent = entry[0] if isinstance(entry, tuple) and entry else None if agent is not None and id(agent) in running_ids: continue # active mid-turn; don't evict, don't substitute evict_plan.append((key, agent)) for key, _ in evict_plan: _cache.pop(key, None) remaining_over_cap = len(_cache) - cap if remaining_over_cap > 0: logger.warning( "Agent cache over cap (%d > %d); %d excess slot(s) held by " "mid-turn agents — will re-check on next insert.", len(_cache), cap, remaining_over_cap, ) for key, agent in evict_plan: logger.info( "Agent cache at cap; evicting LRU session=%s (cache_size=%d)", key, len(_cache), ) if agent is not None: # Commit end-of-session memory extraction, then soft-release, # both on the daemon thread so the (possibly network-bound) # provider call never blocks the held cache lock. The commit # only fires for finalizable-not-yet-expired sessions whose # agent would otherwise vanish before the expiry watcher can # fire on_session_end (#11205, LRU-cap variant). threading.Thread( target=self._commit_then_release_soft, args=(agent, key), daemon=True, name=f"agent-cache-evict-{key[:24]}", ).start() def _sweep_idle_cached_agents(self) -> int: """Evict cached agents whose AIAgent has been idle past the idle TTL. Safe to call from the session expiry watcher without holding the cache lock — acquires it internally. Returns the number of entries evicted. Resource cleanup is scheduled on daemon threads. Agents currently in _running_agents are SKIPPED for the same reason as _enforce_agent_cache_cap: tearing down an active turn's clients mid-flight would crash the request. """ _cache = getattr(self, "_agent_cache", None) _lock = getattr(self, "_agent_cache_lock", None) if _cache is None or _lock is None: return 0 now = time.time() idle_ttl = self._agent_cache_idle_ttl() to_evict: List[tuple] = [] running_ids = { id(a) for _, a in self._running_agent_items() if a is not None and a is not _AGENT_PENDING_SENTINEL } with _lock: for key, entry in list(_cache.items()): agent = entry[0] if isinstance(entry, tuple) and entry else None if agent is None: continue if id(agent) in running_ids: continue # mid-turn — don't tear it down last_activity = getattr(agent, "_last_activity_ts", None) if last_activity is None: continue if (now - last_activity) > idle_ttl: # Check whether the session has actually expired in the # session store. If it hasn't (e.g. daily-reset mode # where the reset fires hours after the user's last # message), keep the agent in cache so the session-store # expiry watcher can still find it and call # on_session_end() with the live transcript. Skipping # eviction here means the agent stays alive until the # session genuinely expires, at which point the watcher # (gateway/run.py _session_expiry_watcher) tears it down # properly. (#11205 follow-up) # # BUT only defer when the watcher will EVER finalize this # session. For a mode == "none" session the watcher never # fires (is_session_finalizable() is False), so deferring # would pin the agent in cache for the gateway's entire # lifetime — the exact leak this idle sweep exists to # relieve. Those sessions fall through to soft eviction # WITHOUT on_session_end, and that is correct: a mode=="none" # session never reaches a session-end boundary, so there is # no missed on_session_end to compensate for. (The finite # case — a session evicted under LRU-cap pressure before it # expires — is instead covered by _commit_memory_before_soft_ # evict on the cap path, which fires on_session_end via the # live agent's memory manager before releasing it.) session_entry = None _store = getattr(self, "session_store", None) try: if _store is not None: _store._ensure_loaded() session_entry = _store._entries.get(key) except Exception: session_entry = None if ( session_entry is not None and _store is not None and _store.is_session_finalizable(session_entry) and not _store._is_session_expired(session_entry) ): continue # keep agent — finite session hasn't expired to_evict.append((key, agent)) for key, _ in to_evict: _cache.pop(key, None) for key, agent in to_evict: logger.info( "Agent cache idle-TTL evict: session=%s (idle=%.0fs)", key, now - getattr(agent, "_last_activity_ts", now), ) threading.Thread( target=self._release_evicted_agent_soft, args=(agent,), daemon=True, name=f"agent-cache-idle-{key[:24]}", ).start() return len(to_evict) # ------------------------------------------------------------------ # Proxy mode: forward messages to a remote Hermes API server # ------------------------------------------------------------------ def _get_proxy_url(self) -> Optional[str]: """Return the proxy URL if proxy mode is configured, else None. Checks GATEWAY_PROXY_URL env var first (convenient for Docker), then ``gateway.proxy_url`` in config.yaml. """ url = os.getenv("GATEWAY_PROXY_URL", "").strip() if url: return url.rstrip("/") cfg = _load_gateway_config() url = (cfg.get("gateway") or {}).get("proxy_url") url = (url or "").strip() if url: return url.rstrip("/") return None def _build_stream_consumer_config( self, source: "SessionSource", scfg: Any, adapter: Any, *, on_missing_cursor: str, ) -> "tuple[Any, Optional[Callable[[], None]]]": """Build the shared ``StreamConsumerConfig`` and the optional Telegram pause-typing closure used by both agent-run paths. ``on_missing_cursor`` controls how platforms whose adapter sets ``SUPPORTS_MESSAGE_EDITING = False`` are handled — both semantics are preserved verbatim from the pre-refactor call sites: - ``"fallback"`` (proxy path): stream anyway with an empty cursor. - ``"raise"`` (in-process agent path): raise ``RuntimeError`` so the caller's ``except`` skips streaming entirely. Returns ``(consumer_cfg, pause_typing_before_finalize)``. """ from gateway.stream_consumer import StreamConsumerConfig _pause_typing_before_finalize = None if source.platform == Platform.TELEGRAM and hasattr(adapter, "pause_typing_for_chat"): def _pause_typing_before_finalize( _adapter=adapter, _chat_id=source.chat_id, ) -> None: _adapter.pause_typing_for_chat(_chat_id) # Platforms that don't support editing sent messages # (e.g. QQ, WeChat) should skip streaming entirely — # without edit support, the consumer sends a partial # first message that can never be updated, resulting in # duplicate messages (partial + final). # (The proxy path instead opts into a cursorless fallback # via on_missing_cursor="fallback".) _adapter_supports_edit = getattr(adapter, "SUPPORTS_MESSAGE_EDITING", True) # Adapters that can't edit messages but provide a native-streaming # transport (e.g. WeCom's msgtype: "stream" via send_stream_frame) # get past the gate — the consumer's native branch delivers the full # turn through that transport. _adapter_supports_native_stream = bool(getattr( adapter, "SUPPORTS_NATIVE_STREAMING", False, )) if ( not _adapter_supports_edit and not _adapter_supports_native_stream and on_missing_cursor == "raise" ): raise RuntimeError("skip streaming for non-editable platform") _effective_cursor = scfg.cursor if _adapter_supports_edit else "" # Some Matrix clients render the streaming cursor # as a visible tofu/white-box artifact. Keep # streaming text on Matrix, but suppress the cursor. _buffer_only = False if source.platform == Platform.MATRIX: _effective_cursor = "" _buffer_only = True # Fresh-final applies to Telegram only — other # platforms either edit in place cheaply (Discord, # Slack) or don't have the timestamp-on-edit / # edit-timestamp-stays-stale problem. # (Ported from openclaw/openclaw#72038.) _fresh_final_secs = ( float(getattr(scfg, "fresh_final_after_seconds", 0.0) or 0.0) if source.platform == Platform.TELEGRAM else 0.0 ) _consumer_cfg = StreamConsumerConfig( edit_interval=scfg.edit_interval, buffer_threshold=scfg.buffer_threshold, cursor=_effective_cursor, buffer_only=_buffer_only, fresh_final_after_seconds=_fresh_final_secs, transport=scfg.transport or "edit", chat_type=getattr(source, "chat_type", "") or "", ) return _consumer_cfg, _pause_typing_before_finalize async def _run_agent_via_proxy( self, message: str, context_prompt: str, history: List[Dict[str, Any]], source: "SessionSource", session_id: str, session_key: str = None, run_generation: Optional[int] = None, event_message_id: Optional[str] = None, ) -> Dict[str, Any]: """Forward the message to a remote Hermes API server instead of running a local AIAgent. When ``GATEWAY_PROXY_URL`` (or ``gateway.proxy_url`` in config.yaml) is set, the gateway becomes a thin relay: it handles platform I/O (encryption, threading, media) and delegates all agent work to the remote server via ``POST /v1/chat/completions`` with SSE streaming. This lets a Docker container handle Matrix E2EE while the actual agent runs on the host with full access to local files, memory, skills, and a unified session store. """ try: from aiohttp import ClientSession as _AioClientSession, ClientTimeout except ImportError: return { "final_response": "⚠️ Proxy mode requires aiohttp. Install with: pip install aiohttp", "messages": [], "api_calls": 0, "tools": [], } proxy_url = self._get_proxy_url() if not proxy_url: return { "final_response": "⚠️ Proxy URL not configured (GATEWAY_PROXY_URL or gateway.proxy_url)", "messages": [], "api_calls": 0, "tools": [], } # Scope-aware read: the proxy key is a per-profile credential; under # multiplex honor the installed scope's verdict (Slack pattern for # the unscoped default-profile loop). try: from agent.secret_scope import UnscopedSecretError, get_secret try: proxy_key = (get_secret("GATEWAY_PROXY_KEY") or "").strip() except UnscopedSecretError: proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() except Exception: proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() def _run_still_current() -> bool: if run_generation is None or not session_key: return True return self._is_session_run_current(session_key, run_generation) # Build messages in OpenAI chat format -------------------------- # # The remote api_server can maintain session continuity via # X-Hermes-Session-Id, so it loads its own history. We only # need to send the current user message. If the remote has # no history for this session yet, include what we have locally # so the first exchange has context. # # We always include the current message. For history, send a # compact version (text-only user/assistant turns) — the remote # handles tool replay and system prompts. api_messages: List[Dict[str, str]] = [] if context_prompt: api_messages.append({"role": "system", "content": context_prompt}) for msg in history: role = msg.get("role") content = msg.get("content") if role in {"user", "assistant"} and content: api_messages.append({"role": role, "content": content}) api_messages.append({"role": "user", "content": message}) # HTTP headers --------------------------------------------------- headers: Dict[str, str] = {"Content-Type": "application/json"} if proxy_key: headers["Authorization"] = f"Bearer {proxy_key}" if session_id: headers["X-Hermes-Session-Id"] = session_id body = { "model": "hermes-agent", "messages": api_messages, "stream": True, } # Set up platform streaming if available ------------------------- _stream_consumer = None _scfg = getattr(getattr(self, "config", None), "streaming", None) if _scfg is None: from gateway.config import StreamingConfig _scfg = StreamingConfig() platform_key = _platform_config_key(source.platform) user_config = _load_gateway_config() from gateway.display_config import resolve_display_setting _plat_streaming = resolve_display_setting( user_config, platform_key, "streaming" ) _streaming_enabled = ( _scfg.enabled and _scfg.transport != "off" if _plat_streaming is None else bool(_plat_streaming) ) _thread_metadata: Optional[Dict[str, Any]] = self._thread_metadata_for_source(source, event_message_id) if _streaming_enabled: try: from gateway.stream_consumer import GatewayStreamConsumer _adapter = self._adapter_for_source(source) if _adapter: _consumer_cfg, _pause_typing_before_finalize = ( self._build_stream_consumer_config( source, _scfg, _adapter, on_missing_cursor="fallback", ) ) _stream_consumer = GatewayStreamConsumer( adapter=_adapter, chat_id=source.chat_id, config=_consumer_cfg, metadata=_thread_metadata, on_before_finalize=_pause_typing_before_finalize, initial_reply_to_id=event_message_id, run_still_current=_run_still_current, ) except Exception as _sc_err: logger.debug("Proxy: could not set up stream consumer: %s", _sc_err) # Run the stream consumer task in the background stream_task = None if _stream_consumer: stream_task = asyncio.create_task(_stream_consumer.run()) # Send typing indicator _adapter = self._adapter_for_source(source) if _adapter: try: await _adapter.send_typing(source.chat_id, metadata=_thread_metadata) except Exception: pass # Make the HTTP request with SSE streaming ----------------------- full_response = "" _start = time.time() try: _timeout = ClientTimeout(total=0, sock_read=1800) async with _AioClientSession(timeout=_timeout) as session: async with session.post( f"{proxy_url}/v1/chat/completions", json=body, headers=headers, ) as resp: if resp.status != 200: error_text = await resp.text() logger.warning( "Proxy error (%d) from %s: %s", resp.status, proxy_url, error_text[:500], ) return { "final_response": f"⚠️ Proxy error ({resp.status}): {error_text[:300]}", "messages": [], "api_calls": 0, "tools": [], } # Parse SSE stream buffer = "" async for chunk in resp.content.iter_any(): if not _run_still_current(): logger.info( "Discarding stale proxy stream for %s — generation %d is no longer current", session_key or "?", run_generation or 0, ) return { "final_response": "", "messages": [], "api_calls": 0, "tools": [], "history_offset": len(history), "session_id": session_id, "response_previewed": False, } text = chunk.decode("utf-8", errors="replace") buffer += text # Process complete SSE lines while "\n" in buffer: line, buffer = buffer.split("\n", 1) line = line.strip() if not line: continue if line.startswith("data: "): data = line[6:] if data.strip() == "[DONE]": break try: obj = json.loads(data) choices = obj.get("choices", []) if choices: delta = choices[0].get("delta", {}) content = delta.get("content", "") if content: full_response += content if _stream_consumer: _stream_consumer.on_delta(content) except json.JSONDecodeError: pass if len(buffer) > _GATEWAY_PROXY_SSE_BUFFER_MAX_CHARS: raise ValueError( "Proxy SSE stream exceeded max buffer size without a line boundary" ) except asyncio.CancelledError: raise except Exception as e: logger.error("Proxy connection error to %s: %s", proxy_url, e) if not full_response: return { "final_response": f"⚠️ Proxy connection error: {e}", "messages": [], "api_calls": 0, "tools": [], } # Partial response — return what we got finally: # Finalize stream consumer if _stream_consumer: _stream_consumer.finish() if stream_task: try: await asyncio.wait_for(stream_task, timeout=5.0) except (asyncio.TimeoutError, asyncio.CancelledError): stream_task.cancel() _elapsed = time.time() - _start if not _run_still_current(): logger.info( "Discarding stale proxy result for %s — generation %d is no longer current", session_key or "?", run_generation or 0, ) return { "final_response": "", "messages": [], "api_calls": 0, "tools": [], "history_offset": len(history), "session_id": session_id, "response_previewed": False, } logger.info( "proxy response: url=%s session=%s time=%.1fs response=%d chars", proxy_url, (session_id or "")[:20], _elapsed, len(full_response), ) return { "final_response": full_response or "(No response from remote agent)", "messages": [ {"role": "user", "content": message}, {"role": "assistant", "content": full_response}, ], "api_calls": 1, "tools": [], "history_offset": len(history), "session_id": session_id, "response_previewed": _stream_consumer is not None and bool(full_response), } # ------------------------------------------------------------------ async def _run_agent( self, message: str, context_prompt: str, history: List[Dict[str, Any]], source: SessionSource, session_id: str, session_key: str = None, run_generation: Optional[int] = None, _interrupt_depth: int = 0, event_message_id: Optional[str] = None, inbound_message_id: Optional[str] = None, channel_prompt: Optional[str] = None, moa_config: Optional[dict] = None, persist_user_message: Optional[Any] = None, persist_user_timestamp: Optional[float] = None, persist_user_display_kind: Optional[str] = None, message_type: Optional[str] = None, ) -> Dict[str, Any]: """Profile-scoping wrapper around the agent run. When multiplexing is active, resolve the inbound source's profile and run the whole turn inside ``_profile_runtime_scope`` so config/skills/ memory resolve to that profile's home AND credentials resolve from that profile's secret scope (never the process-global ``os.environ``). When multiplexing is off this is a transparent pass-through — zero behavior change for single-profile gateways. """ if not getattr(getattr(self, "config", None), "multiplex_profiles", False): return await self._run_agent_inner( message, context_prompt, history, source, session_id, session_key=session_key, run_generation=run_generation, _interrupt_depth=_interrupt_depth, event_message_id=event_message_id, inbound_message_id=inbound_message_id, channel_prompt=channel_prompt, moa_config=moa_config, persist_user_message=persist_user_message, persist_user_timestamp=persist_user_timestamp, persist_user_display_kind=persist_user_display_kind, message_type=message_type, ) profile_home = self._resolve_profile_home_for_source(source) with _profile_runtime_scope(profile_home): return await self._run_agent_inner( message, context_prompt, history, source, session_id, session_key=session_key, run_generation=run_generation, _interrupt_depth=_interrupt_depth, event_message_id=event_message_id, inbound_message_id=inbound_message_id, channel_prompt=channel_prompt, moa_config=moa_config, persist_user_message=persist_user_message, persist_user_timestamp=persist_user_timestamp, persist_user_display_kind=persist_user_display_kind, message_type=message_type, ) def _profile_name_for_source(self, source: SessionSource) -> Optional[str]: """Resolve the profile name for an inbound source via configured routes. Returns ``None`` when multiplexing is off, no routes are configured, or no route matches. Callers (``build_source``, ``_resolve_profile_home_for_source``) treat ``None`` as "use the default/active profile". When ``gateway.profile_routes`` is configured, the most specific matching route wins (guild < channel < thread). See :mod:`gateway.profile_routing` for matching rules. Gated on ``gateway.multiplex_profiles``: routing stamps ``source.profile``, which selects the session-key namespace and batch keys — but the profile-scoped agent run only activates under multiplexing. Without this gate, a configured route with multiplexing off would namespace batch/session keys by profile while the agent still runs in ``agent:main``, splitting the two out of agreement. """ config = getattr(self, "config", None) if not getattr(config, "multiplex_profiles", False): return None routes = getattr(config, "profile_routes", None) if not routes: return None from gateway.profile_routing import ProfileRouteRejected, match_profile_route try: matched = match_profile_route( routes, platform=source.platform.value, guild_id=getattr(source, "guild_id", None), chat_id=source.chat_id, thread_id=getattr(source, "thread_id", None), parent_chat_id=getattr(source, "parent_chat_id", None), ) except Exception: logger.warning( "Profile route matching failed for %s/%s, falling back to default", source.platform, source.chat_id, exc_info=True, ) return None if matched: try: served = {name for name, _home in _multiplex_profile_homes(config)} except Exception as exc: logger.warning( "Rejecting profile route %r because the served-profile set " "could not be resolved", matched.name, exc_info=True, ) raise ProfileRouteRejected(matched.name) from exc if matched.profile not in served: logger.warning( "Rejecting profile route %r: target profile %r is not served", matched.name, matched.profile, ) raise ProfileRouteRejected(matched.name) return matched.profile logger.debug( "No profile route matched: platform=%s chat_id=%s thread_id=%s parent_chat_id=%s", source.platform.value, source.chat_id, getattr(source, "thread_id", None), getattr(source, "parent_chat_id", None), ) return None def _resolve_profile_home_for_source(self, source: SessionSource) -> "Path": """Resolve which profile's HERMES_HOME should serve this inbound source. Resolution order: 1. ``source.profile`` — set by /p// URL prefix, per-credential adapter ownership, OR profile_routes matching at ``build_source`` time. 2. ``_profile_name_for_source`` — re-run routing here as a defensive fallback for sources that bypass ``build_source``. 3. The active profile (the multiplexer's own home). """ from gateway.profile_routing import ProfileRouteRejected from hermes_cli.profiles import ( get_active_profile_name, get_profile_dir, profile_exists, ) from hermes_constants import get_hermes_home # Track whether a profile was explicitly requested (vs. falling back to default) explicit_profile = None try: name = (source.profile or "").strip() if name: explicit_profile = name # User explicitly set this profile if not name: name = self._profile_name_for_source(source) if name: explicit_profile = name # Routing explicitly set this profile if not name: name = get_active_profile_name() or "default" profile_dir = get_profile_dir(name) # Warn if an explicit profile doesn't exist on disk if explicit_profile and not profile_exists(name): logger.warning( "Profile %r does not exist for source %s/%s (guild_id=%s), " "falling back to global HERMES_HOME", explicit_profile, source.platform.value, source.chat_id, getattr(source, "guild_id", None), ) return get_hermes_home() return profile_dir except ProfileRouteRejected: raise except Exception: # Catch normalization errors, path errors, etc. logger.warning( "Failed to resolve profile directory for source %s/%s (guild_id=%s), " "falling back to global HERMES_HOME: %s", source.platform.value, source.chat_id, getattr(source, "guild_id", None), explicit_profile or "(no profile)", exc_info=True, ) return get_hermes_home() async def _run_agent_inner( self, message: str, context_prompt: str, history: List[Dict[str, Any]], source: SessionSource, session_id: str, session_key: str = None, run_generation: Optional[int] = None, _interrupt_depth: int = 0, event_message_id: Optional[str] = None, inbound_message_id: Optional[str] = None, channel_prompt: Optional[str] = None, moa_config: Optional[dict] = None, persist_user_message: Optional[Any] = None, persist_user_timestamp: Optional[float] = None, persist_user_display_kind: Optional[str] = None, message_type: Optional[str] = None, ) -> Dict[str, Any]: """ Run the agent with the given message and context. Returns the full result dict from run_conversation, including: - "final_response": str (the text to send back) - "messages": list (full conversation including tool calls) - "api_calls": int - "completed": bool This is run in a thread pool to not block the event loop. Supports interruption via new messages. """ # ---- Proxy mode: delegate to remote API server ---- if self._get_proxy_url(): return await self._run_agent_via_proxy( message=message, context_prompt=context_prompt, history=history, source=source, session_id=session_id, session_key=session_key, run_generation=run_generation, event_message_id=event_message_id, ) from run_agent import AIAgent import queue def _run_still_current() -> bool: if run_generation is None or not session_key: return True return self._is_session_run_current(session_key, run_generation) user_config = _load_gateway_config() platform_key = _platform_config_key(source.platform) enabled_toolsets = self._resolve_enabled_toolsets_for_source( user_config, source, platform_key ) agent_cfg_local = user_config.get("agent") or {} from agent.skill_utils import parse_config_string_list disabled_toolsets = parse_config_string_list(agent_cfg_local.get("disabled_toolsets")) or None display_config = user_config.get("display", {}) if not isinstance(display_config, dict): display_config = {} # Per-platform display settings — resolve via display_config module # which checks display.platforms.. first, then # display. global, then built-in platform defaults. from gateway.display_config import resolve_display_setting # Apply tool preview length config (0 = no limit) try: from agent.display import set_tool_preview_max_len _tpl = resolve_display_setting(user_config, platform_key, "tool_preview_length", 0) set_tool_preview_max_len(int(_tpl) if _tpl else 0) except Exception: pass # Apply friendly tool labels config (default on) — per-platform aware try: from agent.display import set_friendly_tool_labels _ftl = resolve_display_setting(user_config, platform_key, "friendly_tool_labels", True) set_friendly_tool_labels(bool(_ftl)) except Exception: pass # Tool progress mode — resolved per-platform with env var fallback _resolved_tp = resolve_display_setting(user_config, platform_key, "tool_progress") _env_tp = os.getenv("HERMES_TOOL_PROGRESS_MODE") _display_cfg = display_config if isinstance(display_config, dict) else {} _platforms_cfg = _display_cfg.get("platforms") or {} _platform_cfg = _platforms_cfg.get(platform_key) or {} _legacy_tp_overrides = _display_cfg.get("tool_progress_overrides") or {} _tool_progress_configured = ( "tool_progress" in _display_cfg or ( isinstance(_platform_cfg, dict) and "tool_progress" in _platform_cfg ) or ( isinstance(_legacy_tp_overrides, dict) and platform_key in _legacy_tp_overrides ) ) progress_mode = ( _env_tp if _env_tp and not _tool_progress_configured else (_resolved_tp or _env_tp or "all") ) # Tool progress grouping: "accumulate" (edit one bubble) or "separate" (one msg per tool) progress_grouping = resolve_display_setting(user_config, platform_key, "tool_progress_grouping") or "accumulate" from gateway.status_phrases import choose_status_phrase, resolve_status_phrase_catalog _generic_status_recent: List[str] = [] _generic_status_catalog = resolve_status_phrase_catalog(user_config, platform_key) def _display_surface_mode( setting: str, *, default: bool = False, require_platform_override_for: set[Any] | None = None, allow_generic: bool = False, ) -> str: """Return off|raw|generic for a gateway visibility surface.""" if require_platform_override_for: current_platform = _gateway_platform_value(source.platform) platform_only = { _gateway_platform_value(item) for item in require_platform_override_for } if ( current_platform in platform_only and not _has_platform_display_override(user_config, platform_key, setting) ): return "off" value = resolve_display_setting(user_config, platform_key, setting, default) if isinstance(value, str) and value.strip().lower() == "generic": return "generic" if allow_generic else "off" return "raw" if bool(value) else "off" def _generic_status_phrase(kind: str, *, tool_name: str | None = None, preview: str | None = None, args: Any = None) -> str: try: return choose_status_phrase( kind, tool_name=tool_name, preview=preview, args=args, recent=_generic_status_recent, catalog=_generic_status_catalog, ) except Exception as _phrase_err: logger.debug("generic status phrase selection failed: %s", _phrase_err) return "still on it" if kind in {"heartbeat", "waiting", "long_running", "status"} else "one sec" # Disable tool progress for webhooks - they don't support message editing, # so each progress line would be sent as a separate message. from gateway.config import Platform tool_progress_enabled = progress_mode not in {"off", "log"} and source.platform != Platform.WEBHOOK # Live working-state status for text-rendering typing indicators # (Slack's assistant status line). Independent of tool_progress — # Slack defaults tool_progress off (permanent lines spam channels) # but the status line is ephemeral, so live status stays useful # there. Rendering rides the existing _keep_typing refresh: the # callback only stores a phrase on the adapter, costing zero extra # platform API calls. _live_status_mode = resolve_display_setting( user_config, platform_key, "live_status", "full" ) _live_status_adapter = self._adapter_for_source(source) if not getattr(_live_status_adapter, "supports_status_text", False): _live_status_adapter = None if _live_status_mode == "off": _live_status_adapter = None # "log" mode: tool calls are written to ~/.hermes/logs/tool_calls.log # instead of the chat (#3459 / #3458). Gateway-only by design. log_mode_enabled = progress_mode == "log" and source.platform != Platform.WEBHOOK log_queue: "queue.Queue | None" = queue.Queue() if log_mode_enabled else None # Natural assistant status messages are intentionally independent from # tool progress and token streaming. Users can keep tool_progress quiet # in chat platforms while opting into concise mid-turn updates. interim_assistant_messages_mode = _display_surface_mode( "interim_assistant_messages", default=True, require_platform_override_for={Platform.MATTERMOST}, ) interim_assistant_messages_enabled = ( source.platform != Platform.WEBHOOK and interim_assistant_messages_mode != "off" ) # thinking_progress is independent — if enabled, we need the progress # queue even when tool_progress is off (thinking relay uses same infra). # Mattermost requires a per-platform opt-in: global scratch-text display # is too easy to leak into busy public threads. _thinking_mode = _display_surface_mode( "thinking_progress", default=False, require_platform_override_for={Platform.MATTERMOST}, ) _thinking_enabled = _thinking_mode != "off" # Slack-native task cards (#29483): when the Slack adapter's opt-in # is set, tool progress renders as native plan/task cards via # chat.startStream — the progress queue is needed even though Slack # keeps ordinary text tool_progress off by default (requiring both # flags would silently leave the native feature inactive). _progress_adapter_for_native = self._adapter_for_source(source) _native_slack_task_cards = False if ( source.platform == Platform.SLACK and _progress_adapter_for_native is not None and hasattr(_progress_adapter_for_native, "native_task_cards_enabled") ): try: _native_slack_task_cards = bool( _progress_adapter_for_native.native_task_cards_enabled() ) except Exception: logger.debug("Slack native task-card config check failed", exc_info=True) needs_progress_queue = ( tool_progress_enabled or _thinking_enabled or _native_slack_task_cards ) # Queue for progress messages (thread-safe) progress_queue = queue.Queue() if needs_progress_queue else None last_tool = [None] # Mutable container for tracking in closure last_progress_msg = [None] # Track last message for dedup repeat_count = [0] # How many times the same message repeated # True when the previously enqueued progress line was a terminal # fenced code block — consecutive terminal calls then drop the # repeated "💻 terminal" header and render back-to-back blocks. last_was_terminal_block = [False] # ── Discord voice "verbal ack before tool calls" ──────────────── # When the bot is in a voice channel with the continuous mixer # installed (discord.voice_fx.enabled), speak a short phrase ("let me # look into that") over the ambient idle bed on the FIRST tool call of # the turn. Fires from tool_start_callback (independent of the # tool-progress text gate), at most once per turn. No-op on every # other platform / when not in a voice channel. _voice_ack_fired = [False] _voice_ack_guild: List[Optional[int]] = [None] if source.platform == Platform.DISCORD: _va = self.adapters.get(Platform.DISCORD) # source.chat_id is the linked text channel; resolve the guild whose # voice connection is bound to it (mirrors DiscordAdapter.play_tts). _vtc = getattr(_va, "_voice_text_channels", None) if isinstance(_vtc, dict) and hasattr(_va, "voice_mixer_active"): for _gid, _tc in _vtc.items(): if str(_tc) == str(source.chat_id) and _va.voice_mixer_active(_gid): _voice_ack_guild[0] = _gid break _voice_ack_loop = asyncio.get_running_loop() # voice_ack_callback extracted to TurnRunner.voice_ack_callback # (published onto turn_ctx after the runner is constructed below). # Auto-cleanup of temporary progress bubbles (Telegram + any adapter # that implements ``delete_message``). When enabled via # ``display.platforms..cleanup_progress: true``, message IDs # from the tool-progress / "⏳ Working — N min" / status-callback bubbles # are collected here and deleted after the final response lands. # Failed runs skip cleanup so the bubbles remain as breadcrumbs. _cleanup_progress = bool( resolve_display_setting(user_config, platform_key, "cleanup_progress") ) _cleanup_adapter = self._adapter_for_source(source) if _cleanup_progress else None # getattr, not attribute access — same duck-typed-adapter guard as the # edit_message check in send_progress_messages below: a fake/minimal # adapter without delete_message means "can't delete", not a crash. _cleanup_delete = getattr(type(_cleanup_adapter), "delete_message", None) if _cleanup_adapter is not None else None if _cleanup_adapter is not None and ( _cleanup_delete is None or _cleanup_delete is BasePlatformAdapter.delete_message ): # Adapter doesn't support deletion — silently disable. _cleanup_progress = False _cleanup_adapter = None _cleanup_msg_ids: List[str] = [] # First-touch onboarding latch: fires at most once per run, even if # several tools exceed the threshold. long_tool_hint_fired = [False] _LONG_TOOL_THRESHOLD_S = 30.0 turn_ctx = TurnContext( source=source, _run_still_current=_run_still_current, _live_status_adapter=_live_status_adapter, _live_status_mode=_live_status_mode, _thinking_enabled=_thinking_enabled, progress_mode=progress_mode, progress_grouping=progress_grouping, tool_progress_enabled=tool_progress_enabled, progress_queue=progress_queue, log_queue=log_queue, last_progress_msg=last_progress_msg, last_tool=last_tool, last_was_terminal_block=last_was_terminal_block, repeat_count=repeat_count, long_tool_hint_fired=long_tool_hint_fired, _LONG_TOOL_THRESHOLD_S=_LONG_TOOL_THRESHOLD_S, _cleanup_progress=_cleanup_progress, _cleanup_msg_ids=_cleanup_msg_ids, message=message, AIAgent=AIAgent, resolve_display_setting=resolve_display_setting, user_config=user_config, enabled_toolsets=enabled_toolsets, disabled_toolsets=disabled_toolsets, log_mode_enabled=log_mode_enabled, interim_assistant_messages_enabled=interim_assistant_messages_enabled, needs_progress_queue=needs_progress_queue, _native_slack_task_cards=_native_slack_task_cards, _voice_ack_fired=_voice_ack_fired, _voice_ack_guild=_voice_ack_guild, _voice_ack_loop=_voice_ack_loop, history=history, context_prompt=context_prompt, channel_prompt=channel_prompt, session_id=session_id, session_key=session_key, run_generation=run_generation, _interrupt_depth=_interrupt_depth, event_message_id=event_message_id, inbound_message_id=inbound_message_id, moa_config=moa_config, persist_user_message=persist_user_message, persist_user_timestamp=persist_user_timestamp, persist_user_display_kind=persist_user_display_kind, ) turn_runner = TurnRunner(self, turn_ctx) # Callback invoked by agent on tool lifecycle events — extracted to # TurnRunner.progress_callback (bound method, same signature). turn_ctx.progress_callback = turn_runner.progress_callback turn_ctx.voice_ack_callback = turn_runner.voice_ack_callback turn_ctx.native_tool_start_callback = turn_runner.combined_tool_start_callback turn_ctx.native_tool_complete_callback = ( turn_runner.native_tool_complete_callback ) # Background task to send progress messages # Accumulates tool lines into a single message that gets edited. # # Threading metadata is platform-specific: # - Slack DM threading needs event_message_id fallback (reply thread) # - Telegram forum topics use message_thread_id; Hermes-created private # DM topic lanes require both thread metadata and a reply anchor # - Feishu only honors reply_in_thread when sending a reply, so topic # progress uses the triggering event message as the reply target # - Other platforms should use explicit source.thread_id only # # Slack honours platforms.slack.extra.reply_in_thread=false: if the # user has opted out of threaded replies, don't synthesise a thread # for progress messages either — the very first progress message # would otherwise create a thread that all subsequent replies # (including the final answer) would inherit (#18859). _progress_reply_in_thread = True if source.platform == Platform.SLACK: _slack_adapter_for_progress = self._adapter_for_source(source) if _slack_adapter_for_progress is not None: try: # Relay lane: the adapter owns mode resolution (nested # platforms.relay.extra.slack subset with flat-key # fallback). Native lane: read the flat extra as before. _mode_fn = getattr( _slack_adapter_for_progress, "_effective_reply_in_thread", None, ) if callable(_mode_fn): _progress_reply_in_thread = bool(_mode_fn()) else: _progress_reply_in_thread = bool( _slack_adapter_for_progress.config.extra.get( "reply_in_thread", True ) ) except Exception: _progress_reply_in_thread = True elif str(getattr(source.platform, "value", source.platform) or "").lower() == "buzz": # Buzz honours the same opt-out (reply_to_mode: off / # extra.reply_in_thread: false). When the user asked for flat # channel replies, progress must not synthesise a thread either. _buzz_adapter_for_progress = self._adapter_for_source(source) if _buzz_adapter_for_progress is not None: try: _progress_reply_in_thread = ( getattr(_buzz_adapter_for_progress, "_reply_to_mode", "first") != "off" ) except Exception: _progress_reply_in_thread = True _progress_thread_id = _resolve_progress_thread_id( source.platform, source.thread_id, event_message_id, reply_in_thread=_progress_reply_in_thread, ) # Relay Discord auto-thread lane: a channel-initiating message has no # thread_id at ingest (the thread is born on the connector's FIRST # send). The connector stamps prospective_thread_id (the anchor message # id, == the id of the thread it will create) and auto-threads any # outbound carrying that anchor as reply_to. Without it, the progress / # tool-status bubble is sent flat (no thread, no anchor) and lands in # the PARENT channel while the final reply threads — the search-status # updates leaked outside the thread (staging repro 2026-08-02). Carry # the anchor on the progress send so it routes into the SAME auto-thread. _relay_prospective_thread_id = ( str(getattr(source, "prospective_thread_id", None)) if source.platform == Platform.DISCORD and getattr(source, "delivered_via_upstream_relay", False) and getattr(source, "prospective_thread_id", None) and not source.thread_id else None ) _progress_metadata = ( self._thread_metadata_for_source(source, event_message_id) if _progress_thread_id == source.thread_id else self._thread_metadata_for_target( source.platform, source.chat_id, _progress_thread_id, chat_type=getattr(source, "chat_type", None), reply_to_message_id=event_message_id, ) ) if _progress_thread_id else None if _progress_metadata is None and _relay_prospective_thread_id: # No real thread yet, but the connector will auto-thread on the # reply anchor; carry it so progress joins that thread. _progress_metadata = {"reply_to_message_id": event_message_id} _progress_metadata = _non_conversational_metadata(_progress_metadata, platform=source.platform) if _native_slack_task_cards: # chat.startStream in channels requires the recipient team/user # pair; harmless extras elsewhere, so stamp them whenever known. _progress_metadata = dict(_progress_metadata or {}) if source.scope_id: _progress_metadata.setdefault("recipient_team_id", source.scope_id) _progress_metadata.setdefault("slack_team_id", source.scope_id) if source.user_id: _progress_metadata.setdefault("recipient_user_id", source.user_id) _progress_reply_to = ( event_message_id if ( source.platform in (Platform.FEISHU, Platform.MATTERMOST) and source.thread_id and event_message_id ) or ( # Buzz has no native thread_id; threading is always via reply-to # the triggering event id (channel clutter otherwise). Skipped # when the user opted out of threaded replies. str(getattr(source.platform, "value", source.platform) or "").lower() == "buzz" and event_message_id and _progress_reply_in_thread ) or _relay_prospective_thread_id else None ) async def write_tool_log(): """Drain log_queue and append tool-call lines to tool_calls.log. Only active when ``display.tool_progress`` is ``log``. Uses a RotatingFileHandler (5MB × 3 backups) so the audit log can't grow unbounded, and the shared RedactingFormatter so secrets never land on disk. """ if log_queue is None: return from logging.handlers import RotatingFileHandler from agent.redact import RedactingFormatter log_dir = _hermes_home / "logs" log_dir.mkdir(parents=True, exist_ok=True) file_handler = RotatingFileHandler( log_dir / "tool_calls.log", maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8", ) file_handler.setFormatter(RedactingFormatter("%(message)s")) tool_logger = logging.getLogger(f"hermes.tool_calls.{id(log_queue)}") tool_logger.setLevel(logging.INFO) tool_logger.propagate = False tool_logger.addHandler(file_handler) try: while True: try: tool_logger.info("%s", log_queue.get_nowait()) except queue.Empty: await asyncio.sleep(0.3) except Exception as e: logger.error("write_tool_log error: %s", e) await asyncio.sleep(1) except asyncio.CancelledError: pass finally: # Drain remaining entries before closing so late tool calls # from the final iteration aren't lost. while True: try: tool_logger.info("%s", log_queue.get_nowait()) except queue.Empty: break except Exception: break tool_logger.removeHandler(file_handler) try: file_handler.flush() file_handler.close() except Exception: pass # Extracted to TurnRunner.send_progress_messages. The threading # metadata computed above is published onto the shared TurnContext # exactly where the original closure's captured locals were bound. turn_ctx._progress_metadata = _progress_metadata turn_ctx._progress_reply_to = _progress_reply_to send_progress_messages = turn_runner.send_progress_messages # We need to share the agent instance for interrupt support agent_holder = [None] # Mutable container for the agent instance turn_ctx.agent_holder = agent_holder result_holder = [None] # Mutable container for the result tools_holder = [None] # Mutable container for the tool definitions stream_consumer_holder = [None] # Mutable container for stream consumer # #60671 — streaming PCM audio consumer. Created on the gateway # event-loop thread (NOT inside run_sync's executor worker) so the # outer finalisation / interrupt paths can reference it without a # cross-scope NameError. streaming_tts_consumer_holder: list = [None] turn_ctx.result_holder = result_holder turn_ctx.tools_holder = tools_holder turn_ctx.stream_consumer_holder = stream_consumer_holder turn_ctx.streaming_tts_consumer_holder = streaming_tts_consumer_holder # Bridge sync step_callback → async hooks.emit for agent:step events _loop_for_step = asyncio.get_running_loop() _hooks_ref = self.hooks # Bridge extracted to TurnRunner._step_callback_sync; the loop and # hooks refs bound just above are published at their original site. turn_ctx._loop_for_step = _loop_for_step turn_ctx._hooks_ref = _hooks_ref turn_ctx._step_callback_sync = turn_runner._step_callback_sync # Bridge sync event_callback → async hooks.emit for lifecycle events # (e.g. session:compress fires after context compression splits a session) # Bridge extracted to TurnRunner._event_callback_sync. turn_ctx._event_callback_sync = turn_runner._event_callback_sync # Bridge sync status_callback → async adapter.send for context pressure _status_adapter = self._adapter_for_source(source) _status_chat_id = source.chat_id if source.platform == Platform.FEISHU and source.thread_id and event_message_id: # Feishu topics only keep messages inside the topic when they are # sent via the reply API with reply_in_thread=true. Status/interim, # approval, and stream-consumer paths usually only receive metadata, # so carry the triggering message id as a Feishu-specific fallback. _status_thread_metadata: Optional[Dict[str, Any]] = { "thread_id": _progress_thread_id, "reply_to_message_id": event_message_id, } else: _status_thread_metadata = ( self._thread_metadata_for_source(source, event_message_id) if _progress_thread_id == source.thread_id else self._thread_metadata_for_target( source.platform, source.chat_id, _progress_thread_id, chat_type=getattr(source, "chat_type", None), reply_to_message_id=event_message_id, ) ) if _progress_thread_id else None if _status_thread_metadata is None and _relay_prospective_thread_id: # Relay Discord auto-thread lane (see _progress_metadata above): # carry the reply anchor so status/interim bubbles route into # the same connector-created thread as the final reply. _status_thread_metadata = { "reply_to_message_id": event_message_id } # Bridge extracted to TurnRunner._status_callback_sync; publish the # status wiring computed above onto the shared TurnContext at the # exact original binding site. turn_ctx._status_adapter = _status_adapter turn_ctx._status_chat_id = _status_chat_id turn_ctx._status_thread_metadata = _status_thread_metadata turn_ctx._status_callback_sync = turn_runner._status_callback_sync # ---- Streaming TTS consumer setup (#60671) ---- # Created on the gateway event-loop thread (here, in _run_agent_inner), # NOT inside run_sync's executor worker. This avoids a cross-scope # NameError: the outer interrupt / finalisation paths reference the # consumer via ``streaming_tts_consumer_holder[0]``. # # Gates: voice input, auto-TTS enabled for this chat, adapter # supports streaming, and a usable streaming TTS provider configured. _stts_adapter = self._adapter_for_source(source) _is_voice_input = ( message_type is not None and str(getattr(message_type, "value", message_type)).lower() == "voice" ) if ( _stts_adapter is not None and _is_voice_input and _stts_adapter._should_auto_tts_for_chat(source.chat_id) ): try: from gateway.streaming_tts_consumer import StreamingTTSConsumer from tools.tts_tool import _load_tts_config _tts_cfg = _load_tts_config() _gateway_loop = self._gateway_loop or asyncio.get_event_loop() _stts_consumer = StreamingTTSConsumer( adapter=_stts_adapter, chat_id=source.chat_id, tts_config=_tts_cfg, loop=_gateway_loop, metadata=_status_thread_metadata, ) if _stts_consumer.active: streaming_tts_consumer_holder[0] = _stts_consumer _stts_consumer.start() # else: consumer inactive (no streaming provider) — leave # the holder as None so the whole-file fallback path runs. except Exception as _stts_err: logger.debug("Could not set up streaming TTS consumer: %s", _stts_err) # run_sync extracted to TurnRunner.run_sync (bound method; the # executor call below is unchanged). Its closed-over locals travel # on turn_ctx; `nonlocal message` rebinds became ctx.message writes. run_sync = turn_runner.run_sync # Start progress message sender if enabled. Gate on needs_progress_queue # (tool_progress OR thinking_progress), not tool_progress alone: the # sender drains BOTH tool-progress lines and _thinking scratch bubbles. # With the old tool_progress-only gate, a thinking_progress:true / # tool_progress:off user had the callback queue _thinking messages that # no task ever drained — so they silently never appeared. progress_task = None if needs_progress_queue: progress_task = asyncio.create_task(send_progress_messages()) # Start the tool-call log writer when tool_progress == "log". log_task = None if log_mode_enabled: log_task = asyncio.create_task(write_tool_log()) # Start stream consumer task — polls for consumer creation since it # happens inside run_sync (thread pool) after the agent is constructed. stream_task = None async def _start_stream_consumer(): """Wait for the stream consumer to be created, then run it.""" for _ in range(200): # Up to 10s wait if stream_consumer_holder[0] is not None: await stream_consumer_holder[0].run() return await asyncio.sleep(0.05) stream_task = asyncio.create_task(_start_stream_consumer()) # Track this agent as running for this session (for interrupt support) # We do this in a callback after the agent is created async def track_agent(): # Wait for agent to be created while agent_holder[0] is None: await asyncio.sleep(0.05) if not session_key: return # Only promote the sentinel to the real agent if this run is still # current. If /stop or /new bumped the generation while we were # spinning up, leave the newer run's slot alone — we'll be # discarded by the stale-result check in _handle_message_with_agent. if run_generation is not None and not self._is_session_run_current( session_key, run_generation ): logger.info( "Skipping stale agent promotion for %s — generation %s is no longer current", session_key or "", run_generation, ) return self._session_state(session_key).turn.agent = agent_holder[0] if self._draining: self._update_runtime_status("draining") tracking_task = asyncio.create_task(track_agent()) # Monitor for interrupts from the adapter (new messages arriving). # This is the PRIMARY interrupt path for regular text messages — # Level 1 (base.py) catches them before _handle_message() is reached, # so the Level 2 running_agent.interrupt() path never fires. # The inactivity poll loop below has a BACKUP check in case this # task dies (no error handling = silent death = lost interrupts). _interrupt_detected = asyncio.Event() # shared with backup check async def monitor_for_interrupt(): if not session_key: return while True: await asyncio.sleep(0.2) # Check every 200ms try: # Re-resolve adapter each iteration so reconnects don't # leave us holding a stale reference. _adapter = self._adapter_for_source(source) if not _adapter: continue # Check if adapter has a pending interrupt for this session. # Must use session_key (build_session_key output) — NOT # source.chat_id — because the adapter stores interrupt events # under the full session key. if hasattr(_adapter, 'has_pending_interrupt') and _adapter.has_pending_interrupt(session_key): agent = agent_holder[0] if agent: # Peek at the pending message text WITHOUT consuming it. # The message must remain in _pending_messages so the # post-run dequeue at _dequeue_pending_event() can # retrieve the full MessageEvent (with media metadata). # If we pop here, a race exists: the agent may finish # before checking _interrupt_requested, and the message # is lost — neither the interrupt path nor the dequeue # path finds it. _peek_event = _adapter._pending_messages.get(session_key) pending_text = None if _peek_event is not None: pending_text = _peek_event.text or "" # Transcribe audio media BEFORE signaling the # agent, so voice messages interrupt with the # real transcript instead of an empty string # (or file-path placeholder). Matches the UX # of fresh voice messages including the # optional 🎙️ echo back to the user. _media_urls = getattr(_peek_event, "media_urls", None) or [] if self._pending_event_audio_paths(_peek_event): pending_text, _ = await self._transcribe_and_echo_pending_voice( _peek_event, _adapter, source, pending_text, log_context="Voice-interrupt", metadata={"thread_id": source.thread_id} if source.thread_id else None, ) elif not pending_text and _media_urls: pending_text = _build_media_placeholder(_peek_event) logger.debug("Interrupt detected from adapter, signaling agent...") agent.interrupt(pending_text) _interrupt_detected.set() # Abort streaming TTS on barge-in (#60671). _stts = streaming_tts_consumer_holder[0] if _stts is not None: _stts.abort("barge-in") break except asyncio.CancelledError: raise except Exception as _mon_err: logger.debug("monitor_for_interrupt error (will retry): %s", _mon_err) interrupt_monitor = asyncio.create_task(monitor_for_interrupt()) # Periodic "still working" notifications for long-running tasks. # Fires every N seconds so the user knows the agent hasn't died. # Config: agent.gateway_notify_interval in config.yaml, or # HERMES_AGENT_NOTIFY_INTERVAL env var. Default 180s (3 min). # 0 = disable notifications. _NOTIFY_INTERVAL_RAW = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180) _NOTIFY_INTERVAL = _NOTIFY_INTERVAL_RAW if _NOTIFY_INTERVAL_RAW > 0 else None _long_running_mode = _display_surface_mode( "long_running_notifications", default=True, allow_generic=True, ) if _long_running_mode == "off": _NOTIFY_INTERVAL = None _notify_start = time.time() async def _notify_long_running(): if _NOTIFY_INTERVAL is None: return # Notifications disabled (gateway_notify_interval: 0) _notify_adapter = self._adapter_for_source(source) if not _notify_adapter: return # Track the heartbeat message id so we can edit-in-place on # platforms that support it (Telegram, Discord, Slack, etc.) # instead of spamming a new "Still working" bubble every # interval. Falls back to send-new when edit fails or isn't # supported by the adapter. _heartbeat_msg_id: Optional[str] = None while True: await asyncio.sleep(_NOTIFY_INTERVAL) # Stop heartbeating once this run no longer owns the session # slot or the executor has finished — otherwise a stale # "running: delegate_task" bubble can outlive the run that # spawned it (#12029). _executor_task is a closure var bound # just after this task is scheduled; tolerate the brief window # before then (the first wake is _NOTIFY_INTERVAL away anyway). try: _exec_ref = _executor_task except NameError: _exec_ref = None if not self._should_emit_long_running_notification( session_key, agent_holder[0], _exec_ref ): break _elapsed_mins = int((time.time() - _notify_start) // 60) # Include agent activity context if available. Default # heartbeat is terse: elapsed + current tool. Verbose # iteration counter is gated on busy_ack_detail so users # who want it can opt in per platform. _agent_ref = agent_holder[0] _status_detail = "" _want_iteration_detail = bool( resolve_display_setting( user_config, platform_key, "busy_ack_detail", True, ) ) if _agent_ref and hasattr(_agent_ref, "get_activity_summary"): try: _a = _agent_ref.get_activity_summary() _parts = [] if _want_iteration_detail: _parts.append( f"iteration {_a['api_call_count']}/{_a['max_iterations']}" ) _action = _a.get("current_tool") or _a.get("last_activity_desc") if _action: _parts.append(str(_action)) if _parts: _status_detail = " — " + ", ".join(_parts) except Exception: pass _heartbeat_text = ( _generic_status_phrase("status") if _long_running_mode == "generic" else f"⏳ Working — {_elapsed_mins} min{_status_detail}" ) try: _notify_res = None if _heartbeat_msg_id: try: _notify_res = await _notify_adapter.edit_message( source.chat_id, _heartbeat_msg_id, _heartbeat_text, ) except Exception as _ee: logger.debug("Heartbeat edit failed: %s", _ee) _notify_res = None if not (_notify_res and getattr(_notify_res, "success", False)): _notify_res = await _notify_adapter.send( source.chat_id, _heartbeat_text, metadata=_interim_metadata(_non_conversational_metadata(_status_thread_metadata, platform=source.platform)), ) if getattr(_notify_res, "success", False) and getattr( _notify_res, "message_id", None ): _heartbeat_msg_id = str(_notify_res.message_id) if _cleanup_progress: _cleanup_msg_ids.append(_heartbeat_msg_id) except Exception as _ne: logger.debug("Long-running notification error: %s", _ne) _notify_task = asyncio.create_task(_notify_long_running()) def _stream_confirmed_final_delivery( consumer, final_text: str, *, previewed: bool = False, ) -> bool: """Return True only when the actual final reply reached the user.""" if consumer is None: return False if getattr(consumer, "final_response_sent", False): # A successful finalize call is not proof the *content* was # final: the edit may have carried only the last preview # snapshot while the tail generated between that snapshot and # stream completion never reached any API call (#71643). # Reconcile the recorded turn-final payload against the # completed response; only a demonstrable mismatch (False) # overrides the flag — including payload-less multi-message # split delivery (#78541). None (no record on a non-split # legacy path) keeps the legacy trust so ambiguous-timeout # dedup is not regressed. matcher = getattr(consumer, "delivered_final_matches", None) if callable(matcher): try: if matcher(final_text) is False: return False except Exception: pass return True if previewed: has_delivered_text = getattr(consumer, "has_delivered_text", None) if callable(has_delivered_text): try: return bool(has_delivered_text(final_text)) except Exception: return False return False try: # Run in thread pool to not block. Use an *inactivity*-based # timeout instead of a wall-clock limit: the agent can run for # hours if it's actively calling tools / receiving stream tokens, # but a hung API call or stuck tool with no activity for the # configured duration is caught and killed. (#4815) # # Config: agent.gateway_timeout in config.yaml, or # HERMES_AGENT_TIMEOUT env var (env var takes precedence). # Default 1800s (30 min inactivity). 0 = unlimited. _agent_timeout_raw = _float_env("HERMES_AGENT_TIMEOUT", 1800) _agent_timeout = _agent_timeout_raw if _agent_timeout_raw > 0 else None _agent_warning_raw = _float_env("HERMES_AGENT_TIMEOUT_WARNING", 900) _agent_warning = _agent_warning_raw if _agent_warning_raw > 0 else None _warning_fired = False # A background=true process intentionally survives a successful # turn, so capture existing IDs and reap only children created by # THIS turn if it times out. The daemon watchdog is independent of # asyncio: cgroup memory reclaim may starve the event loop that runs # the normal timeout poll, but it need not also postpone cleanup # until the loop recovers (#76115). from tools.process_registry import process_registry _turn_task_id = session_id or "" _turn_process_baseline = process_registry.snapshot_running_ids(_turn_task_id) turn_ctx.process_task_id = _turn_task_id turn_ctx.process_baseline = _turn_process_baseline _turn_worker_done = threading.Event() _turn_timeout_fired = threading.Event() _turn_cleanup_lock = threading.Lock() # task_id above is session-scoped, not turn-scoped (#76115 # review): gate the eventual reap on this exact claim still # being current, so a replacement turn that starts on the same # session before the watchdog fires doesn't get its own fresh # process killed by this turn's stale baseline. _turn_run_generation = run_generation _turn_is_current = ( (lambda: self._is_session_run_current(session_key, _turn_run_generation)) if _turn_run_generation is not None else (lambda: True) ) def _run_sync_with_timeout_lifecycle(): try: return run_sync() finally: _turn_worker_done.set() # `.turn.agent` on the session state is only reset to # _AGENT_PENDING_SENTINEL when the *next* turn is # claimed (see _session_state(...).turn.agent = ... at # claim time), so a stale reference to this exact agent # instance stays reachable from # _interrupt_and_clear_session() until then. Clearing # the ownership markers here — the instant this turn's # own worker finishes — closes that window: an # explicit /stop landing on the already-finished turn # no longer reaps background work the turn deliberately # left running (#76115). _finished_agent = agent_holder[0] if agent_holder else None if _finished_agent is not None: _finished_agent._gateway_turn_process_task_id = "" _finished_agent._gateway_turn_process_baseline = frozenset() if _agent_timeout is not None: threading.Thread( target=_watch_gateway_turn_inactivity, kwargs={ "agent_holder": agent_holder, "task_id": _turn_task_id, "process_baseline": _turn_process_baseline, "timeout": _agent_timeout, "worker_done": _turn_worker_done, "timeout_fired": _turn_timeout_fired, "cleanup_lock": _turn_cleanup_lock, "poll_interval": 5.0, "is_still_current": _turn_is_current, }, name=f"gateway-turn-watchdog-{_turn_task_id[:12]}", daemon=True, ).start() _executor_task = asyncio.ensure_future( self._run_in_executor_with_context(_run_sync_with_timeout_lifecycle) ) _inactivity_timeout = False _POLL_INTERVAL = 5.0 if _agent_timeout is None: # Unlimited — still poll periodically for backup interrupt # detection in case monitor_for_interrupt() silently died. response = None while True: done, _ = await asyncio.wait( {_executor_task}, timeout=_POLL_INTERVAL ) if done: response = _executor_task.result() break # Backup interrupt check: if the monitor task died or # missed the interrupt, catch it here. if not _interrupt_detected.is_set() and session_key: _backup_adapter = self._adapter_for_source(source) _backup_agent = agent_holder[0] if (_backup_adapter and _backup_agent and hasattr(_backup_adapter, 'has_pending_interrupt') and _backup_adapter.has_pending_interrupt(session_key)): _bp_event = _backup_adapter._pending_messages.get(session_key) _bp_text = _bp_event.text if _bp_event else None if _bp_event is not None: _bp_media_urls = getattr(_bp_event, "media_urls", None) or [] if self._pending_event_audio_paths(_bp_event): _bp_text, _ = await self._transcribe_and_echo_pending_voice( _bp_event, _backup_adapter, source, _bp_text or "", log_context="Voice-backup-interrupt", metadata={"thread_id": source.thread_id} if source.thread_id else None, ) elif not _bp_text and _bp_media_urls: _bp_text = _build_media_placeholder(_bp_event) logger.info( "Backup interrupt detected for session %s " "(monitor task state: %s)", session_key, "done" if interrupt_monitor.done() else "running", ) _backup_agent.interrupt(_bp_text) _interrupt_detected.set() # Abort streaming TTS on barge-in (#60671). _stts = streaming_tts_consumer_holder[0] if _stts is not None: _stts.abort("barge-in") else: # Poll loop: check the agent's built-in activity tracker # (updated by _touch_activity() on every tool call, API # call, and stream delta) every few seconds. response = None while True: done, _ = await asyncio.wait( {_executor_task}, timeout=_POLL_INTERVAL ) if done: # Prefer the real result when the worker finished, # even if the watchdog fired in the same window: the # completed run already persisted its reply to session # history, so surfacing the "agent inactive" diagnostic # here would contradict the stored transcript. This # mirrors _abandon_timed_out_gateway_turn's own # worker_done-wins tiebreak (under cleanup_lock). response = _executor_task.result() break if _turn_timeout_fired.is_set(): _inactivity_timeout = True break # Agent still running — check inactivity. _agent_ref = agent_holder[0] _idle_secs = 0.0 if _agent_ref and hasattr(_agent_ref, "get_activity_summary"): try: _act = _agent_ref.get_activity_summary() _idle_secs = _act.get("seconds_since_activity", 0.0) except Exception: pass # Staged warning: fire once before escalating to full timeout. if (not _warning_fired and _agent_warning is not None and _idle_secs >= _agent_warning): _warning_fired = True _warn_adapter = self._adapter_for_source(source) if _warn_adapter: _elapsed_warn = int(_agent_warning // 60) or 1 _remaining_mins = int((_agent_timeout - _agent_warning) // 60) or 1 try: await _warn_adapter.send( source.chat_id, f"⚠️ No activity for {_elapsed_warn} min. " f"If the agent does not respond soon, it will " f"be timed out in {_remaining_mins} min. " f"You can continue waiting or use /reset.", metadata=_interim_metadata(_status_thread_metadata), ) except Exception as _warn_err: logger.debug("Inactivity warning send error: %s", _warn_err) if _idle_secs >= _agent_timeout: _inactivity_timeout = True threading.Thread( target=_abandon_timed_out_gateway_turn, kwargs={ "agent_holder": agent_holder, "task_id": _turn_task_id, "process_baseline": _turn_process_baseline, "worker_done": _turn_worker_done, "timeout_fired": _turn_timeout_fired, "cleanup_lock": _turn_cleanup_lock, "is_still_current": _turn_is_current, }, name=f"gateway-turn-reaper-{_turn_task_id[:12]}", daemon=True, ).start() break # Backup interrupt check (same as unlimited path). if not _interrupt_detected.is_set() and session_key: _backup_adapter = self._adapter_for_source(source) _backup_agent = agent_holder[0] if (_backup_adapter and _backup_agent and hasattr(_backup_adapter, 'has_pending_interrupt') and _backup_adapter.has_pending_interrupt(session_key)): _bp_event = _backup_adapter._pending_messages.get(session_key) _bp_text = _bp_event.text if _bp_event else None if _bp_event is not None: _bp_media_urls = getattr(_bp_event, "media_urls", None) or [] if self._pending_event_audio_paths(_bp_event): _bp_text, _ = await self._transcribe_and_echo_pending_voice( _bp_event, _backup_adapter, source, _bp_text or "", log_context="Voice-backup-interrupt", metadata={"thread_id": source.thread_id} if source.thread_id else None, ) elif not _bp_text and _bp_media_urls: _bp_text = _build_media_placeholder(_bp_event) logger.info( "Backup interrupt detected for session %s " "(monitor task state: %s)", session_key, "done" if interrupt_monitor.done() else "running", ) _backup_agent.interrupt(_bp_text) _interrupt_detected.set() # Abort streaming TTS on barge-in (#60671). _stts = streaming_tts_consumer_holder[0] if _stts is not None: _stts.abort("barge-in") if _inactivity_timeout: # Build a diagnostic summary from the agent's activity tracker. _timed_out_agent = agent_holder[0] _activity = {} if _timed_out_agent and hasattr(_timed_out_agent, "get_activity_summary"): try: _activity = _timed_out_agent.get_activity_summary() except Exception: pass _last_desc = _activity.get("last_activity_desc", "unknown") _secs_ago = _activity.get("seconds_since_activity", 0) _cur_tool = _activity.get("current_tool") _iter_n = _activity.get("api_call_count", 0) _iter_max = _activity.get("max_iterations", 0) logger.error( "Agent idle for %.0fs (timeout %.0fs) in session %s " "| last_activity=%s | iteration=%s/%s | tool=%s", _secs_ago, _agent_timeout, session_key, _last_desc, _iter_n, _iter_max, _cur_tool or "none", ) # Interrupt the agent if it's still running so the thread # pool worker is freed. if _timed_out_agent: request_hard_interrupt(_timed_out_agent, _INTERRUPT_REASON_TIMEOUT) _timeout_mins = int(_agent_timeout // 60) or 1 # Construct a user-facing message with diagnostic context. _diag_lines = [ f"⏱️ Agent inactive for {_timeout_mins} min — no tool calls " f"or API responses." ] if _cur_tool: _diag_lines.append( f"The agent appears stuck on tool `{_cur_tool}` " f"({_secs_ago:.0f}s since last activity, " f"iteration {_iter_n}/{_iter_max})." ) else: _diag_lines.append( f"Last activity: {_last_desc} ({_secs_ago:.0f}s ago, " f"iteration {_iter_n}/{_iter_max}). " "The agent may have been waiting on an API response." ) _diag_lines.append( "To increase the limit, set agent.gateway_timeout in config.yaml " "(value in seconds, 0 = no limit) and restart the gateway.\n" "Try again, or use /reset to start fresh." ) response = { "final_response": "\n".join(_diag_lines), "messages": result_holder[0].get("messages", []) if result_holder[0] else [], "api_calls": _iter_n, "tools": tools_holder[0] or [], "history_offset": 0, "failed": True, } # Track fallback model state: if the agent switched to a # fallback model during this run, persist it so /model shows # the actually-active model instead of the config default. # Skip eviction when the run failed — evicting a failed agent # forces MCP reinit on the next message for no benefit (the # same error will recur). This was the root cause of #7130: # a bad model ID triggered fallback → eviction → recreation → # MCP reinit → same 400 → loop, burning 91% CPU for hours. _agent = agent_holder[0] _result_for_fb = result_holder[0] _run_failed = _result_for_fb.get("failed") if _result_for_fb else False if _agent is not None and hasattr(_agent, 'model') and not _run_failed: _cfg_model = _resolve_gateway_model() # Normalize _cfg_model the same way AIAgent.__init__ does, so a # vendor-prefixed config value (e.g. "deepseek/deepseek-v4-pro") # matches the agent's stripped model ("deepseek-v4-pro") on # native providers. Without this, _agent.model != _cfg_model is # always true for vendor-prefixed config and the cached agent is # evicted on every successful turn — destroying prompt caching. # Aggregators (openrouter, etc.) keep the vendor/model slug, so # they're left untouched. try: from hermes_cli.model_normalize import ( _AGGREGATOR_PROVIDERS, normalize_model_for_provider, ) _agent_provider = getattr(_agent, 'provider', '') or '' if _agent_provider and _agent_provider not in _AGGREGATOR_PROVIDERS: _cfg_model = normalize_model_for_provider(_cfg_model, _agent_provider) except Exception: pass if _agent.model != _cfg_model and not self._is_intentional_model_switch(session_key, _agent.model): # Fallback activated on a successful run — evict cached # agent so the next message retries the primary model. self._evict_cached_agent(session_key) # Check if we were interrupted OR have a queued message (/queue). result = result_holder[0] adapter = self._adapter_for_source(source) # Finalize the streaming-TTS consumer (#60671). # # finish() is called from the outer event-loop thread (not the # executor worker) so early returns from run_sync are also # finalised. wait_complete() drains queued audio; on timeout # the consumer is aborted unconditionally — if audio was # audible, suppression is preserved so the gateway does not # replay from the beginning; if no audio was audible, the # whole-file fallback path is permitted. _stts = streaming_tts_consumer_holder[0] if _stts is not None: _stts.finish() try: await _stts.wait_complete(timeout=10.0) except Exception as _stts_done_err: logger.debug("streaming TTS wait_complete error: %s", _stts_done_err) if not _stts.done: # Timeout before or after audible audio: abort to free # the consumer task. Audible streams retain suppression; # silent streams remain eligible for whole-file fallback. _stts.abort("streaming TTS finalisation timeout") await _stts.wait_complete(timeout=2.0) if _stts.suppress_whole_file and adapter is not None: _mark_turn = getattr(adapter, "_mark_streaming_tts_completed_turn", None) if callable(_mark_turn): _mark_turn(session_key, run_generation) # Get pending message from adapter. # Use session_key (not source.chat_id) to match adapter's storage keys. pending_event = None pending = None if result and adapter and session_key: pending_event = _dequeue_pending_event(adapter, session_key) # /queue overflow: after consuming the adapter's "next-up" # slot, promote the next queued event into it so the # recursive run's drain will see it. This keeps the slot # occupied for the full FIFO chain, which (a) preserves # order, and (b) causes any mid-chain /queue to correctly # route to overflow rather than jumping the queue. pending_event = self._promote_queued_event(session_key, adapter, pending_event) if result.get("interrupted") and not pending_event and result.get("interrupt_message"): interrupt_message = result.get("interrupt_message") if _is_control_interrupt_message(interrupt_message): logger.info( "Ignoring control interrupt message for session %s: %s", session_key or "?", interrupt_message, ) else: pending = interrupt_message elif pending_event: # Transcribe audio media on the dequeued event BEFORE it is # handed back as the next user turn, so queued/interrupting # voice messages drain with the real transcript instead of # a file-path placeholder. When configured, echo each # transcript back to the user in the same 🎙️ format as # fresh voice messages. _pending_text = pending_event.text or "" _media_urls = getattr(pending_event, "media_urls", None) or [] if self._pending_event_audio_paths(pending_event): pending, _ = await self._transcribe_and_echo_pending_voice( pending_event, adapter, source, _pending_text, log_context="Voice-drain", metadata={"thread_id": source.thread_id} if source.thread_id else None, ) if not pending: pending = _build_media_placeholder(pending_event) else: pending = _pending_text or _build_media_placeholder(pending_event) if pending: logger.debug("Processing queued message after agent completion: '%s...'", pending[:40]) # Leftover /steer: if a steer arrived after the last tool batch # (e.g. during the final API call), the agent couldn't inject it # and returned it in result["pending_steer"]. Deliver it as the # next user turn so it isn't silently dropped. if result and not pending and not pending_event: _leftover_steer = result.get("pending_steer") if _leftover_steer: pending = _leftover_steer logger.debug("Delivering leftover /steer as next turn: '%s...'", pending[:40]) # Safety net: if the pending text is a slash command (e.g. "/stop", # "/new"), discard it — commands should never be passed to the agent # as user input. The primary fix is in base.py (commands bypass the # active-session guard), but this catches edge cases where command # text leaks through the interrupt_message fallback. if pending and pending.strip().startswith("/"): _pending_parts = pending.strip().split(None, 1) _pending_cmd_word = _pending_parts[0][1:].lower() if _pending_parts else "" if _pending_cmd_word: try: from hermes_cli.commands import resolve_command as _rc_pending if _rc_pending(_pending_cmd_word): logger.info( "Discarding command '/%s' from pending queue — " "commands must not be passed as agent input", _pending_cmd_word, ) pending_event = None pending = None except Exception: pass if self._draining and (pending_event or pending): logger.info( "Discarding pending follow-up for session %s during gateway %s", session_key or "?", self._status_action_label(), ) pending_event = None pending = None if pending_event or pending: logger.debug("Processing pending message: '%s...'", pending[:40]) # Clear the adapter's interrupt event so the next _run_agent call # doesn't immediately re-trigger the interrupt before the new agent # even makes its first API call (this was causing an infinite loop). if adapter and hasattr(adapter, '_active_sessions') and session_key and session_key in adapter._active_sessions: adapter._active_sessions[session_key].clear() # Cap recursion depth to prevent resource exhaustion when the # user sends multiple messages while the agent keeps failing. (#816) if _interrupt_depth >= self._MAX_INTERRUPT_DEPTH: logger.warning( "Interrupt recursion depth %d reached for session %s — " "queueing message instead of recursing.", _interrupt_depth, session_key, ) adapter = self._adapter_for_source(source) if adapter and pending_event: merge_pending_message_event(adapter._pending_messages, session_key, pending_event) elif adapter and hasattr(adapter, 'queue_message'): adapter.queue_message(session_key, pending) return result_holder[0] or {"final_response": response, "messages": history} was_interrupted = result.get("interrupted") if not was_interrupted: # Queued message after normal completion — deliver the first # response before processing the queued follow-up. # Skip if streaming already delivered it. _sc = stream_consumer_holder[0] if _sc and stream_task: try: await asyncio.wait_for(stream_task, timeout=5.0) except (asyncio.TimeoutError, asyncio.CancelledError): stream_task.cancel() try: await stream_task except asyncio.CancelledError: pass except Exception as e: logger.debug("Stream consumer wait before queued message failed: %s", e) # The queued branch needs raw ``result`` for interruption, # history, and recursion state, but delivery must use the # finalized task result. The latter contains empty/failure # normalization and any final response processing applied by # _run_agent_task; sending the raw copy bypasses those steps. _delivery_result = response if isinstance(response, dict) else (result or {}) _previewed = bool(_delivery_result.get("response_previewed")) first_response = _delivery_result.get("final_response", "") _already_streamed = _stream_confirmed_final_delivery( _sc, first_response, previewed=_previewed, ) # Apply the same predicate as the normal completed-turn path. # This direct queued-send branch predates intentional-silence # filtering, so without this check it leaks the literal marker. try: from gateway.response_filters import is_intentional_silence_agent_result _intentional_silence = is_intentional_silence_agent_result( _delivery_result, first_response, ) except Exception: _intentional_silence = False if _intentional_silence: logger.info( "Queued follow-up for session %s: suppressing intentional silence marker before continuing.", session_key or "?", ) elif first_response: try: if _already_streamed: logger.info( "Queued follow-up for session %s: final text delivery confirmed; delivering explicit media before continuing.", session_key or "?", ) else: logger.info( "Queued follow-up for session %s: final stream delivery not confirmed; sending first response before continuing.", session_key or "?", ) await self._deliver_queued_first_response( first_response, source=source, adapter=adapter, metadata=_status_thread_metadata, event_message_id=event_message_id, text_already_delivered=_already_streamed, deliver_media=not _delivery_result.get("failed"), stream_consumer=_sc, ) except Exception as e: logger.warning("Failed to send first response before queued message: %s", e) # Release deferred bg-review notifications now that the # first response has been delivered. Pop from the # adapter's callback dict (prevents double-fire in # base.py's finally block) and call it. if getattr(type(adapter), "pop_post_delivery_callback", None) is not None: _bg_cb = adapter.pop_post_delivery_callback( session_key, generation=run_generation, ) if callable(_bg_cb): try: _bg_result = _bg_cb() if inspect.isawaitable(_bg_result): await _bg_result except Exception: pass elif adapter and hasattr(adapter, "_post_delivery_callbacks"): _bg_cb = adapter._post_delivery_callbacks.pop(session_key, None) if callable(_bg_cb): try: _bg_result = _bg_cb() if inspect.isawaitable(_bg_result): await _bg_result except Exception: pass # else: interrupted — discard the interrupted response ("Operation # interrupted." is just noise; the user already knows they sent a # new message). updated_history = result.get("messages", history) next_source = source next_message = pending next_message_id = None next_channel_prompt = None next_session_key = session_key # #60671 — carry the pending event's message_type into the # recursive call so queued voice turns can stream TTS and # re-mark the generation for the final delivered turn. next_message_type = None if pending_event is not None: next_source = getattr(pending_event, "source", None) or source if self._is_goal_continuation_event(pending_event) and not self._goal_still_active_for_session(session_id): logger.info( "Discarding stale goal continuation for session %s — goal is no longer active", session_key or "?", ) return result # Resolve the follow-up's session key BEFORE preparing the # inbound text: _prepare_inbound_message_text buffers native # image paths under the key it is given, and the recursive # _run_agent below consumes them under next_session_key. # The write and consume keys must match or the images drop. try: next_session_key = self._session_key_for_source(next_source) except Exception: logger.debug( "Queued follow-up session-key resolution failed; reusing %s", session_key or "?", exc_info=True, ) next_message = await self._prepare_profile_scoped_inbound_message_text( event=pending_event, source=next_source, history=updated_history, session_key=next_session_key, ) if next_message is None: return result next_message_id = self._reply_anchor_for_event(pending_event) next_channel_prompt = getattr(pending_event, "channel_prompt", None) next_message_type = getattr(pending_event, "message_type", None) # Clear the completed streaming marker from the prior logical # turn so the recursive turn's streaming TTS is not suppressed # by the prior turn's completion (#60671). _clear_adapter = self._adapter_for_source(source) if _clear_adapter is not None and session_key and run_generation is not None: _completed_turns = getattr(_clear_adapter, "_streaming_tts_completed_turns", None) if _completed_turns is not None: _prior_key = getattr(_clear_adapter, "_streaming_tts_turn_key", None) if callable(_prior_key): _pk = _prior_key(session_key, run_generation) if _pk: _completed_turns.discard(_pk) # Restart typing indicator so the user sees activity while # the follow-up turn runs. The outer _process_message_background # typing task is still alive but may be stale. _followup_adapter = self._adapter_for_source(source) if _followup_adapter: try: await _followup_adapter.send_typing( source.chat_id, metadata=_status_thread_metadata, ) except Exception: pass # Re-baseline the cached agent's message_count snapshot before # recursing into the in-band queued (/queue) follow-up turn. # The first turn has completed and flushed its own user + # assistant rows to the SessionDB, so the cross-process # coherence guard (#45966) — which this recursive _run_agent # call re-enters — would otherwise see the grown on-disk count # against the stale build-time snapshot and rebuild the agent # on THIS process's OWN writes, destroying the prompt-cache # prefix #46237 was merged to preserve. The existing # re-baseline in _handle_message_with_agent only runs after the # whole _run_agent chain unwinds — too late for the in-band # follow-up. Use the same (session_key, session_id) the # recursive call runs under so the snapshot matches exactly # what the follow-up's guard will consult. Fail-safe in helper. await self._refresh_agent_cache_message_count(session_key, session_id) followup_result = await self._run_agent( message=next_message, context_prompt=context_prompt, history=updated_history, source=next_source, session_id=session_id, session_key=next_session_key, run_generation=run_generation, _interrupt_depth=_interrupt_depth + 1, event_message_id=next_message_id, channel_prompt=next_channel_prompt, message_type=next_message_type, ) return _preserve_queued_followup_history_offset(result, followup_result) finally: # Stop progress sender, interrupt monitor, and notification task if progress_task: progress_task.cancel() if log_task: log_task.cancel() interrupt_monitor.cancel() _notify_task.cancel() # Wait for stream consumer to finish its final edit if stream_task: # If the agent never created a stream consumer (e.g. non- # streaming code path, or a test stub returning synchronously) # there is nothing to flush — cancel immediately instead of # waiting out the 5s timeout on a task that's just polling for # a consumer that will never arrive. This was a 5-second # cost per non-streaming test run. _has_stream_consumer = ( stream_consumer_holder and stream_consumer_holder[0] is not None ) if not _has_stream_consumer: stream_task.cancel() try: await stream_task except asyncio.CancelledError: pass else: try: await asyncio.wait_for(stream_task, timeout=5.0) except (asyncio.TimeoutError, asyncio.CancelledError): stream_task.cancel() try: await stream_task except asyncio.CancelledError: pass # Unconditional abort + bounded wait for the streaming-TTS # consumer (#60671 hardening). Covers cancellation / exception # paths where the normal finalisation block was skipped. _stts_finally = streaming_tts_consumer_holder[0] if _stts_finally is not None and not _stts_finally.done: _stts_finally.abort("cleanup") try: await _stts_finally.wait_complete(timeout=2.0) except Exception: pass # Clean up tracking tracking_task.cancel() if session_key: # Only release the slot if this run's generation still owns # it. A /stop or /new that bumped the generation while we # were unwinding has already installed its own state; this # guard prevents an old run from clobbering it on the way # out. self._release_running_agent_state( session_key, run_generation=run_generation ) if self._draining: self._update_runtime_status("draining") # Wait for cancelled tasks for task in [progress_task, log_task, interrupt_monitor, tracking_task, _notify_task]: if task: try: await task except asyncio.CancelledError: pass except Exception: # A background task that died of a non-cancellation # error (transport drop in a progress/card publish) # must not abort the cleanup path — everything after # this loop (final-delivery bookkeeping) still runs # (review B7). logger.debug( "background turn task failed during cleanup", exc_info=True, ) # If streaming already delivered the response, mark it so the # caller's send() is skipped (avoiding duplicate messages). # BUT: never suppress delivery when the agent failed — the error # message is new content the user hasn't seen, and it must reach # them even if streaming had sent earlier partial output. # # Also never suppress when the final response is "(empty)" — this # means the model failed to produce content after tool calls (common # with mimo-v2-pro, GLM-5, etc.). The stream consumer may have # sent intermediate text ("Let me search for that…") alongside the # tool call, setting already_sent=True, but that text is NOT the # final answer. Suppressing delivery here leaves the user staring # at silence. (#10xxx — "agent stops after web search") _sc = stream_consumer_holder[0] if isinstance(response, dict) and not response.get("failed"): _final = response.get("final_response") or "" _is_empty_sentinel = not _final or _final == "(empty)" # response_previewed means the interim_assistant_callback already # saw the final text, but only suppress the normal send if that # exact final text was delivered. Unrelated commentary/progress # must not be mistaken for the final response (#14238). _previewed = bool(response.get("response_previewed")) _content_delivered = bool( _sc and getattr(_sc, "final_content_delivered", False) ) # #71643: a *successful* finalize edit can still carry only the # last preview snapshot — deltas generated between that edit and # stream completion never reach any API call, and both suppression # flags are set from the call's success rather than its content. # Reconcile the consumer's recorded turn-final payload against the # completed response: on a demonstrable mismatch (False) neither # final_response_sent nor final_content_delivered may suppress the # normal final send. False also covers payload-less multi-message # split delivery (#78541). None (no record on a non-split legacy # path) keeps legacy trust; the failed-finalize family # (#51828 / #33793) is unaffected because those paths leave the # flags False or record the complete fallback payload. _stale_finalized = False if _content_delivered and not _is_empty_sentinel: _matcher = getattr(_sc, "delivered_final_matches", None) if callable(_matcher): try: _stale_finalized = _matcher(_final) is False except Exception: _stale_finalized = False if _stale_finalized: _content_delivered = False # Plugin hooks (e.g. transform_llm_output) may have appended content # after streaming finished — when the response was transformed, always # send the final version so the appended content reaches the client. _transformed = bool(response.get("response_transformed")) # Only suppress the normal send when the actual final reply reached # the user: the stream consumer streamed it (final_response_sent / # final_content_delivered), or the interim preview delivered that # *exact* final text. Unrelated commentary/progress shown during a # compression/session split must not be mistaken for the final # response (#14238). _streamed = _stream_confirmed_final_delivery( _sc, _final, previewed=_previewed, ) if not _is_empty_sentinel and not _transformed and (_streamed or _content_delivered): logger.info( "Suppressing normal final send for session %s: final delivery already confirmed (streamed=%s previewed=%s content_delivered=%s).", session_key or "?", _streamed, _previewed, _content_delivered, ) response["already_sent"] = True elif not _is_empty_sentinel and not _transformed and _stale_finalized and _sc is not None: # Stale finalize (#71643): the streamed message holds only the # last preview snapshot. Prefer editing it up to the complete # response (same shape as the transformed branch below) so the # user gets one corrected message; on edit failure fall through # with already_sent unset so the normal final send delivers the # complete text. # # Not valid for a multi-message split delivery: there # ``message_id`` is only the LAST chunk, so editing it with the # complete response would repeat every sealed head chunk's text # inside the tail message. Fall through to the normal final send # instead (#78541). _sc_msg_id = _sc.message_id _sc_adapter = getattr(_sc, "adapter", None) if getattr(_sc, "_turn_split_delivery", False): logger.info( "Stale streamed finalize detected for session %s on a multi-message split; skipping the in-place reconciliation edit and delivering the complete response via normal final send (#78541).", session_key or "?", ) elif _sc_msg_id and _sc_msg_id != "__no_edit__" and _sc_adapter is not None: try: _reconcile_res = await _sc_adapter.edit_message( chat_id=source.chat_id, message_id=_sc_msg_id, content=_final, finalize=True, ) if getattr(_reconcile_res, "success", True): response["already_sent"] = True logger.info( "Reconciled stale streamed finalize for session %s: edited message %s with the complete response (#71643).", session_key or "?", _sc_msg_id, ) else: logger.warning( "Stale-finalize reconciliation edit failed for session %s (%s); sending complete response via normal final send.", session_key or "?", getattr(_reconcile_res, "error", None), ) except Exception as _edit_err: logger.warning( "Stale-finalize reconciliation edit failed for session %s: %s; sending complete response via normal final send.", session_key or "?", _edit_err, ) else: logger.info( "Stale streamed finalize detected for session %s with no editable message; delivering complete response via normal final send (#71643).", session_key or "?", ) elif not _is_empty_sentinel and _transformed and _sc is not None: # Plugin hooks transformed the response after streaming — edit the # existing streamed message instead of sending a duplicate. _sc_msg_id = _sc.message_id if _sc_msg_id: try: await _sc.adapter.edit_message( chat_id=source.chat_id, message_id=_sc_msg_id, content=response["final_response"], finalize=True, ) response["already_sent"] = True logger.info( "Edited streamed message %s for session %s to include plugin-transformed content.", _sc_msg_id, session_key or "?", ) except Exception as _edit_err: logger.warning( "Failed to edit streamed message for session %s: %s", session_key or "?", _edit_err, ) elif _sc is not None and not _is_empty_sentinel: # DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed for this # turn but suppression did NOT fire, so the gateway's normal # final-send is about to run. On WeCom this is the exact window # that produced "回复了两条" — a final-frame ack still in flight # (final_content_delivered not yet set) while this send races # ahead. Log the decision inputs so a recurrence can be pinned to # "signal never set" vs "ack-pending race". # See docs/rca-wecom-stream-final-ack-timeout-duplicate.md. logger.warning( "Normal final-send NOT suppressed despite active stream " "consumer for session %s: streamed=%s previewed=%s " "content_delivered=%s transformed=%s final_len=%d — " "possible duplicate send (see wecom ack-timeout RCA).", session_key or "?", _streamed, _previewed, _content_delivered, _transformed, len(_final), ) # Schedule deletion of tracked temporary progress bubbles after the # final response lands. Failed runs skip this so bubbles remain as # breadcrumbs for the user to see what work happened. Only fires on # adapters that support ``delete_message`` (see init above); failures # are swallowed — deletion is best-effort. if ( _cleanup_progress and _cleanup_adapter is not None and _cleanup_msg_ids and session_key and isinstance(response, dict) and not response.get("failed") and hasattr(_cleanup_adapter, "register_post_delivery_callback") ): _ids_snapshot = list(_cleanup_msg_ids) _chat_id_snapshot = source.chat_id _adapter_snapshot = _cleanup_adapter _loop_snapshot = asyncio.get_running_loop() def _cleanup_temp_bubbles() -> None: async def _delete_all() -> None: for _mid in _ids_snapshot: try: await _adapter_snapshot.delete_message( _chat_id_snapshot, _mid ) except Exception: pass try: safe_schedule_threadsafe( _delete_all(), _loop_snapshot, logger=logger, log_message="Temp bubble cleanup scheduling error", ) except Exception: pass try: _cleanup_adapter.register_post_delivery_callback( session_key, _cleanup_temp_bubbles, generation=run_generation, ) except Exception as _rpe: logger.debug("Post-delivery cleanup registration failed: %s", _rpe) return response def _run_planned_stop_watcher( stop_event: threading.Event, runner, loop: asyncio.AbstractEventLoop, shutdown_handler, *, poll_interval: float = 0.5, ) -> None: """Poll for the planned-stop marker and trigger graceful shutdown. On Windows, ``asyncio.add_signal_handler`` raises NotImplementedError for SIGTERM/SIGINT, so the standard signal-driven shutdown path never runs when ``hermes gateway stop`` signals the gateway. The consequence is that the drain loop is skipped — in-flight agent sessions are killed mid-turn and ``resume_pending`` is never set, so the next gateway boot has no idea those sessions need to be auto-resumed (issue #33778, v0.13.0 session-resume feature broken on native Windows). This watcher runs on every platform (cheap, defensive) and bridges the gap on Windows by translating a filesystem marker into the same shutdown-handler invocation a real SIGTERM would have produced on POSIX. The CLI's ``hermes_cli.gateway_windows.stop()`` writes the marker via ``write_planned_stop_marker(pid)`` and then waits for the gateway PID to exit; this watcher is what makes that exit happen cleanly. On POSIX this is a no-op safety net — the signal handler always races us to consuming the marker file because it fires synchronously from the kernel's signal delivery. Args: stop_event: cleared by start_gateway() during normal shutdown to tell the watcher to exit. runner: the GatewayRunner instance; we check ``_running`` and ``_draining`` to avoid triggering shutdown if the gateway is already in one of those states. loop: the asyncio event loop the shutdown handler must run on. shutdown_handler: same callable that's wired to SIGTERM — tolerates a ``None`` signal argument (planned stop case) and consumes the marker via ``consume_planned_stop_marker_for_self()``. poll_interval: seconds between marker checks. 0.5s gives a responsive shutdown without burning CPU. """ from gateway.status import ( _get_planned_stop_marker_path, planned_stop_marker_targets_self, ) marker_path = _get_planned_stop_marker_path() while not stop_event.is_set(): try: if ( marker_path.exists() and not getattr(runner, "_draining", False) and getattr(runner, "_running", False) ): # A marker existing is NOT sufficient — it may have been # written for a PREVIOUS gateway instance (different PID) # and left behind because that process exited before the # CLI's stop() could clean it up. Firing the handler on a # stale/foreign marker drives the gateway into shutdown, # then consume_planned_stop_marker_for_self() correctly # reports a PID mismatch — but by then we're already # stopping, so it's logged as an unexpected "UNKNOWN" exit # and the watchdog crash-loops the gateway (issue #34597, # a regression from PR #33798 which added this watcher # without the PID check). # # Only fire when the marker actually targets us. The probe # is non-destructive on a match (the handler does the # authoritative consume on the loop thread) and self-heals # by unlinking stale/malformed markers so they cannot wedge # a freshly booted gateway. if not planned_stop_marker_targets_self(): stop_event.wait(poll_interval) continue # Drive the same path as a real signal handler. # Pass signal=None — the handler tolerates that and consumes # the marker via consume_planned_stop_marker_for_self, # which also validates target_pid + start_time match us. loop.call_soon_threadsafe(shutdown_handler, None) # Done — the handler will set _draining; we exit on next tick. break except Exception as _e: logger.debug("Planned-stop watcher tick error: %s", _e) stop_event.wait(poll_interval) def _drain_restart_safe_cron_deliveries(adapters, loop, runner=None) -> None: """Drain each profile's worker queue through its matching live adapters.""" from cron import scheduler as cron_scheduler if runner is None: if adapters is not None: cron_scheduler.drain_delivery_queue(adapters, loop) return for profile_name, profile_home in _handoff_watch_scopes(runner): scoped_home = profile_home or get_hermes_home() if profile_name is None: profile_adapters = adapters else: profile_adapters = getattr(runner, "_profile_adapters", {}).get( profile_name ) if profile_adapters is None: continue with _profile_runtime_scope(scoped_home): if profile_name is not None and not profile_adapters and adapters: routes = cron_scheduler._primary_profile_routes_for_current_home() if routes: profile_adapters = cron_scheduler.SharedRouteAdapters( adapters, routes ) cron_scheduler.drain_delivery_queue(profile_adapters, loop) def _start_gateway_housekeeping( stop_event: threading.Event, adapters=None, loop=None, interval: int = 60, cron_provider=None, runner=None, ): """Background thread for gateway-only periodic chores (NOT cron). Split out of the historical ``_start_cron_ticker`` so the cron *trigger* can live behind the ``CronScheduler`` provider (built-in or external) while these gateway-specific chores keep running independently of which provider fires cron. An external scale-to-zero provider has no 60s loop at all, but this housekeeping still wants its hourly cadence — so it owns its own loop. Refreshes the channel directory every 5 minutes and prunes the image/audio/video/document/screenshot caches + expired ``hermes debug share`` pastes once per hour, and polls the curator hourly (its inner gate enforces the real weekly cadence). """ from gateway.platforms.base import ( cleanup_audio_cache, cleanup_document_cache, cleanup_image_cache, cleanup_screenshot_cache, cleanup_video_cache, ) from tools.tool_result_storage import cleanup_spillover_cache from tools.environments.local import cleanup_terminal_temp_cache from tools.bot_mode_dm import cleanup_bot_dm_cache from tools.bot_relay import cleanup_bot_relay_artifacts from hermes_cli.debug import _sweep_expired_pastes IMAGE_CACHE_EVERY = 60 # ticks — once per hour at default 60s interval CHANNEL_DIR_EVERY = 5 # ticks — every 5 minutes PASTE_SWEEP_EVERY = 60 # ticks — once per hour CURATOR_EVERY = 60 # ticks — poll hourly (inner gate handles the real cadence) AUTO_ARCHIVE_EVERY = 60 # ticks — poll hourly (state_meta gate owns the real cadence) MEMORY_TRIM_EVERY = 1 # shared helper cooldown bounds actual allocator work MISFIRE_SWEEP_EVERY = 5 # ticks — every 5 minutes (grace window gates real work) FTS_STALE_RETRY_EVERY = 1 # SessionDB rate-limits the real work (_FTS_STALE_RETRY_SECONDS) # Every platform media cache prunes on the same hourly cadence — one loop # over (name, cleanup_fn), not a copy-pasted try/except per cache. MEDIA_CACHE_CLEANUPS = ( ("Image", cleanup_image_cache), ("Document", cleanup_document_cache), ("Audio", cleanup_audio_cache), ("Video", cleanup_video_cache), ("Screenshot", cleanup_screenshot_cache), ("Spillover", cleanup_spillover_cache), ("Terminal temp", cleanup_terminal_temp_cache), ("Bot DM", cleanup_bot_dm_cache), ("Bot relay", cleanup_bot_relay_artifacts), ) logger.info("Gateway housekeeping started (interval=%ds)", interval) tick_count = 0 while not stop_event.is_set(): tick_count += 1 # Restart-safe cron workers run outside the gateway cgroup and queue # their final send for whichever gateway instance is live. Drain on # the gateway-wide housekeeper rather than the built-in scheduler tick: # external providers do not run that ticker. if adapters is not None or runner is not None: try: _drain_restart_safe_cron_deliveries(adapters, loop, runner) except Exception as exc: logger.debug("Cron durable delivery queue drain error: %s", exc) if tick_count % CHANNEL_DIR_EVERY == 0 and adapters: try: from gateway.channel_directory import build_channel_directory if loop is not None: # build_channel_directory is async (Slack web calls), and # this runs in a background thread. Schedule onto the # gateway event loop and wait briefly for completion so # refresh failures are still logged via the except. fut = safe_schedule_threadsafe( build_channel_directory(adapters), loop, logger=logger, log_message="Channel directory refresh scheduling error", ) if fut is not None: fut.result(timeout=30) except Exception as e: logger.debug("Channel directory refresh error: %s", e) if tick_count % IMAGE_CACHE_EVERY == 0: for cache_name, cleanup_fn in MEDIA_CACHE_CLEANUPS: try: removed = cleanup_fn(max_age_hours=24) if removed: logger.info("%s cache cleanup: removed %d stale file(s)", cache_name, removed) except Exception as e: logger.debug("%s cache cleanup error: %s", cache_name, e) if tick_count % PASTE_SWEEP_EVERY == 0: try: deleted, remaining = _sweep_expired_pastes() if deleted: logger.info( "Paste sweep: deleted %d expired paste(s), %d pending", deleted, remaining, ) except Exception as e: logger.debug("Paste sweep error: %s", e) # Misfire catch-up (external cron providers only): fire jobs whose # scheduled time passed with no external fire delivered — the # backstop for a dead loopback fire hop (gateway restart window, # api_server not bound, scheduler retries exhausted). The helper # no-ops for the built-in ticker and enforces the # cron.misfire_grace_minutes window; the store CAS claim de-dupes # against a late external retry arriving concurrently. if cron_provider is not None and tick_count % MISFIRE_SWEEP_EVERY == 0: try: from cron.scheduler_provider import fire_overdue_jobs caught_up = fire_overdue_jobs( cron_provider, adapters=adapters, loop=loop ) if caught_up: logger.info( "Misfire catch-up: fired %d overdue job(s)", caught_up ) except Exception as e: logger.debug("Misfire catch-up sweep error: %s", e) # Curator — piggy-back on the housekeeping loop so long-running # gateways get weekly skill maintenance without needing restarts. # maybe_run_curator() is internally gated by config.interval_hours # (7 days by default), so CURATOR_EVERY is just the poll rate — the # real work only fires once per config interval. if tick_count % CURATOR_EVERY == 0: try: from agent.curator import maybe_run_curator maybe_run_curator( idle_for_seconds=float("inf"), on_summary=lambda msg: logger.info("curator: %s", msg), ) except Exception as e: logger.debug("Curator tick error: %s", e) # Skill Sync — best-effort periodic pull on the same cadence. # Inert unless the access gate is open and a sync base URL is # configured; never raises. try: from tools.skills_sync_client import maybe_pull_skills maybe_pull_skills() except Exception as e: logger.debug("Sync pull tick error: %s", e) # Org-shared skills. Gated on real org membership (the token must # carry an org role), so a solo account never reaches the network. try: from tools.skills_sync_client import maybe_pull_org_skills maybe_pull_org_skills() except Exception as e: logger.debug("Org sync pull tick error: %s", e) # Stale-session auto-archive — a live timer, so gateways that stay up # for weeks keep sweeping on schedule (the startup hook fires once). # maybe_auto_archive() is gated by sessions.min_interval_hours in # state_meta; this is just the poll rate. Opens its own SessionDB — # SQLite connections are thread-bound and this runs off-loop. if tick_count % AUTO_ARCHIVE_EVERY == 0: try: from hermes_cli.config import load_config as _load_full_config from hermes_state import get_shared_session_db, release_shared_session_db _sess_cfg = (_load_full_config().get("sessions") or {}) if _sess_cfg.get("auto_archive", False): _adb = get_shared_session_db() try: _adb.maybe_auto_archive( idle_days=float(_sess_cfg.get("auto_archive_days", 3)), min_interval_hours=int(_sess_cfg.get("min_interval_hours", 24)), ) finally: from hermes_state import release_or_close release_or_close(_adb) except Exception as e: logger.debug("Auto-archive tick error: %s", e) # Deferred stale-FTS rebuild retry (#100108). A SessionDB that opened # while another process held state.db / the rebuild lock fails closed # and leaves search on the LIKE fallback; a short-lived CLI clears # that on its next open, but the gateway opens once and stays up for # days. Retry here, on the existing tick, against the shared # instances this process already holds: non-blocking admission, no # new thread, rate-limited inside SessionDB. No-op when nothing is # stale (one attribute read per instance). if tick_count % FTS_STALE_RETRY_EVERY == 0: try: from hermes_state_registry import live_shared_session_dbs for _sdb in live_shared_session_dbs(): _retry = getattr(_sdb, "retry_deferred_fts_recovery", None) if callable(_retry) and _retry(): logger.info( "Deferred state.db FTS rebuild completed in-process " "for %s; full-text search restored.", getattr(_sdb, "db_path", "state.db"), ) except Exception as exc: logger.debug("Deferred FTS retry tick error: %s", exc) # This is the long-lived messaging-gateway counterpart to the TUI idle # reaper. The helper is config-gated and rate-limited, so calling it on # the 60s housekeeping cadence does not create a trim storm. if tick_count % MEMORY_TRIM_EVERY == 0: try: from hermes_cli.mem_trim import trim_memory trim_memory(reason="messaging gateway housekeeping") except Exception as exc: # debug, not warning: sibling housekeeping branches all log # failures at debug, and a persistent failure (e.g. broken # import after a partial update) would otherwise warn every # 60s forever. logger.debug( "gateway housekeeping memory trim failed: %s: %s", type(exc).__name__, exc, ) stop_event.wait(timeout=interval) logger.info("Gateway housekeeping stopped") def _start_cron_ticker(stop_event: threading.Event, adapters=None, loop=None, interval: int = 60): """DEPRECATED shim — preserved for backward compatibility. The cron trigger now lives behind the ``CronScheduler`` provider (``cron.scheduler_provider``); the gateway resolves a provider and runs its ``start()`` directly (see ``start_gateway``). This shim runs ONLY the built-in in-process tick loop, exactly as before, for any external caller or test that still references this symbol (e.g. hermes_cli/debug.py). It no longer runs gateway housekeeping — that moved to ``_start_gateway_housekeeping``. """ from cron.scheduler_provider import InProcessCronScheduler InProcessCronScheduler().start(stop_event, adapters=adapters, loop=loop, interval=interval) def _stop_cron_provider(provider) -> None: """Stop a cron provider without letting it choose the gateway exit code.""" try: provider.stop() except SystemExit as exc: logger.warning( "Cron provider stop() attempted to exit the gateway with code %s; ignoring", exc.code, ) except Exception as exc: logger.debug("Cron provider stop() error: %s", exc) # Upper bound for cooperatively draining the cron ticker on shutdown. The cron # thread delivers via ``safe_schedule_threadsafe`` and blocks on # ``future.result(timeout=60)`` (see cron/scheduler.py::_deliver_result), so a # single in-flight delivery unblocks within ~60s. The extra margin covers the # hop back through run_one_job's bookkeeping. _CRON_SHUTDOWN_DRAIN_TIMEOUT = 65.0 # Upper bound for cooperatively draining the housekeeping ticker on shutdown. # Housekeeping periodically refreshes the channel directory via # ``safe_schedule_threadsafe(build_channel_directory(...), loop)`` and blocks on # ``fut.result(timeout=30)`` (see ``_start_gateway_housekeeping``) — the same # loop-scheduled-future pattern as cron. So the cooperative bound must cover # that 30s future (plus margin) rather than the old 5s join, otherwise a # channel-directory refresh in flight at shutdown gets abandoned mid-resolve. # Unlike a dropped cron delivery this is not user-facing (it self-heals on the # next tick), but bounding it correctly keeps the drain honest. _HOUSEKEEPING_SHUTDOWN_DRAIN_TIMEOUT = 35.0 async def _await_thread_exit( thread: Optional[threading.Thread], timeout: float, poll: float = 0.1 ) -> bool: """Wait for a daemon thread to exit WITHOUT blocking the event loop. A synchronous ``thread.join()`` here would freeze the event loop — fatal for the cron ticker, whose in-flight delivery is a coroutine scheduled onto *this* loop via ``safe_schedule_threadsafe``. Blocking the loop deadlocks that delivery (the loop can never run it), so ``join(timeout=5)`` always times out and the message is silently dropped on restart (#58818). Polling ``is_alive()`` with ``await asyncio.sleep`` keeps the loop running so the pending delivery completes, then the ticker sees ``stop_event`` and exits. Returns True if the thread exited within ``timeout``. """ if thread is None: return True deadline = asyncio.get_running_loop().time() + max(0.0, timeout) while thread.is_alive() and asyncio.get_running_loop().time() < deadline: await asyncio.sleep(poll) return not thread.is_alive() async def _shutdown_mcp_servers_nonblocking(timeout: float = 5.0) -> bool: """Close MCP servers off-loop with a bounded wait (#82874). ``shutdown_mcp_servers()`` is synchronous and can block for its full internal 15s future wait when the MCP loop and its stdio children are torn down concurrently (every process in the tree gets SIGTERM at once under a container/supervisor stop). Calling it directly from the gateway event-loop thread freezes the loop for that whole window, so supervisors with a shorter kill grace (s6-overlay defaults to 3s) SIGKILL the gateway before ``lifecycle_ledger.mark_exited()`` runs and every subsequent boot reports a phantom unclean death. Run it on a daemon thread instead and poll with ``_await_thread_exit`` so the loop keeps servicing teardown. If it does not finish within ``timeout`` we proceed with shutdown; the daemon thread is left to finish (or die with the process) in the background. Returns True when the MCP shutdown completed within the budget. """ def _do() -> None: try: from tools.mcp_tool import shutdown_mcp_servers shutdown_mcp_servers() except Exception: logger.debug("MCP shutdown raised", exc_info=True) thread = threading.Thread(target=_do, name="mcp-shutdown", daemon=True) thread.start() done = await _await_thread_exit(thread, timeout=timeout) if not done: logger.warning( "MCP shutdown did not finish within %.1fs; continuing gateway " "teardown (background thread will be reaped at process exit)", timeout, ) return done def _shutdown_gateway_health_export(runner: Any) -> None: """Idempotently drain and detach Gateway Health OTLP export.""" runtime = getattr(runner, "_gateway_health_export_runtime", None) if runtime is None: return runner._gateway_health_export_runtime = None try: runtime.shutdown() except Exception: logger.debug("gateway health OTLP export shutdown failed", exc_info=True) def _gateway_stderr_formatter() -> logging.Formatter: """Return the redacting formatter used by the gateway stderr stream.""" from agent.redact import RedactingFormatter return RedactingFormatter("%(asctime)s %(levelname)s %(name)s: %(message)s") # ownership guard inserted below (PR #93084) def _replace_target_belongs_to_other_profile(existing_pid: int) -> bool: """Return True when ``--replace`` must refuse to signal ``existing_pid``. The PID file is HERMES_HOME-scoped, but a poisoned/stale record can point at another profile's LIVE gateway; signaling it starts the cross-profile SIGTERM restart loop this guard exists to prevent (#89315). This is a destructive-action authority check, so ownership is decided by the persisted identity record ALONE — exact ``_same_hermes_home`` equality — and only while that record stays bound to the live target by exact PID + start-time identity: * The authorizing record is whichever source produced the PID for this destructive decision (PID file, gateway lock record, or runtime-status fallback). A readable live argv carries no HERMES_HOME (it travels in the environment), so it can never prove home ownership; it is used only as an additional CONSISTENCY check — token-exact profile flags that clearly contradict our home refuse the signal even when the record agrees. * Missing, legacy, conflicting, stale-bound, or unprovable identity → refuse (fail closed). Same-home targets keep replacing normally; every refusal path here only narrows what the legacy start_time check alone used to allow. """ try: from gateway.status import ( _get_pid_path, _get_process_hermes_home, _get_process_start_time, _pid_from_record, _read_pid_record, _record_looks_like_gateway, _read_process_cmdline, _same_hermes_home, ) our_home = _get_process_hermes_home() # ── Authorize from the persisted identity record ────────────── # Bound claim: the record must describe THIS pid with THIS live # start time, otherwise it is stale/poisoned and proves nothing. record = _read_pid_record(_get_pid_path()) if not isinstance(record, dict) or not _record_looks_like_gateway(record): logger.warning( "Refusing --replace: no valid gateway pid record to prove " "ownership of PID %s.", existing_pid, ) return True record_pid = _pid_from_record(record) if record_pid != existing_pid: logger.warning( "Refusing --replace: pid record names %s, not target %s.", record_pid, existing_pid, ) return True recorded_start = record.get("start_time") if not isinstance(recorded_start, int) or isinstance(recorded_start, bool): return True if _get_process_start_time(existing_pid) != recorded_start: logger.warning( "Refusing --replace: pid record start-time does not match " "the live process %s (stale/PID-reuse record).", existing_pid, ) return True recorded_home = record.get("hermes_home") if not isinstance(recorded_home, str) or not recorded_home.strip(): # Legacy record without hermes_home cannot prove ownership. logger.warning( "Refusing --replace: pid record predates hermes_home " "stampings; ownership of PID %s unprovable.", existing_pid, ) return True if not _same_hermes_home(recorded_home, our_home): logger.error( "Refusing --replace: pid record belongs to a different " "HERMES_HOME (%s, ours %s). Remove the stale PID record or " "stop the owning profile explicitly.", recorded_home, our_home, ) return True # ── Readable-argv consistency check (never authority) ───────── # An explicit profile flag / HERMES_HOME= on the argv that clearly # contradicts our home refuses even though the record agreed; a bare # or matching argv adds nothing either way. try: live_cmdline = _read_process_cmdline(existing_pid) except Exception: live_cmdline = None # consistency probe failure → record decides if live_cmdline and _looks_like_profile_conflict_from_cmdline( live_cmdline, our_home ): logger.error( "Refusing --replace: target PID %s command line explicitly " "advertises a different profile than HERMES_HOME %s.", existing_pid, our_home, ) return True return False except Exception: # Destructive action + unknown ownership => fail closed (#89315). logger.warning( "cross-profile --replace ownership probe failed for PID %s; " "refusing to signal", existing_pid, exc_info=True, ) return True def _looks_like_profile_conflict_from_cmdline(command: str, our_home) -> bool: """Token-exact contradiction check between a target argv and our home. Authority lives in the pid record; this only catches argv that EXPLICITLY advertises a different profile than ours. Substring matching is not identity: ``--profile timothy`` must NOT read as profile ``tim``. Returns False whenever the argv does not clearly contradict our home. """ from gateway.status import _profile_name_for_home profile_name = _profile_name_for_home(our_home) try: tokens = shlex.split(command) except ValueError: tokens = command.split() def _flag_value(flag: str) -> Optional[str]: """Value of ``--flag X`` / ``--flag=X`` occurrences, token-exact.""" values = [] i = 0 while i < len(tokens): tok = tokens[i] if tok == flag and i + 1 < len(tokens): values.append(tokens[i + 1]) i += 2 continue if tok.startswith(flag + "="): values.append(tok[len(flag) + 1:]) i += 1 return values[-1] if values else None def _env_home_value() -> Optional[str]: """HERMES_HOME= env-style assignment on the argv, token-exact.""" prefix = "HERMES_HOME=" for tok in reversed(tokens): if tok.startswith(prefix): return tok[len(prefix):] return None if profile_name is not None and profile_name != "default": # Our home is a named profile: any explicit DIFFERENT named profile # on the argv contradicts it. Bare argv stays consistent (legacy # default-gateway argv never carried profile flags). for flag in ("--profile", "-p"): value = _flag_value(flag) if value is not None and value != profile_name: return True home_value = _flag_value("--hermes-home") or _env_home_value() if home_value is not None and os.path.normcase(os.path.normpath(home_value)) != os.path.normcase(os.path.normpath(str(our_home))): return True return False # Our home is the default/root: ANY explicit named-profile flag on the # argv contradicts it. if _flag_value("--profile") is not None or _flag_value("-p") is not None: return True home_value = _flag_value("--hermes-home") or _env_home_value() if home_value is not None and os.path.normcase(os.path.normpath(home_value)) != os.path.normcase(os.path.normpath(str(our_home))): return True return False async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = False, verbosity: Optional[int] = 0) -> bool: """ Start the gateway and run until interrupted. This is the main entry point for running the gateway. Returns True if the gateway ran successfully, False if it failed to start. A False return causes a non-zero exit code so systemd can auto-restart. Args: config: Optional gateway configuration override. replace: If True, kill any existing gateway instance before starting. Useful for systemd services to avoid restart-loop deadlocks when the previous process hasn't fully exited yet. """ # Enable interactive exec approval for dangerous commands on messaging # platforms. Set here (not at module import) so incidental imports of # gateway.run from CLI/tool code do not poison HERMES_EXEC_ASK. os.environ["HERMES_EXEC_ASK"] = "1" from hermes_cli.resource_limits import apply_nofile_soft_limit apply_nofile_soft_limit() # Snapshot the checkout revision now, while sys.modules still matches disk, # so a later `git pull` under this long-lived process can be detected (and # risky work like model switching refused) instead of crashing on a stale # in-memory module. from gateway.code_skew import record_boot_fingerprint record_boot_fingerprint() # ── Duplicate-instance guard ────────────────────────────────────── # Prevent two gateways from running under the same HERMES_HOME. # The PID file is scoped to HERMES_HOME, so future multi-profile # setups (each profile using a distinct HERMES_HOME) will naturally # allow concurrent instances without tripping this guard. from gateway.status import ( acquire_gateway_runtime_lock, get_running_pid, get_process_start_time, release_gateway_runtime_lock, remove_pid_file, terminate_pid, ) existing_pid = get_running_pid() if existing_pid is not None and existing_pid != os.getpid(): if replace: # Cross-profile ownership gate (#89315): never signal a live # process we cannot prove belongs to this HERMES_HOME. A poisoned # PID record steering --replace at another profile's gateway is # exactly the restart-loop shape this flow must not allow. if _replace_target_belongs_to_other_profile(existing_pid): from gateway.status import _get_process_hermes_home logger.error( "Refusing --replace: PID %d cannot be proven to belong " "to this profile's gateway (HERMES_HOME %s). Remove the " "stale PID record or stop the owning profile explicitly.", existing_pid, _get_process_hermes_home(), ) return False existing_start_time = get_process_start_time(existing_pid) logger.info( "Replacing existing gateway instance (PID %d) with --replace.", existing_pid, ) # Record a takeover marker so the target's shutdown handler # recognises its SIGTERM as a planned takeover and exits 0 # (rather than exit 1, which would trigger systemd's # Restart=on-failure and start a flap loop against us). # Best-effort — proceed even if the write fails. try: from gateway.status import write_takeover_marker write_takeover_marker(existing_pid) except Exception as e: logger.debug("Could not write takeover marker: %s", e) # Snapshot the old gateway's child processes BEFORE signalling it: # once it exits, orphans are reparented and can no longer be found # by a parent walk. On POSIX, adapter subprocesses that outlive # the gateway keep holding scoped token locks and block the # replacement (Windows terminate_pid(force=True) already # tree-kills via taskkill /T). Best-effort — [] on any failure. try: from gateway.status import _snapshot_gateway_children _old_gateway_children = _snapshot_gateway_children(existing_pid) except Exception: _old_gateway_children = [] try: terminate_pid(existing_pid, force=False) except ProcessLookupError: pass # Already gone except (PermissionError, OSError): logger.error( "Permission denied killing PID %d. Cannot replace.", existing_pid, ) # Marker is scoped to a specific target; clean it up on # give-up so it doesn't grief an unrelated future shutdown. try: from gateway.status import clear_takeover_marker clear_takeover_marker() except Exception: pass return False # Wait up to 10 seconds for the old process to exit. # ``os.kill(pid, 0)`` on Windows is NOT a no-op — use the # handle-based existence check instead. from gateway.status import _pid_exists old_gateway_exited = False for _ in range(20): if not _pid_exists(existing_pid): old_gateway_exited = True break # Process is gone # start_gateway is async: a blocking sleep here froze the # event loop (signal handlers, health checks, every other # coroutine) for up to 10s per replacement (#36163). await asyncio.sleep(0.5) else: # Still alive after 10s — force kill logger.warning( "Old gateway (PID %d) did not exit after SIGTERM, sending SIGKILL.", existing_pid, ) try: terminate_pid( existing_pid, force=True, expected_start_time=existing_start_time, ) except ProcessLookupError: old_gateway_exited = True except (PermissionError, OSError): pass # Confirm the force-kill actually reaped the process before we # clear its PID file / scoped locks. SIGKILL can fail to take # (e.g. an uninterruptible-sleep or zombie-reaping parent), and # if we blindly clear the metadata and start a fresh instance # we end up with two live gateways fighting over the same # token — the duplicate-gateway failure in #19471. if not old_gateway_exited: for _ in range(20): if not _pid_exists(existing_pid): old_gateway_exited = True break # Async context — never block the loop (#36163). await asyncio.sleep(0.25) if not old_gateway_exited: logger.error( "Old gateway (PID %d) still appears alive after SIGKILL; " "aborting replacement to avoid a duplicate gateway.", existing_pid, ) try: from gateway.status import clear_takeover_marker clear_takeover_marker() except Exception: pass return False # Old gateway confirmed dead — reap any orphaned child processes # it left behind (POSIX; mirrors Windows taskkill /T tree-kill). # Orphaned adapter subprocesses would otherwise keep holding # scoped token locks against us. Best-effort, never raises. try: from gateway.status import reap_gateway_children reap_gateway_children( _old_gateway_children, parent_pid=existing_pid ) except Exception: logger.debug( "Child reap for replaced gateway PID %d failed", existing_pid, exc_info=True, ) remove_pid_file() # remove_pid_file() is a no-op when the PID doesn't match. # Force-unlink to cover the old-process-crashed case. try: (get_hermes_home() / "gateway.pid").unlink(missing_ok=True) except Exception: pass # Clean up any takeover marker the old process didn't consume # (e.g. SIGKILL'd before its shutdown handler could read it). try: from gateway.status import clear_takeover_marker clear_takeover_marker() except Exception: pass # Also release all scoped locks left by the old process. # Stopped (Ctrl+Z) processes don't release locks on exit, # leaving stale lock files that block the new gateway from starting. try: from gateway.status import release_all_scoped_locks _released = release_all_scoped_locks( owner_pid=existing_pid, owner_start_time=existing_start_time, ) if _released: logger.info("Released %d stale scoped lock(s) from old gateway.", _released) except Exception: pass else: hermes_home = str(get_hermes_home()) logger.error( "Another gateway instance is already running (PID %d, HERMES_HOME=%s). " "Use 'hermes gateway restart' to replace it, or 'hermes gateway stop' first.", existing_pid, hermes_home, ) print( f"\n❌ Gateway already running (PID {existing_pid}).\n" f" Use 'hermes gateway restart' to replace it,\n" f" or 'hermes gateway stop' to kill it first.\n" f" Or use 'hermes gateway run --replace' to auto-replace.\n" ) return False # Sync bundled skills on gateway start (fast -- skips unchanged) try: from tools.skills_sync import sync_skills sync_skills(quiet=True) except Exception: pass # Centralized logging — agent.log (INFO+), errors.log (WARNING+), # and gateway.log (INFO+, gateway-component records only). # Idempotent, so repeated calls from AIAgent.__init__ won't duplicate. from hermes_logging import setup_logging, _safe_stderr setup_logging(hermes_home=_hermes_home, mode="gateway") # Startup security posture audit — warn-on-load, never blocks. Surfaces # root / weak-SSH / ephemeral-container / unauthenticated-listener posture # so operators get the "you're exposed" signal the June 2026 MCP-config # persistence campaign victims never had. try: from hermes_cli.security_audit_startup import log_startup_security_warnings _audit_cfg = None try: from hermes_cli.config import read_raw_config _audit_cfg = read_raw_config() except Exception: _audit_cfg = None log_startup_security_warnings(hermes_home=_hermes_home, config=_audit_cfg) except Exception as _audit_exc: logger.debug("Startup security audit failed (non-fatal): %s", _audit_exc) # Optional stderr handler — level driven by -v/-q flags on the CLI. # verbosity=None (-q/--quiet): no stderr output # verbosity=0 (default): WARNING and above # verbosity=1 (-v): INFO and above # verbosity=2+ (-vv/-vvv): DEBUG if verbosity is not None: _stderr_level = {0: logging.WARNING, 1: logging.INFO}.get(verbosity, logging.DEBUG) _stderr_handler = logging.StreamHandler(_safe_stderr()) _stderr_handler.setLevel(_stderr_level) _stderr_handler.setFormatter(_gateway_stderr_formatter()) logging.getLogger().addHandler(_stderr_handler) # Lower root logger level if needed so DEBUG records can reach the handler if _stderr_level < logging.getLogger().level: logging.getLogger().setLevel(_stderr_level) runner = GatewayRunner(config) # Multiplex: swap the launch-home file handlers for per-profile routers so # each profile's records land in its own logs/ (#82936). Must run after # the runner resolved the (possibly None) config and after setup_logging. _enable_multiplex_log_routing(runner.config) # ``--replace`` is explicit startup authority, not a durable reconnect # policy. GatewayRunner scopes this bit to cold adapter connects and clears # it before the background reconnect watcher starts. runner._platform_lock_takeover_on_start = bool(replace) # Track whether an unexpected signal initiated the shutdown. When an # unexpected SIGTERM kills the gateway, we exit non-zero so service # managers can revive the process. Planned stop paths write a marker # before signalling us so they can exit cleanly instead. _signal_initiated_shutdown = False # Set up signal handlers def shutdown_signal_handler(received_signal=None): nonlocal _signal_initiated_shutdown # Planned --replace takeover check: when a sibling gateway is # taking over via --replace, it wrote a marker naming this PID # before sending SIGTERM. If present, treat the signal as a # planned shutdown and exit 0 so systemd's Restart=on-failure # doesn't revive us (which would flap-fight the replacer when # both services are enabled, e.g. hermes.service + hermes- # gateway.service from pre-rename installs). planned_takeover = False try: from gateway.status import consume_takeover_marker_for_self planned_takeover = consume_takeover_marker_for_self() except Exception as e: logger.debug("Takeover marker check failed: %s", e) # Planned stop check: service managers and `hermes gateway stop` # also send SIGTERM, which is indistinguishable from an unexpected # external kill unless the CLI marks it first. SIGINT comes from an # interactive Ctrl+C and is likewise an intentional foreground stop. planned_stop = False if received_signal == signal.SIGINT: planned_stop = True elif not planned_takeover: try: from gateway.status import consume_planned_stop_marker_for_self planned_stop = consume_planned_stop_marker_for_self() except Exception as e: logger.debug("Planned stop marker check failed: %s", e) # Fast (<10ms) snapshot of who's asking us to shut down — runs # synchronously inside the asyncio signal handler, so we keep it # purely stdlib + /proc reads, no subprocesses. See PR #15826 # (May 2026): the previous implementation called `ps aux` here # synchronously, blocking the event loop for up to 3s while # adapter teardown couldn't begin. try: from gateway.shutdown_forensics import ( format_context_for_log, snapshot_shutdown_context, spawn_async_diagnostic, ) _shutdown_ctx = snapshot_shutdown_context(received_signal) except Exception as _e: _shutdown_ctx = None logger.debug("snapshot_shutdown_context failed: %s", _e) if planned_takeover: logger.info( "Received %s as a planned --replace takeover — exiting cleanly", _shutdown_ctx["signal"] if _shutdown_ctx else "SIGTERM", ) elif planned_stop: logger.info( "Received %s as a planned gateway stop — exiting cleanly", _shutdown_ctx["signal"] if _shutdown_ctx else "SIGTERM/SIGINT", ) else: _signal_initiated_shutdown = True # Mirror onto the runner so _stop_impl can suppress the # gateway_state=stopped persist for unexpected signals # (container/s6 SIGTERM on restart, OOM, bare kill) — see # issue #42675. Operator-initiated stops set a planned-stop # marker first, land in the `planned_stop` branch above, and # leave this flag False so they DO persist "stopped". runner._signal_initiated_shutdown = True logger.info( "Received %s — initiating shutdown", _shutdown_ctx["signal"] if _shutdown_ctx else "SIGTERM/SIGINT", ) # Always log who/what triggered the signal — most useful single # line when diagnosing "the gateway keeps dying" tickets. Format # is one line, key=value, parent_cmdline last (often long). if _shutdown_ctx is not None: try: logger.warning( "Shutdown context: %s", format_context_for_log(_shutdown_ctx) ) except Exception as _e: logger.debug("format_context_for_log failed: %s", _e) # Spawn the heavyweight diagnostic (ps auxf, pstree, dmesg) in # a detached subprocess so it can finish writing to disk even # if our cgroup is being torn down. Bounded by an internal # timeout; never blocks the event loop here. try: _diag_log = _hermes_home / "logs" / "gateway-shutdown-diag.log" spawn_async_diagnostic( _diag_log, _shutdown_ctx["signal"], timeout_seconds=5.0 ) except Exception as _e: logger.debug("spawn_async_diagnostic failed: %s", _e) asyncio.create_task(runner.stop()) def restart_signal_handler(): runner.request_restart(detached=False, via_service=True) loop = asyncio.get_running_loop() # Install a loop-level exception handler that swallows transient # network errors from background tasks. Issues #31066 / #31110: # an unhandled ``telegram.error.TimedOut`` (or peer NetworkError / # httpx connection error) in any awaited coroutine would propagate # to the loop and kill the gateway process, taking down every # profile attached to the same runner. systemd then restarts the # service after ~5s but the active conversation turn is lost. # # The fix is intentionally narrow: only well-known transient # network errors are swallowed (and logged with full traceback so # the originating call site is still discoverable). Anything else # is forwarded to the default handler so real bugs still surface. loop.set_exception_handler(_gateway_loop_exception_handler) if threading.current_thread() is threading.main_thread(): for sig in (signal.SIGINT, signal.SIGTERM): try: loop.add_signal_handler(sig, shutdown_signal_handler, sig) # windows-footgun: ok — wrapped in try/except NotImplementedError for Windows except NotImplementedError: pass if hasattr(signal, "SIGUSR1"): try: loop.add_signal_handler(signal.SIGUSR1, restart_signal_handler) # windows-footgun: ok — POSIX signal, guarded by hasattr above + try/except NotImplementedError except NotImplementedError: pass else: logger.info("Skipping signal handlers (not running in main thread).") # Windows fallback: asyncio.add_signal_handler raises NotImplementedError # on Windows, so `hermes gateway stop`'s SIGTERM (which Python maps to # TerminateProcess on Windows) never invokes shutdown_signal_handler. # That means the drain loop never runs, mark_resume_pending never fires, # and sessions are silently lost across restarts (issue #33778). # # The fix is a marker-polling thread: `hermes gateway stop` writes the # planned-stop marker BEFORE killing, and this thread notices it and # drives the same shutdown path the signal handler would have. Runs # on every platform (cheap, defensive) so non-signal-bearing # environments (Windows native, sandboxed CI runners that mask # SIGTERM) still get a clean drain. _planned_stop_watcher_stop = threading.Event() _planned_stop_watcher_thread = threading.Thread( target=_run_planned_stop_watcher, args=(_planned_stop_watcher_stop, runner, loop, shutdown_signal_handler), daemon=True, name="planned-stop-watcher", ) _planned_stop_watcher_thread.start() # Claim the PID file BEFORE bringing up any platform adapters. # This closes the --replace race window: two concurrent `gateway run # --replace` invocations both pass the termination-wait above, but # only the winner of the O_CREAT|O_EXCL race below will ever open # Telegram polling, Discord gateway sockets, etc. The loser exits # cleanly before touching any external service. import atexit from gateway.status import write_pid_file, remove_pid_file, get_running_pid _current_pid = get_running_pid() if _current_pid is not None and _current_pid != os.getpid(): logger.error( "Another gateway instance (PID %d) started during our startup. " "Exiting to avoid double-running.", _current_pid ) return False if not acquire_gateway_runtime_lock(): logger.error( "Gateway runtime lock is already held by another instance. Exiting." ) return False try: write_pid_file() except FileExistsError: release_gateway_runtime_lock() logger.error( "PID file race lost to another gateway instance. Exiting." ) return False atexit.register(remove_pid_file) atexit.register(release_gateway_runtime_lock) # Control socket (#92091 step 1) — the gateway-owned identify/status # surface. Started immediately after the PID-file claim: winning that # O_EXCL race is the moment this process becomes the authoritative # gateway for its HERMES_HOME, so from here on "does a socket answer?" # is a truthful liveness/identity query for updater and fleet consumers. # Strictly non-fatal: a bind failure only means consumers fall back to # the process-scan/state-file layer, exactly as before this feature. _control_server = None try: from gateway.control_socket import GatewayControlServer # pause-for-update (#92091 step 2): the updater asks this gateway to # drain in-flight turns and exit cleanly — releasing every venv file # handle — instead of being tree-killed mid-turn. Same drain path as # SIGUSR1/service restarts (request_restart(via_service=True)); the # updater (or the service manager) relaunches after the code swap. # The handler runs on the socket's executor thread, so the restart # request is marshalled onto the loop thread; the ACK returns the # drain budget so the caller knows how long to wait for exit. _main_loop = asyncio.get_running_loop() def _pause_for_update_handler() -> dict: try: from hermes_cli.gateway import _get_restart_drain_timeout _drain = float(_get_restart_drain_timeout()) except Exception: _drain = 30.0 accepted_box: list[bool] = [] _done = threading.Event() def _request() -> None: try: accepted_box.append( runner.request_restart(detached=False, via_service=True) ) finally: _done.set() _main_loop.call_soon_threadsafe(_request) _done.wait(timeout=5.0) accepted = bool(accepted_box and accepted_box[0]) return { "pausing": accepted, "already_stopping": not accepted, "pid": os.getpid(), "drain_timeout": _drain, } _control_server = GatewayControlServer( verb_handlers={"pause-for-update": _pause_for_update_handler} ) if not await _control_server.start(): _control_server = None else: atexit.register(_control_server.cleanup_files) except Exception as _cs_exc: logger.debug("Control socket startup failed (non-fatal): %s", _cs_exc) _control_server = None # Lifecycle ledger (NS-608): report if the previous gateway life died # uncleanly (SIGKILL / OOM / VM death — no exit path ran), then claim # the sentinel for this life. Placed after the PID-file/lock claim so # only the authoritative gateway for this HERMES_HOME touches the # sentinel — a --replace loser exiting above must not clobber it. try: from gateway.lifecycle_ledger import record_startup as _lifecycle_record_startup _lifecycle_record_startup() except Exception as _lc_exc: logger.debug("Lifecycle ledger startup record failed: %s", _lc_exc) try: from hermes_cli.nous_auth_keepalive import start_nous_auth_keepalive start_nous_auth_keepalive() except Exception as exc: logger.debug("Nous auth keepalive did not start: %s", exc) _ensure_windows_gateway_venv_imports() # MCP tool discovery — run in an executor so the asyncio event loop # stays responsive even when a configured MCP server is slow or # unreachable. discover_mcp_tools() uses a blocking 120s wait # internally; calling it from the loop thread would freeze platform # heartbeats (Discord shard, Telegram polling) until it returned. # See #16856. try: await _discover_gateway_mcp_tools(runner.config) except Exception as e: logger.debug("MCP tool discovery failed: %s", e) # Start the gateway try: success = await runner.start() except BaseException: _shutdown_gateway_health_export(runner) raise if not success: _shutdown_gateway_health_export(runner) return False # Recover any pending messages flushed during a previous shutdown (#72680). try: from gateway.shutdown_flush import recover_pending_to_db recovered = recover_pending_to_db() if recovered: logger.info( "Recovered %d pending message(s) from shutdown flush", recovered, ) except Exception: pass if runner.should_exit_cleanly: _shutdown_gateway_health_export(runner) if runner.exit_reason: logger.error("Gateway exiting cleanly: %s", runner.exit_reason) # A clean exit that carries an explicit exit code (e.g. a fatal # config error stamped with GATEWAY_FATAL_CONFIG_EXIT_CODE) must # propagate that code to the process so the s6 finish script can # translate it (78 → 125) and stop the supervisor restart loop. # Without this, the early `return True` below makes main() exit 0, # the finish script's `[ "$1" = "78" ]` check never matches, and # s6 crash-loops the gateway anyway (#51228). if runner.exit_code is not None: raise SystemExit(runner.exit_code) return True if not runner._running: # Startup was intentionally aborted by restart/shutdown before entering # running mode; preserve that lifecycle path without starting cron. try: await runner.wait_for_shutdown() if runner.should_exit_with_failure: if runner.exit_reason: logger.error("Gateway exiting with failure: %s", runner.exit_reason) return False try: await _shutdown_mcp_servers_nonblocking() except Exception: pass if runner.exit_code is not None: raise SystemExit(runner.exit_code) return True finally: _shutdown_gateway_health_export(runner) # Start the background cron scheduler via the resolved provider so # scheduled jobs fire automatically. The built-in provider is the # historical in-process 60s ticker; an external provider (e.g. chronos) # may arm a schedule and return. Pass the event loop so cron delivery can # use live adapters (E2EE support). from cron.scheduler_provider import ( InProcessCronScheduler, resolve_cron_scheduler, scheduler_for_profile_mode, ) cron_stop = threading.Event() multiplex_cron = bool(getattr(runner.config, "multiplex_profiles", False)) cron_provider = scheduler_for_profile_mode( resolve_cron_scheduler(), multiplex_profiles=multiplex_cron, ) cron_start_kwargs: Dict[str, Any] = {"adapters": runner.adapters, "loop": asyncio.get_running_loop()} # Multiplex profiles: tell the built-in ticker which profile homes to # tick so secondary-profile cron jobs actually fire (#69377). # Without this, only the process-global HERMES_HOME (default profile) # is iterated and every secondary profile's cron store is silently # ignored — jobs show as "scheduled" with a valid next_run_at but # never execute because no ticker owns that store. if ( isinstance(cron_provider, InProcessCronScheduler) and multiplex_cron ): try: profile_homes = _multiplex_profile_homes(runner.config) if profile_homes: cron_start_kwargs["profile_homes"] = profile_homes # Per-profile adapters so each profile's cron output is # delivered via its own bot/adapter instead of the default # profile's. cron_start_kwargs["profile_adapters"] = getattr( runner, "_profile_adapters", None ) # runner.adapters belongs to the default profile, which # profiles_to_serve() names "default" in its multiplex list. # Thread that identity so the ticker reserves the shared adapters # for the default profile alone and never routes a secondary's # cron through the default bot (even before its adapter connects, # when profile_adapters[name] is still absent/empty). cron_start_kwargs["default_profile"] = "default" logger.info( "Cron scheduler will tick %d profile(s) under multiplex: %s", len(profile_homes), [p[0] if isinstance(p, tuple) else p for p in profile_homes], ) except Exception as exc: logger.warning( "Could not resolve profile homes for multiplex cron: %s", exc, ) # External cron providers own their remote scheduling contract. Only the # in-process ticker polls local due jobs, so only it receives the local # external-drain dispatch gate. if isinstance(cron_provider, InProcessCronScheduler): cron_start_kwargs["can_dispatch"] = lambda: not ( runner._draining or runner._external_drain_active ) cron_thread = threading.Thread( target=cron_provider.start, args=(cron_stop,), kwargs=cron_start_kwargs, daemon=True, name="cron-scheduler", ) cron_thread.start() # Preflight tell for the hosted fire path: an external cron provider # (Chronos) delivers scheduled fires over HTTP to THIS process's # api_server adapter on loopback. If that adapter never came up (most # commonly API_SERVER_KEY missing from this process's environment — # e.g. a gateway relaunched outside its supervisor without the profile # env), every scheduled fire will fail with ConnectError at the # dashboard forwarder while manual runs keep working, which users # reliably misread as a job bug. Say it loudly ONCE at startup, when # it is fixable, instead of letting the first miss say it at 2am. if not isinstance(cron_provider, InProcessCronScheduler): try: _has_api_server = Platform.API_SERVER in (runner.adapters or {}) except Exception: _has_api_server = True # never let the tell break startup if not _has_api_server: logger.warning( "Cron provider '%s' is active but the api_server adapter is " "NOT running in this gateway — scheduled fires arrive over " "loopback HTTP and will all fail (jobs only run when " "triggered manually). Most common cause: API_SERVER_KEY is " "missing from this gateway process's environment. Restart " "the gateway through its supervisor (`hermes gateway " "restart`) so the profile env loads.", getattr(cron_provider, "name", "external"), ) # Gateway-only periodic housekeeping (channel dir, cache cleanup, paste # sweep, curator) — runs independently of which cron provider is active. # Shares cron_stop as the shutdown signal. housekeeping_thread = threading.Thread( target=_start_gateway_housekeeping, args=(cron_stop,), kwargs={ "adapters": runner.adapters, "loop": asyncio.get_running_loop(), "cron_provider": cron_provider, "runner": runner, }, daemon=True, name="gateway-housekeeping", ) housekeeping_thread.start() # READY is emitted only after adapters, cron, and housekeeping have all # reached their running boundary. Missing config/systemd runtime state # leaves the watchdog disabled without changing gateway behavior. start_watchdog = getattr(runner, "_start_systemd_watchdog", None) if callable(start_watchdog): start_watchdog() # Wait for shutdown await runner.wait_for_shutdown() # Stop the control socket first: once shutdown begins this process is no # longer a truthful "the gateway is serving here" answer, and a successor # (--replace / supervisor respawn) must be able to bind. Early-exit paths # above don't reach this; their process exit runs the atexit # cleanup_files hook, and a successor clears any stale socket on bind. if _control_server is not None: try: await _control_server.stop() except Exception: logger.debug("Control socket stop failed (non-fatal)", exc_info=True) try: from hermes_cli.nous_auth_keepalive import stop_nous_auth_keepalive stop_nous_auth_keepalive() except Exception: pass if runner.should_exit_with_failure: if runner.exit_reason: logger.error("Gateway exiting with failure: %s", runner.exit_reason) return False # Stop cron scheduler + housekeeping cleanly. # # These MUST be awaited cooperatively, not join()ed. A cron delivery in # flight when the gateway restarts is a coroutine scheduled onto THIS event # loop (safe_schedule_threadsafe); the ticker thread is blocked on its # future.result(). A synchronous cron_thread.join() would block the loop, # so that delivery could never run — it timed out and the message was # silently dropped (#58818). Awaiting keeps the loop alive so the in-flight # delivery finishes before we tear down. cron_stop.set() _stop_cron_provider(cron_provider) if not await _await_thread_exit(cron_thread, timeout=_CRON_SHUTDOWN_DRAIN_TIMEOUT): logger.warning( "Cron ticker did not exit within %.0fs of shutdown — an in-flight " "delivery may have been dropped.", _CRON_SHUTDOWN_DRAIN_TIMEOUT, ) await _await_thread_exit( housekeeping_thread, timeout=_HOUSEKEEPING_SHUTDOWN_DRAIN_TIMEOUT ) # Stop the planned-stop watcher (daemon=True so this is belt-and-suspenders). _planned_stop_watcher_stop.set() _planned_stop_watcher_thread.join(timeout=2) # Close MCP server connections (off-loop, bounded — #82874) try: await _shutdown_mcp_servers_nonblocking() except Exception: pass if runner.exit_code is not None: raise SystemExit(runner.exit_code) # When an unexpected SIGTERM caused the shutdown and it wasn't a planned # restart (/restart, /update, SIGUSR1), exit non-zero so systemd's # Restart=on-failure revives the process. This covers: # - hermes update killing the gateway mid-work # - External kill commands # - WSL2/container runtime sending unexpected signals # `hermes gateway stop` and interactive Ctrl+C are handled above as # planned stops and should not trigger service-manager revival. if _signal_initiated_shutdown and not runner._restart_requested: logger.info( "Exiting with code 1 (signal-initiated shutdown without restart " "request) so systemd Restart=on-failure can revive the gateway." ) return False # → sys.exit(1) in the caller # Older restart paths may reach here without ``runner.exit_code`` set. # Keep the historical non-zero fallback for service-managed restarts. if runner._restart_via_service: logger.info( "Exiting with code 75 (service-restart requested) so the service " "manager relaunches the gateway." ) raise SystemExit(75) return True def _guard_corrupt_user_config() -> None: """Fail closed when the active profile's config.yaml cannot be parsed. The gateway is a fully non-interactive surface: nobody is present to repair a corrupt ``config.yaml``, and silently continuing on built-in defaults lets provider auto-detection adopt credentials from ``.env`` that the config never named (issue #81952). Same policy and escape hatch (``HERMES_IGNORE_USER_CONFIG=1``) as the non-interactive CLI guard in ``hermes_cli/main.py``. """ from hermes_cli.config import ( InvalidUserConfigError, require_parseable_user_config, ) try: require_parseable_user_config() except InvalidUserConfigError as exc: print(f"Error: {exc}", file=sys.stderr) raise SystemExit(2) from exc def main(): """CLI entry point for the gateway.""" # Refuse to start on a corrupt config.yaml — before any config-dependent # startup (watchdog, DB opens, provider resolution). See _guard docstring. _guard_corrupt_user_config() # Advertise the agent harness to child processes (AI_AGENT is the # cross-agent standard; HERMES_AGENT the Hermes-specific marker — see # _advertise_agent_env in hermes_cli/main.py, kept inline here to avoid # importing that module's startup side effects). The value must equal our # public agent-harness registry id (``hermes-agent``) — standard-var # matching is exact. setdefault so an outer harness is never clobbered. os.environ.setdefault("AI_AGENT", "hermes-agent") os.environ.setdefault("HERMES_AGENT", "true") # Positive process identity: ledger registration + Windows job-object # self-attach, so update-time reapers can identify this gateway (and its # child tree dies with it on Windows). Best-effort — never blocks startup. try: from hermes_cli.process_identity import ( attach_self_to_kill_on_close_job, register_self, ) register_self("gateway") attach_self_to_kill_on_close_job() except Exception: pass # Startup-liveness watchdog (OOF-298): armed before config load, DB # opens, and the rest of pre-loop startup so a deadlock in that window # still gets the process respawned by the service supervisor instead of # wedging as a live-PID zombie. (Import-time coverage for the standard # ``hermes gateway run`` path is provided even earlier, by the argv # fast-path in hermes_cli.main.) Disarmed by GatewayRunner once the # event loop is confirmed live. try: from gateway.startup_watchdog import arm_startup_watchdog arm_startup_watchdog() except Exception: pass # Force UTF-8 stdio on Windows — gateway logs and startup banner would # otherwise UnicodeEncodeError on cp1252 consoles. No-op on POSIX. try: from hermes_cli.stdio import configure_windows_stdio configure_windows_stdio() except Exception: pass import argparse parser = argparse.ArgumentParser(description="Hermes Gateway - Multi-platform messaging") parser.add_argument("--config", "-c", help="Path to gateway config file") parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") args = parser.parse_args() config = None if args.config: import yaml with open(args.config, encoding="utf-8") as f: data = yaml.safe_load(f) or {} config = GatewayConfig.from_dict(data) # start_gateway() performs the full graceful teardown (adapters # disconnected, sessions saved + flushed, SQLite closed, cron/MCP stopped, # PID file + runtime lock released) before it returns OR raises SystemExit # with an explicit code. Force-exit afterwards so a wedged non-daemon worker # thread (e.g. a ThreadPoolExecutor tool/LLM call blocked with no timeout) # cannot block interpreter finalization (Py_FinalizeEx joins all non-daemon # threads, incl. concurrent.futures' _python_exit) and strand the gateway # half-shut down with the supervisor unable to restart it (#53107). # # SystemExit is caught explicitly: start_gateway raises it on the # clean-fatal-config (#51228), planned-restart, and service-restart paths, # all of which complete teardown first. Routing those codes through the # same os._exit backstop means EVERY exit path is wedge-proof, not just the # boolean-return ones. try: success = asyncio.run(start_gateway(config)) exit_code = 0 if success else 1 except SystemExit as e: # e.code may be None (→ 0), an int, or a str (→ 1, like CPython). if e.code is None: exit_code = 0 elif isinstance(e.code, int): exit_code = e.code else: exit_code = 1 _exit_after_graceful_shutdown(exit_code) def _exit_after_graceful_shutdown(exit_code: int) -> None: """Flush stdio, release the PID file + runtime lock, then hard-exit. Graceful teardown is already complete by the time this runs, so there is nothing left that needs a clean interpreter shutdown. We deliberately use ``os._exit`` (not ``sys.exit``): ``sys.exit`` raises ``SystemExit``, which triggers ``Py_FinalizeEx`` → ``wait_for_thread_shutdown`` and joins every non-daemon thread — exactly the hang (#53107) a wedged tool-worker causes. ``os._exit`` bypasses ``atexit`` handlers, so we cannot rely on the ``atexit``-registered ``remove_pid_file`` / ``release_gateway_runtime_lock`` (registered in ``start_gateway``) to run. The full-shutdown path releases both explicitly in ``_stop_impl``, but the EARLY exit paths — clean-fatal-config (#51228) and startup-aborted-before-running — raise ``SystemExit`` right after ``runner.start()`` without going through ``_stop_impl``, so on those paths ``atexit`` was the only thing releasing them. Now that those paths are routed through this backstop (#53107), release both here explicitly. Both calls are idempotent — ``remove_pid_file`` only unlinks a PID file that belongs to this process, and ``release_gateway_runtime_lock`` no-ops when the lock is already released — so this is a no-op on the normal shutdown path and the actual cleanup on the early-exit paths. Logging IS drained here: the rotating file handlers are driven by an async ``QueueListener`` on a dedicated thread (see ``hermes_logging._register_queued_handler``), so records emitted right before shutdown may still be sitting in the in-memory queue. ``os._exit`` below bypasses ``atexit``, so the ``atexit``-registered listener drain never runs on this path — we drain explicitly (bounded, via ``drain_log_queue``) or lose the last log lines (including the shutdown reason on the early-exit paths). Stdio is flushed too. """ for stream in (sys.stdout, sys.stderr): try: stream.flush() except Exception: pass # Release PID + runtime lock BEFORE the log drain: the drain is bounded but # could still take up to its timeout on a wedged disk, and these locks must # never be stranded. os._exit skips atexit, and the early SystemExit exit # paths never run _stop_impl, so release here (idempotent). try: from gateway.status import remove_pid_file, release_gateway_runtime_lock remove_pid_file() release_gateway_runtime_lock() except Exception: pass # Mark this life cleanly exited in the lifecycle sentinel (NS-608). This # is the single funnel every graceful exit passes through, so the next # boot's unclean-death detector only fires for genuine SIGKILL/OOM/VM # deaths. Ownership-guarded internally: a --replace old life won't # clobber the replacement's freshly claimed "running" sentinel. try: from gateway.lifecycle_ledger import mark_exited mark_exited(exit_code, reason="graceful_shutdown") except Exception: pass # Drain the async log queue: os._exit bypasses atexit, so the listener's # atexit drain won't fire. Use drain_log_queue() (bounded, no restart), NOT # flush_log_queue(): if the listener is wedged on the rotation lock — the # exact failure this async-logging change survives — an unbounded stop() # join would re-freeze the shutdown. drain_log_queue() no-ops when logging # never initialized a queue (very early aborts), so this is always safe. try: from hermes_logging import drain_log_queue drain_log_queue(timeout=1.0) except Exception: pass os._exit(exit_code) if __name__ == "__main__": main()