"""Out-of-loop shutdown and event-loop liveness backstops (#66892, #69089).

When the asyncio loop freezes mid-drain, every asyncio-based recovery path is
structurally unable to fire: the drain deadline, status rewrites, and forensics
all need the same loop that is stuck. launchd/systemd KeepAlive only restarts a
*dead* process, so a wedged-but-alive gateway sits as a zombie until manual
SIGKILL.

This module provides:

1. A plain OS-thread shutdown watchdog armed at ``stop()``. If shutdown has not
   completed within ``restart_drain_timeout + grace``, it dumps all-thread
   stacks via ``faulthandler`` plus a metadata snapshot, then ``os._exit`` so
   the service manager can revive the process.
2. An event-loop heartbeat file at ``<HERMES_HOME>/state/gateway.heartbeat`` so
   external supervision can distinguish "process alive" from "loop frozen"
   (``gateway_state.json`` alone can't — it only rewrites on transitions/turns).
3. A lifetime thread watchdog that can still diagnose and hard-exit when the
   event loop is too frozen to run its own heartbeat or timeout callbacks.
4. A self-rescheduling floor timer that keeps the loop selector's timeout
   finite, giving existing async recovery tasks a chance to resume.
"""

from __future__ import annotations

import asyncio
import faulthandler
import json
import logging
import os
import sys
import threading
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable, Dict, Optional

from gateway.restart import GATEWAY_SERVICE_RESTART_EXIT_CODE
from hermes_constants import get_hermes_home
from utils import atomic_json_write

logger = logging.getLogger(__name__)

# Extra leash beyond ``agent.restart_drain_timeout`` so a slow-but-progressing
# drain is not cut short. Matches the issue #66892 suggested hardening.
DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S = 60.0
DEFAULT_HEARTBEAT_INTERVAL_S = 30.0
DEFAULT_LOOP_FLOOR_TIMER_INTERVAL_S = 5.0
DEFAULT_LOOP_WATCHDOG_INTERVAL_S = 30.0
DEFAULT_LOOP_WATCHDOG_TIMEOUT_S = 10.0
# 3 sustained misses (~90-120s of loop block) escalate. The false-positive
# class that motivated raising this (the watchdog's own on-loop heartbeat
# fsync stalling the loop it monitors) is fixed at the root by the off-loop
# heartbeat write + two-witness probe (#90502), so the default stays tight
# for genuine wedges. Deployments with legitimately slow loops can tune via
# gateway.loop_watchdog_* in config.yaml.
DEFAULT_LOOP_WATCHDOG_MAX_STRIKES = 3
_HEARTBEAT_RELATIVE = ("state", "gateway.heartbeat")
_WATCHDOG_DUMP_RELATIVE = ("logs", "gateway-shutdown-watchdog.log")


class _LoopFloorTimerHandle:
    """Cancelable owner for the currently scheduled selector floor timer."""

    def __init__(self, loop: asyncio.AbstractEventLoop, interval: float):
        self._loop = loop
        self._interval = interval
        self._cancelled = False
        self._timer: Optional[asyncio.TimerHandle] = None
        self._schedule()

    def _schedule(self) -> None:
        self._timer = self._loop.call_later(self._interval, self._tick)

    def _tick(self) -> None:
        if not self._cancelled:
            self._schedule()

    def cancel(self) -> None:
        self._cancelled = True
        if self._timer is not None:
            self._timer.cancel()


class _LoopLivenessWatchdogHandle:
    """Small lifecycle handle for the daemon liveness thread."""

    def __init__(self, stop_event: threading.Event, thread: threading.Thread):
        self._stop_event = stop_event
        self._thread = thread

    def stop(self) -> None:
        self._stop_event.set()

    def join(self, timeout: Optional[float] = None) -> None:
        self._thread.join(timeout=timeout)

    def is_alive(self) -> bool:
        return self._thread.is_alive()


def _arm_loop_floor_timer(
    loop: asyncio.AbstractEventLoop,
    interval: float = DEFAULT_LOOP_FLOOR_TIMER_INTERVAL_S,
) -> _LoopFloorTimerHandle:
    """Keep at least one timer pending so selector waits remain bounded."""
    try:
        resolved_interval = float(interval)
        if resolved_interval <= 0:
            raise ValueError
    except (TypeError, ValueError):
        resolved_interval = DEFAULT_LOOP_FLOOR_TIMER_INTERVAL_S
    return _LoopFloorTimerHandle(loop, resolved_interval)


def start_loop_liveness_watchdog(
    loop: asyncio.AbstractEventLoop,
    *,
    probe_interval: float = DEFAULT_LOOP_WATCHDOG_INTERVAL_S,
    probe_timeout: float = DEFAULT_LOOP_WATCHDOG_TIMEOUT_S,
    max_strikes: int = DEFAULT_LOOP_WATCHDOG_MAX_STRIKES,
    exit_code: int = GATEWAY_SERVICE_RESTART_EXIT_CODE,
) -> Optional[_LoopLivenessWatchdogHandle]:
    """Start an out-of-loop watchdog that hard-exits after missed probes.

    The guard is on by default; operators opt out with
    ``gateway.loop_watchdog: false`` in config.yaml (enforced by the caller,
    ``GatewayRunner._start_loop_liveness_guards`` — this module stays
    config-agnostic so bare-loop tests can drive it directly).
    """
    interval = probe_interval
    timeout = probe_timeout
    strikes_limit = max_strikes
    stop_event = threading.Event()

    def _wait_for_probe(probe_event: threading.Event) -> Optional[bool]:
        deadline = time.monotonic() + timeout
        while True:
            if stop_event.is_set():
                return None
            remaining = deadline - time.monotonic()
            if remaining <= 0:
                return probe_event.is_set()
            if probe_event.wait(timeout=min(remaining, 0.05)):
                return True

    def _watchdog() -> None:
        strikes = 0
        while not stop_event.wait(timeout=interval):
            probe_event = threading.Event()
            try:
                loop.call_soon_threadsafe(probe_event.set)
            except RuntimeError:
                # A normally closed loop cannot be probed and no longer needs
                # a process-liveness backstop.
                return
            except Exception:
                logger.debug(
                    "Failed to schedule gateway loop liveness probe", exc_info=True
                )
                return

            responded = _wait_for_probe(probe_event)
            if responded is None:
                return
            if responded:
                strikes = 0
                continue

            if stop_event.is_set():
                return
            strikes += 1
            if strikes < strikes_limit:
                continue

            if stop_event.is_set():
                return
            try:
                logger.critical(
                    "Gateway event loop missed %d consecutive liveness probes; "
                    "dumping all thread stacks and exiting with code %d so the "
                    "service supervisor can restart it.",
                    strikes,
                    exit_code,
                )
            except Exception:
                pass
            try:
                faulthandler.dump_traceback(all_threads=True)
            except Exception:
                logger.debug("Loop liveness faulthandler dump failed", exc_info=True)
            if stop_event.is_set():
                return
            # Record the watchdog exit in the lifecycle sentinel so the next
            # boot reports "watchdog hard-exit" instead of misclassifying
            # this as an unclean SIGKILL/OOM death (NS-608).
            try:
                from gateway.lifecycle_ledger import mark_exited
                mark_exited(exit_code, reason="loop_liveness_watchdog")
            except Exception:
                pass
            os._exit(exit_code)
            return

    thread = threading.Thread(
        target=_watchdog,
        daemon=True,
        name="gateway-loop-liveness-watchdog",
    )
    try:
        thread.start()
    except Exception:
        logger.debug("Failed to start gateway loop liveness watchdog", exc_info=True)
        return None
    return _LoopLivenessWatchdogHandle(stop_event, thread)


def _process_hermes_home() -> Path:
    """HERMES_HOME for process-level identity files (ignore profile overrides)."""
    val = os.environ.get("HERMES_HOME", "").strip()
    if val:
        return Path(val)
    return get_hermes_home()


def get_loop_heartbeat_path(home: Optional[Path] = None) -> Path:
    """Return ``<HERMES_HOME>/state/gateway.heartbeat``."""
    base = home if home is not None else _process_hermes_home()
    return base.joinpath(*_HEARTBEAT_RELATIVE)


def get_loop_tick_socket_path(
    home: Optional[Path] = None, pid: Optional[int] = None
) -> Path:
    """Return the loop-scheduling witness socket for ``pid``.

    ``<HERMES_HOME>/state/gateway.loop-tick.<pid>.sock`` — PID-suffixed so a
    leftover node from a previous process can never be mistaken for this
    gateway's witness. Served by the gateway loop itself (see
    ``_tick_socket_handler``): an answer is direct proof that the loop is
    dispatching, which is exactly the property the heartbeat file lost when
    its write moved off-loop (#90502).
    """
    base = home if home is not None else _process_hermes_home()
    return base.joinpath(
        "state", f"gateway.loop-tick.{int(pid if pid is not None else os.getpid())}.sock"
    )


def get_shutdown_watchdog_dump_path(home: Optional[Path] = None) -> Path:
    """Return the faulthandler / metadata dump path for a fired watchdog."""
    base = home if home is not None else _process_hermes_home()
    return base.joinpath(*_WATCHDOG_DUMP_RELATIVE)


def write_loop_heartbeat(
    *,
    pid: Optional[int] = None,
    start_time: Optional[float] = None,
    home: Optional[Path] = None,
    extra: Optional[Dict[str, Any]] = None,
) -> Path:
    """Atomically rewrite the loop-liveness heartbeat file.

    ``start_time`` is the gateway process start (``time.time()`` epoch seconds)
    so supervisors can detect PID reuse. Best-effort — never raises.
    """
    path = get_loop_heartbeat_path(home)
    payload: Dict[str, Any] = {
        "pid": int(pid if pid is not None else os.getpid()),
        "updated_at": datetime.now(timezone.utc).isoformat(),
        "monotonic": time.monotonic(),
    }
    if start_time is not None:
        payload["start_time"] = float(start_time)
    # Embed a cheap memory sample (own RSS + MemAvailable + swap) so the
    # heartbeat doubles as a rolling pre-death telemetry snapshot: after an
    # unclean death (SIGKILL/OOM/VM loss) the last heartbeat is the closest
    # surviving record of memory pressure — see gateway.lifecycle_ledger
    # (NS-608).  Best-effort; <1ms of /proc reads on Linux, {} elsewhere.
    try:
        from gateway.lifecycle_ledger import sample_memory

        mem = sample_memory()
        if mem:
            payload["mem"] = mem
    except Exception:
        pass
    if extra:
        payload.update(extra)
    try:
        atomic_json_write(path, payload, indent=None)
    except Exception:
        logger.debug("Failed to write gateway loop heartbeat", exc_info=True)
    return path


def resolve_shutdown_watchdog_delay(
    drain_timeout: float,
    *,
    grace_s: float = DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S,
) -> float:
    """Return the wall-clock leash for the shutdown watchdog thread."""
    try:
        drain = max(float(drain_timeout), 0.0)
    except (TypeError, ValueError):
        drain = 0.0
    try:
        grace = max(float(grace_s), 0.0)
    except (TypeError, ValueError):
        grace = DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S
    return drain + grace


def _write_watchdog_dump(
    dump_path: Path,
    *,
    delay_s: float,
    snapshot: Optional[Dict[str, Any]],
) -> None:
    """Best-effort faulthandler + metadata dump before hard-exit."""
    try:
        dump_path.parent.mkdir(parents=True, exist_ok=True)
    except OSError:
        return

    header = {
        "event": "shutdown_watchdog_fired",
        "pid": os.getpid(),
        "delay_s": delay_s,
        "fired_at": datetime.now(timezone.utc).isoformat(),
        "snapshot": snapshot or {},
    }
    try:
        with open(dump_path, "a", encoding="utf-8") as fh:
            fh.write(json.dumps(header, default=str) + "\n")
            fh.write("--- faulthandler dump (all threads) ---\n")
            fh.flush()
            try:
                faulthandler.dump_traceback(file=fh, all_threads=True)
            except Exception:
                fh.write("(faulthandler.dump_traceback failed)\n")
            fh.write("--- end dump ---\n")
            fh.flush()
    except Exception:
        pass

    # Also dump to stderr so journald/launchd capture it even if the file
    # write failed (wedged disk was one of the #66892 hypotheses).
    try:
        sys.stderr.write(
            f"Gateway shutdown watchdog fired after {delay_s:.0f}s "
            f"(pid={os.getpid()}); dumping all thread stacks.\n"
        )
        sys.stderr.flush()
        faulthandler.dump_traceback(all_threads=True)
    except Exception:
        pass


def arm_shutdown_watchdog(
    delay_s: float,
    *,
    done_event: Optional[threading.Event] = None,
    snapshot_fn: Optional[Callable[[], Dict[str, Any]]] = None,
    exit_code: int = 1,
    dump_path: Optional[Path] = None,
    name: str = "gateway-shutdown-watchdog",
) -> threading.Event:
    """Arm a daemon-thread hard-exit backstop for a wedged shutdown path.

    If ``done_event`` is set before ``delay_s`` elapses, the thread exits
    quietly (normal / progressing shutdown completed). Otherwise it dumps
    diagnostics and calls ``os._exit(exit_code)``.

    Never raises. Returns the ``done_event`` (creating one when omitted) so
    the caller can disarm on successful completion.
    """
    done = done_event if done_event is not None else threading.Event()
    try:
        delay = max(float(delay_s), 0.0)
    except (TypeError, ValueError):
        delay = DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S

    if delay <= 0:
        return done

    def _watchdog() -> None:
        # Wait with interruptible chunks so a late disarm doesn't need the
        # full remaining sleep to observe done_event.
        deadline = time.monotonic() + delay
        while time.monotonic() < deadline:
            remaining = deadline - time.monotonic()
            if done.wait(timeout=min(remaining, 1.0)):
                return
        if done.is_set():
            return

        snapshot: Optional[Dict[str, Any]] = None
        if snapshot_fn is not None:
            try:
                snapshot = snapshot_fn()
            except Exception as exc:
                snapshot = {"snapshot_error": repr(exc)}

        target = dump_path if dump_path is not None else get_shutdown_watchdog_dump_path()
        _write_watchdog_dump(target, delay_s=delay, snapshot=snapshot)

        try:
            logger.critical(
                "Shutdown watchdog fired after %.0fs — forcing process exit "
                "(asyncio drain path appears wedged; see %s)",
                delay,
                target,
            )
        except Exception:
            pass

        for stream in (sys.stdout, sys.stderr):
            try:
                stream.flush()
            except Exception:
                pass
        # Mirror _exit_after_graceful_shutdown: release PID file + runtime
        # lock BEFORE the log drain (locks must never be stranded), then
        # drain the async log queue so the logger.critical above actually
        # reaches the file before os._exit bypasses atexit. (#66892)
        try:
            from gateway.status import remove_pid_file, release_gateway_runtime_lock
            remove_pid_file()
            release_gateway_runtime_lock()
        except Exception:
            pass
        try:
            from hermes_logging import drain_log_queue
            drain_log_queue(timeout=1.0)
        except Exception:
            pass
        # Record the watchdog exit so the next boot's unclean-death detector
        # reports "shutdown watchdog fired" instead of SIGKILL/OOM (NS-608).
        try:
            from gateway.lifecycle_ledger import mark_exited
            mark_exited(exit_code, reason="shutdown_watchdog")
        except Exception:
            pass
        os._exit(exit_code)

    try:
        threading.Thread(target=_watchdog, daemon=True, name=name).start()
    except Exception:
        logger.debug("Failed to arm shutdown watchdog", exc_info=True)
    return done


async def _tick_socket_handler(
    reader: asyncio.StreamReader, writer: asyncio.StreamWriter
) -> None:
    """Answer a liveness ping with one byte.

    Runs on the gateway loop: the reply is produced only while the loop is
    actually dispatching, so a successful read is a witness of loop
    schedulability that no executor thread and no filesystem stall can
    refresh. A UNIX-socket write is a socket-buffer copy — no fsync, no
    disk I/O — so the witness keeps working on the exact filesystem that
    stalls the heartbeat write. Best-effort; never raises.
    """
    try:
        writer.write(b"1")
        await writer.drain()
    except Exception:
        pass
    finally:
        try:
            writer.close()
        except Exception:
            pass


async def loop_heartbeat_forever(
    *,
    interval_s: float = DEFAULT_HEARTBEAT_INTERVAL_S,
    start_time: Optional[float] = None,
    home: Optional[Path] = None,
    should_continue: Optional[Callable[[], bool]] = None,
) -> None:
    """Rewrite the loop heartbeat file on a cadence until cancelled / gated off.

    Runs as an asyncio task on the gateway loop — if the loop freezes, this task
    stops and the file mtime/updated_at goes stale for external monitors. That
    property is load-bearing and is preserved below: the write is still
    *initiated* by the loop, so a frozen loop still lets the file age.

    The write itself is handed to a thread, because it is not free. It ends in
    ``atomic_json_write`` -> ``os.fsync``, and on a filesystem that stalls, that
    fsync blocks whatever thread runs it. Doing it inline meant the loop-liveness
    watchdog's own heartbeat could block the loop it exists to monitor: the probe
    times out at ``DEFAULT_LOOP_WATCHDOG_TIMEOUT_S`` (10s) and gives up after
    ``DEFAULT_LOOP_WATCHDOG_MAX_STRIKES`` (3), a ~90-120s budget, while a WSL2
    VHDX under io pressure was measured stalling a trivial stat-and-fsync probe
    at p99 31s and max 112s. So the watchdog killed the loop for being
    unresponsive at the moment it was blocked inside the watchdog's own write.

    Awaited, not fire-and-forget: an unawaited task would keep the file fresh
    while the loop was wedged, which is exactly the signal the docstring above
    promises. And a single in-flight write at a time, so a 112s stall cannot pile
    up one queued thread per interval behind it.

    Because the write is now off-loop, file freshness is no longer *proof* of
    loop schedulability: a stalled write or a saturated executor can age the file
    while the loop runs, and a write that lands after the loop froze can keep it
    fresh. The file therefore stops being sufficient authority on its own. This
    task also arms a loop-scheduling witness — a UNIX socket answered by the
    loop itself (``_tick_socket_handler``) — and records whether it is armed in
    the heartbeat payload (``loop_tick_socket``). External probes must require
    the witness to agree with file staleness before classifying a loop as
    wedged; see ``hermes_cli.gateway.probe_gateway_loop_liveness`` for the
    two-witness contract.
    """
    try:
        interval = max(float(interval_s), 1.0)
    except (TypeError, ValueError):
        interval = DEFAULT_HEARTBEAT_INTERVAL_S

    # Arm the loop-scheduling witness. Best-effort: a failed bind (permissions,
    # path length) must not abort the gateway or the file heartbeat — it only
    # disables the witness, and the payload flag tells probes that staleness is
    # no longer sufficient authority to escalate.
    #
    # Windows (non-POSIX generally): asyncio AF_UNIX support is POSIX-only, so
    # the AF_UNIX arm below is gated to POSIX — an ungated call raised
    # AttributeError on every native-Windows gateway start (#96956). Instead of
    # leaving the witness permanently absent there, the non-POSIX arm binds a
    # TCP loopback server on 127.0.0.1 with an OS-assigned port and publishes
    # the port in the heartbeat payload (``loop_tick_tcp_port``) so probes know
    # where to connect. Same protocol, same loop-owned semantics. If that bind
    # fails, the payload records loop_tick_socket=False and probes classify
    # UNKNOWN, never WEDGED — the graceful-drain backstop stays in place. (WSL2
    # — the #90502 incident environment — is Linux and arms the socket.)
    tick_server = None
    tick_socket_path = None
    tick_tcp_port = None
    try:
        if os.name == "posix":
            tick_socket_path = get_loop_tick_socket_path(home)
            tick_socket_path.parent.mkdir(parents=True, exist_ok=True)
            # Re-bind over a leftover node from a dead process (os._exit(75) /
            # SIGKILL skip the finally-unlink; PID reuse re-lands on this
            # PID-suffixed path) is handled by asyncio itself:
            # create_unix_server os.remove()s an existing socket node before
            # binding — guarded by test_producer_rebinds_over_stale_socket_node.
            # What asyncio does NOT do is clean up SIBLING nodes from other
            # dead PIDs, so sweep those to keep state/ from accumulating
            # gateway.loop-tick.*.sock nodes across crash-restart cycles.
            # POSIX-only: os.kill(pid, 0) is a liveness probe here, but on
            # Windows os.kill calls TerminateProcess for non-CTRL signals —
            # and AF_UNIX server nodes are never created there anyway.
            try:
                for _stale in tick_socket_path.parent.glob(
                    "gateway.loop-tick.*.sock"
                ):
                    if _stale == tick_socket_path:
                        continue
                    try:
                        _stale_pid = int(_stale.name.split(".")[-2])
                    except (ValueError, IndexError):
                        _stale.unlink(missing_ok=True)
                        continue
                    try:
                        os.kill(_stale_pid, 0)  # windows-footgun: ok — inside os.name == "posix" gate
                    except OSError:
                        _stale.unlink(missing_ok=True)
            except Exception:
                logger.debug(
                    "stale loop-tick socket sweep failed", exc_info=True
                )
            tick_server = await asyncio.start_unix_server(
                _tick_socket_handler, path=str(tick_socket_path)
            )
        else:
            # Windows / non-POSIX: no AF_UNIX support, so use a TCP loopback
            # server on 127.0.0.1 as the loop-scheduling witness instead.
            # Same protocol (connect → read one byte "1"), same semantics
            # (pure in-memory, zero disk I/O, answered only when the loop
            # is dispatching). Port is dynamic (assigned by the OS) and
            # published via the heartbeat payload so external probes know
            # where to connect.
            tick_server = await asyncio.start_server(
                _tick_socket_handler, host="127.0.0.1", port=0
            )
            # Get the actual port assigned by the OS
            _sock_addrs = tick_server.sockets if hasattr(tick_server, "sockets") else []
            for _s in _sock_addrs:
                try:
                    _sname = _s.getsockname()
                    if isinstance(_sname, tuple) and len(_sname) >= 2:
                        tick_tcp_port = int(_sname[1])
                        break
                except Exception:
                    pass
    except Exception:
        tick_server = None
        tick_tcp_port = None
        logger.warning(
            "Loop tick socket unavailable — liveness probes will have no "
            "loop-scheduling witness and will not escalate on a stale heartbeat",
            exc_info=True,
        )

    async def _write_off_loop() -> None:
        # write_loop_heartbeat never raises, so a failure here is an executor
        # problem (shutdown, saturation) and must not kill the heartbeat task.
        try:
            await asyncio.to_thread(
                write_loop_heartbeat,
                start_time=start_time,
                home=home,
                extra={
                    "loop_tick_socket": tick_server is not None,
                    "loop_tick_tcp_port": tick_tcp_port,
                },
            )
        except asyncio.CancelledError:
            raise
        except Exception:
            logger.debug("Loop heartbeat write failed off-loop", exc_info=True)

    try:
        # Immediate first write so monitors see a fresh file as soon as the
        # gateway is running, not after the first interval.
        await _write_off_loop()
        while True:
            if should_continue is not None and not should_continue():
                return
            await asyncio.sleep(interval)
            if should_continue is not None and not should_continue():
                return
            await _write_off_loop()
    finally:
        if tick_server is not None:
            tick_server.close()
            try:
                await tick_server.wait_closed()
            except Exception:
                pass
            if tick_socket_path is not None:
                try:
                    tick_socket_path.unlink(missing_ok=True)
                except Exception:
                    pass