"""Restart supervised gateway profiles from a clean Python generation.

The normal update command keeps executing in the interpreter that started before
``git pull``.  This module is deliberately small: it imports no gateway code
itself and launches the regular per-profile gateway command in a new
interpreter.  It is used only after the in-process restart phase has raised, so
that the recovery path cannot inherit the stale ``sys.modules`` graph that
caused the failure.

Outcome vocabulary (deliberately conservative):

- ``verified``          — the relaunch command exited 0 AND the profile's
  systemd unit was independently observed ``active`` afterwards.  This is the
  only outcome that may claim supervisor coverage.
- ``relaunch_attempted`` — the relaunch command exited 0 but no independent
  supervisor observation was possible (non-systemd supervisor, ``systemctl``
  missing, or the unit probe was inconclusive).  ``rc == 0`` from
  ``gateway restart`` is not proof that the new code generation is running,
  so this outcome must never be treated as verified coverage.
- ``failed``            — the relaunch command errored, timed out, or exited
  non-zero.

The pass covers two runtime families, because the in-process restart phase
covers both and an abort can strand either one (#92145):

- **gateway profiles**, relaunched through the existing per-profile
  ``hermes_cli.main -p <profile> gateway restart`` command; and
- **``hermes-serve*`` systemd units**, restarted directly through
  ``systemctl``.  ``hermes serve`` is not a gateway profile and has no
  per-profile relaunch command, but it is the runtime that hosts
  ``tui_gateway.server``: the process the original report saw answering every
  chat turn with an ``ImportError`` for a symbol that existed on disk.  The
  unit family is enumerated from systemd itself rather than from the update
  inventory, so a manually launched or Desktop-owned ``hermes serve`` — which
  has no relaunch authority — can never enter this path.

Serve-unit identity is always ``<scope>/<unit>`` (``user/hermes-serve``,
``system/hermes-serve``).  The two managers can each own a unit of the same
name and they are different processes: an identity that projects the scope
away lets one settled unit suppress recovery of the other, and lets one
scope's outcome speak for the other's.  Scope is therefore never dropped and
reconstructed — not in the skip payload, not in ``verified``/``failed``, not
in the receipt.
"""

from __future__ import annotations

import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import time
from collections.abc import Callable, Iterable, Mapping
from typing import Any

_RECOVERY_ENV = "HERMES_UPDATE_RESTART_RECOVERY"
_GATEWAY_MARKERS = ("_HERMES_GATEWAY", "HERMES_GATEWAY", "HERMES_GATEWAY_MODE")
_PROFILE_RESTART_TIMEOUT = 90
_VERIFY_TIMEOUT = 15
_PROFILE_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
_SUPERVISOR_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,31}$")
_UNIT_RE = re.compile(r"^hermes-serve(-[a-z0-9][a-z0-9_-]{0,63})?\.service$")
_SERVE_UNIT_PATTERN = "hermes-serve*"
_SCOPE_LABELS = ("user", "system")
_UNIT_RESTART_TIMEOUT = 60
_UNIT_SETTLE_ATTEMPTS = 10
_UNIT_SETTLE_DELAY = 1.0


def _profile_command(profile: str) -> list[str]:
    """Build a parameterized restart command for exactly one profile."""
    return [
        sys.executable,
        "-m",
        "hermes_cli.main",
        "-p",
        profile,
        "gateway",
        "restart",
    ]


def _child_environment() -> dict[str, str]:
    """Return an environment that cannot self-identify as the gateway owner."""
    env = os.environ.copy()
    for marker in _GATEWAY_MARKERS:
        env.pop(marker, None)
    env[_RECOVERY_ENV] = "1"
    return env


def _run_profile_restart(
    profile: str,
    *,
    run: Callable[..., Any],
) -> bool:
    """Run one profile restart without inheriting the updater's process state."""
    kwargs: dict[str, Any] = {
        "stdin": subprocess.DEVNULL,
        "capture_output": True,
        "text": True,
        "encoding": "utf-8",
        "errors": "replace",
        "check": False,
        "timeout": _PROFILE_RESTART_TIMEOUT,
        "env": _child_environment(),
    }
    if os.name == "nt":
        kwargs["creationflags"] = (
            getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
            | getattr(subprocess, "DETACHED_PROCESS", 0)
        )
    else:
        kwargs["start_new_session"] = True

    try:
        result = run(_profile_command(profile), **kwargs)
    except (OSError, subprocess.TimeoutExpired):
        return False
    return getattr(result, "returncode", 1) == 0


def _systemd_unit_candidates(profile: str) -> tuple[str, ...]:
    """Unit names the existing systemd gateway lifecycle produces per profile."""
    if profile == "default":
        return (
            "hermes-gateway.service",
            "gateway.service",
            "gateway-default.service",
        )
    return (
        f"hermes-gateway-{profile}.service",
        f"gateway-{profile}.service",
    )


def _systemd_verified_active(profile: str, *, run: Callable[..., Any]) -> bool:
    """Return True only when systemd itself reports the profile's unit active.

    This is the observation that separates ``verified`` from
    ``relaunch_attempted``.  Any failure here (no ``systemctl``, probe error,
    unit not ``active``) means we could NOT verify — never that the restart
    failed.
    """
    systemctl = shutil.which("systemctl")
    if not systemctl:
        return False
    for unit in _systemd_unit_candidates(profile):
        try:
            result = run(
                [systemctl, "--user", "is-active", unit],
                capture_output=True,
                text=True,
                encoding="utf-8",
                errors="replace",
                check=False,
                timeout=_VERIFY_TIMEOUT,
            )
        except (OSError, subprocess.TimeoutExpired):
            continue
        if (
            getattr(result, "returncode", 1) == 0
            and (getattr(result, "stdout", "") or "").strip() == "active"
        ):
            return True
    return False


def restart_profiles(
    profiles: Iterable[str],
    *,
    supervisors: Mapping[str, str] | None = None,
    run: Callable[..., Any] = subprocess.run,
) -> dict[str, list[str]]:
    """Restart the supplied profiles and return per-profile terminal results.

    The caller supplies only profiles whose inventory identified a service
    supervisor.  Manual gateways are intentionally excluded before this module
    is called: killing one without a relaunch authority would turn stale code
    into an outage.

    A profile only lands in ``verified`` when its supervisor is systemd and
    ``systemctl --user is-active`` independently confirms the unit after the
    relaunch command succeeded.  Every other zero-exit relaunch is reported as
    ``relaunch_attempted`` — the code cannot observe supervisor coverage for
    those paths and must not claim it.
    """
    supervisors = supervisors or {}
    normalized = sorted(
        {profile for profile in profiles if isinstance(profile, str) and profile}
    )
    verified: list[str] = []
    relaunch_attempted: list[str] = []
    failed: list[str] = []
    for profile in normalized:
        if not _run_profile_restart(profile, run=run):
            failed.append(profile)
            continue
        if supervisors.get(profile) == "systemd" and _systemd_verified_active(
            profile, run=run
        ):
            verified.append(profile)
        else:
            relaunch_attempted.append(profile)
    return {
        "verified": verified,
        "relaunch_attempted": relaunch_attempted,
        "failed": failed,
    }


def _systemctl_scopes() -> list[tuple[str, list[str]]]:
    """``systemctl`` invocations for the user and system scopes, or nothing.

    Mirrors the scope pair the in-process restart phase walks. ``systemctl``
    is resolved through ``shutil.which`` so this module never has to import
    any Hermes platform helper — importing the freshly pulled tree is exactly
    what aborted the phase that called us.

    Each scope is returned with its label because ``hermes-serve.service`` in
    the user manager and ``hermes-serve.service`` in the system manager are
    two different processes. Every identity this module produces or consumes
    stays qualified by that label; the bare unit name is never the key.
    """
    systemctl = shutil.which("systemctl")
    if not systemctl or sys.platform != "linux":
        return []
    return [("user", [systemctl, "--user"]), ("system", [systemctl])]


def _listed_serve_units(scope: list[str], *, run: Callable[..., Any]) -> list[str]:
    """Serve units systemd knows about in one scope, validated by name."""
    try:
        result = run(
            scope
            + [
                "list-units",
                _SERVE_UNIT_PATTERN,
                "--plain",
                "--no-legend",
                "--no-pager",
            ],
            capture_output=True,
            text=True,
            encoding="utf-8",
            errors="replace",
            check=False,
            timeout=_VERIFY_TIMEOUT,
        )
    except (OSError, subprocess.TimeoutExpired):
        return []
    units: list[str] = []
    for line in (getattr(result, "stdout", "") or "").splitlines():
        parts = line.split()
        if not parts:
            continue
        # The glob is a systemd pattern, not a name gate: `hermes-serve*` also
        # matches the unrelated `hermes-server.service`. Require the exact
        # base unit or the hyphenated profile family, same shape as the
        # in-process phase's own name gate.
        if _UNIT_RE.fullmatch(parts[0]) and parts[0] not in units:
            units.append(parts[0])
    return units


def _unit_property(
    scope: list[str], unit: str, prop: str, *, run: Callable[..., Any]
) -> str | None:
    """One ``systemctl show`` property, or ``None`` when it cannot be read."""
    try:
        result = run(
            scope + ["show", unit, f"--property={prop}", "--value"],
            capture_output=True,
            text=True,
            encoding="utf-8",
            errors="replace",
            check=False,
            timeout=_VERIFY_TIMEOUT,
        )
    except (OSError, subprocess.TimeoutExpired):
        return None
    if getattr(result, "returncode", 1) != 0:
        return None
    return (getattr(result, "stdout", "") or "").strip()


def _unit_main_pid(scope: list[str], unit: str, *, run: Callable[..., Any]) -> int:
    """The unit's ``MainPID``; ``0`` when absent or unreadable."""
    raw = _unit_property(scope, unit, "MainPID", run=run)
    try:
        return int(raw or 0)
    except ValueError:
        return 0


def _unit_is_active(scope: list[str], unit: str, *, run: Callable[..., Any]) -> bool:
    try:
        result = run(
            scope + ["is-active", unit],
            capture_output=True,
            text=True,
            encoding="utf-8",
            errors="replace",
            check=False,
            timeout=_VERIFY_TIMEOUT,
        )
    except (OSError, subprocess.TimeoutExpired):
        return False
    return (getattr(result, "stdout", "") or "").strip() == "active"


def _serve_unit_replaced(
    scope: list[str],
    unit: str,
    previous_pid: int,
    *,
    run: Callable[..., Any],
    sleep: Callable[[float], Any],
) -> bool:
    """Did the unit come back on a NEW main process?

    ``restart`` returning 0 is not evidence: the whole point of #92145 is that
    a live process can keep serving the pre-update generation while every
    status command reports success. A changed ``MainPID`` on an ``active``
    unit is the observation that the old interpreter — and its stale
    ``sys.modules`` — is gone.
    """
    for attempt in range(_UNIT_SETTLE_ATTEMPTS):
        if attempt:
            sleep(_UNIT_SETTLE_DELAY)
        if not _unit_is_active(scope, unit, run=run):
            continue
        current = _unit_main_pid(scope, unit, run=run)
        if current > 0 and current != previous_pid:
            return True
    return False


def _qualified(scope_label: str, base: str) -> str:
    """The only identity this module reports for a serve unit."""
    return f"{scope_label}/{base}"


def _normalized_skips(
    skip_units: Iterable[Any],
) -> tuple[set[tuple[str, str]], set[str]]:
    """Split already-settled units into scope-qualified and legacy entries.

    Qualified entries (``{"scope": "user", "unit": "hermes-serve"}`` or the
    equivalent ``"user/hermes-serve"`` string) suppress exactly one process.

    A bare ``"hermes-serve"`` carries no scope and therefore cannot say WHICH
    of two same-named processes was settled. It is honoured across both scopes
    because that is all the information it contains — the caller in this tree
    always sends the qualified shape, and this branch exists only for a payload
    written by a pre-update interpreter that had no scope to send.
    """
    qualified: set[tuple[str, str]] = set()
    legacy: set[str] = set()
    for entry in skip_units or ():
        if isinstance(entry, Mapping):
            scope_label = str(entry.get("scope") or "")
            base = str(entry.get("unit") or "").removesuffix(".service")
        else:
            scope_label, sep, unit = str(entry).partition("/")
            if not sep:
                scope_label, unit = "", scope_label
            base = unit.removesuffix(".service")
        if not base:
            continue
        if scope_label in _SCOPE_LABELS:
            qualified.add((scope_label, base))
        else:
            legacy.add(base)
    return qualified, legacy


def restart_serve_units(
    *,
    skip_units: Iterable[Any] = (),
    run: Callable[..., Any] = subprocess.run,
    sleep: Callable[[float], Any] = time.sleep,
) -> dict[str, list[str]]:
    """Restart every active ``hermes-serve*`` systemd unit from this process.

    ``hermes serve`` hosts ``tui_gateway.server`` and is restarted by the
    in-process phase alongside the gateway units, but it is not a gateway
    profile: no ``gateway restart`` command reaches it. When the phase aborts
    part-way — systemd lists ``hermes-gateway.service`` before
    ``hermes-serve.service``, so the gateway is typically already done — the
    serve unit is the one left holding generation-N modules over a
    generation-N+1 checkout.

    Units are enumerated from systemd, never from the update inventory. That
    keeps the relaunch authority requirement structural: a manually launched
    or Desktop-owned ``hermes serve`` owns no unit and therefore cannot be
    touched here.

    Every identity in and out of this function is scope-qualified. The user
    manager and the system manager can each own a ``hermes-serve.service``,
    and they are different processes: projecting the scope away would let one
    already-settled unit suppress recovery of the other, and would let one
    scope's success describe the other's outcome.

    Returns ``{"verified": [...], "failed": [...]}`` whose entries are
    ``<scope>/<base unit>`` (no ``.service`` suffix), e.g.
    ``user/hermes-serve``.
    """
    skipped_qualified, skipped_legacy = _normalized_skips(skip_units)
    # (scope, base unit) -> replaced?  A unit name can exist in BOTH the user
    # and the system scope; each is a separate process and each is proven,
    # reported and accounted for on its own.
    outcomes: dict[tuple[str, str], bool] = {}
    seen: set[tuple[str, str]] = set()
    for scope_label, scope in _systemctl_scopes():
        for unit in _listed_serve_units(scope, run=run):
            base = unit.removesuffix(".service")
            target = (scope_label, base)
            if target in seen or target in skipped_qualified or base in skipped_legacy:
                continue
            seen.add(target)
            if not _unit_is_active(scope, unit, run=run):
                # Not running: nothing is serving a stale generation from it.
                continue
            previous_pid = _unit_main_pid(scope, unit, run=run)
            if previous_pid <= 0:
                # Active with no readable main process: a replacement cannot
                # be observed, so it cannot be claimed. Restarting blind and
                # reporting success is the failure mode this module exists to
                # remove.
                outcomes[target] = False
                continue
            try:
                result = run(
                    scope + ["--no-ask-password", "restart", unit],
                    capture_output=True,
                    text=True,
                    encoding="utf-8",
                    errors="replace",
                    check=False,
                    timeout=_UNIT_RESTART_TIMEOUT,
                )
            except (OSError, subprocess.TimeoutExpired):
                outcomes[target] = False
                continue
            if getattr(result, "returncode", 1) != 0:
                # Includes the unprivileged system-scope case. We do not probe
                # for sudo here: an unverifiable unit must read as failed so
                # the update stays explicitly incomplete.
                outcomes[target] = False
                continue
            outcomes[target] = _serve_unit_replaced(
                scope, unit, previous_pid, run=run, sleep=sleep
            )
    return {
        "verified": sorted(
            _qualified(*target) for target, ok in outcomes.items() if ok
        ),
        "failed": sorted(
            _qualified(*target) for target, ok in outcomes.items() if not ok
        ),
    }


def _parse_payload(stream) -> tuple[list[str], dict[str, str], bool, list[str]]:
    payload = json.load(stream)
    profiles = payload.get("profiles") if isinstance(payload, dict) else None
    if not isinstance(profiles, list):
        raise ValueError("recovery payload must contain a profiles list")
    if any(
        not isinstance(profile, str) or not _PROFILE_ID_RE.fullmatch(profile)
        for profile in profiles
    ):
        raise ValueError("recovery profiles contain an invalid profile id")
    raw_supervisors = payload.get("supervisors") if isinstance(payload, dict) else None
    supervisors: dict[str, str] = {}
    if raw_supervisors is not None:
        if not isinstance(raw_supervisors, dict) or any(
            not isinstance(profile, str)
            or not isinstance(supervisor, str)
            or not _PROFILE_ID_RE.fullmatch(profile)
            or not _SUPERVISOR_RE.fullmatch(supervisor)
            for profile, supervisor in raw_supervisors.items()
        ):
            raise ValueError("recovery supervisors map is invalid")
        supervisors = dict(raw_supervisors)
    raw_serve = payload.get("serve_units") if isinstance(payload, dict) else None
    recover_serve = False
    skip_units: list[str] = []
    if raw_serve is not None:
        if not isinstance(raw_serve, dict):
            raise ValueError("recovery serve_units block is invalid")
        recover_serve = bool(raw_serve.get("recover"))
        raw_skip = raw_serve.get("skip") or []
        if not isinstance(raw_skip, list) or any(
            not isinstance(entry, (str, dict)) for entry in raw_skip
        ):
            raise ValueError("recovery serve_units skip list is invalid")
        for entry in raw_skip:
            # A skip entry names one already-settled process. The qualified
            # shape carries the systemd scope, because `hermes-serve.service`
            # can exist in BOTH managers and settling one says nothing about
            # the other. A bare string is the legacy shape a pre-update
            # interpreter can still send; it is kept, and read as
            # scope-agnostic by `restart_serve_units`.
            if isinstance(entry, dict):
                scope_label = entry.get("scope")
                unit = entry.get("unit")
                if not isinstance(unit, str):
                    raise ValueError("recovery serve_units skip list is invalid")
                if not isinstance(scope_label, str):
                    scope_label = ""
            else:
                scope_label, _, unit = entry.rpartition("/")
            # Only the shapes systemd can actually produce for this family; a
            # skip entry is a name filter, never a command argument. An
            # unrecognized scope drops the entry rather than raising: dropping
            # a skip can only ever cause one more restart-and-verify, while
            # honouring an unreadable one could suppress recovery of a stale
            # process.
            if scope_label and scope_label not in _SCOPE_LABELS:
                continue
            if not _UNIT_RE.fullmatch(
                unit if unit.endswith(".service") else f"{unit}.service"
            ):
                continue
            base = unit.removesuffix(".service")
            skip_units.append(f"{scope_label}/{base}" if scope_label else base)
    return profiles, supervisors, recover_serve, skip_units


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--stdin",
        action="store_true",
        help=argparse.SUPPRESS,
    )
    args = parser.parse_args(argv)
    if not args.stdin:
        parser.error("this command is an internal update-recovery entry point")

    try:
        profiles, supervisors, recover_serve, skip_units = _parse_payload(sys.stdin)
        result = restart_profiles(profiles, supervisors=supervisors)
        result["serve_units"] = (
            restart_serve_units(skip_units=skip_units)
            if recover_serve
            else {"verified": [], "failed": []}
        )
    except (ValueError, json.JSONDecodeError) as exc:
        print(
            json.dumps(
                {
                    "error": str(exc),
                    "verified": [],
                    "relaunch_attempted": [],
                    "failed": [],
                    "serve_units": {"verified": [], "failed": []},
                }
            )
        )
        return 2

    print(json.dumps(result, sort_keys=True))
    if result["failed"] or result["serve_units"]["failed"]:
        return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(main())