Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,649 @@
|
||||
"""Out-of-loop shutdown and event-loop liveness backstops (#66892, #69089).
|
||||
|
||||
When the asyncio loop freezes mid-drain, every asyncio-based recovery path is
|
||||
structurally unable to fire: the drain deadline, status rewrites, and forensics
|
||||
all need the same loop that is stuck. launchd/systemd KeepAlive only restarts a
|
||||
*dead* process, so a wedged-but-alive gateway sits as a zombie until manual
|
||||
SIGKILL.
|
||||
|
||||
This module provides:
|
||||
|
||||
1. A plain OS-thread shutdown watchdog armed at ``stop()``. If shutdown has not
|
||||
completed within ``restart_drain_timeout + grace``, it dumps all-thread
|
||||
stacks via ``faulthandler`` plus a metadata snapshot, then ``os._exit`` so
|
||||
the service manager can revive the process.
|
||||
2. An event-loop heartbeat file at ``<HERMES_HOME>/state/gateway.heartbeat`` so
|
||||
external supervision can distinguish "process alive" from "loop frozen"
|
||||
(``gateway_state.json`` alone can't — it only rewrites on transitions/turns).
|
||||
3. A lifetime thread watchdog that can still diagnose and hard-exit when the
|
||||
event loop is too frozen to run its own heartbeat or timeout callbacks.
|
||||
4. A self-rescheduling floor timer that keeps the loop selector's timeout
|
||||
finite, giving existing async recovery tasks a chance to resume.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import faulthandler
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from gateway.restart import GATEWAY_SERVICE_RESTART_EXIT_CODE
|
||||
from hermes_constants import get_hermes_home
|
||||
from utils import atomic_json_write
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Extra leash beyond ``agent.restart_drain_timeout`` so a slow-but-progressing
|
||||
# drain is not cut short. Matches the issue #66892 suggested hardening.
|
||||
DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S = 60.0
|
||||
DEFAULT_HEARTBEAT_INTERVAL_S = 30.0
|
||||
DEFAULT_LOOP_FLOOR_TIMER_INTERVAL_S = 5.0
|
||||
DEFAULT_LOOP_WATCHDOG_INTERVAL_S = 30.0
|
||||
DEFAULT_LOOP_WATCHDOG_TIMEOUT_S = 10.0
|
||||
# 3 sustained misses (~90-120s of loop block) escalate. The false-positive
|
||||
# class that motivated raising this (the watchdog's own on-loop heartbeat
|
||||
# fsync stalling the loop it monitors) is fixed at the root by the off-loop
|
||||
# heartbeat write + two-witness probe (#90502), so the default stays tight
|
||||
# for genuine wedges. Deployments with legitimately slow loops can tune via
|
||||
# gateway.loop_watchdog_* in config.yaml.
|
||||
DEFAULT_LOOP_WATCHDOG_MAX_STRIKES = 3
|
||||
_HEARTBEAT_RELATIVE = ("state", "gateway.heartbeat")
|
||||
_WATCHDOG_DUMP_RELATIVE = ("logs", "gateway-shutdown-watchdog.log")
|
||||
|
||||
|
||||
class _LoopFloorTimerHandle:
|
||||
"""Cancelable owner for the currently scheduled selector floor timer."""
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, interval: float):
|
||||
self._loop = loop
|
||||
self._interval = interval
|
||||
self._cancelled = False
|
||||
self._timer: Optional[asyncio.TimerHandle] = None
|
||||
self._schedule()
|
||||
|
||||
def _schedule(self) -> None:
|
||||
self._timer = self._loop.call_later(self._interval, self._tick)
|
||||
|
||||
def _tick(self) -> None:
|
||||
if not self._cancelled:
|
||||
self._schedule()
|
||||
|
||||
def cancel(self) -> None:
|
||||
self._cancelled = True
|
||||
if self._timer is not None:
|
||||
self._timer.cancel()
|
||||
|
||||
|
||||
class _LoopLivenessWatchdogHandle:
|
||||
"""Small lifecycle handle for the daemon liveness thread."""
|
||||
|
||||
def __init__(self, stop_event: threading.Event, thread: threading.Thread):
|
||||
self._stop_event = stop_event
|
||||
self._thread = thread
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
|
||||
def join(self, timeout: Optional[float] = None) -> None:
|
||||
self._thread.join(timeout=timeout)
|
||||
|
||||
def is_alive(self) -> bool:
|
||||
return self._thread.is_alive()
|
||||
|
||||
|
||||
def _arm_loop_floor_timer(
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
interval: float = DEFAULT_LOOP_FLOOR_TIMER_INTERVAL_S,
|
||||
) -> _LoopFloorTimerHandle:
|
||||
"""Keep at least one timer pending so selector waits remain bounded."""
|
||||
try:
|
||||
resolved_interval = float(interval)
|
||||
if resolved_interval <= 0:
|
||||
raise ValueError
|
||||
except (TypeError, ValueError):
|
||||
resolved_interval = DEFAULT_LOOP_FLOOR_TIMER_INTERVAL_S
|
||||
return _LoopFloorTimerHandle(loop, resolved_interval)
|
||||
|
||||
|
||||
def start_loop_liveness_watchdog(
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
*,
|
||||
probe_interval: float = DEFAULT_LOOP_WATCHDOG_INTERVAL_S,
|
||||
probe_timeout: float = DEFAULT_LOOP_WATCHDOG_TIMEOUT_S,
|
||||
max_strikes: int = DEFAULT_LOOP_WATCHDOG_MAX_STRIKES,
|
||||
exit_code: int = GATEWAY_SERVICE_RESTART_EXIT_CODE,
|
||||
) -> Optional[_LoopLivenessWatchdogHandle]:
|
||||
"""Start an out-of-loop watchdog that hard-exits after missed probes.
|
||||
|
||||
The guard is on by default; operators opt out with
|
||||
``gateway.loop_watchdog: false`` in config.yaml (enforced by the caller,
|
||||
``GatewayRunner._start_loop_liveness_guards`` — this module stays
|
||||
config-agnostic so bare-loop tests can drive it directly).
|
||||
"""
|
||||
interval = probe_interval
|
||||
timeout = probe_timeout
|
||||
strikes_limit = max_strikes
|
||||
stop_event = threading.Event()
|
||||
|
||||
def _wait_for_probe(probe_event: threading.Event) -> Optional[bool]:
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
if stop_event.is_set():
|
||||
return None
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return probe_event.is_set()
|
||||
if probe_event.wait(timeout=min(remaining, 0.05)):
|
||||
return True
|
||||
|
||||
def _watchdog() -> None:
|
||||
strikes = 0
|
||||
while not stop_event.wait(timeout=interval):
|
||||
probe_event = threading.Event()
|
||||
try:
|
||||
loop.call_soon_threadsafe(probe_event.set)
|
||||
except RuntimeError:
|
||||
# A normally closed loop cannot be probed and no longer needs
|
||||
# a process-liveness backstop.
|
||||
return
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Failed to schedule gateway loop liveness probe", exc_info=True
|
||||
)
|
||||
return
|
||||
|
||||
responded = _wait_for_probe(probe_event)
|
||||
if responded is None:
|
||||
return
|
||||
if responded:
|
||||
strikes = 0
|
||||
continue
|
||||
|
||||
if stop_event.is_set():
|
||||
return
|
||||
strikes += 1
|
||||
if strikes < strikes_limit:
|
||||
continue
|
||||
|
||||
if stop_event.is_set():
|
||||
return
|
||||
try:
|
||||
logger.critical(
|
||||
"Gateway event loop missed %d consecutive liveness probes; "
|
||||
"dumping all thread stacks and exiting with code %d so the "
|
||||
"service supervisor can restart it.",
|
||||
strikes,
|
||||
exit_code,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
faulthandler.dump_traceback(all_threads=True)
|
||||
except Exception:
|
||||
logger.debug("Loop liveness faulthandler dump failed", exc_info=True)
|
||||
if stop_event.is_set():
|
||||
return
|
||||
# Record the watchdog exit in the lifecycle sentinel so the next
|
||||
# boot reports "watchdog hard-exit" instead of misclassifying
|
||||
# this as an unclean SIGKILL/OOM death (NS-608).
|
||||
try:
|
||||
from gateway.lifecycle_ledger import mark_exited
|
||||
mark_exited(exit_code, reason="loop_liveness_watchdog")
|
||||
except Exception:
|
||||
pass
|
||||
os._exit(exit_code)
|
||||
return
|
||||
|
||||
thread = threading.Thread(
|
||||
target=_watchdog,
|
||||
daemon=True,
|
||||
name="gateway-loop-liveness-watchdog",
|
||||
)
|
||||
try:
|
||||
thread.start()
|
||||
except Exception:
|
||||
logger.debug("Failed to start gateway loop liveness watchdog", exc_info=True)
|
||||
return None
|
||||
return _LoopLivenessWatchdogHandle(stop_event, thread)
|
||||
|
||||
|
||||
def _process_hermes_home() -> Path:
|
||||
"""HERMES_HOME for process-level identity files (ignore profile overrides)."""
|
||||
val = os.environ.get("HERMES_HOME", "").strip()
|
||||
if val:
|
||||
return Path(val)
|
||||
return get_hermes_home()
|
||||
|
||||
|
||||
def get_loop_heartbeat_path(home: Optional[Path] = None) -> Path:
|
||||
"""Return ``<HERMES_HOME>/state/gateway.heartbeat``."""
|
||||
base = home if home is not None else _process_hermes_home()
|
||||
return base.joinpath(*_HEARTBEAT_RELATIVE)
|
||||
|
||||
|
||||
def get_loop_tick_socket_path(
|
||||
home: Optional[Path] = None, pid: Optional[int] = None
|
||||
) -> Path:
|
||||
"""Return the loop-scheduling witness socket for ``pid``.
|
||||
|
||||
``<HERMES_HOME>/state/gateway.loop-tick.<pid>.sock`` — PID-suffixed so a
|
||||
leftover node from a previous process can never be mistaken for this
|
||||
gateway's witness. Served by the gateway loop itself (see
|
||||
``_tick_socket_handler``): an answer is direct proof that the loop is
|
||||
dispatching, which is exactly the property the heartbeat file lost when
|
||||
its write moved off-loop (#90502).
|
||||
"""
|
||||
base = home if home is not None else _process_hermes_home()
|
||||
return base.joinpath(
|
||||
"state", f"gateway.loop-tick.{int(pid if pid is not None else os.getpid())}.sock"
|
||||
)
|
||||
|
||||
|
||||
def get_shutdown_watchdog_dump_path(home: Optional[Path] = None) -> Path:
|
||||
"""Return the faulthandler / metadata dump path for a fired watchdog."""
|
||||
base = home if home is not None else _process_hermes_home()
|
||||
return base.joinpath(*_WATCHDOG_DUMP_RELATIVE)
|
||||
|
||||
|
||||
def write_loop_heartbeat(
|
||||
*,
|
||||
pid: Optional[int] = None,
|
||||
start_time: Optional[float] = None,
|
||||
home: Optional[Path] = None,
|
||||
extra: Optional[Dict[str, Any]] = None,
|
||||
) -> Path:
|
||||
"""Atomically rewrite the loop-liveness heartbeat file.
|
||||
|
||||
``start_time`` is the gateway process start (``time.time()`` epoch seconds)
|
||||
so supervisors can detect PID reuse. Best-effort — never raises.
|
||||
"""
|
||||
path = get_loop_heartbeat_path(home)
|
||||
payload: Dict[str, Any] = {
|
||||
"pid": int(pid if pid is not None else os.getpid()),
|
||||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"monotonic": time.monotonic(),
|
||||
}
|
||||
if start_time is not None:
|
||||
payload["start_time"] = float(start_time)
|
||||
# Embed a cheap memory sample (own RSS + MemAvailable + swap) so the
|
||||
# heartbeat doubles as a rolling pre-death telemetry snapshot: after an
|
||||
# unclean death (SIGKILL/OOM/VM loss) the last heartbeat is the closest
|
||||
# surviving record of memory pressure — see gateway.lifecycle_ledger
|
||||
# (NS-608). Best-effort; <1ms of /proc reads on Linux, {} elsewhere.
|
||||
try:
|
||||
from gateway.lifecycle_ledger import sample_memory
|
||||
|
||||
mem = sample_memory()
|
||||
if mem:
|
||||
payload["mem"] = mem
|
||||
except Exception:
|
||||
pass
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
try:
|
||||
atomic_json_write(path, payload, indent=None)
|
||||
except Exception:
|
||||
logger.debug("Failed to write gateway loop heartbeat", exc_info=True)
|
||||
return path
|
||||
|
||||
|
||||
def resolve_shutdown_watchdog_delay(
|
||||
drain_timeout: float,
|
||||
*,
|
||||
grace_s: float = DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S,
|
||||
) -> float:
|
||||
"""Return the wall-clock leash for the shutdown watchdog thread."""
|
||||
try:
|
||||
drain = max(float(drain_timeout), 0.0)
|
||||
except (TypeError, ValueError):
|
||||
drain = 0.0
|
||||
try:
|
||||
grace = max(float(grace_s), 0.0)
|
||||
except (TypeError, ValueError):
|
||||
grace = DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S
|
||||
return drain + grace
|
||||
|
||||
|
||||
def _write_watchdog_dump(
|
||||
dump_path: Path,
|
||||
*,
|
||||
delay_s: float,
|
||||
snapshot: Optional[Dict[str, Any]],
|
||||
) -> None:
|
||||
"""Best-effort faulthandler + metadata dump before hard-exit."""
|
||||
try:
|
||||
dump_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
except OSError:
|
||||
return
|
||||
|
||||
header = {
|
||||
"event": "shutdown_watchdog_fired",
|
||||
"pid": os.getpid(),
|
||||
"delay_s": delay_s,
|
||||
"fired_at": datetime.now(timezone.utc).isoformat(),
|
||||
"snapshot": snapshot or {},
|
||||
}
|
||||
try:
|
||||
with open(dump_path, "a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(header, default=str) + "\n")
|
||||
fh.write("--- faulthandler dump (all threads) ---\n")
|
||||
fh.flush()
|
||||
try:
|
||||
faulthandler.dump_traceback(file=fh, all_threads=True)
|
||||
except Exception:
|
||||
fh.write("(faulthandler.dump_traceback failed)\n")
|
||||
fh.write("--- end dump ---\n")
|
||||
fh.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Also dump to stderr so journald/launchd capture it even if the file
|
||||
# write failed (wedged disk was one of the #66892 hypotheses).
|
||||
try:
|
||||
sys.stderr.write(
|
||||
f"Gateway shutdown watchdog fired after {delay_s:.0f}s "
|
||||
f"(pid={os.getpid()}); dumping all thread stacks.\n"
|
||||
)
|
||||
sys.stderr.flush()
|
||||
faulthandler.dump_traceback(all_threads=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def arm_shutdown_watchdog(
|
||||
delay_s: float,
|
||||
*,
|
||||
done_event: Optional[threading.Event] = None,
|
||||
snapshot_fn: Optional[Callable[[], Dict[str, Any]]] = None,
|
||||
exit_code: int = 1,
|
||||
dump_path: Optional[Path] = None,
|
||||
name: str = "gateway-shutdown-watchdog",
|
||||
) -> threading.Event:
|
||||
"""Arm a daemon-thread hard-exit backstop for a wedged shutdown path.
|
||||
|
||||
If ``done_event`` is set before ``delay_s`` elapses, the thread exits
|
||||
quietly (normal / progressing shutdown completed). Otherwise it dumps
|
||||
diagnostics and calls ``os._exit(exit_code)``.
|
||||
|
||||
Never raises. Returns the ``done_event`` (creating one when omitted) so
|
||||
the caller can disarm on successful completion.
|
||||
"""
|
||||
done = done_event if done_event is not None else threading.Event()
|
||||
try:
|
||||
delay = max(float(delay_s), 0.0)
|
||||
except (TypeError, ValueError):
|
||||
delay = DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S
|
||||
|
||||
if delay <= 0:
|
||||
return done
|
||||
|
||||
def _watchdog() -> None:
|
||||
# Wait with interruptible chunks so a late disarm doesn't need the
|
||||
# full remaining sleep to observe done_event.
|
||||
deadline = time.monotonic() + delay
|
||||
while time.monotonic() < deadline:
|
||||
remaining = deadline - time.monotonic()
|
||||
if done.wait(timeout=min(remaining, 1.0)):
|
||||
return
|
||||
if done.is_set():
|
||||
return
|
||||
|
||||
snapshot: Optional[Dict[str, Any]] = None
|
||||
if snapshot_fn is not None:
|
||||
try:
|
||||
snapshot = snapshot_fn()
|
||||
except Exception as exc:
|
||||
snapshot = {"snapshot_error": repr(exc)}
|
||||
|
||||
target = dump_path if dump_path is not None else get_shutdown_watchdog_dump_path()
|
||||
_write_watchdog_dump(target, delay_s=delay, snapshot=snapshot)
|
||||
|
||||
try:
|
||||
logger.critical(
|
||||
"Shutdown watchdog fired after %.0fs — forcing process exit "
|
||||
"(asyncio drain path appears wedged; see %s)",
|
||||
delay,
|
||||
target,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
stream.flush()
|
||||
except Exception:
|
||||
pass
|
||||
# Mirror _exit_after_graceful_shutdown: release PID file + runtime
|
||||
# lock BEFORE the log drain (locks must never be stranded), then
|
||||
# drain the async log queue so the logger.critical above actually
|
||||
# reaches the file before os._exit bypasses atexit. (#66892)
|
||||
try:
|
||||
from gateway.status import remove_pid_file, release_gateway_runtime_lock
|
||||
remove_pid_file()
|
||||
release_gateway_runtime_lock()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
from hermes_logging import drain_log_queue
|
||||
drain_log_queue(timeout=1.0)
|
||||
except Exception:
|
||||
pass
|
||||
# Record the watchdog exit so the next boot's unclean-death detector
|
||||
# reports "shutdown watchdog fired" instead of SIGKILL/OOM (NS-608).
|
||||
try:
|
||||
from gateway.lifecycle_ledger import mark_exited
|
||||
mark_exited(exit_code, reason="shutdown_watchdog")
|
||||
except Exception:
|
||||
pass
|
||||
os._exit(exit_code)
|
||||
|
||||
try:
|
||||
threading.Thread(target=_watchdog, daemon=True, name=name).start()
|
||||
except Exception:
|
||||
logger.debug("Failed to arm shutdown watchdog", exc_info=True)
|
||||
return done
|
||||
|
||||
|
||||
async def _tick_socket_handler(
|
||||
reader: asyncio.StreamReader, writer: asyncio.StreamWriter
|
||||
) -> None:
|
||||
"""Answer a liveness ping with one byte.
|
||||
|
||||
Runs on the gateway loop: the reply is produced only while the loop is
|
||||
actually dispatching, so a successful read is a witness of loop
|
||||
schedulability that no executor thread and no filesystem stall can
|
||||
refresh. A UNIX-socket write is a socket-buffer copy — no fsync, no
|
||||
disk I/O — so the witness keeps working on the exact filesystem that
|
||||
stalls the heartbeat write. Best-effort; never raises.
|
||||
"""
|
||||
try:
|
||||
writer.write(b"1")
|
||||
await writer.drain()
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
writer.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def loop_heartbeat_forever(
|
||||
*,
|
||||
interval_s: float = DEFAULT_HEARTBEAT_INTERVAL_S,
|
||||
start_time: Optional[float] = None,
|
||||
home: Optional[Path] = None,
|
||||
should_continue: Optional[Callable[[], bool]] = None,
|
||||
) -> None:
|
||||
"""Rewrite the loop heartbeat file on a cadence until cancelled / gated off.
|
||||
|
||||
Runs as an asyncio task on the gateway loop — if the loop freezes, this task
|
||||
stops and the file mtime/updated_at goes stale for external monitors. That
|
||||
property is load-bearing and is preserved below: the write is still
|
||||
*initiated* by the loop, so a frozen loop still lets the file age.
|
||||
|
||||
The write itself is handed to a thread, because it is not free. It ends in
|
||||
``atomic_json_write`` -> ``os.fsync``, and on a filesystem that stalls, that
|
||||
fsync blocks whatever thread runs it. Doing it inline meant the loop-liveness
|
||||
watchdog's own heartbeat could block the loop it exists to monitor: the probe
|
||||
times out at ``DEFAULT_LOOP_WATCHDOG_TIMEOUT_S`` (10s) and gives up after
|
||||
``DEFAULT_LOOP_WATCHDOG_MAX_STRIKES`` (3), a ~90-120s budget, while a WSL2
|
||||
VHDX under io pressure was measured stalling a trivial stat-and-fsync probe
|
||||
at p99 31s and max 112s. So the watchdog killed the loop for being
|
||||
unresponsive at the moment it was blocked inside the watchdog's own write.
|
||||
|
||||
Awaited, not fire-and-forget: an unawaited task would keep the file fresh
|
||||
while the loop was wedged, which is exactly the signal the docstring above
|
||||
promises. And a single in-flight write at a time, so a 112s stall cannot pile
|
||||
up one queued thread per interval behind it.
|
||||
|
||||
Because the write is now off-loop, file freshness is no longer *proof* of
|
||||
loop schedulability: a stalled write or a saturated executor can age the file
|
||||
while the loop runs, and a write that lands after the loop froze can keep it
|
||||
fresh. The file therefore stops being sufficient authority on its own. This
|
||||
task also arms a loop-scheduling witness — a UNIX socket answered by the
|
||||
loop itself (``_tick_socket_handler``) — and records whether it is armed in
|
||||
the heartbeat payload (``loop_tick_socket``). External probes must require
|
||||
the witness to agree with file staleness before classifying a loop as
|
||||
wedged; see ``hermes_cli.gateway.probe_gateway_loop_liveness`` for the
|
||||
two-witness contract.
|
||||
"""
|
||||
try:
|
||||
interval = max(float(interval_s), 1.0)
|
||||
except (TypeError, ValueError):
|
||||
interval = DEFAULT_HEARTBEAT_INTERVAL_S
|
||||
|
||||
# Arm the loop-scheduling witness. Best-effort: a failed bind (permissions,
|
||||
# path length) must not abort the gateway or the file heartbeat — it only
|
||||
# disables the witness, and the payload flag tells probes that staleness is
|
||||
# no longer sufficient authority to escalate.
|
||||
#
|
||||
# Windows (non-POSIX generally): asyncio AF_UNIX support is POSIX-only, so
|
||||
# the AF_UNIX arm below is gated to POSIX — an ungated call raised
|
||||
# AttributeError on every native-Windows gateway start (#96956). Instead of
|
||||
# leaving the witness permanently absent there, the non-POSIX arm binds a
|
||||
# TCP loopback server on 127.0.0.1 with an OS-assigned port and publishes
|
||||
# the port in the heartbeat payload (``loop_tick_tcp_port``) so probes know
|
||||
# where to connect. Same protocol, same loop-owned semantics. If that bind
|
||||
# fails, the payload records loop_tick_socket=False and probes classify
|
||||
# UNKNOWN, never WEDGED — the graceful-drain backstop stays in place. (WSL2
|
||||
# — the #90502 incident environment — is Linux and arms the socket.)
|
||||
tick_server = None
|
||||
tick_socket_path = None
|
||||
tick_tcp_port = None
|
||||
try:
|
||||
if os.name == "posix":
|
||||
tick_socket_path = get_loop_tick_socket_path(home)
|
||||
tick_socket_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Re-bind over a leftover node from a dead process (os._exit(75) /
|
||||
# SIGKILL skip the finally-unlink; PID reuse re-lands on this
|
||||
# PID-suffixed path) is handled by asyncio itself:
|
||||
# create_unix_server os.remove()s an existing socket node before
|
||||
# binding — guarded by test_producer_rebinds_over_stale_socket_node.
|
||||
# What asyncio does NOT do is clean up SIBLING nodes from other
|
||||
# dead PIDs, so sweep those to keep state/ from accumulating
|
||||
# gateway.loop-tick.*.sock nodes across crash-restart cycles.
|
||||
# POSIX-only: os.kill(pid, 0) is a liveness probe here, but on
|
||||
# Windows os.kill calls TerminateProcess for non-CTRL signals —
|
||||
# and AF_UNIX server nodes are never created there anyway.
|
||||
try:
|
||||
for _stale in tick_socket_path.parent.glob(
|
||||
"gateway.loop-tick.*.sock"
|
||||
):
|
||||
if _stale == tick_socket_path:
|
||||
continue
|
||||
try:
|
||||
_stale_pid = int(_stale.name.split(".")[-2])
|
||||
except (ValueError, IndexError):
|
||||
_stale.unlink(missing_ok=True)
|
||||
continue
|
||||
try:
|
||||
os.kill(_stale_pid, 0) # windows-footgun: ok — inside os.name == "posix" gate
|
||||
except OSError:
|
||||
_stale.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"stale loop-tick socket sweep failed", exc_info=True
|
||||
)
|
||||
tick_server = await asyncio.start_unix_server(
|
||||
_tick_socket_handler, path=str(tick_socket_path)
|
||||
)
|
||||
else:
|
||||
# Windows / non-POSIX: no AF_UNIX support, so use a TCP loopback
|
||||
# server on 127.0.0.1 as the loop-scheduling witness instead.
|
||||
# Same protocol (connect → read one byte "1"), same semantics
|
||||
# (pure in-memory, zero disk I/O, answered only when the loop
|
||||
# is dispatching). Port is dynamic (assigned by the OS) and
|
||||
# published via the heartbeat payload so external probes know
|
||||
# where to connect.
|
||||
tick_server = await asyncio.start_server(
|
||||
_tick_socket_handler, host="127.0.0.1", port=0
|
||||
)
|
||||
# Get the actual port assigned by the OS
|
||||
_sock_addrs = tick_server.sockets if hasattr(tick_server, "sockets") else []
|
||||
for _s in _sock_addrs:
|
||||
try:
|
||||
_sname = _s.getsockname()
|
||||
if isinstance(_sname, tuple) and len(_sname) >= 2:
|
||||
tick_tcp_port = int(_sname[1])
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
tick_server = None
|
||||
tick_tcp_port = None
|
||||
logger.warning(
|
||||
"Loop tick socket unavailable — liveness probes will have no "
|
||||
"loop-scheduling witness and will not escalate on a stale heartbeat",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
async def _write_off_loop() -> None:
|
||||
# write_loop_heartbeat never raises, so a failure here is an executor
|
||||
# problem (shutdown, saturation) and must not kill the heartbeat task.
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
write_loop_heartbeat,
|
||||
start_time=start_time,
|
||||
home=home,
|
||||
extra={
|
||||
"loop_tick_socket": tick_server is not None,
|
||||
"loop_tick_tcp_port": tick_tcp_port,
|
||||
},
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.debug("Loop heartbeat write failed off-loop", exc_info=True)
|
||||
|
||||
try:
|
||||
# Immediate first write so monitors see a fresh file as soon as the
|
||||
# gateway is running, not after the first interval.
|
||||
await _write_off_loop()
|
||||
while True:
|
||||
if should_continue is not None and not should_continue():
|
||||
return
|
||||
await asyncio.sleep(interval)
|
||||
if should_continue is not None and not should_continue():
|
||||
return
|
||||
await _write_off_loop()
|
||||
finally:
|
||||
if tick_server is not None:
|
||||
tick_server.close()
|
||||
try:
|
||||
await tick_server.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
if tick_socket_path is not None:
|
||||
try:
|
||||
tick_socket_path.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
Reference in New Issue
Block a user