Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

gateway/memory_status.py

@@ -0,0 +1,203 @@
+"""Memory status rollup for ``/api/status`` (NS-656).
+
+The gateway already *produces* every memory-pressure signal a user would
+want to know about, but all of it dies in log files:
+
+* :func:`gateway.shutdown_watchdog.write_loop_heartbeat` embeds a
+  :func:`gateway.lifecycle_ledger.sample_memory` snapshot (gateway RSS +
+  system MemAvailable/MemTotal + swap) in ``state/gateway.heartbeat``
+  every 30 seconds.
+* :func:`gateway.lifecycle_ledger.record_startup` detects an unclean
+  previous death and flags ``suspected_oom`` — but only into
+  ``gateway-exit-diag.log`` and a WARNING line.
+* ``gateway/agent_cache_pressure.py`` evicts transcripts under pressure,
+  again log-only.
+
+So a hosted agent can be OOM-killed hourly (the BlueAtlas incident,
+NS-608) while its dashboard and the NAS agent card both look perfectly
+healthy.  This module is the read side that closes the gap: it distills
+the *already-persisted* heartbeat + lifecycle sentinel into a compact,
+public-safe block that ``/api/status`` can serve to the dashboard SPA
+and the NAS availability sweep — no new sampling, no IPC with the
+gateway process, just two small file reads.
+
+Public-safety note: ``/api/status`` is an unauthenticated liveness probe
+(``PUBLIC_API_PATHS``), which is exactly why NAS can consume it.  This
+block therefore carries only coarse numbers (MB granularity), enums, and
+booleans — the same disclosure class as the existing ``active_agents``
+count and ``nous_session_valid`` field (which was added for the same
+NAS-sweep audience).
+
+Everything here is best-effort and read-only: a missing/corrupt file
+degrades to ``pressure="unknown"`` rather than raising into the status
+endpoint.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+# Pressure thresholds on system MemAvailable.  ``critical`` deliberately
+# mirrors the lifecycle ledger's OOM-suspicion heuristics
+# (:data:`gateway.lifecycle_ledger._LOW_MEM_AVAILABLE_KIB` /
+# ``_LOW_MEM_AVAILABLE_FRACTION``): if a memory level would make a
+# subsequent unclean death "suspected OOM", the user should already have
+# been warned at that level while the process was still alive.
+_CRITICAL_AVAILABLE_KIB = 64 * 1024  # < 64 MiB available
+_CRITICAL_AVAILABLE_FRACTION = 0.05  # < 5% of MemTotal
+_ELEVATED_AVAILABLE_KIB = 128 * 1024  # < 128 MiB available
+_ELEVATED_AVAILABLE_FRACTION = 0.15  # < 15% of MemTotal
+
+# A heartbeat older than this no longer describes the present.  The writer
+# cadence is 30s (DEFAULT_HEARTBEAT_INTERVAL_S); 150s of slack tolerates a
+# briefly stalled loop without letting a long-dead gateway's last sample
+# masquerade as current pressure.
+_HEARTBEAT_FRESH_TTL_S = 150.0
+
+_KIB_PER_MB = 1024
+
+
+def _mb(kib: Any) -> Optional[int]:
+    if isinstance(kib, bool) or not isinstance(kib, int) or kib < 0:
+        return None
+    return kib // _KIB_PER_MB
+
+
+def _parse_iso(value: Any) -> Optional[datetime]:
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        parsed = datetime.fromisoformat(value)
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed
+
+
+def classify_pressure(
+    available_kib: Any, total_kib: Any
+) -> str:
+    """Map a MemAvailable/MemTotal pair to ``ok``/``elevated``/``critical``.
+
+    ``unknown`` when the sample is missing or malformed — the caller must
+    not treat "we could not read it" as "memory is fine".
+    """
+    if (
+        isinstance(available_kib, bool)
+        or not isinstance(available_kib, int)
+        or available_kib < 0
+    ):
+        return "unknown"
+    fraction: Optional[float] = None
+    if (
+        not isinstance(total_kib, bool)
+        and isinstance(total_kib, int)
+        and total_kib > 0
+    ):
+        fraction = available_kib / total_kib
+    if available_kib < _CRITICAL_AVAILABLE_KIB or (
+        fraction is not None and fraction < _CRITICAL_AVAILABLE_FRACTION
+    ):
+        return "critical"
+    if available_kib < _ELEVATED_AVAILABLE_KIB or (
+        fraction is not None and fraction < _ELEVATED_AVAILABLE_FRACTION
+    ):
+        return "elevated"
+    return "ok"
+
+
+def _read_heartbeat(home: Optional[Path]) -> Optional[Dict[str, Any]]:
+    try:
+        from gateway.lifecycle_ledger import _read_json
+        from gateway.shutdown_watchdog import get_loop_heartbeat_path
+
+        return _read_json(get_loop_heartbeat_path(home))
+    except Exception:
+        return None
+
+
+def _read_sentinel(home: Optional[Path]) -> Optional[Dict[str, Any]]:
+    try:
+        from gateway.lifecycle_ledger import (
+            _read_json,
+            get_lifecycle_sentinel_path,
+        )
+
+        return _read_json(get_lifecycle_sentinel_path(home))
+    except Exception:
+        return None
+
+
+def collect_memory_status(
+    home: Optional[Path] = None,
+    *,
+    now: Optional[datetime] = None,
+) -> Dict[str, Any]:
+    """Build the ``memory`` block for ``/api/status``.
+
+    ``home`` scopes the read to a profile's HERMES_HOME (the status
+    endpoint's ``?profile=`` handling passes it through); ``None`` means
+    the active profile.  ``now`` is injectable for tests.
+
+    Always returns a dict — a gateway that is down, has never written a
+    heartbeat, or whose files are corrupt yields
+    ``{"pressure": "unknown", ...}`` with whatever fields could still be
+    recovered.  Never raises.
+    """
+    moment = now or datetime.now(timezone.utc)
+    status: Dict[str, Any] = {
+        "pressure": "unknown",
+        "gateway_rss_mb": None,
+        "system_total_mb": None,
+        "system_available_mb": None,
+        "swap_used_mb": None,
+        "sampled_at": None,
+        "last_boot_unclean": False,
+        "last_boot_suspected_oom": False,
+        # Identity of the CURRENT gateway life (the sentinel's started_at).
+        # A suspected-OOM restart writes a fresh sentinel, so this changes on
+        # every boot — the dashboard keys banner dismissal on it so that
+        # acknowledging one OOM restart does not mute reports of the NEXT
+        # one (the hourly-restart-loop case is exactly the one that matters).
+        "boot_id": None,
+    }
+
+    heartbeat = _read_heartbeat(home)
+    if heartbeat:
+        sampled_at = _parse_iso(heartbeat.get("updated_at"))
+        mem = heartbeat.get("mem")
+        if isinstance(mem, dict):
+            status["gateway_rss_mb"] = _mb(mem.get("rss_kib"))
+            status["system_total_mb"] = _mb(mem.get("mem_total_kib"))
+            status["system_available_mb"] = _mb(mem.get("mem_available_kib"))
+            status["swap_used_mb"] = _mb(mem.get("swap_used_kib"))
+            if sampled_at is not None:
+                status["sampled_at"] = sampled_at.isoformat()
+                age_s = (moment - sampled_at).total_seconds()
+                if 0 <= age_s <= _HEARTBEAT_FRESH_TTL_S:
+                    status["pressure"] = classify_pressure(
+                        mem.get("mem_available_kib"),
+                        mem.get("mem_total_kib"),
+                    )
+                # else: stale sample — numbers are still reported (they are
+                # honest about *when* via sampled_at) but pressure stays
+                # "unknown" so a dead gateway's final gasp cannot render a
+                # live "critical" banner forever.
+
+    sentinel = _read_sentinel(home)
+    if sentinel:
+        status["last_boot_unclean"] = bool(sentinel.get("prior_unclean_exit"))
+        status["last_boot_suspected_oom"] = bool(
+            sentinel.get("prior_suspected_oom")
+        )
+        started_at = sentinel.get("started_at")
+        if isinstance(started_at, str) and started_at:
+            status["boot_id"] = started_at
+
+    return status