Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

gateway/scale_to_zero.py

@@ -0,0 +1,314 @@
+"""Scale-to-zero idle detection + dormant-quiesce for the gateway (Phase 0).
+
+This is the gateway-side BEHAVIOUR layer that consumes the relay scale-to-zero
+PRIMITIVES (gateway-gateway Phase 5: the buffered-flip, the durable per-instance
+buffer, the wakeUrl poke, the reconnect supervisor). It owns the *decision* to go
+idle, drives the relay transport's ``go_dormant()`` (D12), and then SUSPENDS the
+machine itself through the local Fly Machines API socket. Wake stays platform-side:
+autostart-on-wakeUrl (decisions.md Q3=C′).
+
+Why the gateway self-suspends instead of relying on ``autostop:"suspend"``: Fly
+Proxy judges idle exclusively on INBOUND proxied connections — it cannot see an
+in-flight agent turn (outbound-only LLM traffic) and there is no way for the app
+to signal "not ready to suspend". Mid-2026 the proxy also stopped treating open
+OUTBOUND sockets as activity, so the relay WebSocket no longer masks the race:
+Fly would suspend a machine mid-job, and could suspend BEFORE ``go_dormant()``
+flipped the relay destination (the buffered-event black hole). Owning the suspend
+call closes both: it only ever fires after the idle predicate (no running agents,
+no live background work, inbound-quiet) holds AND the dormant quiesce completed.
+
+Design constraints (decisions.md):
+  - Per-instance enable is gated SOLELY by the NAS "Labs" toggle, carried to the
+    gateway as the ``HERMES_SCALE_TO_ZERO`` env stamp (D11/Q8=A). NOT a user
+    config key; ``scale_to_zero.idle_timeout_minutes`` IS config.yaml (D2).
+  - Arm only when messaging is relay-only or absent (D1/F6) AND a wakeUrl is
+    registered (§3.4(1)) AND the flag is set.
+  - Idle = no in-flight agent turn AND no inbound for N min AND no live
+    background work (D2/D3/F7).
+  - The quiesce uses ``go_dormant()`` (socket closed + supervisor preserved),
+    NEVER the stop/restart drain or ``disconnect()`` (F12/F14). The process stays
+    alive; Fly freezes+resumes it.
+  - ``mark_resume_pending`` is deliberately NOT called here (D13 — suspend
+    preserves RAM; revive only if we move to autostop:"stop" or see kills).
+
+The pure helpers (``parse_idle_timeout_seconds``, ``scale_to_zero_enabled``,
+``messaging_is_relay_only_or_absent``, ``is_idle``, ``should_arm``) take plain
+inputs so they unit-test without a live gateway.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import socket
+from typing import Any, Iterable, Optional
+
+logger = logging.getLogger(__name__)
+
+# Env flag stamped by NAS when the scaleToZero Labs toggle is on (D11/Q8=A),
+# mirroring how the `relay` feature stamps GATEWAY_RELAY_URL. Truthy values only.
+SCALE_TO_ZERO_ENV = "HERMES_SCALE_TO_ZERO"
+
+# Fly-injected machine identity (present on every Fly machine). Used by the
+# self-suspend call; both must be present for self_suspend_available().
+FLY_APP_NAME_ENV = "FLY_APP_NAME"
+FLY_MACHINE_ID_ENV = "FLY_MACHINE_ID"
+
+# The local flaps (Fly Machines API) unix socket, available inside every Fly
+# machine. A POST to /v1/apps/{app}/machines/{id}/suspend snapshots RAM and
+# suspends THIS machine — the Fly-endorsed replacement for proxy autostop when
+# the app must own the idle decision (https://fly.io/docs/reference/suspend-resume/).
+FLY_API_SOCKET = "/.fly/api"
+
+
+# config.yaml default (D2). Behavioural setting -> config, not env.
+# 2 minutes: with the gateway owning the suspend (idle predicate covers agent
+# turns, cron, API runs, and background work; the relay drains + flips before
+# the freeze), a short window is safe — real work always blocks the suspend and
+# resume-from-suspend is sub-second, so the only cost of waking "too eagerly"
+# after a quiet spell is a Fly-proxied poke away. Longer windows just bill idle
+# RAM. Raise per-instance via gateway.scale_to_zero.idle_timeout_minutes.
+DEFAULT_IDLE_TIMEOUT_MINUTES = 2
+
+_TRUTHY = {"1", "true", "yes", "on"}
+
+
+def scale_to_zero_enabled(environ: Optional[dict] = None) -> bool:
+    """Whether the per-instance Labs toggle is on (the HERMES_SCALE_TO_ZERO stamp).
+
+    D11/Q8=A: this env flag is the SOLE per-instance enable signal reaching the
+    gateway. Absent/blank/falsey -> disabled (fail-safe default off).
+    """
+    env = environ if environ is not None else os.environ
+    return str(env.get(SCALE_TO_ZERO_ENV, "")).strip().lower() in _TRUTHY
+
+
+def parse_idle_timeout_seconds(
+    cfg_value: Any, default_minutes: int = DEFAULT_IDLE_TIMEOUT_MINUTES
+) -> float:
+    """Coerce ``scale_to_zero.idle_timeout_minutes`` (config.yaml, D2) to seconds.
+
+    Degrades to the default on any non-numeric / non-positive value (never raises,
+    never returns <= 0 — a zero/negative timeout would make the gateway go dormant
+    instantly, which is never the intent).
+    """
+    try:
+        minutes = float(cfg_value)
+    except (TypeError, ValueError):
+        minutes = float(default_minutes)
+    if minutes <= 0:
+        minutes = float(default_minutes)
+    return minutes * 60.0
+
+
+def messaging_is_relay_only_or_absent(platforms: Iterable[Any]) -> bool:
+    """True iff the only connected messaging platform is RELAY, or there is none
+    (a Chronos-only / no-platform agent) — the F6/D1 structural precondition.
+
+    A directly-connected platform (Discord/Telegram/Slack/...) holds a live
+    socket and cannot scale to zero, so its presence disarms the feature. We
+    compare by the platform's ``.value``/name to avoid importing the enum here
+    (keeps this module import-light and unit-testable).
+    """
+    names = {_platform_name(p) for p in platforms}
+    names.discard("relay")
+    return len(names) == 0
+
+
+def _platform_name(platform: Any) -> str:
+    value = getattr(platform, "value", platform)
+    return str(value).strip().lower()
+
+
+def should_arm(
+    *,
+    enabled: bool,
+    relay_only_or_absent: bool,
+    wake_url: Optional[str],
+) -> bool:
+    """Whether to start the idle watcher at all (D1/D11/§3.4(1)).
+
+    ALL must hold: the Labs flag is on, messaging is relay-only/absent, and a
+    wakeUrl is registered (a suspended instance with no reachable wake target is
+    a black hole — §3.4(1)). Any unmet -> the watcher never starts (no idle
+    timer, no dormancy), so a non-opted instance behaves exactly as today.
+    """
+    return bool(enabled) and bool(relay_only_or_absent) and bool(wake_url)
+
+
+def is_idle(
+    *,
+    active_work_count: int,
+    seconds_since_last_inbound: float,
+    idle_timeout_seconds: float,
+    has_live_background_work: bool,
+) -> bool:
+    """The idle predicate (D2/D3/F7). Pure — composes the three conjuncts.
+
+    Idle iff: no counted active work (in-flight agent turns + cron jobs +
+    API-server runs — the caller aggregates every foreground work source),
+    no inbound within the timeout window, and no live background work
+    (backgrounded delegate_task / kanban / bg terminal). Any active work
+    keeps the gateway awake — suspending mid-flight would lose it.
+
+    ``active_work_count`` deliberately names the BROAD aggregate, not just
+    agents: a caller passing only ``len(_running_agents)`` reopens the
+    mid-cron-job suspend hole. Callers that cannot read a work source must
+    fail AWAKE (pass a positive sentinel), never fail to 0.
+    """
+    if active_work_count > 0:
+        return False
+    if has_live_background_work:
+        return False
+    return seconds_since_last_inbound >= idle_timeout_seconds
+
+
+# Dashboard-client liveness marker. The dashboard process (tui_gateway/ws.py,
+# a DIFFERENT process from the gateway on hosted instances) touches this file
+# on every /api/ws connect and inbound frame — the desktop app, web dashboard
+# and TUI all send `gateway.ping` every 15s (apps/shared/src/json-rpc-gateway.ts,
+# ui-tui/src/gatewayClient.ts). The gateway folds the mtime into its inbound
+# clock, so an open client holds the box awake exactly like a chat message does
+# and gets the same idle_timeout grace after it disconnects. Without this the
+# box suspends under the open client, the client's reconnect loop re-pokes the
+# Fly-proxied hostname, autostart resumes it, and the instance flaps every ~60s
+# (13 of 72 active opted-in prod instances, 2026-09-02).
+#
+# There is deliberately NO staleness cutoff here: the mtime is a timestamp of
+# real inbound, and is_idle already decides whether it is recent enough. A
+# lingering marker cannot pin the box — once it is older than idle_timeout it
+# no longer counts, exactly like an old _last_inbound_at.
+DASHBOARD_CLIENT_HEARTBEAT_REL = os.path.join("state", "dashboard_clients.heartbeat")
+
+
+def dashboard_client_heartbeat_path(hermes_home: Optional[os.PathLike | str] = None):
+    """Path of the dashboard-client liveness marker under HERMES_HOME."""
+    from pathlib import Path
+
+    if hermes_home is None:
+        from hermes_constants import get_hermes_home
+
+        hermes_home = get_hermes_home()
+    return Path(hermes_home) / DASHBOARD_CLIENT_HEARTBEAT_REL
+
+
+def touch_dashboard_client_heartbeat(path: Optional[os.PathLike | str] = None) -> bool:
+    """Mark "a dashboard client is attached right now". Best-effort, never raises."""
+    try:
+        p = dashboard_client_heartbeat_path() if path is None else path
+        os.makedirs(os.path.dirname(p), exist_ok=True)
+        with open(p, "a", encoding="utf-8"):
+            pass
+        os.utime(p, None)
+        return True
+    except Exception:  # noqa: BLE001 - liveness garnish must never break the WS
+        logger.debug("scale-to-zero: dashboard heartbeat touch failed", exc_info=True)
+        return False
+
+
+def dashboard_client_last_seen(
+    path: Optional[os.PathLike | str] = None,
+    *,
+    now: Optional[float] = None,
+) -> Optional[float]:
+    """Epoch seconds a dashboard client last sent a WS frame, or None if never.
+
+    Missing marker -> None (the steady state on a box nobody has the dashboard
+    open on — NOT fail-awake, or every instance would never sleep). An
+    unreadable marker -> ``now`` (fail-awake: an unreadable source counts as
+    activity, same rule as the work counters in ``is_idle``).
+    """
+    import time
+
+    current = time.time() if now is None else now
+    p = dashboard_client_heartbeat_path() if path is None else path
+    try:
+        # Clamp to now: a wall-clock step-back (NTP) can leave the mtime in the
+        # future, which would push idle out by the step size for no reason.
+        return min(os.stat(p).st_mtime, current)
+    except FileNotFoundError:
+        return None
+    except OSError:
+        return current
+
+
+def self_suspend_available(environ: Optional[dict] = None) -> bool:
+    """Whether this process can suspend its own machine via the flaps socket.
+
+    True iff the Fly-injected machine identity is present AND the local Machines
+    API socket exists. Off-Fly (local dev, Azure ACA, tests) this is False and
+    the watcher skips the quiesce entirely: the platform owns the freeze, so
+    the gateway stays connected until it lands.
+    """
+    env = environ if environ is not None else os.environ
+    return bool(
+        str(env.get(FLY_APP_NAME_ENV, "")).strip()
+        and str(env.get(FLY_MACHINE_ID_ENV, "")).strip()
+        and os.path.exists(FLY_API_SOCKET)
+    )
+
+
+def suspend_self(
+    environ: Optional[dict] = None,
+    *,
+    socket_path: str = FLY_API_SOCKET,
+    timeout: float = 10.0,
+) -> bool:
+    """POST /v1/apps/{app}/machines/{id}/suspend on the local flaps socket.
+
+    Fly's in-machine Machines API needs no token — the socket itself is the
+    credential. Equivalent to:
+        curl --unix-socket /.fly/api -X POST \\
+          http://flaps/v1/apps/$FLY_APP_NAME/machines/$FLY_MACHINE_ID/suspend
+
+    Returns True when flaps accepted the request (2xx). The caller should treat
+    this as fire-and-forget: on success the kernel freezes this process shortly
+    after, so there may be nothing meaningful to run afterwards. Never raises —
+    a failed suspend just leaves the machine running (fail-awake, never
+    fail-frozen), which costs money but loses no work.
+
+    stdlib-only on purpose: a plain unix-socket HTTP/1.1 request, no httpx/
+    requests dependency in the hot path and no async plumbing to freeze
+    mid-await.
+    """
+    env = environ if environ is not None else os.environ
+    app = str(env.get(FLY_APP_NAME_ENV, "")).strip()
+    machine_id = str(env.get(FLY_MACHINE_ID_ENV, "")).strip()
+    if not app or not machine_id:
+        logger.warning("scale-to-zero: suspend_self called without Fly machine identity")
+        return False
+    request = (
+        f"POST /v1/apps/{app}/machines/{machine_id}/suspend HTTP/1.1\r\n"
+        "Host: flaps\r\n"
+        "Content-Length: 0\r\n"
+        "Connection: close\r\n"
+        "\r\n"
+    )
+    try:
+        with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
+            sock.settimeout(timeout)
+            sock.connect(socket_path)
+            sock.sendall(request.encode("ascii"))
+            response = b""
+            while len(response) < 65536:
+                chunk = sock.recv(4096)
+                if not chunk:
+                    break
+                response += chunk
+    except OSError as exc:
+        logger.warning("scale-to-zero: flaps suspend request failed: %s", exc)
+        return False
+    status_line = response.split(b"\r\n", 1)[0].decode("ascii", "replace")
+    parts = status_line.split()
+    ok = len(parts) >= 2 and parts[1].isdigit() and 200 <= int(parts[1]) < 300
+    if ok:
+        logger.info("scale-to-zero: machine suspend accepted by flaps (%s)", status_line)
+    else:
+        body = response.split(b"\r\n\r\n", 1)[-1][:500].decode("utf-8", "replace")
+        logger.warning(
+            "scale-to-zero: flaps suspend rejected: %s %s",
+            status_line,
+            json.dumps(body)[:500],
+        )
+    return ok