Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/gateway/test_scale_to_zero_dashboard_client.py

@@ -0,0 +1,284 @@
+"""Scale-to-zero: an attached dashboard/desktop/TUI WS client counts as activity.
+
+Background (2026-09-02 fleet audit): 13 of 72 active opted-in prod instances
+flapped suspend -> proxy-wake every ~60s. The gateway only stamped
+``_last_inbound_at`` for messaging inbound, so it suspended under an open
+dashboard client; the client's reconnect loop re-poked the Fly-proxied hostname
+and autostart resumed the box. The dashboard runs in a separate process on
+hosted instances, so the signal crosses over as a marker-file mtime.
+
+These tests exercise the REAL seams — the pure helpers with a real temp
+HERMES_HOME, GatewayRunner._scale_to_zero_is_idle's composition, and
+tui_gateway.ws.handle_ws — rather than stubbing the collection under test
+(the F25 / #84327 lesson: bugs live at the call site, not in the pure predicate).
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+
+import pytest
+
+from gateway import scale_to_zero as s2z
+from gateway.run import GatewayRunner
+
+
+@pytest.fixture
+def hermes_home(tmp_path, monkeypatch):
+    monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+    return tmp_path
+
+
+def _stat_denying(target):
+    """os.stat replacement that denies ONLY the marker path (stdlib callers unaffected)."""
+    real = os.stat
+
+    def _stat(path, *a, **k):
+        if os.fspath(path) == os.fspath(target):
+            raise PermissionError("nope")
+        return real(path, *a, **k)
+
+    return _stat
+
+
+# --- pure helpers -----------------------------------------------------------
+
+
+def test_heartbeat_path_lives_under_hermes_home_state(hermes_home):
+    p = s2z.dashboard_client_heartbeat_path()
+    assert p == hermes_home / "state" / "dashboard_clients.heartbeat"
+
+
+def test_last_seen_missing_marker_is_none_not_fail_awake(hermes_home):
+    # Steady state for a box nobody has the dashboard open on: must read as
+    # "no client", otherwise no instance would ever suspend.
+    assert s2z.dashboard_client_last_seen() is None
+
+
+def test_touch_creates_state_dir_and_marker(hermes_home):
+    assert s2z.touch_dashboard_client_heartbeat() is True
+    p = s2z.dashboard_client_heartbeat_path()
+    assert p.exists()
+    seen = s2z.dashboard_client_last_seen()
+    assert seen is not None and abs(seen - time.time()) < 5
+
+
+def test_last_seen_returns_raw_mtime_without_staleness_cutoff(hermes_home):
+    # No liveness cutoff here on purpose: is_idle decides recency. A 1h-old
+    # marker still reports its mtime; the gateway then finds it outside
+    # idle_timeout, same as an old _last_inbound_at.
+    s2z.touch_dashboard_client_heartbeat()
+    p = s2z.dashboard_client_heartbeat_path()
+    mtime = os.stat(p).st_mtime
+    assert s2z.dashboard_client_last_seen(now=mtime + 10) == mtime
+    assert s2z.dashboard_client_last_seen(now=mtime + 3600) == mtime
+
+
+def test_last_seen_future_mtime_is_clamped_to_now(hermes_home):
+    # A wall-clock step-back can leave the marker in the future; it must not
+    # extend the idle window past "now".
+    s2z.touch_dashboard_client_heartbeat()
+    p = s2z.dashboard_client_heartbeat_path()
+    future = time.time() + 600
+    os.utime(p, (future, future))
+    now = time.time()
+    assert s2z.dashboard_client_last_seen(now=now) == now
+
+
+def test_last_seen_unreadable_marker_fails_awake(hermes_home, monkeypatch):
+    s2z.touch_dashboard_client_heartbeat()
+
+    monkeypatch.setattr(s2z.os, "stat", _stat_denying(s2z.dashboard_client_heartbeat_path()))
+    now = 1_000_000.0
+    # Unreadable (not missing) => counts as activity right now.
+    assert s2z.dashboard_client_last_seen(now=now) == now
+
+
+def test_touch_never_raises(hermes_home, monkeypatch):
+    monkeypatch.setattr(s2z.os, "utime", lambda *a, **k: (_ for _ in ()).throw(OSError("ro")))
+    assert s2z.touch_dashboard_client_heartbeat() is False
+
+
+# --- gateway side: the idle predicate composition ---------------------------
+
+
+def _runner(monkeypatch, *, last_inbound_at):
+    r = GatewayRunner.__new__(GatewayRunner)
+    r._running = True
+    r._last_inbound_at = last_inbound_at
+    r._running_agents = {}
+    r._background_tasks = set()
+    r.adapters = {}
+    monkeypatch.setattr(r, "_scale_to_zero_idle_timeout_seconds", lambda: 120.0, raising=False)
+    monkeypatch.setattr(r, "_scale_to_zero_has_live_background_work", lambda: False, raising=False)
+    monkeypatch.setattr("cron.scheduler.get_running_job_ids", lambda: [])
+    return r
+
+
+def test_idle_without_dashboard_client_unchanged(hermes_home, monkeypatch):
+    r = _runner(monkeypatch, last_inbound_at=time.time() - 600)
+    assert r._scale_to_zero_is_idle() is True
+
+
+def test_attached_dashboard_client_blocks_idle(hermes_home, monkeypatch):
+    r = _runner(monkeypatch, last_inbound_at=time.time() - 600)
+    s2z.touch_dashboard_client_heartbeat()
+    assert r._scale_to_zero_is_idle() is False
+
+
+def test_client_gets_the_same_idle_grace_as_a_message(hermes_home, monkeypatch):
+    """Last WS frame 100s ago with a 120s idle_timeout => still inside the
+    window => NOT idle. This is the 2-minute-after-the-app-closes contract; an
+    earlier draft cut the marker off at 45s and suspended ~50s after
+    disconnect (observed live on staging)."""
+    r = _runner(monkeypatch, last_inbound_at=time.time() - 600)
+    s2z.touch_dashboard_client_heartbeat()
+    p = s2z.dashboard_client_heartbeat_path()
+    old = time.time() - 100
+    os.utime(p, (old, old))
+    assert r._scale_to_zero_is_idle() is False
+
+
+def test_client_gone_longer_than_idle_timeout_is_idle(hermes_home, monkeypatch):
+    r = _runner(monkeypatch, last_inbound_at=time.time() - 600)
+    s2z.touch_dashboard_client_heartbeat()
+    p = s2z.dashboard_client_heartbeat_path()
+    old = time.time() - 121
+    os.utime(p, (old, old))
+    assert r._scale_to_zero_is_idle() is True
+
+
+def test_marker_predating_gateway_inbound_does_not_matter(hermes_home, monkeypatch):
+    # Ancient marker from a client that left hours ago, gateway idle 600s.
+    r = _runner(monkeypatch, last_inbound_at=time.time() - 600)
+    s2z.touch_dashboard_client_heartbeat()
+    p = s2z.dashboard_client_heartbeat_path()
+    old = time.time() - 7200
+    os.utime(p, (old, old))
+    assert r._scale_to_zero_is_idle() is True
+
+
+def test_dashboard_client_seen_recently_extends_inbound_clock(hermes_home, monkeypatch):
+    # Marker 30s old: inbound clock moves to 30s ago, which is
+    # inside the 120s window => not idle, even though the gateway's own
+    # _last_inbound_at is ancient.
+    r = _runner(monkeypatch, last_inbound_at=time.time() - 600)
+    s2z.touch_dashboard_client_heartbeat()
+    p = s2z.dashboard_client_heartbeat_path()
+    t = time.time() - 30
+    os.utime(p, (t, t))
+    assert r._scale_to_zero_is_idle() is False
+
+
+def test_newer_gateway_inbound_wins_over_older_marker(hermes_home, monkeypatch):
+    r = _runner(monkeypatch, last_inbound_at=time.time() - 5)
+    monkeypatch.setattr(r, "_scale_to_zero_idle_timeout_seconds", lambda: 10.0, raising=False)
+    s2z.touch_dashboard_client_heartbeat()
+    p = s2z.dashboard_client_heartbeat_path()
+    t = time.time() - 40
+    os.utime(p, (t, t))
+    # Picking the marker (40s > 10s) would read idle; the chat message (5s) wins.
+    assert r._scale_to_zero_is_idle() is False
+    # _last_inbound_at itself is not mutated by the read.
+    assert time.time() - r._last_inbound_at < 10
+
+
+def test_unreadable_marker_keeps_gateway_awake(hermes_home, monkeypatch):
+    r = _runner(monkeypatch, last_inbound_at=time.time() - 600)
+    s2z.touch_dashboard_client_heartbeat()
+    monkeypatch.setattr(s2z.os, "stat", _stat_denying(s2z.dashboard_client_heartbeat_path()))
+    assert r._scale_to_zero_is_idle() is False
+
+
+# --- dashboard side: the real handle_ws path touches the marker -------------
+
+
+def test_handle_ws_connect_touches_marker(hermes_home, monkeypatch):
+    from tui_gateway import server, ws as ws_mod
+
+    monkeypatch.setattr(server, "_start_backend_heartbeat_refresher", lambda: None)
+    monkeypatch.setattr(server, "_schedule_startup_orphan_sweep", lambda: None, raising=False)
+    monkeypatch.setattr(server, "resolve_skin", lambda: "default")
+    monkeypatch.setattr(server, "_ensure_skin_watcher", lambda: None)
+    monkeypatch.setattr(server, "register_live_transport", lambda *_a, **_k: None)
+    monkeypatch.setattr(server, "_WS_ORPHAN_REAP_GRACE_S", 0)
+    monkeypatch.setattr(ws_mod, "_dashboard_client_touched_at", 0.0)
+
+    class FakeWS:
+        async def accept(self):
+            pass
+
+        async def send_text(self, line):
+            pass
+
+        async def receive_text(self):
+            raise ws_mod._WebSocketDisconnect()
+
+        async def close(self):
+            pass
+
+    assert s2z.dashboard_client_last_seen() is None
+    asyncio.run(ws_mod.handle_ws(FakeWS()))
+    seen = s2z.dashboard_client_last_seen()
+    assert seen is not None and abs(seen - time.time()) < 5
+
+
+def test_handle_ws_inbound_frames_refresh_marker(hermes_home, monkeypatch):
+    from tui_gateway import server, ws as ws_mod
+
+    monkeypatch.setattr(server, "_start_backend_heartbeat_refresher", lambda: None)
+    monkeypatch.setattr(server, "_schedule_startup_orphan_sweep", lambda: None, raising=False)
+    monkeypatch.setattr(server, "resolve_skin", lambda: "default")
+    monkeypatch.setattr(server, "_ensure_skin_watcher", lambda: None)
+    monkeypatch.setattr(server, "register_live_transport", lambda *_a, **_k: None)
+    monkeypatch.setattr(server, "_WS_ORPHAN_REAP_GRACE_S", 0)
+    monkeypatch.setattr(ws_mod, "_dashboard_client_touched_at", 0.0)
+    # Disable the throttle so each frame is observable.
+    monkeypatch.setattr(ws_mod, "_DASHBOARD_CLIENT_TOUCH_MIN_INTERVAL_S", 0.0)
+
+    frames = ['{"jsonrpc":"2.0","method":"gateway.ping","id":1}'] * 2
+    touches: list[float] = []
+    real_touch = s2z.touch_dashboard_client_heartbeat
+
+    def _spy(path=None):
+        touches.append(time.time())
+        return real_touch(path)
+
+    monkeypatch.setattr(s2z, "touch_dashboard_client_heartbeat", _spy)
+
+    class FakeWS:
+        async def accept(self):
+            pass
+
+        async def send_text(self, line):
+            pass
+
+        async def receive_text(self):
+            if frames:
+                return frames.pop()
+            raise ws_mod._WebSocketDisconnect()
+
+        async def close(self):
+            pass
+
+    asyncio.run(ws_mod.handle_ws(FakeWS()))
+    # 1 on connect + 1 per inbound frame.
+    assert len(touches) == 3
+    assert s2z.dashboard_client_last_seen() is not None
+
+
+def test_note_activity_is_throttled(hermes_home, monkeypatch):
+    from tui_gateway import ws as ws_mod
+
+    calls = {"n": 0}
+    monkeypatch.setattr(
+        s2z, "touch_dashboard_client_heartbeat", lambda path=None: calls.__setitem__("n", calls["n"] + 1) or True
+    )
+    monkeypatch.setattr(ws_mod, "_dashboard_client_touched_at", 0.0)
+    ws_mod._note_dashboard_client_activity(force=True)
+    ws_mod._note_dashboard_client_activity()
+    ws_mod._note_dashboard_client_activity()
+    assert calls["n"] == 1
+    ws_mod._note_dashboard_client_activity(force=True)
+    assert calls["n"] == 2