aiturk-hermes-ide/tests/gateway/test_handoff_watcher_resilience.py

"""Handoff watcher resilience: no head-of-line blocking, no stranded rows.

Two failure modes raised by adversarial review of the multi-profile handoff
work, both fixed here and pinned by these tests.

1. HEAD-OF-LINE BLOCKING. ``_process_handoff`` runs a full agent turn plus
   platform delivery. Awaiting it inline meant one slow handoff in profile A
   stopped the watcher from even POLLING B, C and D. Since the CLI gives up
   after 60s, a perfectly good handoff could time out purely because another
   profile's was ahead of it. Dispatch is now fire-and-forget.

2. STRANDED ``running`` ROWS. Only the watcher sets ``running``, for the span
   of one in-process dispatch. A gateway that dies mid-dispatch leaves the row
   there forever — and ``request_handoff`` refuses a NEW request unless the
   state is NULL/completed/failed, so that session can never hand off again,
   silently. Startup now reclaims those rows to ``failed``.
"""

import asyncio
import types
from pathlib import Path

import pytest

from gateway import run


def _running_flag(ticks):
    """A ``_running`` stand-in that is True for ``ticks`` reads, then False."""
    states = iter([True] * ticks + [False])

    class _Running:
        def __bool__(_self):
            try:
                return next(states)
            except StopIteration:
                return False

    return _Running()


class _SlowDB:
    """One pending row; ``_process_handoff`` for it never finishes on its own."""

    def __init__(self):
        self.polls = 0
        self.claimed = []

    async def list_pending_handoffs(self):
        self.polls += 1
        return [{"id": "slow-row"}]

    async def claim_handoff(self, sid):
        # Real claim is atomic pending→running: it succeeds exactly once.
        if sid in self.claimed:
            return False
        self.claimed.append(sid)
        return True

    async def complete_handoff(self, sid):
        return None

    async def fail_handoff(self, sid, err):
        return None


@pytest.mark.asyncio
async def test_slow_handoff_does_not_block_later_polls(monkeypatch):
    """A handoff that never returns must not stop the poll loop.

    Mutation-survivable by construction: ``_process_handoff`` blocks on an
    Event that is only set AFTER the watcher has been given room to keep
    polling. Restoring the inline ``await self._process_handoff`` deadlocks
    tick 1 — the watcher never reaches tick 2, ``release`` is never set, and
    ``wait_for`` raises TimeoutError.

    Note the sleep stub must yield control (``asyncio.sleep(0)``); a stub that
    returns immediately without yielding lets an inline dispatch monopolise
    the loop and masks the bug.
    """
    monkeypatch.setattr(run, "_handoff_watch_scopes", lambda _r: [(None, None)])

    # Capture the real sleep BEFORE patching: the stub runs as
    # ``run.asyncio.sleep``, which is the same module object the test imports,
    # so calling ``asyncio.sleep`` inside it would recurse into itself.
    _real_sleep = asyncio.sleep

    async def _yield_sleep(_seconds):
        await _real_sleep(0)

    monkeypatch.setattr(run.asyncio, "sleep", _yield_sleep)

    db = _SlowDB()
    started = asyncio.Event()
    release = asyncio.Event()

    async def _process_handoff(row, profile_name=None):
        started.set()
        await release.wait()

    fake = types.SimpleNamespace()
    fake._session_db = db
    fake._running = _running_flag(3)
    fake._process_handoff = _process_handoff

    async def _watch():
        await run.GatewayRunner._handoff_watcher(fake, interval=0.0, drain_timeout=0.01)

    task = asyncio.ensure_future(_watch())
    await asyncio.wait_for(started.wait(), timeout=5)

    # Give the loop real turns to poll while the handoff is stuck.
    for _ in range(20):
        await _real_sleep(0)

    polls_while_stuck = db.polls
    release.set()
    # The watcher may already have exited its loop and be draining; either way
    # it must finish once the stuck handoff is released.
    try:
        await asyncio.wait_for(task, timeout=5)
    except asyncio.TimeoutError:
        task.cancel()
        raise AssertionError("watcher did not finish after the handoff was released")

    assert polls_while_stuck >= 2, (
        "poll loop must keep polling while a handoff is in flight; "
        f"polls={polls_while_stuck}"
    )


@pytest.mark.asyncio
async def test_inflight_row_is_not_claimed_twice(monkeypatch):
    """A row already dispatched must be skipped by later ticks."""
    monkeypatch.setattr(run, "_handoff_watch_scopes", lambda _r: [(None, None)])

    async def _no_sleep(_seconds):
        return None

    monkeypatch.setattr(run.asyncio, "sleep", _no_sleep)

    db = _SlowDB()
    calls = []

    async def _process_handoff(row, profile_name=None):
        calls.append(row["id"])
        await asyncio.sleep(3600)

    fake = types.SimpleNamespace()
    fake._session_db = db
    fake._running = _running_flag(4)
    fake._process_handoff = _process_handoff

    coro = run.GatewayRunner._handoff_watcher(fake, interval=0.0, drain_timeout=0.01)
    await asyncio.wait_for(coro, timeout=5)

    assert calls == ["slow-row"], f"dispatched more than once: {calls}"


class _ReclaimDB:
    """Records the reclaim call and reports nothing pending."""

    def __init__(self, stale_ids=("dead-row",)):
        self.stale_ids = list(stale_ids)
        self.reclaim_calls = []

    async def reclaim_stale_running_handoffs(self, error):
        self.reclaim_calls.append(error)
        return self.stale_ids

    async def list_pending_handoffs(self):
        return []


@pytest.mark.asyncio
async def test_startup_reclaims_rows_stranded_in_running(monkeypatch):
    """Rows left 'running' by a dead gateway are failed at startup.

    Without this, ``request_handoff`` keeps rejecting new requests for that
    session forever and nothing tells the user why.
    """
    monkeypatch.setattr(run, "_handoff_watch_scopes", lambda _r: [(None, None)])

    async def _no_sleep(_seconds):
        return None

    monkeypatch.setattr(run.asyncio, "sleep", _no_sleep)

    db = _ReclaimDB()
    fake = types.SimpleNamespace()
    fake._session_db = db
    fake._running = _running_flag(1)

    async def _process_handoff(row, profile_name=None):
        return None

    fake._process_handoff = _process_handoff

    coro = run.GatewayRunner._handoff_watcher(fake, interval=0.0, drain_timeout=0.01)
    await asyncio.wait_for(coro, timeout=5)

    assert len(db.reclaim_calls) == 1, "reclaim must run exactly once per store"
    assert "/handoff" in db.reclaim_calls[0], (
        "the recorded error should tell the user how to retry"
    )


@pytest.mark.asyncio
async def test_reclaim_runs_per_profile_store(monkeypatch):
    """Every served profile's store gets reclaimed, not just the root's."""
    scopes = [
        (None, None),
        ("bala", Path("/h/profiles/bala")),
        ("medicina", Path("/h/profiles/medicina")),
    ]
    monkeypatch.setattr(run, "_handoff_watch_scopes", lambda _r: scopes)

    class _Scope:
        def __init__(self, home):
            self.home = home

        async def __aenter__(self):
            return self

        async def __aexit__(self, *exc):
            return False

    monkeypatch.setattr(run, "_async_profile_runtime_scope", _Scope)

    async def _no_sleep(_seconds):
        return None

    monkeypatch.setattr(run.asyncio, "sleep", _no_sleep)

    db = _ReclaimDB(stale_ids=[])
    fake = types.SimpleNamespace()
    fake._session_db = db
    fake._running = _running_flag(1)

    async def _process_handoff(row, profile_name=None):
        return None

    fake._process_handoff = _process_handoff

    coro = run.GatewayRunner._handoff_watcher(fake, interval=0.0, drain_timeout=0.01)
    await asyncio.wait_for(coro, timeout=5)

    assert len(db.reclaim_calls) == 3, (
        f"expected one reclaim per scope (root + 2 profiles), got {len(db.reclaim_calls)}"
    )


@pytest.mark.asyncio
async def test_reclaim_tolerates_store_without_the_method(monkeypatch):
    """An older/duck-typed store must not abort watcher startup."""
    monkeypatch.setattr(run, "_handoff_watch_scopes", lambda _r: [(None, None)])

    async def _no_sleep(_seconds):
        return None

    monkeypatch.setattr(run.asyncio, "sleep", _no_sleep)

    class _OldDB:
        def __init__(self):
            self.polls = 0

        async def list_pending_handoffs(self):
            self.polls += 1
            return []

    db = _OldDB()
    fake = types.SimpleNamespace()
    fake._session_db = db
    fake._running = _running_flag(1)

    async def _process_handoff(row, profile_name=None):
        return None

    fake._process_handoff = _process_handoff

    coro = run.GatewayRunner._handoff_watcher(fake, interval=0.0, drain_timeout=0.01)
    await asyncio.wait_for(coro, timeout=5)

    assert db.polls == 1, "the watcher must still poll when reclaim is unavailable"


def test_reclaim_stale_running_handoffs_flips_only_running_rows(tmp_path):
    """DB-level: only 'running' rows are touched, and their ids are returned."""
    from hermes_state import SessionDB

    db = SessionDB(db_path=tmp_path / "state.db")
    for sid, state in (
        ("dead", "running"),
        ("queued", "pending"),
        ("done", "completed"),
    ):
        db.create_session(sid, "cli")
        db._execute_write(
            lambda conn, s=sid, st=state: conn.execute(
                "UPDATE sessions SET handoff_state = ? WHERE id = ?", (st, s)
            )
        )

    reclaimed = db.reclaim_stale_running_handoffs("gateway died")

    assert reclaimed == ["dead"]
    assert db.get_handoff_state("dead")["state"] == "failed"
    assert db.get_handoff_state("dead")["error"] == "gateway died"
    assert db.get_handoff_state("queued")["state"] == "pending"
    assert db.get_handoff_state("done")["state"] == "completed"


def test_reclaimed_session_can_request_handoff_again(tmp_path):
    """The point of the reclaim: the session is usable again.

    ``request_handoff`` only accepts NULL/completed/failed, so a stranded
    'running' row is what permanently locks the session out.
    """
    from hermes_state import SessionDB

    db = SessionDB(db_path=tmp_path / "state.db")
    db.create_session("stuck", "cli")
    db._execute_write(
        lambda conn: conn.execute(
            "UPDATE sessions SET handoff_state = 'running' WHERE id = 'stuck'"
        )
    )

    assert db.request_handoff("stuck", "telegram") is False, (
        "precondition: a stranded 'running' row blocks new handoff requests"
    )

    db.reclaim_stale_running_handoffs("gateway died")

    assert db.request_handoff("stuck", "telegram") is True, (
        "after reclaim the session must be able to hand off again"
    )