Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/gateway/test_kanban_reconcile_orphans.py

@@ -0,0 +1,178 @@
+"""Tests: orphaned-card reconciliation for the kanban dispatcher.
+
+Tracked-state vs. reality divergence: a task can sit in ``status='running'``
+with broken claim bookkeeping — ``claim_lock IS NULL`` or ``claim_expires IS
+NULL`` (crash mid-claim, manual SQL, DB restore, partial migration). None of
+the existing recovery paths ever touch such a card:
+
+- ``release_stale_claims`` requires ``claim_expires IS NOT NULL``;
+- ``detect_crashed_workers`` requires a host-local ``claim_lock`` prefix and
+  a recorded ``worker_pid``;
+- ``detect_stale_running`` is disabled by default (``stale_timeout=0``).
+
+Result: a zombie card that shows Running forever. ``reconcile_orphaned_running``
+is the reconciliation pass: it finds those orphans, requeues them to ``ready``
+with an explanatory note, and logs a ``reconciled`` event. Wired into
+``dispatch_once`` each tick, gated by ``kanban.reconcile_orphans`` (config.yaml,
+default on) at the gateway watcher layer.
+
+Inspired by openai/symphony's tracker reconciliation (Apache-2.0), idea-level.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from hermes_cli import kanban_db as kb
+
+
+@pytest.fixture
+def kanban_home(tmp_path, monkeypatch):
+    home = tmp_path / ".hermes"
+    home.mkdir()
+    monkeypatch.setenv("HERMES_HOME", str(home))
+    monkeypatch.setenv("HERMES_KANBAN_HOME", str(home))
+    monkeypatch.setenv("HERMES_KANBAN_CRASH_GRACE_SECONDS", "0")
+    monkeypatch.setattr(Path, "home", lambda: tmp_path)
+    db_path = kb.kanban_db_path(board="default")
+    kb._INITIALIZED_PATHS.discard(str(db_path.resolve()))
+    kb.init_db()
+    return home
+
+
+@pytest.fixture
+def conn(kanban_home):
+    with kb.connect() as c:
+        yield c
+
+
+def _orphan_running(conn, tid, *, claim_lock=None, claim_expires=None,
+                    worker_pid=None):
+    """Force a task into running with (partially) broken claim bookkeeping."""
+    conn.execute(
+        "UPDATE tasks SET status='running', claim_lock=?, claim_expires=?, "
+        "worker_pid=? WHERE id=?",
+        (claim_lock, claim_expires, worker_pid, tid),
+    )
+    conn.commit()
+
+
+class TestReconcileOrphanedRunning:
+    def test_null_claim_lock_orphan_requeued(self, conn):
+        """running + claim_lock NULL → requeued to ready with a note."""
+        tid = kb.create_task(conn, title="zombie", assignee="w")
+        _orphan_running(conn, tid)
+
+        reconciled = kb.reconcile_orphaned_running(conn)
+
+        assert reconciled == [tid]
+        row = conn.execute(
+            "SELECT status, claim_lock, claim_expires, worker_pid "
+            "FROM tasks WHERE id=?", (tid,),
+        ).fetchone()
+        assert row["status"] == "ready"
+        assert row["claim_lock"] is None
+        assert row["claim_expires"] is None
+        assert row["worker_pid"] is None
+
+    def test_null_claim_expires_orphan_requeued(self, conn):
+        """running + claim_lock set but claim_expires NULL is also invisible
+        to release_stale_claims — reconciliation must catch it."""
+        host = kb._claimer_id().split(":", 1)[0]
+        tid = kb.create_task(conn, title="half-claim", assignee="w")
+        _orphan_running(conn, tid, claim_lock=f"{host}:dead")
+
+        reconciled = kb.reconcile_orphaned_running(conn)
+
+        assert reconciled == [tid]
+        assert conn.execute(
+            "SELECT status FROM tasks WHERE id=?", (tid,)
+        ).fetchone()["status"] == "ready"
+
+    def test_reconciled_event_and_note_logged(self, conn):
+        tid = kb.create_task(conn, title="zombie", assignee="w")
+        _orphan_running(conn, tid)
+
+        kb.reconcile_orphaned_running(conn)
+
+        events = kb.list_events(conn, tid)
+        recon = [e for e in events if e.kind == "reconciled"]
+        assert len(recon) == 1
+        assert recon[0].payload["reason"] == "orphaned_running"
+        comments = kb.list_comments(conn, tid)
+        assert any("reconcil" in (c.body or "").lower() for c in comments)
+
+    def test_healthy_running_task_untouched(self, conn):
+        """A properly claimed running task is NOT an orphan."""
+        tid = kb.create_task(conn, title="healthy", assignee="w")
+        kb.claim_task(conn, tid)
+
+        assert kb.reconcile_orphaned_running(conn) == []
+        assert conn.execute(
+            "SELECT status FROM tasks WHERE id=?", (tid,)
+        ).fetchone()["status"] == "running"
+
+    def test_live_worker_pid_defers_reconcile(self, conn):
+        """If the orphan row still records a live PID on this host, don't
+        requeue beside a possibly-alive worker — defer to the next tick."""
+        tid = kb.create_task(conn, title="maybe-alive", assignee="w")
+        sleeper = subprocess.Popen(["sleep", "30"])
+        try:
+            _orphan_running(conn, tid, worker_pid=sleeper.pid)
+            assert kb.reconcile_orphaned_running(conn) == []
+            assert conn.execute(
+                "SELECT status FROM tasks WHERE id=?", (tid,)
+            ).fetchone()["status"] == "running"
+        finally:
+            sleeper.terminate()
+            sleeper.wait()
+
+    def test_dead_worker_pid_orphan_requeued(self, conn):
+        """Orphan with a recorded but dead PID is reconciled."""
+        tid = kb.create_task(conn, title="dead-pid", assignee="w")
+        dead = subprocess.Popen(["true"])
+        dead.wait()
+        _orphan_running(conn, tid, worker_pid=dead.pid)
+
+        assert kb.reconcile_orphaned_running(conn) == [tid]
+
+    def test_non_running_statuses_ignored(self, conn):
+        for status in ("todo", "ready", "blocked", "done"):
+            tid = kb.create_task(conn, title=f"s-{status}", assignee="w")
+            conn.execute(
+                "UPDATE tasks SET status=?, claim_lock=NULL, "
+                "claim_expires=NULL WHERE id=?", (status, tid),
+            )
+        conn.commit()
+        assert kb.reconcile_orphaned_running(conn) == []
+
+
+class TestDispatchOnceReconciles:
+    def test_dispatch_once_reconciles_orphans(self, conn):
+        tid = kb.create_task(conn, title="zombie", assignee="w")
+        _orphan_running(conn, tid)
+
+        result = kb.dispatch_once(conn, spawn_fn=lambda *a, **k: (True, ""),
+                                  dry_run=True)
+
+        assert tid in result.reconciled_orphans
+        assert conn.execute(
+            "SELECT status FROM tasks WHERE id=?", (tid,)
+        ).fetchone()["status"] == "ready"
+
+    def test_dispatch_once_reconcile_can_be_disabled(self, conn):
+        """kanban.reconcile_orphans=false plumbs through as
+        reconcile_orphans=False and skips the pass."""
+        tid = kb.create_task(conn, title="zombie", assignee="w")
+        _orphan_running(conn, tid)
+
+        result = kb.dispatch_once(conn, spawn_fn=lambda *a, **k: (True, ""),
+                                  dry_run=True, reconcile_orphans=False)
+
+        assert result.reconciled_orphans == []
+        assert conn.execute(
+            "SELECT status FROM tasks WHERE id=?", (tid,)
+        ).fetchone()["status"] == "running"