Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/hermes_cli/test_update_cron_deadlock_guard.py

@@ -0,0 +1,167 @@
+"""Regression tests for #100179: cron-update three-way restart deadlock.
+
+When `hermes update` runs INSIDE the gateway's own process tree (the
+hermes-auto-update cron job), waiting for that gateway to exit is a
+circular wait:
+
+  gateway waits on all in-flight work units (#77184)
+    -> cron agent session waits on the `hermes update` process
+      -> `hermes update` waits on the gateway to exit  [back to A]
+
+The wedged-loop probe cannot break it: the cron session posts activity
+every ~180s (process-tool poll return), so it is never marked wedged and
+the gateway burns the full 1800s force-drain cap.
+
+The fix: when the target gateway PID is an ancestor of this process,
+fire-and-forget (SIGUSR1 + return) instead of drain-waiting.
+"""
+
+from unittest.mock import patch
+
+import pytest
+
+linux_only = pytest.mark.linux_only
+
+
+class TestAncestorDetectionGuard:
+    """_is_pid_ancestor_of_current_process is the deadlock discriminator."""
+
+    def test_own_pid_is_ancestor(self):
+        import os
+
+        from hermes_cli.gateway import _is_pid_ancestor_of_current_process
+
+        assert _is_pid_ancestor_of_current_process(os.getpid()) is True
+
+    def test_parent_pid_is_ancestor(self):
+        import os
+
+        from hermes_cli.gateway import _is_pid_ancestor_of_current_process
+
+        ppid = os.getppid()
+        if ppid <= 1:
+            pytest.skip("no meaningful parent in this environment")
+        assert _is_pid_ancestor_of_current_process(ppid) is True
+
+    def test_unrelated_pid_is_not_ancestor(self):
+        from hermes_cli.gateway import _is_pid_ancestor_of_current_process
+
+        # PID 0 / negative are never ancestors; a very high unlikely PID isn't
+        # either. Use the documented zero/negative contract for determinism.
+        assert _is_pid_ancestor_of_current_process(0) is False
+        assert _is_pid_ancestor_of_current_process(-5) is False
+
+
+@linux_only
+class TestSelfRestartFireAndForget:
+    """_request_gateway_self_restart signals without waiting for exit."""
+
+    def test_refuses_non_ancestor_pid(self):
+        from hermes_cli.gateway import _request_gateway_self_restart
+
+        # A non-ancestor must be refused — signalling an unrelated gateway
+        # and returning immediately would skip its drain entirely.
+        assert _request_gateway_self_restart(0) is False
+
+    def test_signals_ancestor_and_returns_immediately(self):
+        """The ancestor path sends SIGUSR1 and does NOT poll for exit."""
+        import os
+        import signal as _signal
+
+        from hermes_cli import gateway as gw
+
+        sent = []
+
+        def _fake_kill(pid, sig):
+            sent.append((pid, sig))
+
+        with patch.object(gw.os, "kill", side_effect=_fake_kill), patch.object(
+            gw, "_wait_for_pid_exit",
+            side_effect=AssertionError(
+                "fire-and-forget must NOT wait for the gateway to exit — "
+                "that wait is the #100179 deadlock"
+            ),
+        ):
+            ok = gw._request_gateway_self_restart(os.getpid())
+
+        assert ok is True
+        assert sent == [(os.getpid(), _signal.SIGUSR1)]
+
+    def test_graceful_restart_does_wait(self):
+        """Contrast: the non-ancestor path DOES drain-wait (unchanged)."""
+        import signal as _signal
+
+        from hermes_cli import gateway as gw
+
+        waited = []
+
+        with patch.object(gw.os, "kill"), patch.object(
+            gw, "_wait_for_pid_exit",
+            side_effect=lambda pid, t: waited.append((pid, t)) or True,
+        ):
+            ok = gw._graceful_restart_via_sigusr1(4242, drain_timeout=7.0)
+
+        assert ok is True
+        assert waited == [(4242, 7.0)], "drain path must still wait for exit"
+
+
+class TestDrainOrSignalTriage:
+    """_drain_or_signal_gateway_for_update routes the three cases correctly."""
+
+    def _patched(self, monkeypatch, *, ancestor, wedged):
+        from hermes_cli import gateway as gw
+
+        calls = {"self_restart": [], "escalate": [], "drain": []}
+        monkeypatch.setattr(
+            gw, "_is_pid_ancestor_of_current_process", lambda pid: ancestor
+        )
+        monkeypatch.setattr(
+            gw,
+            "probe_gateway_loop_liveness",
+            lambda pid: gw.GATEWAY_LOOP_WEDGED if wedged else "alive",
+        )
+        monkeypatch.setattr(
+            gw,
+            "_request_gateway_self_restart",
+            lambda pid: calls["self_restart"].append(pid) or True,
+        )
+        monkeypatch.setattr(
+            gw, "_escalate_wedged_gateway", lambda pid: calls["escalate"].append(pid)
+        )
+        monkeypatch.setattr(
+            gw,
+            "_graceful_restart_via_sigusr1",
+            lambda pid, drain_timeout: calls["drain"].append((pid, drain_timeout))
+            or True,
+        )
+        return calls
+
+    def test_ancestor_gateway_is_signalled_fire_and_forget(self, monkeypatch):
+        """In-tree gateway (#100179): SIGUSR1 request, NO drain wait."""
+        from hermes_cli.update_cmd import _drain_or_signal_gateway_for_update
+
+        calls = self._patched(monkeypatch, ancestor=True, wedged=False)
+        assert _drain_or_signal_gateway_for_update(1234, 900.0, "svc") is True
+        assert calls["self_restart"] == [1234]
+        assert calls["drain"] == [], "drain-waiting on an ancestor IS the deadlock"
+        assert calls["escalate"] == []
+
+    def test_wedged_gateway_is_escalated(self, monkeypatch):
+        """Provably-dead loop (#81642): bounded escalation, no drain wait."""
+        from hermes_cli.update_cmd import _drain_or_signal_gateway_for_update
+
+        calls = self._patched(monkeypatch, ancestor=False, wedged=True)
+        assert _drain_or_signal_gateway_for_update(1234, 900.0, "svc") is True
+        assert calls["escalate"] == [1234]
+        assert calls["drain"] == []
+        assert calls["self_restart"] == []
+
+    def test_live_out_of_tree_gateway_still_drain_waits(self, monkeypatch):
+        """Normal out-of-tree update keeps the full graceful drain semantics."""
+        from hermes_cli.update_cmd import _drain_or_signal_gateway_for_update
+
+        calls = self._patched(monkeypatch, ancestor=False, wedged=False)
+        assert _drain_or_signal_gateway_for_update(1234, 900.0, "svc") is True
+        assert calls["drain"] == [(1234, 900.0)]
+        assert calls["self_restart"] == []
+        assert calls["escalate"] == []