Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/gateway/test_runner_fatal_adapter.py

@@ -0,0 +1,242 @@
+import asyncio
+from unittest.mock import AsyncMock
+
+import pytest
+
+from gateway.config import GatewayConfig, Platform, PlatformConfig
+from gateway.platforms.base import BasePlatformAdapter, MessageEvent, SendResult
+from gateway.run import GatewayRunner
+from gateway.session import SessionSource, build_session_key
+
+
+class _FatalAdapter(BasePlatformAdapter):
+    def __init__(self):
+        super().__init__(PlatformConfig(enabled=True, token="token"), Platform.TELEGRAM)
+
+    async def connect(self, *, is_reconnect: bool = False) -> bool:
+        self._set_fatal_error(
+            "telegram_token_lock",
+            "Another local Hermes gateway is already using this Telegram bot token.",
+            retryable=False,
+        )
+        return False
+
+    async def disconnect(self) -> None:
+        self._mark_disconnected()
+
+    async def send(self, chat_id, content, reply_to=None, metadata=None):
+        raise NotImplementedError
+
+    async def get_chat_info(self, chat_id):
+        return {"id": chat_id}
+
+
+class _RuntimeRetryableAdapter(BasePlatformAdapter):
+    def __init__(self):
+        super().__init__(PlatformConfig(enabled=True, token="token"), Platform.WHATSAPP)
+
+    async def connect(self, *, is_reconnect: bool = False) -> bool:
+        return True
+
+    async def disconnect(self) -> None:
+        self._mark_disconnected()
+
+    async def send(self, chat_id, content, reply_to=None, metadata=None):
+        raise NotImplementedError
+
+    async def get_chat_info(self, chat_id):
+        return {"id": chat_id}
+
+
+class _ReplacementDeliveryAdapter(BasePlatformAdapter):
+    def __init__(self):
+        super().__init__(
+            PlatformConfig(enabled=True, token="token", typing_indicator=False),
+            Platform.DISCORD,
+        )
+        self.sent: list[str] = []
+        self.connected = True
+
+    async def connect(self, *, is_reconnect: bool = False) -> bool:
+        return True
+
+    async def disconnect(self) -> None:
+        self.connected = False
+
+    async def send(self, chat_id, content, reply_to=None, metadata=None):
+        if not self.connected:
+            return SendResult(success=False, error="Not connected")
+        self.sent.append(content)
+        return SendResult(success=True, message_id=f"m-{len(self.sent)}")
+
+    async def send_typing(self, chat_id, metadata=None) -> None:
+        return None
+
+    async def get_chat_info(self, chat_id):
+        return {"id": chat_id}
+
+
+@pytest.mark.asyncio
+async def test_runner_queues_retryable_runtime_fatal_for_reconnection(monkeypatch, tmp_path):
+    """Retryable runtime fatal errors queue the platform for reconnection
+    AND keep the gateway alive — the background reconnect watcher recovers
+    the platform when the underlying issue clears.  (Previously this
+    exited-with-failure to trigger a systemd restart; that converted
+    transient failures into infinite restart loops.)
+    """
+    config = GatewayConfig(
+        platforms={
+            Platform.WHATSAPP: PlatformConfig(enabled=True, token="token")
+        },
+        sessions_dir=tmp_path / "sessions",
+    )
+    runner = GatewayRunner(config)
+    adapter = _RuntimeRetryableAdapter()
+    adapter._set_fatal_error(
+        "whatsapp_bridge_exited",
+        "WhatsApp bridge process exited unexpectedly (code 1).",
+        retryable=True,
+    )
+
+    runner.adapters = {Platform.WHATSAPP: adapter}
+    runner.delivery_router.adapters = runner.adapters
+    runner.stop = AsyncMock()
+
+    await runner._handle_adapter_fatal_error(adapter)
+
+    # Gateway stays alive — watcher will retry in background
+    runner.stop.assert_not_awaited()
+    assert runner._exit_with_failure is False
+    assert Platform.WHATSAPP in runner._failed_platforms
+    assert runner._failed_platforms[Platform.WHATSAPP]["attempts"] == 0
+
+
+@pytest.mark.asyncio
+async def test_retryable_fatal_queues_reconnect_after_cancellation_swallowing_disconnect(
+    monkeypatch, tmp_path
+):
+    """A wedged old adapter cannot block runner-owned reconnect recovery."""
+    monkeypatch.setenv("HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT", "0.01")
+    config = GatewayConfig(
+        platforms={Platform.WHATSAPP: PlatformConfig(enabled=True, token="token")},
+        sessions_dir=tmp_path / "sessions",
+    )
+    runner = GatewayRunner(config)
+    adapter = _RuntimeRetryableAdapter()
+    adapter._set_fatal_error("transport_stale", "transport stale", retryable=True)
+    runner.adapters = {Platform.WHATSAPP: adapter}
+    runner.delivery_router.adapters = runner.adapters
+
+    started = asyncio.Event()
+    release = asyncio.Event()
+    finished = asyncio.Event()
+
+    async def swallow_cancellation():
+        started.set()
+        while not release.is_set():
+            try:
+                await release.wait()
+            except asyncio.CancelledError:
+                continue
+        finished.set()
+
+    monkeypatch.setattr(adapter, "disconnect", swallow_cancellation)
+    operation = asyncio.create_task(runner._handle_adapter_fatal_error(adapter))
+    await started.wait()
+    done, _pending = await asyncio.wait({operation}, timeout=0.2)
+    try:
+        assert operation in done
+        assert runner.adapters == {}
+        assert Platform.WHATSAPP in runner._failed_platforms
+        assert runner._failed_platforms[Platform.WHATSAPP]["attempts"] == 0
+    finally:
+        release.set()
+        await asyncio.wait({operation}, timeout=0.2)
+        await asyncio.wait_for(finished.wait(), timeout=0.2)
+
+
+@pytest.mark.asyncio
+async def test_retryable_fatal_queues_before_disconnect_returns(monkeypatch, tmp_path):
+    """#80598: reconnect queue must populate before disconnect finishes.
+
+    After a long network outage, Telegram disconnect can wedge on a half-dead
+    socket. If the fatal handler only queues after disconnect returns, the
+    watcher never learns about the failure and the gateway stays permanently
+    deaf. Queue first; teardown is best-effort after.
+    """
+    config = GatewayConfig(
+        platforms={Platform.WHATSAPP: PlatformConfig(enabled=True, token="token")},
+        sessions_dir=tmp_path / "sessions",
+    )
+    runner = GatewayRunner(config)
+    adapter = _RuntimeRetryableAdapter()
+    adapter._set_fatal_error(
+        "telegram_network_error",
+        "Telegram polling could not reconnect after 10 network error retries.",
+        retryable=True,
+    )
+    runner.adapters = {Platform.WHATSAPP: adapter}
+    runner.delivery_router.adapters = runner.adapters
+    runner.stop = AsyncMock()
+
+    disconnect_entered = asyncio.Event()
+    release = asyncio.Event()
+
+    async def blocking_disconnect():
+        # Disconnect has started — the platform must already be queued.
+        assert Platform.WHATSAPP in runner._failed_platforms
+        disconnect_entered.set()
+        await release.wait()
+
+    monkeypatch.setattr(adapter, "disconnect", blocking_disconnect)
+    operation = asyncio.create_task(runner._handle_adapter_fatal_error(adapter))
+    await asyncio.wait_for(disconnect_entered.wait(), timeout=0.5)
+    try:
+        assert runner.adapters == {}
+        assert Platform.WHATSAPP in runner._failed_platforms
+        assert runner._failed_platforms[Platform.WHATSAPP]["attempts"] == 0
+        runner.stop.assert_not_awaited()
+    finally:
+        release.set()
+        await asyncio.wait_for(operation, timeout=0.5)
+
+
+@pytest.mark.asyncio
+async def test_fatal_handler_outer_timeout_still_queues_platform(monkeypatch, tmp_path):
+    """#80598: outer deadline must queue even if the impl task never returns."""
+    monkeypatch.setenv("HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT", "0.05")
+    config = GatewayConfig(
+        platforms={Platform.WHATSAPP: PlatformConfig(enabled=True, token="token")},
+        sessions_dir=tmp_path / "sessions",
+    )
+    runner = GatewayRunner(config)
+    adapter = _RuntimeRetryableAdapter()
+    adapter._set_fatal_error("transport_stale", "transport stale", retryable=True)
+    runner.adapters = {Platform.WHATSAPP: adapter}
+    runner.delivery_router.adapters = runner.adapters
+    runner.stop = AsyncMock()
+
+    started = asyncio.Event()
+    release = asyncio.Event()
+
+    async def wedged_impl(_adapter):
+        # Simulate a hang before/without reaching the reconnect queue.
+        runner.adapters.pop(Platform.WHATSAPP, None)
+        runner.delivery_router.adapters = runner.adapters
+        started.set()
+        await release.wait()
+
+    monkeypatch.setattr(runner, "_handle_adapter_fatal_error_impl", wedged_impl)
+    operation = asyncio.create_task(runner._handle_adapter_fatal_error(adapter))
+    await started.wait()
+    # Outer budget is disconnect_timeout + min(2, max(0.05, timeout)) ≈ 0.1s.
+    done, _pending = await asyncio.wait({operation}, timeout=1.0)
+    try:
+        assert operation in done
+        assert Platform.WHATSAPP in runner._failed_platforms
+        runner.stop.assert_not_awaited()
+    finally:
+        release.set()
+        await asyncio.wait({operation}, timeout=0.2)
+
+