Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/gateway/test_53175_cleanup_off_loop.py

@@ -0,0 +1,168 @@
+"""Regression test for #53175: gateway event loop wedged by synchronous
+agent-resource cleanup run inline from loop coroutines.
+
+#35994 fixed the /new reset path, but the same synchronous
+``_cleanup_agent_resources`` (agent.close() tears down terminal sandboxes /
+browser daemons / background processes; shutdown_memory_provider() may do
+SQLite / network IO via a memory plugin) was still called INLINE on the event
+loop from three other places:
+
+  * ``_session_expiry_watcher`` (the 5-minute idle sweep) — live loop
+  * ``_handle_message_with_agent`` cache-hygiene re-eviction — live loop
+  * ``_finalize_shutdown_agents`` / ``stop()`` idle-cache loop — shutdown
+
+A wedged provider on any of these froze the whole loop: the bot went silent,
+the runtime-status ``updated_at`` heartbeat stopped advancing (the symptom the
+reporter's watchdog keyed on), and SIGTERM could not be serviced (requiring
+``kill -9``).
+
+The fix routes all four call sites through ``_cleanup_agent_resources_off_loop``
+which offloads to a worker thread under a bounded ``asyncio.wait_for``, so the
+loop is never blocked and a stuck teardown degrades gracefully.
+
+These tests drive that shared helper directly — it is the single chokepoint
+every fixed call site now uses.
+"""
+import asyncio
+import logging
+import threading
+from contextvars import copy_context
+from types import SimpleNamespace
+
+import pytest
+
+
+def _make_runner():
+    """Bare GatewayRunner with a real thread-pool-backed executor helper."""
+    from gateway.run import GatewayRunner
+
+    runner = object.__new__(GatewayRunner)
+    from concurrent.futures import ThreadPoolExecutor
+
+    executor = ThreadPoolExecutor(max_workers=2)
+    runner._get_executor = lambda: executor
+
+    async def _run_in_executor_with_context(func, *args):
+        loop = asyncio.get_running_loop()
+        ctx = copy_context()
+        return await loop.run_in_executor(executor, lambda: ctx.run(func, *args))
+
+    runner._run_in_executor_with_context = _run_in_executor_with_context
+    return runner, executor
+
+
+def _agent_with_close(close_fn):
+    return SimpleNamespace(
+        close=close_fn,
+        shutdown_memory_provider=lambda *a, **k: None,
+        _session_messages=None,
+    )
+
+
+@pytest.mark.asyncio
+async def test_cleanup_off_loop_does_not_block_event_loop():
+    """A slow agent.close() must NOT freeze the loop. A concurrent heartbeat
+    keeps ticking WHILE close() blocks in its worker thread — proving the
+    cleanup was offloaded, not run inline (which would freeze the loop and
+    stall the runtime-status updated_at heartbeat, #53175)."""
+    runner, executor = _make_runner()
+    close_started = threading.Event()
+    release = threading.Event()
+
+    def slow_close():
+        close_started.set()
+        release.wait(timeout=5)  # block the WORKER thread, not the loop
+
+    agent = _agent_with_close(slow_close)
+
+    ticks = {"n": 0}
+    stop = threading.Event()
+
+    async def _heartbeat():
+        while not stop.is_set():
+            ticks["n"] += 1
+            await asyncio.sleep(0.005)
+
+    hb = asyncio.create_task(_heartbeat())
+    cleanup_task = asyncio.create_task(
+        runner._cleanup_agent_resources_off_loop(agent, context="test")
+    )
+
+    for _ in range(200):
+        if close_started.is_set():
+            break
+        await asyncio.sleep(0.005)
+    assert close_started.is_set(), "close() never ran"
+
+    ticks_at_block = ticks["n"]
+    await asyncio.sleep(0.1)
+    ticks_during_block = ticks["n"] - ticks_at_block
+
+    release.set()
+    await cleanup_task
+    stop.set()
+    await hb
+    executor.shutdown(wait=False)
+
+    assert ticks_during_block >= 5, (
+        f"event loop was blocked during agent cleanup (#53175): only "
+        f"{ticks_during_block} ticks while close() was running"
+    )
+
+
+@pytest.mark.asyncio
+async def test_cleanup_off_loop_times_out_gracefully(caplog):
+    """A cleanup that exceeds the bounded timeout logs a warning and returns —
+    the caller (sweep / shutdown / hygiene) proceeds rather than hanging."""
+    runner, executor = _make_runner()
+
+    async def _instant_timeout(aw, timeout=None):
+        if asyncio.iscoroutine(aw):
+            aw.close()
+        raise asyncio.TimeoutError
+
+    import gateway.run as _run
+
+    agent = _agent_with_close(lambda: None)
+    with caplog.at_level(logging.WARNING, logger="gateway.run"):
+        # Patch the wait_for the helper uses so we don't actually wait 30s.
+        orig = _run.asyncio.wait_for
+        _run.asyncio.wait_for = _instant_timeout
+        try:
+            await runner._cleanup_agent_resources_off_loop(agent, context="sweep")
+        finally:
+            _run.asyncio.wait_for = orig
+    executor.shutdown(wait=False)
+
+    assert any(
+        "exceeded" in r.message and "#53175" in r.message for r in caplog.records
+    ), "expected the timeout warning to be logged"
+
+
+@pytest.mark.asyncio
+async def test_cleanup_off_loop_swallows_executor_failure(caplog):
+    """If the offloaded cleanup raises, the helper logs and returns — a
+    teardown failure must never abort the loop coroutine that triggered it."""
+    runner, executor = _make_runner()
+
+    def boom():
+        raise RuntimeError("provider shutdown blew up")
+
+    # _cleanup_agent_resources swallows its own internal errors, so to reach
+    # the helper's except branch make the offloaded call itself raise.
+    def _boom_cleanup(agent):
+        raise RuntimeError("boom")
+
+    runner._cleanup_agent_resources = _boom_cleanup
+
+    with caplog.at_level(logging.WARNING, logger="gateway.run"):
+        await runner._cleanup_agent_resources_off_loop(
+            _agent_with_close(boom), context="shutdown finalize"
+        )
+    executor.shutdown(wait=False)
+
+    assert any(
+        "failed" in r.message and "#53175" in r.message for r in caplog.records
+    ), "expected the cleanup-failure warning to be logged"
+
+