154 lines
6.3 KiB
Python
154 lines
6.3 KiB
Python
"""TelegramAdapter polling-stall watchdog (#92991).
|
|
|
|
A wedged getUpdates long-poll can be invisible to every other probe: the
|
|
TCP connection dies mid-read (CLOSE-WAIT behind a TUN/proxy route flip),
|
|
``updater.running`` stays True, ``get_me()`` on the general request path
|
|
stays healthy, and — while no messages are queued server-side —
|
|
``pending_update_count`` stays 0. The gateway then goes silently deaf and
|
|
only a full restart recovers it.
|
|
|
|
``_check_polling_stall`` closes that hole: Telegram answers a long-poll
|
|
within ~50s, so a poller with no successful getUpdates round-trip for
|
|
``_POLLING_STALL_TIMEOUT`` seconds is unambiguously wedged, and the check
|
|
escalates loudly through the existing reconnect ladder
|
|
(``_handle_polling_network_error``). ``_polling_heartbeat_loop`` runs the
|
|
check every probe, so steady-state wedges are caught without any Bot API
|
|
call.
|
|
"""
|
|
import asyncio
|
|
import time as _time
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from gateway.config import PlatformConfig
|
|
from plugins.platforms.telegram.adapter import TelegramAdapter
|
|
|
|
|
|
def _make_adapter(*, stalled_seconds: float) -> TelegramAdapter:
|
|
"""Build a polling-mode adapter whose long-poll last succeeded
|
|
``stalled_seconds`` ago, with a healthy general request path and an
|
|
EMPTY server-side queue (so the pending-count probe stays blind)."""
|
|
adapter = TelegramAdapter(PlatformConfig(enabled=True, token="***"))
|
|
adapter._webhook_mode = False
|
|
adapter._app = MagicMock()
|
|
adapter._app.updater.running = True
|
|
bot = MagicMock()
|
|
bot.get_me = AsyncMock()
|
|
bot.get_webhook_info = AsyncMock(
|
|
return_value=MagicMock(pending_update_count=0)
|
|
)
|
|
adapter._app.bot = bot
|
|
adapter._bot = bot
|
|
adapter._polling_generation = 2
|
|
now = _time.monotonic()
|
|
adapter._polling_generation_started_monotonic = now - 500
|
|
adapter._polling_last_progress_monotonic = now - stalled_seconds
|
|
return adapter
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_recent_progress_does_not_escalate():
|
|
"""A healthy poller (fresh round-trip) must never trip the watchdog."""
|
|
adapter = _make_adapter(stalled_seconds=1)
|
|
with patch.object(adapter, "_handle_polling_network_error", new=AsyncMock()) as rec:
|
|
await adapter._check_polling_stall()
|
|
assert adapter._polling_error_task is None
|
|
rec.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_stalled_long_poll_escalates_to_reconnect_ladder():
|
|
"""#92991: with an empty queue and healthy get_me(), only the stall
|
|
timestamp can detect the wedged consumer — and it must."""
|
|
adapter = _make_adapter(stalled_seconds=400)
|
|
recovery = AsyncMock()
|
|
with patch.object(adapter, "_handle_polling_network_error", new=recovery):
|
|
await adapter._check_polling_stall()
|
|
task = adapter._polling_error_task
|
|
assert task is not None
|
|
await task
|
|
recovery.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_generation_with_no_progress_ever_uses_generation_age():
|
|
"""A generation that never completes one round-trip still trips the
|
|
watchdog once its age passes the stall threshold (verifier fallback)."""
|
|
adapter = _make_adapter(stalled_seconds=0)
|
|
adapter._polling_last_progress_monotonic = None
|
|
recovery = AsyncMock()
|
|
with patch.object(adapter, "_handle_polling_network_error", new=recovery):
|
|
await adapter._check_polling_stall()
|
|
task = adapter._polling_error_task
|
|
assert task is not None
|
|
await task
|
|
recovery.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_stall_ignored_while_recovery_in_flight():
|
|
"""An in-flight reconnect owns recovery; the watchdog must not pile on."""
|
|
adapter = _make_adapter(stalled_seconds=400)
|
|
inflight = MagicMock()
|
|
inflight.done.return_value = False
|
|
adapter._polling_error_task = inflight
|
|
with patch.object(adapter, "_handle_polling_network_error", new=AsyncMock()) as rec:
|
|
await adapter._check_polling_stall()
|
|
rec.assert_not_called()
|
|
assert adapter._polling_error_task is inflight
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_stall_check_skipped_in_webhook_mode():
|
|
"""Webhook mode has no long-poll socket to wedge."""
|
|
adapter = _make_adapter(stalled_seconds=400)
|
|
adapter._webhook_mode = True
|
|
with patch.object(adapter, "_handle_polling_network_error", new=AsyncMock()) as rec:
|
|
await adapter._check_polling_stall()
|
|
rec.assert_not_called()
|
|
assert adapter._polling_error_task is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_heartbeat_detects_wedged_long_poll_with_empty_queue():
|
|
"""End-to-end (#92991): drive the lifetime heartbeat loop against a
|
|
wedged poller with a healthy general path and an empty queue. Before the
|
|
stall watchdog, this setup produces total silence — no probe fires and
|
|
no recovery is ever scheduled. After it, the first stall observation
|
|
escalates through the reconnect ladder."""
|
|
adapter = _make_adapter(stalled_seconds=400)
|
|
real_sleep = asyncio.sleep
|
|
|
|
async def fast_sleep(delay, *args, **kwargs):
|
|
await real_sleep(0)
|
|
|
|
with patch("asyncio.sleep", new=fast_sleep):
|
|
with patch.object(adapter, "_handle_polling_network_error", new=AsyncMock()) as rec:
|
|
loop_task = asyncio.ensure_future(adapter._polling_heartbeat_loop())
|
|
try:
|
|
# Run probe cycles until the stall watchdog escalates (the
|
|
# fix) — bounded so the pre-fix state fails cleanly instead
|
|
# of hanging.
|
|
for _ in range(50):
|
|
await real_sleep(0)
|
|
if adapter._polling_error_task is not None:
|
|
break
|
|
# Let the loop observe the teardown flag and exit on its own.
|
|
# Never cancel(): CPython 3.11's wait_for can swallow task
|
|
# cancellation while its inner future is already done, which
|
|
# leaves the busy loop un-cancellable and the test spinning
|
|
# forever.
|
|
adapter._polling_teardown_started = True
|
|
await asyncio.wait_for(loop_task, 10)
|
|
finally:
|
|
if not loop_task.done():
|
|
loop_task.cancel()
|
|
task = adapter._polling_error_task
|
|
assert task is not None, (
|
|
"heartbeat probes saw a wedged long-poll (no getUpdates progress for "
|
|
"400s) but scheduled no recovery"
|
|
)
|
|
await task
|
|
rec.assert_awaited_once()
|