"""Photon adapter resilience to transient Spectrum/Envoy upstream overflow. Covers the three behaviors that let the adapter ride through a Photon "reset reason: overflow" event instead of degrading delivery and silently dying (issue #50185): 1. ``_is_retryable_error`` classifies the Envoy/sidecar overflow strings as retryable so ``_send_with_retry`` actually engages its backoff loop. 2. ``send_typing`` is rate-gated per chat, and ``stop_typing`` resets the gate so the next turn's typing indicator fires immediately. 3. ``_supervise_sidecar`` detects an unexpected sidecar exit and raises a ``retryable=True`` fatal so the gateway reconnect watcher revives the platform — instead of returning silently and leaving ``_inbound_loop`` spinning against a dead port. 4. ``_monitor_sidecar_health`` promotes degraded upstream stream health reported by ``/healthz`` into the same retryable reconnect path. No Node sidecar is spawned and no ports are bound. """ from __future__ import annotations import asyncio from typing import Any, Dict import pytest from gateway.config import PlatformConfig from gateway.platforms.base import SendResult from plugins.platforms.photon import adapter as photon_adapter from plugins.platforms.photon.adapter import PhotonAdapter def _make_adapter(monkeypatch: pytest.MonkeyPatch) -> PhotonAdapter: monkeypatch.setenv("PHOTON_PROJECT_ID", "test-project-id") monkeypatch.setenv("PHOTON_PROJECT_SECRET", "test-project-secret") cfg = PlatformConfig(enabled=True, token="", extra={}) return PhotonAdapter(cfg) # -- Gap 1: retryable classification of overflow errors --------------------- @pytest.mark.parametrize( "error", [ "UNAVAILABLE: internal sidecar error", "upstream connect error or disconnect/reset before headers", "reset reason: overflow", # Case-insensitive: real strings arrive with mixed case. "Internal Sidecar Error", ], ) def test_overflow_strings_classified_retryable(error: str) -> None: assert PhotonAdapter._is_retryable_error(error) is True def test_unrelated_error_not_retryable() -> None: # A genuine permanent failure must NOT be retried. assert PhotonAdapter._is_retryable_error("400 bad request: invalid spaceId") is False assert PhotonAdapter._is_retryable_error(None) is False def test_base_network_patterns_still_match() -> None: # The override delegates to the base classifier first, so generic # network strings keep working. assert PhotonAdapter._is_retryable_error("ConnectError: connection refused") is True def test_structured_non_retryable_sidecar_error_not_legacy_retried() -> None: error = str( photon_adapter.PhotonSidecarError( path="/send", status_code=500, error="internal sidecar error", error_class="auth_or_config", retryable=False, ) ) assert PhotonAdapter._is_retryable_error(error) is False @pytest.mark.asyncio async def test_send_with_retry_uses_structured_retryable_flag( monkeypatch: pytest.MonkeyPatch, ) -> None: adapter = _make_adapter(monkeypatch) calls = 0 sleeps: list[float] = [] async def _fake_sleep(delay: float) -> None: sleeps.append(delay) async def _fake_sidecar_call(path: str, body: Dict[str, Any]) -> Dict[str, Any]: nonlocal calls calls += 1 if calls == 1: raise photon_adapter.PhotonSidecarError( path=path, status_code=500, error="temporary upstream failure", error_class="upstream_transient", retryable=True, ) return {"ok": True, "messageId": "m-2"} monkeypatch.setattr(photon_adapter.asyncio, "sleep", _fake_sleep) monkeypatch.setattr(adapter, "_sidecar_call", _fake_sidecar_call) result = await adapter._send_with_retry( "space-1", "hello", max_retries=1, base_delay=0.25 ) assert result.success is True assert result.message_id == "m-2" assert calls == 2 assert sleeps == [0.25] # -- Gap 2: typing-indicator cooldown --------------------------------------- @pytest.mark.asyncio async def test_typing_cooldown_suppresses_rapid_repeats( monkeypatch: pytest.MonkeyPatch, ) -> None: adapter = _make_adapter(monkeypatch) calls: list[Dict[str, Any]] = [] async def _fake_call(path: str, payload: Dict[str, Any]) -> Any: calls.append(payload) return {"ok": True} monkeypatch.setattr(adapter, "_sidecar_call", _fake_call) # First call fires; immediate repeats are suppressed by the cooldown. await adapter.send_typing("chat-1") await adapter.send_typing("chat-1") await adapter.send_typing("chat-1") assert len(calls) == 1 @pytest.mark.asyncio async def test_stop_typing_resets_cooldown( monkeypatch: pytest.MonkeyPatch, ) -> None: adapter = _make_adapter(monkeypatch) starts = 0 async def _fake_call(path: str, payload: Dict[str, Any]) -> Any: nonlocal starts if payload.get("state") == "start": starts += 1 return {"ok": True} monkeypatch.setattr(adapter, "_sidecar_call", _fake_call) # A start, then a stop (end of turn), then a start for the next turn must # fire immediately — the cooldown only suppresses rapid consecutive starts # without an intervening stop. await adapter.send_typing("chat-1") await adapter.stop_typing("chat-1") await adapter.send_typing("chat-1") assert starts == 2 # -- Gap 3: sidecar crash detection ----------------------------------------- class _EofStdout: """A proc.stdout whose readline() reports immediate EOF (dead sidecar).""" def readline(self) -> bytes: return b"" class _DeadProc: """Minimal subprocess.Popen stand-in for a sidecar that has exited.""" def __init__(self, exit_code: int = 1) -> None: self.stdout = _EofStdout() self.stdin = None self._exit_code = exit_code def poll(self) -> int: return self._exit_code @pytest.mark.asyncio async def test_unexpected_sidecar_exit_raises_retryable_fatal( monkeypatch: pytest.MonkeyPatch, ) -> None: adapter = _make_adapter(monkeypatch) # Simulate a live session whose sidecar then dies underneath it. adapter._inbound_running = True notified: list[bool] = [] async def _fake_notify() -> None: notified.append(True) monkeypatch.setattr(adapter, "_notify_fatal_error", _fake_notify) await adapter._supervise_sidecar(_DeadProc(exit_code=137)) # type: ignore[arg-type] assert adapter.has_fatal_error is True assert adapter.fatal_error_code == "SIDECAR_CRASHED" # retryable=True routes the platform into the reconnect watcher rather # than crashing the whole gateway. assert adapter.fatal_error_retryable is True assert adapter._running is False # The notification is dispatched onto its own task rather than awaited on # the supervisor's stack, so that disconnect() cancelling the supervisor # cannot kill the handoff. Let that task run before asserting delivery. await _drain_pending_tasks() assert notified == [True] @pytest.mark.asyncio async def test_clean_shutdown_does_not_raise_fatal( monkeypatch: pytest.MonkeyPatch, ) -> None: adapter = _make_adapter(monkeypatch) # disconnect() sets _inbound_running = False before stopping the sidecar, # so the detection block must NOT fire on a clean shutdown. adapter._inbound_running = False notified: list[bool] = [] async def _fake_notify() -> None: notified.append(True) monkeypatch.setattr(adapter, "_notify_fatal_error", _fake_notify) await adapter._supervise_sidecar(_DeadProc(exit_code=0)) # type: ignore[arg-type] assert adapter.has_fatal_error is False assert notified == [] @pytest.mark.asyncio async def test_degraded_stream_health_raises_retryable_fatal( monkeypatch: pytest.MonkeyPatch, ) -> None: adapter = _make_adapter(monkeypatch) adapter._inbound_running = True adapter._sidecar_health_interval = 0.0 async def _fake_call(path: str, payload: Dict[str, Any]) -> Any: assert path == "/healthz" return { "ok": True, "stream": { "ok": False, "state": "degraded", "degradedForMs": 120000, "lastIssue": "[spectrum.stream] stream interrupted; reconnecting", }, } notified: list[bool] = [] async def _fake_notify() -> None: notified.append(True) adapter._inbound_running = False monkeypatch.setattr(adapter, "_sidecar_call", _fake_call) monkeypatch.setattr(adapter, "_notify_fatal_error", _fake_notify) await adapter._monitor_sidecar_health() assert adapter.has_fatal_error is True assert adapter.fatal_error_code == "UPSTREAM_STREAM_DEGRADED" assert adapter.fatal_error_retryable is True # Dispatched detached (see _dispatch_fatal_notification) so the health # task's own teardown cannot cancel the handoff; drain before asserting. await _drain_pending_tasks() assert notified == [True] async def _drain_pending_tasks(limit: int = 50) -> None: """Let detached fatal-notification tasks finish before asserting on them. ``_dispatch_fatal_notification`` deliberately does not await the notification (that is what kept ``disconnect()`` from cancelling its own caller), so a test that drives ``_monitor_sidecar_health`` / ``_supervise_sidecar`` directly returns before the notification has run. """ for _ in range(limit): pending = [ t for t in asyncio.all_tasks() if t is not asyncio.current_task() and not t.done() ] if not pending: return await asyncio.wait(pending, timeout=1.0) # -- Gap 5: self-cancellation race in _stop_sidecar() (issue #73159) -------- # # The tests above mock out _notify_fatal_error() entirely, so the real # integration chain (supervisor task -> _notify_fatal_error() -> # disconnect() -> _stop_sidecar() -> cancel the supervisor task) is never # exercised end to end. That chain is exactly where the bug lives: when # _notify_fatal_error() is a real callback that calls adapter.disconnect(), # _stop_sidecar() is invoked FROM WITHIN the currently-running supervisor # task, and cancelling self._sidecar_supervisor_task there cancels the very # task executing the fatal-error handler -- aborting it (via CancelledError, # a BaseException the handler's `except Exception` guards don't catch) # before the Gateway's reconnect-queue step ever runs. class _FakeStoppedProc: """Minimal proc stand-in so _stop_sidecar() reaches its finally block (it early-returns entirely when self._sidecar_proc is None) without spawning a real subprocess or doing any real I/O.""" def __init__(self) -> None: self.stdin = None def wait(self, timeout: float | None = None) -> int: return 0 @pytest.mark.asyncio async def test_supervisor_task_survives_self_triggered_disconnect( monkeypatch: pytest.MonkeyPatch, ) -> None: """The real chain: _supervise_sidecar() (running as the actual self._sidecar_supervisor_task) detects the crash and calls a REAL _notify_fatal_error() that calls adapter.disconnect() -- which reaches _stop_sidecar() from inside the task it's about to try to cancel. Before the fix: this raises CancelledError out of _supervise_sidecar(), so the task ends up in the "cancelled" state and reconnect_queued below is never set (mirroring how the real Gateway's fatal-error handler, which runs the reconnect-queue logic AFTER disconnect() returns, never gets there either). After the fix: disconnect() completes normally, _supervise_sidecar() returns normally, and the task is NOT cancelled. """ adapter = _make_adapter(monkeypatch) adapter._inbound_running = True # A minimal fake proc so _stop_sidecar() reaches its finally block # (the real process-management behavior is covered separately by # test_sidecar_lifecycle.py) -- this test is purely about the # self-cancellation race. adapter._sidecar_proc = _FakeStoppedProc() reconnect_queued: list[bool] = [] async def _real_notify_fatal_error() -> None: # Stand-in for the Gateway's actual fatal-error handler: it calls # adapter.disconnect() (real chain: disconnect -> _stop_sidecar, # which used to self-cancel), then -- only if that completes # without the CancelledError escaping -- proceeds to queue the # platform for background reconnection. await adapter.disconnect() reconnect_queued.append(True) monkeypatch.setattr(adapter, "_notify_fatal_error", _real_notify_fatal_error) async def _run_supervisor(): await adapter._supervise_sidecar(_DeadProc(exit_code=75)) task = asyncio.ensure_future(_run_supervisor()) adapter._sidecar_supervisor_task = task # Must complete cleanly -- must NOT raise CancelledError out to us. await task assert task.cancelled() is False, ( "The supervisor task must not end up cancelled by its own " "fatal-error handling chain" ) assert adapter.has_fatal_error is True assert adapter.fatal_error_code == "SIDECAR_CRASHED" assert reconnect_queued == [True], ( "The reconnect-queue step (everything after disconnect() returns " "in the real Gateway handler) must actually run -- this is the " "exact step issue #73159 reports as silently skipped" ) # _stop_sidecar() must have cleared the task reference either way. assert adapter._sidecar_supervisor_task is None @pytest.mark.asyncio async def test_external_disconnect_still_cancels_supervisor( monkeypatch: pytest.MonkeyPatch, ) -> None: """The OTHER call path -- external cleanup (Gateway shutdown, an explicit /platform disconnect) -- calls _stop_sidecar() from a DIFFERENT task than the supervisor. That legitimate case must still cancel a still-running supervisor task exactly as before.""" adapter = _make_adapter(monkeypatch) adapter._sidecar_proc = _FakeStoppedProc() supervisor_ran_forever = asyncio.Event() async def _hangs_forever(): try: supervisor_ran_forever.set() await asyncio.sleep(3600) except asyncio.CancelledError: raise task = asyncio.ensure_future(_hangs_forever()) adapter._sidecar_supervisor_task = task await supervisor_ran_forever.wait() # Called from THIS (different) task -- the external-cleanup case. await adapter._stop_sidecar() # cancel() only schedules the CancelledError; await it so the task # actually settles into the cancelled state before asserting. try: await task except asyncio.CancelledError: pass assert task.cancelled() is True, ( "External cleanup must still cancel a running supervisor task" ) assert adapter._sidecar_supervisor_task is None # -- target_not_allowed: shared/free-tier outbound-send restriction ---------- # # Spectrum throws AuthenticationError("Target not allowed for this project") # from space.send when a shared/free-tier line initiates an outbound send to # a new target. The sidecar classifies it as the structured code # `target_not_allowed`; the adapter must treat it as permanent in BOTH # _send_with_retry and _standalone_send, surfacing the canonical user-facing # message instead of raw upstream error text (issues #50971 / #51897). def test_target_not_allowed_maps_to_canonical_message() -> None: err = photon_adapter._sidecar_error_from_response( "/send", 500, '{"ok":false,"error":"internal sidecar error",' '"error_class":"target_not_allowed","retryable":false}', ) assert err.error_class == "target_not_allowed" assert err.retryable is False assert err.error == photon_adapter._TARGET_NOT_ALLOWED_MESSAGE # No raw upstream text may leak through the structured code path. assert "Target not allowed for this project" not in str(err) @pytest.mark.asyncio async def test_standalone_send_classifies_target_not_allowed( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "token") class _Resp: status_code = 500 text = ( '{"ok":false,"error":"internal sidecar error",' '"error_class":"target_not_allowed","retryable":false}' ) @staticmethod def json() -> Dict[str, Any]: return { "ok": False, "error": "internal sidecar error", "error_class": "target_not_allowed", "retryable": False, } class _FakeClient: def __init__(self, *a: Any, **k: Any) -> None: pass async def __aenter__(self) -> "_FakeClient": return self async def __aexit__(self, *a: Any) -> bool: return False async def post(self, *a: Any, **k: Any) -> _Resp: return _Resp() monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _FakeClient) result = await photon_adapter._standalone_send( PlatformConfig(enabled=True, extra={}), "space-1", "hello", ) assert result.get("error") == photon_adapter._TARGET_NOT_ALLOWED_MESSAGE assert result.get("error_class") == "target_not_allowed" assert result.get("retryable") is False assert "Target not allowed for this project" not in str(result)