"""Regression tests for #48820 (4th repro): job-object teardown killed the post-update respawned gateway silently, and the updater printed "✓ Restarting Windows gateway profile(s)" anyway. Two fixes under test: 1. ``_spawn_gateway_restart_watcher``'s inlined watcher source must (a) route the respawned gateway's stray stdout/stderr to ``logs/gateway-stdio.log`` (it was ``DEVNULL`` — a gateway killed by parent Job Object teardown left ZERO trace anywhere), and (b) stamp ``_HERMES_GATEWAY_BREAKAWAY`` =1/0 on the respawn env exactly like the canonical ``gateway_windows._spawn_detached``, so the lifecycle/exit-diag records show whether the gateway escaped the parent's Job Object. 2. ``_resume_windows_gateways_after_update`` must verify a stable gateway process actually exists (via ``gateway_windows._wait_for_gateway_ready``) before printing the ✓ — a truthy launch return only proves the watcher process was created, not that the respawned gateway survived the updater's Job Object teardown. """ from unittest.mock import patch import pytest import hermes_cli.gateway as gateway import hermes_cli.gateway_windows as gateway_windows import hermes_cli.main as hm from hermes_cli._subprocess_compat import _WINDOWS_GATEWAY_BREAKAWAY_ENV from hermes_cli.update_cmd import _resume_windows_gateways_after_update # --------------------------------------------------------------------------- # 1. Watcher template contract # --------------------------------------------------------------------------- def _captured_watcher_source(monkeypatch) -> str: """Spawn the watcher with a mocked Popen and return the inlined -c source.""" captured = {} def fake_popen(argv, **kwargs): captured["argv"] = argv captured["kwargs"] = kwargs class _P: pid = 12345 return _P() monkeypatch.setattr(gateway.subprocess, "Popen", fake_popen) assert gateway._spawn_gateway_restart_watcher( 999999, ["python", "-m", "hermes_cli.main", "gateway", "run"] ) argv = captured["argv"] assert argv[1] == "-c" return argv[2] class TestWatcherRespawnTemplate: def test_respawn_stdio_routed_to_sidecar_log_not_devnull(self, monkeypatch): """DEVNULL swallowed the dying gateway's last words (#48820 4th repro: 'Zero trace anywhere ... because the watcher respawns with stdout=DEVNULL, stderr=DEVNULL').""" src = _captured_watcher_source(monkeypatch) assert "gateway-stdio.log" in src, ( "watcher respawn must route stray stdout/stderr to the same " "sidecar log _spawn_detached uses, so a gateway killed moments " "after respawn leaves a trace" ) # DEVNULL remains only as the fallback when the log dir is # unavailable — the popen kwargs must not be hardwired to it. assert '"stdout": _stdio_target' in src assert '"stderr": _stdio_target' in src def test_respawn_stamps_breakaway_state_like_spawn_detached( self, monkeypatch ): """The respawned gateway must carry _HERMES_GATEWAY_BREAKAWAY=1 on the primary (breakaway) spawn and =0 on the no-breakaway fallback, mirroring gateway_windows._spawn_detached — without the stamp, a job-teardown kill is indistinguishable from any other silent death in the exit diagnostics.""" src = _captured_watcher_source(monkeypatch) assert "_WINDOWS_GATEWAY_BREAKAWAY_ENV" in src assert _WINDOWS_GATEWAY_BREAKAWAY_ENV == "_HERMES_GATEWAY_BREAKAWAY" # Primary stamps "1", the OSError fallback stamps "0". assert '_WINDOWS_GATEWAY_BREAKAWAY_ENV: "1"' in src assert '_WINDOWS_GATEWAY_BREAKAWAY_ENV: "0"' in src def test_respawn_source_compiles(self, monkeypatch): """The inlined -c template is built via str.format over a dedented literal — guard against brace/indentation regressions.""" src = _captured_watcher_source(monkeypatch) compile(src, "", "exec") def test_watcher_fallback_retry_preserved(self, monkeypatch): """The ERROR_ACCESS_DENIED retry without breakaway must survive.""" src = _captured_watcher_source(monkeypatch) assert "windows_detach_flags_without_breakaway" in src # --------------------------------------------------------------------------- # 2. Post-update resume liveness gate # --------------------------------------------------------------------------- def _token(profiles: dict) -> dict: return { "resume_needed": True, "profiles": profiles, "unmapped_pids": [], "unmapped": [], } class TestResumeLivenessGate: @pytest.fixture(autouse=True) def _windows(self, monkeypatch): monkeypatch.setattr(hm, "_is_windows", lambda: True) monkeypatch.setattr(hm, "_refresh_windows_gateway_launchers", lambda: None) monkeypatch.setattr( gateway, "launch_detached_profile_gateway_restart", lambda *_a: True ) monkeypatch.setattr( gateway, "launch_detached_gateway_restart_by_cmdline", lambda *_a: True ) def test_dead_respawn_fails_the_resume_instead_of_printing_check( self, monkeypatch ): """No stable gateway after the relaunch → the resume raises (update marked incomplete) instead of printing '✓ Restarting'. This is the exact #48820 3rd/4th-repro hole: spawn succeeded, gateway died within seconds, success was reported, platforms were offline for 12.5 hours.""" monkeypatch.setattr( gateway_windows, "_wait_for_gateway_ready", lambda **_kw: [] ) token = _token({"default": 1111}) printed = [] with patch("builtins.print", side_effect=lambda *a, **k: printed.append(a)): with pytest.raises(RuntimeError, match="not verified alive"): _resume_windows_gateways_after_update(token) text = " ".join(str(a) for a in printed) assert "✓ Restarting" not in text assert "could not be verified" in text # The profile stays on the token so retry/reporting still sees it. assert token["profiles"] == {"default": 1111} assert token["resume_needed"] is True def test_live_respawn_prints_check_and_writes_attestation(self, monkeypatch): monkeypatch.setattr( gateway_windows, "_wait_for_gateway_ready", lambda **_kw: [777] ) attested = {} monkeypatch.setattr( gateway_windows, "_write_start_attestation", lambda pids, via: attested.update(pids=pids, via=via), ) token = _token({"default": 1111}) printed = [] with patch("builtins.print", side_effect=lambda *a, **k: printed.append(a)): _resume_windows_gateways_after_update(token) text = " ".join(str(a) for a in printed) assert "✓ Restarting" in text assert attested == {"pids": [777], "via": "post-update relaunch"} assert token["resume_needed"] is False def test_liveness_poll_scans_all_profiles(self, monkeypatch): """The resume relaunches the whole fleet; the verification must not be scoped to the active profile.""" seen = {} def fake_wait(**kwargs): seen.update(kwargs) return [777] monkeypatch.setattr(gateway_windows, "_wait_for_gateway_ready", fake_wait) monkeypatch.setattr( gateway_windows, "_write_start_attestation", lambda *_a, **_kw: None ) with patch("builtins.print"): _resume_windows_gateways_after_update(_token({"work": 2222})) assert seen.get("all_profiles") is True