"""LIVE Windows E2E for #48820 (4th repro): Job-Object teardown vs the gateway restart watcher, with real processes on a real windows-latest runner. Three live proofs (no mocks of the code under test): 1. ``TestJobObjectMechanismLive`` — the mechanism everything rests on: a child spawned with ``windows_detach_flags()`` (CREATE_BREAKAWAY_FROM_JOB) from inside a kill-on-close Job Object SURVIVES the job teardown, while a child spawned with ``windows_detach_flags_without_breakaway()`` is killed by it. This is exactly the reporter's suspected kill path. 2. ``TestWatcherRespawnLive`` — drives the REAL ``hermes_cli.gateway._spawn_gateway_restart_watcher`` end to end with a real stub gateway process, against a temp HERMES_HOME: - the respawned process's stderr must land in ``logs/gateway-stdio.log`` (on unfixed main it went to DEVNULL: a job-teardown kill left ZERO trace); - the respawn env must carry ``_HERMES_GATEWAY_BREAKAWAY=1`` (the stamp that makes a later job-teardown death diagnosable in exit-diag). 3. ``TestResumeVerificationLive`` — the user-visible symptom: the updater's ``_resume_windows_gateways_after_update`` must NOT print "✓ Restarting Windows gateway profile(s)" when the relaunched gateway is dead. The relaunch chain runs for real; the "gateway" is a stub that exits immediately (standing in for the job-teardown kill). On unfixed main the ✓ is printed anyway; after the fix the resume raises "not verified alive". """ from __future__ import annotations import ctypes import os import subprocess import sys import time from ctypes import wintypes from pathlib import Path import pytest pytestmark = [ pytest.mark.windows_only, pytest.mark.skipif(sys.platform != "win32", reason="native Windows only"), ] _REPO_ROOT = Path(__file__).resolve().parents[2] JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 JOB_OBJECT_LIMIT_BREAKAWAY_OK = 0x00000800 JobObjectExtendedLimitInformation = 9 PROCESS_ALL_ACCESS = 0x001FFFFF class IO_COUNTERS(ctypes.Structure): _fields_ = [ ("ReadOperationCount", ctypes.c_ulonglong), ("WriteOperationCount", ctypes.c_ulonglong), ("OtherOperationCount", ctypes.c_ulonglong), ("ReadTransferCount", ctypes.c_ulonglong), ("WriteTransferCount", ctypes.c_ulonglong), ("OtherTransferCount", ctypes.c_ulonglong), ] class JOBOBJECT_BASIC_LIMIT_INFORMATION(ctypes.Structure): _fields_ = [ ("PerProcessUserTimeLimit", ctypes.c_longlong), ("PerJobUserTimeLimit", ctypes.c_longlong), ("LimitFlags", wintypes.DWORD), ("MinimumWorkingSetSize", ctypes.c_size_t), ("MaximumWorkingSetSize", ctypes.c_size_t), ("ActiveProcessLimit", wintypes.DWORD), ("Affinity", ctypes.c_size_t), ("PriorityClass", wintypes.DWORD), ("SchedulingClass", wintypes.DWORD), ] class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(ctypes.Structure): _fields_ = [ ("BasicLimitInformation", JOBOBJECT_BASIC_LIMIT_INFORMATION), ("IoInfo", IO_COUNTERS), ("ProcessMemoryLimit", ctypes.c_size_t), ("JobMemoryLimit", ctypes.c_size_t), ("PeakProcessMemoryUsed", ctypes.c_size_t), ("PeakJobMemoryUsed", ctypes.c_size_t), ] def _make_kill_on_close_job(allow_breakaway: bool) -> int: kernel32 = ctypes.windll.kernel32 job = kernel32.CreateJobObjectW(None, None) assert job, "CreateJobObjectW failed" info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION() flags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE if allow_breakaway: flags |= JOB_OBJECT_LIMIT_BREAKAWAY_OK info.BasicLimitInformation.LimitFlags = flags ok = kernel32.SetInformationJobObject( job, JobObjectExtendedLimitInformation, ctypes.byref(info), ctypes.sizeof(info), ) assert ok, "SetInformationJobObject failed" return job def _assign_to_job(job: int, proc: subprocess.Popen) -> None: kernel32 = ctypes.windll.kernel32 ok = kernel32.AssignProcessToJobObject(job, int(proc._handle)) assert ok, f"AssignProcessToJobObject failed (winerror={ctypes.GetLastError()})" def _pid_alive(pid: int) -> bool: kernel32 = ctypes.windll.kernel32 PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 h = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid) if not h: return False try: code = wintypes.DWORD() kernel32.GetExitCodeProcess(h, ctypes.byref(code)) return code.value == 259 # STILL_ACTIVE finally: kernel32.CloseHandle(h) _SLEEPER = "import time; time.sleep(120)" def _wait_for(predicate, timeout_s: float = 30.0, interval_s: float = 0.25): deadline = time.monotonic() + timeout_s while time.monotonic() < deadline: if predicate(): return True time.sleep(interval_s) return False class TestJobObjectMechanismLive: """Real Job Objects, real children — the #48820 kill mechanism.""" def _driver_source(self, flags_helper: str, pid_file: str) -> str: # The driver runs INSIDE the job and spawns a grandchild "gateway" # with the flag bundle under test, then exits — mirroring the # updater/watcher exiting while its job tears down. return ( "import subprocess, sys, pathlib\n" "sys.path.insert(0, r'%s')\n" "from hermes_cli._subprocess_compat import (\n" " windows_detach_flags, windows_detach_flags_without_breakaway)\n" "flags = %s()\n" "p = subprocess.Popen([sys.executable, '-c', %r],\n" " creationflags=flags, stdin=subprocess.DEVNULL,\n" " stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n" "pathlib.Path(r'%s').write_text(str(p.pid), encoding='utf-8')\n" ) % (str(_REPO_ROOT), flags_helper, _SLEEPER, pid_file) def _run_in_job(self, tmp_path: Path, flags_helper: str) -> int: pid_file = tmp_path / f"{flags_helper}.pid" job = _make_kill_on_close_job(allow_breakaway=True) kernel32 = ctypes.windll.kernel32 try: driver = subprocess.Popen( [ sys.executable, "-c", # Handshake: wait until the test has assigned us to the # job before spawning the grandchild. "import pathlib, sys, time\n" f"go = pathlib.Path(r'{tmp_path / 'go.marker'}')\n" "deadline = time.monotonic() + 30\n" "while not go.exists():\n" " assert time.monotonic() < deadline, 'no go marker'\n" " time.sleep(0.1)\n" + self._driver_source(flags_helper, str(pid_file)), ], cwd=str(_REPO_ROOT), ) _assign_to_job(job, driver) (tmp_path / "go.marker").write_text("go", encoding="utf-8") assert _wait_for(pid_file.exists), "driver never wrote the pid file" gw_pid = int(pid_file.read_text(encoding="utf-8")) assert _wait_for(lambda: driver.poll() is not None), ( "driver did not exit" ) assert _pid_alive(gw_pid), "grandchild died before job teardown" # THE teardown: closing the last job handle fires # KILL_ON_JOB_CLOSE against every process still in the job. kernel32.CloseHandle(job) job = None time.sleep(2.0) return gw_pid finally: (tmp_path / "go.marker").unlink(missing_ok=True) if job: kernel32.CloseHandle(job) def test_breakaway_child_survives_job_teardown(self, tmp_path): pid = self._run_in_job(tmp_path, "windows_detach_flags") try: assert _pid_alive(pid), ( "CREATE_BREAKAWAY_FROM_JOB child must survive the parent " "job's kill-on-close teardown" ) finally: subprocess.run( ["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True ) def test_non_breakaway_child_killed_by_job_teardown(self, tmp_path): """The #48820 kill path, reproduced live: no breakaway → the job's teardown reaps the freshly spawned gateway.""" pid = self._run_in_job(tmp_path, "windows_detach_flags_without_breakaway") try: assert not _pid_alive(pid), ( "child without breakaway must be killed by kill-on-close " "job teardown — this is the silent gateway death of #48820" ) finally: subprocess.run( ["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True ) class TestWatcherRespawnLive: """Drive the real ``_spawn_gateway_restart_watcher`` with real processes.""" def _run_watcher_cycle(self, tmp_path: Path, monkeypatch) -> tuple[Path, Path]: monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home")) (tmp_path / "home").mkdir(parents=True, exist_ok=True) marker = tmp_path / "respawned.marker" # The stub "gateway": records its breakaway stamp env, screams on # stderr (so the stdio sidecar has something to capture), then exits. stub = ( "import os, pathlib, sys\n" f"pathlib.Path(r'{marker}').write_text(\n" " os.environ.get('_HERMES_GATEWAY_BREAKAWAY', 'MISSING'),\n" " encoding='utf-8')\n" "print('stub-gateway-stderr-trace', file=sys.stderr)\n" ) # A real old-pid that exits immediately — the watcher's poll loop # sees it die and respawns. old = subprocess.Popen([sys.executable, "-c", "pass"]) old.wait(timeout=30) import hermes_cli.gateway as gateway assert gateway._spawn_gateway_restart_watcher( old.pid, [sys.executable, "-c", stub] ), "watcher spawn returned False" assert _wait_for(marker.exists, timeout_s=60), ( "watcher never respawned the stub gateway" ) stdio_log = tmp_path / "home" / "logs" / "gateway-stdio.log" return marker, stdio_log def test_respawn_stamps_breakaway_and_leaves_stdio_trace( self, tmp_path, monkeypatch ): marker, stdio_log = self._run_watcher_cycle(tmp_path, monkeypatch) # (a) Breakaway stamp: on unfixed main the respawn env carried no # stamp, so a job-teardown death was undiagnosable. stamp = marker.read_text(encoding="utf-8").strip() assert stamp in {"1", "0"}, ( f"respawned gateway must carry the breakaway stamp, got {stamp!r}" ) # (b) Stdio trace: on unfixed main stderr went to DEVNULL — a dying # gateway left zero trace (#48820 4th repro). assert _wait_for( lambda: stdio_log.exists() and "stub-gateway-stderr-trace" in stdio_log.read_text(encoding="utf-8", errors="replace"), timeout_s=30, ), "respawned gateway stderr must land in logs/gateway-stdio.log" class TestResumeVerificationLive: """The user-visible lie: '✓ Restarting' printed for a dead gateway.""" def test_dead_relaunch_is_not_reported_as_success(self, tmp_path, monkeypatch): monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home")) (tmp_path / "home").mkdir(parents=True, exist_ok=True) import hermes_cli.gateway as gateway import hermes_cli.main as hm from hermes_cli.update_cmd import _resume_windows_gateways_after_update # Peripheral only: don't regenerate launcher scripts into the temp home. monkeypatch.setattr(hm, "_refresh_windows_gateway_launchers", lambda: None) # Real relaunch chain, real watcher, real spawn — but the respawned # "gateway" exits immediately, standing in for the Job-Object # teardown kill. It never registers in the process table as a # gateway, exactly like the dead pid 48452 / 50456 of #48820. def _relaunch(profile, old_pid): dead = subprocess.Popen([sys.executable, "-c", "pass"]) dead.wait(timeout=30) return gateway._spawn_gateway_restart_watcher( dead.pid, [sys.executable, "-c", "pass"] ) monkeypatch.setattr( gateway, "launch_detached_profile_gateway_restart", _relaunch ) token = { "resume_needed": True, "profiles": {"default": 999999}, "unmapped_pids": [], "unmapped": [], } printed: list = [] real_print = print monkeypatch.setattr( "builtins.print", lambda *a, **k: printed.append(" ".join(map(str, a))) ) try: with pytest.raises(RuntimeError, match="not verified alive"): _resume_windows_gateways_after_update(token) finally: monkeypatch.setattr("builtins.print", real_print) text = "\n".join(printed) assert "✓ Restarting" not in text, ( "the updater must not vouch for a gateway that is not alive " f"(#48820). Printed:\n{text}" ) assert "could not be verified" in text