"""Regression for #41403 — ``hermes update`` must restart ALL macOS launchd gateways. The macOS branch of the update's fleet-restart step only restarted the invoking profile's LaunchAgent (``get_launchd_label()`` is profile-scoped). Sibling ``ai.hermes.gateway-`` services kept running pre-update modules cached in ``sys.modules`` and died on their next agent turn once the new code lazily imported a symbol the old module generation didn't have (``ImportError: cannot import name ...`` — or, with a wider version gap, ``TypeError``/``AttributeError`` on changed call signatures with garbled tracebacks, because the source files on disk no longer match the loaded code objects). Also covers the launchd-domain review feedback on PR #41403: every sibling interaction (liveness discovery, kickstart, fresh-PID verification) must be domain-explicit — ``_launchd_domain()`` caches the *current* profile's domain, and a sibling bootstrapped in the other supported domain (``gui/`` vs ``user/``) would otherwise be probed or kickstarted in a domain it does not live in. """ from __future__ import annotations import subprocess import sys import pytest import hermes_cli.gateway as gw import hermes_cli.profiles from hermes_cli.gateway import ( _locate_launchd_gateway_service, _parse_launchd_pid_from_print_output, _probe_launchd_domain_for_label, launchd_gateway_labels_for_install, ) from hermes_cli.update_cmd import ( _restart_macos_launchd_gateways, _warn_incomplete_gateway_fleet_restart, ) pytestmark = pytest.mark.skipif( sys.platform == "win32", reason="launchd fleet restart is macOS-only; helpers use POSIX os.getuid", ) UID = 501 PRINT_RUNNING = ( "system/com.example = {\n" "\tactive count = 1\n" "\tstate = running\n" "\tpid = 4242\n" "\tprogram = /usr/bin/true\n" "}\n" ) PRINT_LOADED_NOT_RUNNING = ( "system/com.example = {\n" "\tactive count = 0\n" "\tstate = not running\n" "\tprogram = /usr/bin/true\n" "}\n" ) @pytest.fixture(autouse=True) def _fixed_uid(monkeypatch): monkeypatch.setattr(gw.os, "getuid", lambda: UID) def _completed(returncode: int = 0, stdout: str = "") -> subprocess.CompletedProcess: return subprocess.CompletedProcess( args=[], returncode=returncode, stdout=stdout, stderr="" ) class _Profile: def __init__(self, name, is_default=False): self.name = name self.is_default = is_default class TestLaunchdGatewayLabelsForInstall: def test_labels_derive_from_this_installs_profiles(self, monkeypatch): """The fleet is THIS install's profiles, root first — never a glob of the shared per-user LaunchAgents dir. A sandboxed HERMES_HOME (tests, side-by-side installs) must not enumerate — and restart — another install's services, and the hermetic test suite must not see the dev machine's real fleet.""" monkeypatch.setattr( hermes_cli.profiles, "list_profiles", lambda: [ _Profile("tfl-wiki"), _Profile("default", is_default=True), _Profile("merit-ops"), _Profile("Bad Name!"), # cannot map to a service suffix — skipped ], ) assert launchd_gateway_labels_for_install() == [ "ai.hermes.gateway", "ai.hermes.gateway-merit-ops", "ai.hermes.gateway-tfl-wiki", ] def test_no_profiles_means_no_fleet(self, monkeypatch): monkeypatch.setattr(hermes_cli.profiles, "list_profiles", lambda: []) assert launchd_gateway_labels_for_install() == [] class TestParseLaunchdPidFromPrintOutput: def test_running_service_pid(self): assert _parse_launchd_pid_from_print_output(PRINT_RUNNING) == 4242 def test_loaded_but_not_running_has_no_pid(self): assert _parse_launchd_pid_from_print_output(PRINT_LOADED_NOT_RUNNING) is None class TestLocateLaunchdGatewayService: def test_domains_resolve_per_label_not_from_cache(self, monkeypatch): """The #41403 review defect: sibling domains are independent.""" gui_loaded = {"ai.hermes.gateway-a"} def fake_run(cmd, **kwargs): assert cmd[:2] == ["launchctl", "print"] domain, _, label = cmd[2].rpartition("/") in_gui = domain == f"gui/{UID}" and label in gui_loaded in_user = domain == f"user/{UID}" and label not in gui_loaded if in_gui or in_user: return _completed(0, PRINT_RUNNING) return _completed(113) monkeypatch.setattr(gw.subprocess, "run", fake_run) # Simulate a prior current-profile resolution having populated the # process-wide cache — per-label lookups must not consult it. monkeypatch.setattr(gw, "_resolved_launchd_domain", f"gui/{UID}") assert _locate_launchd_gateway_service("ai.hermes.gateway-a") == ( f"gui/{UID}", 4242, ) assert _locate_launchd_gateway_service("ai.hermes.gateway-b") == ( f"user/{UID}", 4242, ) def test_loaded_without_live_process(self, monkeypatch): monkeypatch.setattr( gw.subprocess, "run", lambda *a, **k: _completed(0, PRINT_LOADED_NOT_RUNNING), ) assert _locate_launchd_gateway_service("ai.hermes.gateway-x") == ( f"gui/{UID}", None, ) def test_not_loaded_in_either_domain(self, monkeypatch): monkeypatch.setattr(gw.subprocess, "run", lambda *a, **k: _completed(113)) assert _locate_launchd_gateway_service("ai.hermes.gateway-x") == (None, None) def test_timeout_propagates_to_caller(self, monkeypatch): """A wedged launchctl must surface as a failure, not read as 'unloaded' — the update path owns per-label failure accounting.""" def fake_run(cmd, **kwargs): raise subprocess.TimeoutExpired(cmd=cmd, timeout=5) monkeypatch.setattr(gw.subprocess, "run", fake_run) with pytest.raises(subprocess.TimeoutExpired): _locate_launchd_gateway_service("ai.hermes.gateway-x") class TestProbeLaunchdDomainForLabel: def test_unloaded_label_falls_back_to_managername(self, monkeypatch): def fake_run(cmd, **kwargs): if cmd[:2] == ["launchctl", "print"]: raise subprocess.CalledProcessError(113, cmd) if cmd == ["launchctl", "managername"]: return _completed(0, "Aqua\n") raise AssertionError(f"unexpected command {cmd}") monkeypatch.setattr(gw.subprocess, "run", fake_run) assert _probe_launchd_domain_for_label("ai.hermes.gateway-x") == f"gui/{UID}" def test_unloaded_label_defaults_to_user_domain(self, monkeypatch): def fake_run(cmd, **kwargs): if cmd[:2] == ["launchctl", "print"]: raise subprocess.CalledProcessError(113, cmd) if cmd == ["launchctl", "managername"]: return _completed(0, "Background\n") raise AssertionError(f"unexpected command {cmd}") monkeypatch.setattr(gw.subprocess, "run", fake_run) assert _probe_launchd_domain_for_label("ai.hermes.gateway-x") == f"user/{UID}" class TestGetServicePidsScoping: def _wire(self, monkeypatch): monkeypatch.setattr(gw, "is_macos", lambda: True) monkeypatch.setattr(gw, "supports_systemd_services", lambda: False) monkeypatch.setattr(gw, "get_launchd_label", lambda: "ai.hermes.gateway") monkeypatch.setattr( gw, "launchd_gateway_labels_for_install", lambda: ["ai.hermes.gateway", "ai.hermes.gateway-a", "ai.hermes.gateway-b"], ) located = { "ai.hermes.gateway": (f"gui/{UID}", 100), "ai.hermes.gateway-a": (f"gui/{UID}", 200), "ai.hermes.gateway-b": (None, None), # not bootstrapped } monkeypatch.setattr( gw, "_locate_launchd_gateway_service", lambda label: located[label] ) def test_all_profiles_returns_every_gateway_service_pid(self, monkeypatch): """The update sweep's exclude-set must protect ALL freshly-restarted services, not only the invoking profile's (else the sweep SIGTERMs gateways launchd just respawned).""" self._wire(monkeypatch) assert gw._get_service_pids(all_profiles=True) == {100, 200} def test_default_stays_scoped_to_current_profile(self, monkeypatch): """Regression guard: default-scope callers (gateway status, cron, stop_profile_gateway's orphan reaper) must NOT start seeing sibling service PIDs — the reaper SIGTERM/SIGKILLs what they feed it.""" self._wire(monkeypatch) assert gw._get_service_pids() == {100} def test_find_gateway_pids_passes_profile_scope_through(self, monkeypatch): calls: list[bool] = [] monkeypatch.setattr( gw, "_get_service_pids", lambda all_profiles=False: (calls.append(all_profiles), set())[1], ) monkeypatch.setattr(gw, "_scan_gateway_pids", lambda *a, **k: []) monkeypatch.setattr(gw, "supports_systemd_services", lambda: True) gw.find_gateway_pids(all_profiles=False) gw.find_gateway_pids(all_profiles=True) assert calls == [False, True] def _fleet(monkeypatch, tmp_path, *, current, labels, located, registered=None, plist_exists=True, drain_results=None, kick_errors=None, wait_results=None, current_supervised=True): """Wire a fake launchd fleet through hermes_cli.gateway seams. ``located`` maps label -> (domain, pid) as ``_locate_launchd_gateway_service`` would return it (values may also be exceptions to raise). ``registered`` maps label -> bool for the current-profile ``launchctl list`` gate and defaults to "located in some domain". Returns a SimpleNamespace of recorder lists: rec.kickstarts, rec.drains, rec.current_restarts, rec.waits, locates, registered_checks. """ from types import SimpleNamespace rec = SimpleNamespace( kickstarts=[], drains=[], current_restarts=[], waits=[], locates=[], registered_checks=[], current_verifies=[], ) plist = tmp_path / f"{current}.plist" if plist_exists: plist.write_text("") def fake_locate(label): rec.locates.append(label) value = located[label] if isinstance(value, Exception): raise value return value def fake_registered(label): rec.registered_checks.append(label) if registered is not None: return registered[label] value = located.get(label) return ( value is not None and not isinstance(value, Exception) and value[0] is not None ) monkeypatch.setattr(gw, "get_launchd_label", lambda: current) monkeypatch.setattr(gw, "get_launchd_plist_path", lambda: plist) monkeypatch.setattr(gw, "launchd_gateway_labels_for_install", lambda: list(labels)) monkeypatch.setattr(gw, "_locate_launchd_gateway_service", fake_locate) monkeypatch.setattr(gw, "_launchd_service_registered", fake_registered) monkeypatch.setattr( gw, "_graceful_restart_via_sigusr1", lambda pid, drain_timeout: (rec.drains.append(pid), (drain_results or {}).get(pid, False))[1], ) def fake_kickstart(label, domain): err = (kick_errors or {}).get(label) if err is not None: raise err rec.kickstarts.append(f"{domain}/{label}") monkeypatch.setattr(gw, "_launchd_kickstart", fake_kickstart) def fake_wait(label, old_pid, timeout, domain): rec.waits.append(f"{domain}/{label}") return (wait_results or {}).get(label, True) monkeypatch.setattr(gw, "_wait_for_launchd_service_pid", fake_wait) monkeypatch.setattr( gw, "launchd_restart", lambda: rec.current_restarts.append(current) ) # The current profile is now verified the same way siblings are: a # successful launchd_restart() only counts once launchd reports it is # supervising the job (#88848). Stubbed here so the fleet cases keep # asserting on routing rather than on a real launchctl probe. def fake_verify_current(*, label=None, **_kw): rec.current_verifies.append(label) return current_supervised monkeypatch.setattr( gw, "wait_for_launchd_gateway_supervision", fake_verify_current ) return rec class TestRestartMacosLaunchdGateways: def test_current_delegates_and_siblings_kickstart_in_own_domains( self, monkeypatch, tmp_path ): """Current profile keeps launchd_restart(); every sibling (including the root gateway when a named profile invokes the update) is kickstarted — and verified — in the domain IT was located in.""" current = "ai.hermes.gateway-merit-ops" rec = _fleet( monkeypatch, tmp_path, current=current, labels=["ai.hermes.gateway", current, "ai.hermes.gateway-user-scoped"], located={ "ai.hermes.gateway": (f"gui/{UID}", 100), current: (f"gui/{UID}", 200), "ai.hermes.gateway-user-scoped": (f"user/{UID}", 300), }, ) restarted: list[str] = [] failed: list[str] = [] _restart_macos_launchd_gateways(restarted, failed, drain_budget=0.0) assert rec.current_restarts == [current] assert rec.kickstarts == [ f"gui/{UID}/ai.hermes.gateway", f"user/{UID}/ai.hermes.gateway-user-scoped", ] assert rec.waits == [ f"gui/{UID}/ai.hermes.gateway", f"user/{UID}/ai.hermes.gateway-user-scoped", ] assert restarted == [ current, "ai.hermes.gateway", "ai.hermes.gateway-user-scoped", ] assert failed == [] # Siblings were drained before the hard kickstart. assert set(rec.drains) == {100, 300} def test_current_profile_without_plist_makes_no_launchctl_calls( self, monkeypatch, tmp_path ): """Upstream gate order preserved: no plist → the current profile is skipped without ANY launchctl interaction (no registered probe, no locate) — and definitely without inventing a failure. Siblings are still processed.""" current = "ai.hermes.gateway" rec = _fleet( monkeypatch, tmp_path, current=current, labels=[current, "ai.hermes.gateway-a"], located={"ai.hermes.gateway-a": (f"gui/{UID}", 200)}, plist_exists=False, ) restarted: list[str] = [] failed: list[str] = [] _restart_macos_launchd_gateways(restarted, failed, drain_budget=0.0) assert rec.current_restarts == [] assert current not in rec.registered_checks assert current not in rec.locates assert restarted == ["ai.hermes.gateway-a"] assert failed == [] def test_current_profile_registered_but_unlocatable_still_restarts( self, monkeypatch, tmp_path ): """macOS-26 quirk: a label can be `launchctl list`-registered while both explicit gui/user `launchctl print` probes fail (domain doesn't support service management). The gate must use the registered predicate and hand off to launchd_restart(), which owns the domain-unsupported fallback — locate is for siblings only.""" current = "ai.hermes.gateway" rec = _fleet( monkeypatch, tmp_path, current=current, labels=[current], located={current: (None, None)}, registered={current: True}, ) restarted: list[str] = [] failed: list[str] = [] _restart_macos_launchd_gateways(restarted, failed, drain_budget=0.0) assert rec.current_restarts == [current] assert current not in rec.locates assert restarted == [current] assert failed == [] def test_unbootstrapped_sibling_is_skipped_not_failed( self, monkeypatch, tmp_path ): rec = _fleet( monkeypatch, tmp_path, current="ai.hermes.gateway", labels=["ai.hermes.gateway", "ai.hermes.gateway-idle"], located={ "ai.hermes.gateway": (f"gui/{UID}", 100), "ai.hermes.gateway-idle": (None, None), }, ) restarted: list[str] = [] failed: list[str] = [] _restart_macos_launchd_gateways(restarted, failed, drain_budget=0.0) assert rec.kickstarts == [] assert restarted == ["ai.hermes.gateway"] assert failed == [] def test_loaded_but_not_running_sibling_is_kickstarted( self, monkeypatch, tmp_path ): """A bootstrapped service with no live process still holds the old code path for its next launch trigger — kickstart it (no drain).""" rec = _fleet( monkeypatch, tmp_path, current="ai.hermes.gateway", labels=["ai.hermes.gateway", "ai.hermes.gateway-dormant"], located={ "ai.hermes.gateway": (f"gui/{UID}", 100), "ai.hermes.gateway-dormant": (f"gui/{UID}", None), }, ) restarted: list[str] = [] failed: list[str] = [] _restart_macos_launchd_gateways(restarted, failed, drain_budget=0.0) assert rec.drains == [] assert rec.kickstarts == [f"gui/{UID}/ai.hermes.gateway-dormant"] assert restarted == ["ai.hermes.gateway", "ai.hermes.gateway-dormant"] assert failed == [] def test_graceful_drain_with_keepalive_respawn_skips_kickstart( self, monkeypatch, tmp_path ): """When SIGUSR1 rec.drains the sibling and KeepAlive already respawned it on a fresh PID, a second hard kickstart would kill the new process.""" rec = _fleet( monkeypatch, tmp_path, current="ai.hermes.gateway", labels=["ai.hermes.gateway", "ai.hermes.gateway-a"], located={ "ai.hermes.gateway": (f"gui/{UID}", 100), "ai.hermes.gateway-a": (f"gui/{UID}", 200), }, drain_results={200: True}, ) restarted: list[str] = [] failed: list[str] = [] _restart_macos_launchd_gateways(restarted, failed, drain_budget=0.0) assert rec.drains == [200] assert rec.kickstarts == [] assert rec.waits == [f"gui/{UID}/ai.hermes.gateway-a"] assert restarted == ["ai.hermes.gateway", "ai.hermes.gateway-a"] assert failed == [] def test_kickstart_failure_is_recorded_and_rest_continue( self, monkeypatch, tmp_path ): rec = _fleet( monkeypatch, tmp_path, current="ai.hermes.gateway", labels=[ "ai.hermes.gateway", "ai.hermes.gateway-bad", "ai.hermes.gateway-good", ], located={ "ai.hermes.gateway": (f"gui/{UID}", 100), "ai.hermes.gateway-bad": (f"gui/{UID}", 200), "ai.hermes.gateway-good": (f"gui/{UID}", 300), }, kick_errors={ "ai.hermes.gateway-bad": subprocess.CalledProcessError( 5, ["launchctl", "kickstart"] ) }, ) restarted: list[str] = [] failed: list[str] = [] _restart_macos_launchd_gateways(restarted, failed, drain_budget=0.0) assert failed == ["ai.hermes.gateway-bad"] assert rec.kickstarts == [f"gui/{UID}/ai.hermes.gateway-good"] assert restarted == ["ai.hermes.gateway", "ai.hermes.gateway-good"] def test_timeout_during_discovery_is_failed_and_rest_continue( self, monkeypatch, tmp_path ): """A wedged launchctl during liveness discovery must be accounted as a failure (the sibling may still be on old code), not silently skipped — and must not abort the remaining fleet (#68523 parity).""" rec = _fleet( monkeypatch, tmp_path, current="ai.hermes.gateway", labels=[ "ai.hermes.gateway", "ai.hermes.gateway-wedged", "ai.hermes.gateway-after", ], located={ "ai.hermes.gateway": (f"gui/{UID}", 100), "ai.hermes.gateway-wedged": subprocess.TimeoutExpired( cmd=["launchctl", "print"], timeout=5 ), "ai.hermes.gateway-after": (f"gui/{UID}", 300), }, ) restarted: list[str] = [] failed: list[str] = [] _restart_macos_launchd_gateways(restarted, failed, drain_budget=0.0) assert failed == ["ai.hermes.gateway-wedged"] assert rec.kickstarts == [f"gui/{UID}/ai.hermes.gateway-after"] assert restarted == ["ai.hermes.gateway", "ai.hermes.gateway-after"] def test_timeout_during_kickstart_is_failed_and_rest_continue( self, monkeypatch, tmp_path ): rec = _fleet( monkeypatch, tmp_path, current="ai.hermes.gateway", labels=[ "ai.hermes.gateway", "ai.hermes.gateway-wedged", "ai.hermes.gateway-after", ], located={ "ai.hermes.gateway": (f"gui/{UID}", 100), "ai.hermes.gateway-wedged": (f"gui/{UID}", 200), "ai.hermes.gateway-after": (f"gui/{UID}", 300), }, kick_errors={ "ai.hermes.gateway-wedged": subprocess.TimeoutExpired( cmd=["launchctl", "kickstart"], timeout=90 ) }, ) restarted: list[str] = [] failed: list[str] = [] _restart_macos_launchd_gateways(restarted, failed, drain_budget=0.0) assert failed == ["ai.hermes.gateway-wedged"] assert rec.kickstarts == [f"gui/{UID}/ai.hermes.gateway-after"] assert restarted == ["ai.hermes.gateway", "ai.hermes.gateway-after"] def test_sibling_that_never_comes_back_is_failed(self, monkeypatch, tmp_path): rec = _fleet( monkeypatch, tmp_path, current="ai.hermes.gateway", labels=["ai.hermes.gateway", "ai.hermes.gateway-zombie"], located={ "ai.hermes.gateway": (f"gui/{UID}", 100), "ai.hermes.gateway-zombie": (f"gui/{UID}", 200), }, wait_results={"ai.hermes.gateway-zombie": False}, ) restarted: list[str] = [] failed: list[str] = [] _restart_macos_launchd_gateways(restarted, failed, drain_budget=0.0) assert restarted == ["ai.hermes.gateway"] assert failed == ["ai.hermes.gateway-zombie"] class TestWaitForLaunchdServicePid: def test_returns_true_once_pid_changes(self, monkeypatch): pids = iter([200, 200, 4242]) monkeypatch.setattr( gw, "_launchd_print_service_pid", lambda domain, label: (True, next(pids)), ) monkeypatch.setattr(gw.time, "sleep", lambda _s: None) assert gw._wait_for_launchd_service_pid( "ai.hermes.gateway-x", old_pid=200, timeout=5.0, domain=f"gui/{UID}" ) def test_returns_false_when_pid_never_changes(self, monkeypatch): clock = iter(float(i) for i in range(100)) monkeypatch.setattr(gw.time, "monotonic", lambda: next(clock)) monkeypatch.setattr(gw.time, "sleep", lambda _s: None) monkeypatch.setattr( gw, "_launchd_print_service_pid", lambda domain, label: (True, 200), ) assert not gw._wait_for_launchd_service_pid( "ai.hermes.gateway-x", old_pid=200, timeout=3.0, domain=f"gui/{UID}" ) class TestIncompleteWarningMentionsLaunchctl: def test_launchd_labels_get_launchctl_hint(self, capsys): _warn_incomplete_gateway_fleet_restart(["ai.hermes.gateway-merit-ops"]) out = capsys.readouterr().out assert "Update incomplete" in out assert "launchctl kickstart -k" in out def test_systemd_units_keep_systemctl_hint(self, capsys): _warn_incomplete_gateway_fleet_restart(["hermes-gateway-coder"]) out = capsys.readouterr().out assert "systemctl" in out assert "launchctl" not in out