301 lines
12 KiB
Python
301 lines
12 KiB
Python
"""Plan-vs-execution reconciliation (#91277 Phase 2: restart via declared mechanism).
|
|
|
|
Pins:
|
|
- _restart_mechanism returns machine-readable ids; describe_restart_mechanism
|
|
derives display strings (policy table is data, not prose).
|
|
- match_runtime_outcomes classifies every planned runtime against the restart
|
|
phase's bookkeeping: restarted / stopped / failed / unaccounted.
|
|
- report_unaccounted_runtimes escalates (returns True) ONLY on unaccounted
|
|
rows — the silent-miss tripwire.
|
|
"""
|
|
|
|
from hermes_cli.update_inventory import (
|
|
RuntimeRecord,
|
|
UpdatePlan,
|
|
_restart_mechanism,
|
|
describe_restart_mechanism,
|
|
match_runtime_outcomes,
|
|
report_unaccounted_runtimes,
|
|
)
|
|
|
|
|
|
def _plan(*runtimes: RuntimeRecord) -> UpdatePlan:
|
|
plan = UpdatePlan()
|
|
plan.runtimes = list(runtimes)
|
|
return plan
|
|
|
|
|
|
def _rt(profile: str, pid: int, supervisor: str = "manual") -> RuntimeRecord:
|
|
return RuntimeRecord(
|
|
kind="gateway",
|
|
profile=profile,
|
|
pid=pid,
|
|
supervisor=supervisor,
|
|
restart_via=_restart_mechanism(supervisor, profile),
|
|
)
|
|
|
|
|
|
def test_mechanism_ids_are_machine_readable_and_described():
|
|
assert _restart_mechanism("systemd", "default") == "systemd"
|
|
assert _restart_mechanism("launchd", "work") == "launchd"
|
|
assert _restart_mechanism("desktop", "default") == "desktop"
|
|
assert _restart_mechanism("manual", "work") == "manual"
|
|
assert _restart_mechanism("windows-service", "default") == "windows-service"
|
|
# display derives FROM the id
|
|
assert "systemctl" in describe_restart_mechanism("systemd", "default")
|
|
assert "kickstart" in describe_restart_mechanism("launchd", "work")
|
|
assert "-p work" in describe_restart_mechanism("manual", "work")
|
|
assert describe_restart_mechanism("manual", "default") == "hermes gateway restart"
|
|
assert "sc.exe" in describe_restart_mechanism("windows-service", "default")
|
|
|
|
|
|
def test_windows_service_supervisor_classification():
|
|
from hermes_cli.update_inventory import _detect_supervisor_for_pid
|
|
|
|
# An SCM-owned gateway PID classifies as windows-service even when the
|
|
# generic service-PID probe also knows the pid.
|
|
assert (
|
|
_detect_supervisor_for_pid(41, set(), {41}) == "windows-service"
|
|
)
|
|
assert (
|
|
_detect_supervisor_for_pid(41, {41}, {41}) == "windows-service"
|
|
)
|
|
# Without SCM ownership the existing classification is untouched.
|
|
assert _detect_supervisor_for_pid(42, set(), set()) == "manual"
|
|
assert _detect_supervisor_for_pid(42, set(), None) == "manual"
|
|
|
|
|
|
def test_windows_service_runtime_reconciles_via_service_profiles():
|
|
# The update path merges the pause token's service_profiles into
|
|
# relaunched_profiles after sc.exe start — a restarted SCM gateway
|
|
# must not trip the unaccounted tripwire.
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_rt("default", 500, supervisor="windows-service")),
|
|
restarted_services=["hermes-gateway"], relaunched_profiles=["default"],
|
|
externally_supervised_profiles=[], killed_pids=set(), failed_units=[],
|
|
)
|
|
assert outcomes == [
|
|
{"kind": "gateway", "profile": "default", "pid": 500,
|
|
"mechanism": "windows-service", "outcome": "restarted"}
|
|
]
|
|
assert report_unaccounted_runtimes(outcomes) is False
|
|
|
|
|
|
def test_windows_service_runtime_unaccounted_when_restart_fails():
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_rt("work", 501, supervisor="windows-service")),
|
|
restarted_services=[], relaunched_profiles=[],
|
|
externally_supervised_profiles=[], killed_pids=set(), failed_units=[],
|
|
)
|
|
assert outcomes[0]["mechanism"] == "windows-service"
|
|
assert outcomes[0]["outcome"] == "unaccounted"
|
|
assert report_unaccounted_runtimes(outcomes) is True
|
|
|
|
|
|
def test_relaunched_profile_is_restarted():
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_rt("default", 100)),
|
|
restarted_services=[], relaunched_profiles=["default"],
|
|
externally_supervised_profiles=[], killed_pids=set(), failed_units=[],
|
|
)
|
|
assert outcomes == [
|
|
{"kind": "gateway", "profile": "default", "pid": 100,
|
|
"mechanism": "manual", "outcome": "restarted"}
|
|
]
|
|
assert report_unaccounted_runtimes(outcomes) is False
|
|
|
|
|
|
def test_killed_pid_is_stopped():
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_rt("work", 200)),
|
|
restarted_services=[], relaunched_profiles=[],
|
|
externally_supervised_profiles=[], killed_pids={200}, failed_units=[],
|
|
)
|
|
assert outcomes[0]["outcome"] == "stopped"
|
|
|
|
|
|
def test_failed_unit_is_failed():
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_rt("work", 300, supervisor="systemd")),
|
|
restarted_services=[], relaunched_profiles=[],
|
|
externally_supervised_profiles=[], killed_pids=set(),
|
|
failed_units=["hermes-gateway-work.service"],
|
|
)
|
|
assert outcomes[0]["outcome"] == "failed"
|
|
|
|
|
|
def test_restarted_service_unit_matches_profile():
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_rt("default", 400, supervisor="systemd")),
|
|
restarted_services=["hermes-gateway.service"], relaunched_profiles=[],
|
|
externally_supervised_profiles=[], killed_pids=set(), failed_units=[],
|
|
)
|
|
assert outcomes[0]["outcome"] == "restarted"
|
|
|
|
|
|
def test_untouched_runtime_is_unaccounted_and_escalates(capsys):
|
|
"""The tripwire: plan saw it, NO bookkeeping mentions it."""
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_rt("coder", 500)),
|
|
restarted_services=["hermes-gateway.service"],
|
|
relaunched_profiles=["default"],
|
|
externally_supervised_profiles=[], killed_pids={123}, failed_units=[],
|
|
)
|
|
assert outcomes[0]["outcome"] == "unaccounted"
|
|
assert report_unaccounted_runtimes(outcomes) is True
|
|
out = capsys.readouterr().out
|
|
assert "never touched" in out
|
|
assert "coder" in out and "500" in out
|
|
assert "hermes -p <profile> gateway restart" in out
|
|
|
|
|
|
def test_external_supervisor_counts_as_restarted():
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_rt("default", 600, supervisor="desktop")),
|
|
restarted_services=[], relaunched_profiles=[],
|
|
externally_supervised_profiles=["default"], killed_pids=set(),
|
|
failed_units=[],
|
|
)
|
|
assert outcomes[0]["outcome"] == "restarted"
|
|
|
|
|
|
def test_unmanaged_serve_runtime_under_default_profile_is_unaccounted():
|
|
"""#100479: an sshd-spawned `serve --isolated` has no systemd unit and
|
|
shares the default profile with the gateway. A gateway-only restart
|
|
must not be read as covering it — it must trip the tripwire instead."""
|
|
serve_runtime = RuntimeRecord(
|
|
kind="serve",
|
|
profile="default",
|
|
pid=900,
|
|
supervisor="manual-serve",
|
|
restart_via=_restart_mechanism("manual-serve", "default"),
|
|
)
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_rt("default", 100, supervisor="systemd"), serve_runtime),
|
|
restarted_services=["hermes-gateway"], relaunched_profiles=[],
|
|
externally_supervised_profiles=[], killed_pids=set(), failed_units=[],
|
|
)
|
|
by_pid = {o["pid"]: o["outcome"] for o in outcomes}
|
|
assert by_pid[100] == "restarted"
|
|
assert by_pid[900] == "unaccounted"
|
|
assert report_unaccounted_runtimes(outcomes) is True
|
|
|
|
|
|
def _serve(profile: str, pid: int, kind: str = "serve") -> RuntimeRecord:
|
|
return RuntimeRecord(
|
|
kind=kind,
|
|
profile=profile,
|
|
pid=pid,
|
|
supervisor="manual-serve",
|
|
restart_via=_restart_mechanism("manual-serve", profile),
|
|
)
|
|
|
|
|
|
def test_serve_never_borrows_relaunched_or_external_gateway_profile():
|
|
"""Sibling site of #100479: the relaunched_profiles / external-supervisor
|
|
bookkeeping is gateway vocabulary too. A manual gateway relaunch under
|
|
``default`` (or a named profile) says nothing about a serve that shares
|
|
the profile name."""
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_rt("default", 100), _serve("default", 900),
|
|
_rt("work", 101), _serve("work", 901, kind="dashboard")),
|
|
restarted_services=[], relaunched_profiles=["default"],
|
|
externally_supervised_profiles=["work"], killed_pids=set(), failed_units=[],
|
|
)
|
|
by_pid = {o["pid"]: o["outcome"] for o in outcomes}
|
|
assert by_pid == {
|
|
100: "restarted", 900: "unaccounted", 101: "restarted", 901: "unaccounted"
|
|
}
|
|
|
|
|
|
def test_named_profile_serve_does_not_match_gateway_profile_unit():
|
|
"""``hermes-gateway-work.service`` restarted must not credit the ``work``
|
|
serve — the old substring match (``"work" in unit``) did exactly that."""
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_rt("work", 101, supervisor="systemd"), _serve("work", 901)),
|
|
restarted_services=["hermes-gateway-work.service"], relaunched_profiles=[],
|
|
externally_supervised_profiles=[], killed_pids=set(), failed_units=[],
|
|
)
|
|
by_pid = {o["pid"]: o["outcome"] for o in outcomes}
|
|
assert by_pid == {101: "restarted", 901: "unaccounted"}
|
|
|
|
|
|
def test_serve_reconciles_against_its_own_unit_vocabulary():
|
|
"""A serve IS covered when a ``hermes-serve*`` unit for its profile was
|
|
restarted (or failed) — scope-qualified identities included."""
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_serve("default", 900), _serve("work", 901),
|
|
_serve("ops", 902, kind="dashboard"), _serve("qa", 903)),
|
|
restarted_services=["hermes-gateway", "user/hermes-serve",
|
|
"hermes-serve-work.service", "hermes-dashboard-ops"],
|
|
relaunched_profiles=[], externally_supervised_profiles=[],
|
|
killed_pids=set(), failed_units=["hermes-serve-qa.service"],
|
|
)
|
|
by_pid = {o["pid"]: o["outcome"] for o in outcomes}
|
|
assert by_pid == {900: "restarted", 901: "restarted", 902: "restarted", 903: "failed"}
|
|
# exact names: ``work`` must not claim ``hermes-serve-workbench``
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_serve("work", 901)),
|
|
restarted_services=["hermes-serve-workbench.service"], relaunched_profiles=[],
|
|
externally_supervised_profiles=[], killed_pids=set(), failed_units=[],
|
|
)
|
|
assert outcomes[0]["outcome"] == "unaccounted"
|
|
|
|
|
|
def test_serve_outcome_follows_incarnation_probe_when_provided():
|
|
"""With the (pid, create_time) survivor probe result, liveness decides:
|
|
a pre-update serve that is gone was replaced (restarted); one still
|
|
alive is unaccounted — even when a hermes-serve unit was restarted."""
|
|
plan = _plan(_serve("default", 900), _serve("default", 901, kind="dashboard"))
|
|
outcomes = match_runtime_outcomes(
|
|
plan, restarted_services=["hermes-serve.service"], relaunched_profiles=[],
|
|
externally_supervised_profiles=[], killed_pids=set(), failed_units=[],
|
|
stale_serve_pids={900},
|
|
)
|
|
by_pid = {o["pid"]: o["outcome"] for o in outcomes}
|
|
assert by_pid == {900: "unaccounted", 901: "restarted"}
|
|
# killed pid still wins as "stopped"; probe None => fail closed
|
|
outcomes = match_runtime_outcomes(
|
|
plan, restarted_services=[], relaunched_profiles=[],
|
|
externally_supervised_profiles=[], killed_pids={901}, failed_units=[],
|
|
stale_serve_pids=None,
|
|
)
|
|
by_pid = {o["pid"]: o["outcome"] for o in outcomes}
|
|
assert by_pid == {900: "unaccounted", 901: "stopped"}
|
|
|
|
|
|
def test_unaccounted_serve_report_names_serve_remedy_not_gateway_restart(capsys):
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(_serve("default", 900)),
|
|
restarted_services=["hermes-gateway"], relaunched_profiles=[],
|
|
externally_supervised_profiles=[], killed_pids=set(), failed_units=[],
|
|
)
|
|
assert report_unaccounted_runtimes(outcomes) is True
|
|
out = capsys.readouterr().out
|
|
assert "serve [default] pid 900" in out
|
|
assert "hermes-serve.service" in out
|
|
assert "hermes gateway restart" not in out
|
|
|
|
|
|
def test_mixed_fleet_only_the_missed_one_escalates(capsys):
|
|
outcomes = match_runtime_outcomes(
|
|
_plan(
|
|
_rt("default", 700, supervisor="systemd"),
|
|
_rt("work", 701),
|
|
_rt("ghost", 702),
|
|
),
|
|
restarted_services=["hermes-gateway.service"],
|
|
relaunched_profiles=["work"],
|
|
externally_supervised_profiles=[], killed_pids=set(), failed_units=[],
|
|
)
|
|
by_profile = {o["profile"]: o["outcome"] for o in outcomes}
|
|
assert by_profile == {
|
|
"default": "restarted", "work": "restarted", "ghost": "unaccounted"
|
|
}
|
|
assert report_unaccounted_runtimes(outcomes) is True
|
|
out = capsys.readouterr().out
|
|
missed_block = out.split("never touched")[1]
|
|
assert "ghost" in missed_block
|
|
assert "[default]" not in missed_block
|
|
assert "[work]" not in missed_block
|