143 lines
4.9 KiB
Python
143 lines
4.9 KiB
Python
"""Fleet matrix DOWN-state coverage (Phase-1 verification gap, #91277).
|
|
|
|
A gateway that was ALIVE at update start, got stopped by the restart phase,
|
|
and never came back used to produce NO matrix row at all — the check passed
|
|
silently on a fleet that lost messaging. Now it yields a `down` row and the
|
|
matrix returns True (escalate/exit 1). Rollout safety: `down` requires the
|
|
pid to be in the caller's pre-restart snapshot, so stale state files from
|
|
long-dead gateways never false-positive.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
|
|
import hermes_cli.update_receipt as ur
|
|
|
|
|
|
def _setup(monkeypatch, tmp_path, record: dict):
|
|
home = tmp_path / ".hermes"
|
|
home.mkdir(exist_ok=True)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.build_info.get_code_identity",
|
|
lambda refresh=False: {"sha": "HEADSHA", "version": "1.0"},
|
|
)
|
|
monkeypatch.setattr("hermes_cli.profiles._get_default_hermes_home", lambda: home)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.profiles._get_profiles_root", lambda: tmp_path / "no-profiles"
|
|
)
|
|
monkeypatch.setattr("gateway.control_socket.identify_gateway", lambda h, **k: None)
|
|
(home / "gateway_state.json").write_text(json.dumps(record), encoding="utf-8")
|
|
return home
|
|
|
|
|
|
_DEAD_PID = 999999899 # never a live pid
|
|
|
|
|
|
def test_killed_never_replaced_gateway_is_a_down_row(monkeypatch, tmp_path):
|
|
_setup(
|
|
monkeypatch,
|
|
tmp_path,
|
|
{"pid": _DEAD_PID, "gateway_state": "running", "kind": "hermes-gateway",
|
|
"code_version": "0.20.5"},
|
|
)
|
|
fleet = ur.collect_fleet_versions(pre_restart_pids=[_DEAD_PID])
|
|
assert len(fleet) == 1
|
|
assert fleet[0]["state"] == "down"
|
|
assert fleet[0]["pid"] == _DEAD_PID
|
|
# and the matrix escalates on it
|
|
assert ur.print_fleet_version_matrix(fleet) is True
|
|
|
|
|
|
def test_dead_pid_not_in_pre_restart_snapshot_keeps_no_row(monkeypatch, tmp_path):
|
|
"""Stale state file from a long-dead gateway: rollout-safe no-row."""
|
|
_setup(
|
|
monkeypatch,
|
|
tmp_path,
|
|
{"pid": _DEAD_PID, "gateway_state": "running", "kind": "hermes-gateway"},
|
|
)
|
|
assert ur.collect_fleet_versions(pre_restart_pids=[12345]) == []
|
|
assert ur.collect_fleet_versions(pre_restart_pids=None) == []
|
|
assert ur.collect_fleet_versions() == []
|
|
|
|
|
|
def test_cleanly_stopped_gateway_is_not_down(monkeypatch, tmp_path):
|
|
for benign_state in ("stopped", "startup_failed"):
|
|
_setup(
|
|
monkeypatch,
|
|
tmp_path,
|
|
{"pid": _DEAD_PID, "gateway_state": benign_state, "kind": "hermes-gateway"},
|
|
)
|
|
assert ur.collect_fleet_versions(pre_restart_pids=[_DEAD_PID]) == []
|
|
|
|
|
|
def test_recycled_pid_is_not_reported_stale(monkeypatch, tmp_path):
|
|
"""A dead gateway's PID reused by an unrelated process (#93258) must not
|
|
be reported STALE just because *some* process now answers to that PID.
|
|
"""
|
|
from gateway.status import _get_process_start_time
|
|
|
|
reused_pid = os.getpid()
|
|
wrong_start_time = (_get_process_start_time(reused_pid) or 0) + 12345
|
|
_setup(
|
|
monkeypatch,
|
|
tmp_path,
|
|
{
|
|
"pid": reused_pid,
|
|
"start_time": wrong_start_time,
|
|
"gateway_state": "running",
|
|
"code_sha": "OLDSHA",
|
|
"kind": "hermes-gateway",
|
|
},
|
|
)
|
|
fleet = ur.collect_fleet_versions(pre_restart_pids=[reused_pid])
|
|
assert len(fleet) == 1
|
|
assert fleet[0]["state"] == "down"
|
|
|
|
|
|
def test_matching_start_time_is_still_live(monkeypatch, tmp_path):
|
|
"""A record whose start_time matches the live process is not recycled."""
|
|
from gateway.status import _get_process_start_time
|
|
|
|
pid = os.getpid()
|
|
_setup(
|
|
monkeypatch,
|
|
tmp_path,
|
|
{
|
|
"pid": pid,
|
|
"start_time": _get_process_start_time(pid),
|
|
"gateway_state": "running",
|
|
"code_sha": "HEADSHA",
|
|
"kind": "hermes-gateway",
|
|
},
|
|
)
|
|
fleet = ur.collect_fleet_versions(pre_restart_pids=[pid])
|
|
assert len(fleet) == 1
|
|
assert fleet[0]["state"] == "current"
|
|
|
|
|
|
def test_live_gateway_rows_unchanged(monkeypatch, tmp_path):
|
|
"""The live-pid path is untouched by the down-state addition."""
|
|
_setup(
|
|
monkeypatch,
|
|
tmp_path,
|
|
{"pid": os.getpid(), "gateway_state": "running", "code_sha": "HEADSHA",
|
|
"kind": "hermes-gateway"},
|
|
)
|
|
fleet = ur.collect_fleet_versions(pre_restart_pids=[os.getpid()])
|
|
assert len(fleet) == 1
|
|
assert fleet[0]["state"] == "current"
|
|
assert ur.print_fleet_version_matrix(fleet) is False
|
|
|
|
|
|
def test_down_and_stale_both_escalate_with_remediation(capsys):
|
|
fleet = [
|
|
{"profile": "default", "pid": 1, "code_sha": "OLD", "state": "stale"},
|
|
{"profile": "coder", "pid": 2, "code_sha": None, "state": "down"},
|
|
]
|
|
assert ur.print_fleet_version_matrix(fleet) is True
|
|
out = capsys.readouterr().out
|
|
assert "STALE" in out
|
|
assert "DOWN" in out
|
|
assert "hermes gateway restart" in out
|