521 lines
18 KiB
Python
521 lines
18 KiB
Python
"""Interrupted-update fleet-restart obligation (#95294 parts 1+2).
|
|
|
|
A ``hermes update`` killed after git pull advanced HEAD but before the
|
|
fleet restart left running gateways on stale code. The next update said
|
|
"Already up to date" and skipped restart. These tests cover:
|
|
|
|
- ``fleet_restart_pending`` marker written after HEAD advances, cleared
|
|
after a successful (or no-op) fleet restart
|
|
- interrupt between pull and restart leaves the marker
|
|
- next ``hermes update`` with git already up to date still runs the
|
|
pending restart when the marker OR a skewed unfinished latest.json is
|
|
present
|
|
|
|
No live gateway, no network. Git and restart are mocked.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from hermes_cli import main as hermes_main
|
|
from hermes_cli import update_cmd
|
|
from hermes_constants import get_hermes_home
|
|
|
|
|
|
def _make_head_moved_side_effect(pre_sha="abc123", post_sha="def456"):
|
|
"""Simulate git commands where HEAD advances from pre_sha to post_sha."""
|
|
calls = {"n": 0}
|
|
|
|
def side_effect(cmd, **kwargs):
|
|
joined = " ".join(str(c) for c in cmd)
|
|
|
|
if "rev-parse" in joined and "--abbrev-ref" in joined:
|
|
return SimpleNamespace(returncode=0, stdout="main\n", stderr="")
|
|
|
|
if "rev-list" in joined:
|
|
return SimpleNamespace(returncode=0, stdout="3\n", stderr="")
|
|
|
|
if joined.endswith("rev-parse HEAD"):
|
|
if calls["n"] == 0:
|
|
calls["n"] += 1
|
|
return SimpleNamespace(returncode=0, stdout=f"{pre_sha}\n", stderr="")
|
|
return SimpleNamespace(returncode=0, stdout=f"{post_sha}\n", stderr="")
|
|
|
|
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
|
|
|
return side_effect
|
|
|
|
|
|
def _make_up_to_date_side_effect(sha="abc123"):
|
|
"""Simulate git commands where origin is already at HEAD."""
|
|
|
|
def side_effect(cmd, **kwargs):
|
|
joined = " ".join(str(c) for c in cmd)
|
|
|
|
if "rev-parse" in joined and "--abbrev-ref" in joined:
|
|
return SimpleNamespace(returncode=0, stdout="main\n", stderr="")
|
|
|
|
if "rev-list" in joined:
|
|
return SimpleNamespace(returncode=0, stdout="0\n", stderr="")
|
|
|
|
if joined.endswith("rev-parse HEAD"):
|
|
return SimpleNamespace(returncode=0, stdout=f"{sha}\n", stderr="")
|
|
|
|
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
|
|
|
return side_effect
|
|
|
|
|
|
def _patch_update_deps(monkeypatch, tmp_path, run_side_effect):
|
|
"""Patch ``_cmd_update_impl`` helpers. Mirrors test_update_head_moved_gate."""
|
|
monkeypatch.setattr(hermes_main.subprocess, "run", run_side_effect)
|
|
monkeypatch.setattr(hermes_main, "PROJECT_ROOT", tmp_path)
|
|
(tmp_path / ".git").mkdir()
|
|
monkeypatch.setattr(hermes_main, "_resolve_update_branch", lambda args: "main")
|
|
monkeypatch.setattr(hermes_main, "_is_windows", lambda: False)
|
|
monkeypatch.setattr(
|
|
hermes_main,
|
|
"_get_origin_url",
|
|
lambda *a, **k: "https://github.com/NousResearch/hermes-agent.git",
|
|
)
|
|
monkeypatch.setattr(hermes_main, "_is_fork", lambda *a, **k: False)
|
|
monkeypatch.setattr(
|
|
hermes_main, "_stash_local_changes_if_needed", lambda *a, **k: None
|
|
)
|
|
monkeypatch.setattr(hermes_main, "_clear_bytecode_cache", lambda *a, **k: 0)
|
|
monkeypatch.setattr(
|
|
hermes_main, "_record_bytecode_fingerprint", lambda *a, **k: None
|
|
)
|
|
monkeypatch.setattr(hermes_main, "_run_pre_update_backup", lambda *a, **k: None)
|
|
monkeypatch.setattr(
|
|
hermes_main, "_pause_windows_gateways_for_update", lambda: None
|
|
)
|
|
monkeypatch.setattr(
|
|
hermes_main, "_resume_windows_gateways_after_update", lambda *a, **k: None
|
|
)
|
|
monkeypatch.setattr(hermes_main, "_write_update_incomplete_marker", lambda: None)
|
|
monkeypatch.setattr(hermes_main, "_clear_update_incomplete_marker", lambda: None)
|
|
monkeypatch.setattr(
|
|
hermes_main, "_finish_dashboard_update_cleanup", lambda *a, **k: None
|
|
)
|
|
monkeypatch.setattr(
|
|
update_cmd, "_finish_dashboard_update_cleanup", lambda *a, **k: None
|
|
)
|
|
monkeypatch.setattr(hermes_main, "_build_web_ui", lambda *a, **k: None)
|
|
monkeypatch.setattr(
|
|
update_cmd, "_venv_core_imports_healthy", lambda: (True, "")
|
|
)
|
|
monkeypatch.setattr(update_cmd, "_update_node_dependencies", lambda: [])
|
|
monkeypatch.setattr(update_cmd, "_purge_stale_hermes_modules", lambda: None)
|
|
monkeypatch.setattr(hermes_main, "_purge_stale_hermes_modules", lambda: None)
|
|
|
|
import hermes_cli.gateway as hermes_gateway
|
|
|
|
monkeypatch.setattr(
|
|
hermes_gateway, "find_gateway_pids", lambda **_kwargs: []
|
|
)
|
|
monkeypatch.setattr(hermes_gateway, "supports_systemd_services", lambda: False)
|
|
monkeypatch.setattr(
|
|
hermes_gateway, "find_profile_gateway_processes", lambda *a, **k: []
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.update_receipt.collect_fleet_versions",
|
|
lambda **k: [],
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.update_inventory.collect_runtime_inventory",
|
|
lambda: SimpleNamespace(runtimes=[], to_dict=lambda: {}),
|
|
)
|
|
|
|
|
|
def _update_args():
|
|
return SimpleNamespace(branch=None, yes=False, force=False, force_venv=False)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Marker helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_marker_round_trip_under_hermes_home():
|
|
path = update_cmd._fleet_restart_pending_marker_path()
|
|
assert path.parent == get_hermes_home()
|
|
assert path.name == "fleet_restart_pending"
|
|
assert not path.exists()
|
|
|
|
update_cmd._write_fleet_restart_pending_marker(expected_sha="abc123")
|
|
assert path.is_file()
|
|
body = path.read_text(encoding="utf-8")
|
|
assert "started=" in body
|
|
assert "pid=" in body
|
|
assert "expected_sha=abc123" in body
|
|
|
|
update_cmd._clear_fleet_restart_pending_marker()
|
|
assert not path.exists()
|
|
|
|
|
|
def test_pending_needed_when_marker_exists():
|
|
update_cmd._write_fleet_restart_pending_marker()
|
|
assert update_cmd._pending_fleet_restart_needed() is True
|
|
update_cmd._clear_fleet_restart_pending_marker()
|
|
assert update_cmd._pending_fleet_restart_needed() is False
|
|
|
|
|
|
def test_pending_needed_when_unfinished_receipt_runtime_sha_skews(monkeypatch):
|
|
disk_sha = "e" * 40
|
|
old_sha = "7" * 40
|
|
monkeypatch.setattr(update_cmd, "_current_checkout_sha", lambda: disk_sha)
|
|
|
|
receipt_dir = get_hermes_home() / "logs" / "update_receipts"
|
|
receipt_dir.mkdir(parents=True)
|
|
(receipt_dir / "latest.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"exit_code": 1,
|
|
"stop_reason": "KeyboardInterrupt: ",
|
|
"outcome": "failed",
|
|
"plan": {
|
|
"expected_sha": disk_sha,
|
|
"runtimes": [
|
|
{
|
|
"kind": "gateway",
|
|
"profile": "default",
|
|
"pid": 2111768,
|
|
"supervisor": "systemd",
|
|
"code_sha": old_sha,
|
|
"restart_via": "systemd",
|
|
}
|
|
],
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
assert update_cmd._pending_fleet_restart_needed() is True
|
|
|
|
|
|
def test_successful_receipt_with_pre_update_plan_shas_does_not_retrigger(
|
|
monkeypatch,
|
|
):
|
|
"""A completed update's plan.runtimes are pre-pull SHAs — not a catch-up."""
|
|
disk_sha = "n" * 40
|
|
old_sha = "o" * 40
|
|
monkeypatch.setattr(update_cmd, "_current_checkout_sha", lambda: disk_sha)
|
|
|
|
receipt_dir = get_hermes_home() / "logs" / "update_receipts"
|
|
receipt_dir.mkdir(parents=True)
|
|
(receipt_dir / "latest.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"exit_code": 0,
|
|
"outcome": "success",
|
|
"plan": {
|
|
"expected_sha": old_sha,
|
|
"runtimes": [
|
|
{
|
|
"kind": "gateway",
|
|
"profile": "default",
|
|
"pid": 1,
|
|
"code_sha": old_sha,
|
|
}
|
|
],
|
|
},
|
|
"fleet": [
|
|
{
|
|
"profile": "default",
|
|
"pid": 2,
|
|
"code_sha": disk_sha,
|
|
"state": "current",
|
|
}
|
|
],
|
|
"gateway_restart": {"incomplete": False},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
assert update_cmd._pending_fleet_restart_needed() is False
|
|
|
|
|
|
def test_stale_fleet_matrix_on_latest_receipt_is_pending(monkeypatch):
|
|
disk_sha = "n" * 40
|
|
monkeypatch.setattr(update_cmd, "_current_checkout_sha", lambda: disk_sha)
|
|
|
|
receipt_dir = get_hermes_home() / "logs" / "update_receipts"
|
|
receipt_dir.mkdir(parents=True)
|
|
(receipt_dir / "latest.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"outcome": "partial",
|
|
"exit_code": 1,
|
|
"fleet": [
|
|
{
|
|
"profile": "default",
|
|
"pid": 9,
|
|
"code_sha": "s" * 40,
|
|
"state": "stale",
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
assert update_cmd._pending_fleet_restart_needed() is True
|
|
|
|
|
|
def test_run_pending_restart_true_when_no_gateways(monkeypatch, capsys):
|
|
monkeypatch.setattr(
|
|
"hermes_cli.gateway.find_gateway_pids", lambda **k: []
|
|
)
|
|
monkeypatch.setattr(hermes_main, "_purge_stale_hermes_modules", lambda: None)
|
|
|
|
assert update_cmd._run_pending_fleet_restart() is True
|
|
assert "nothing to restart" in capsys.readouterr().out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# cmd_update integration (mocked git / restart)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_marker_written_after_pull_cleared_after_successful_restart(
|
|
monkeypatch, tmp_path, capsys
|
|
):
|
|
args = _update_args()
|
|
_patch_update_deps(monkeypatch, tmp_path, _make_head_moved_side_effect())
|
|
|
|
wrote = []
|
|
orig = update_cmd._write_fleet_restart_pending_marker
|
|
|
|
def _spy(*, expected_sha=""):
|
|
orig(expected_sha=expected_sha)
|
|
wrote.append(update_cmd._fleet_restart_pending_marker_path().is_file())
|
|
|
|
monkeypatch.setattr(update_cmd, "_write_fleet_restart_pending_marker", _spy)
|
|
|
|
hermes_main.cmd_update(args)
|
|
|
|
assert wrote == [True], "marker must exist immediately after HEAD advances"
|
|
assert not update_cmd._fleet_restart_pending_marker_path().exists()
|
|
out = capsys.readouterr().out
|
|
assert "✓ Code updated!" in out
|
|
|
|
|
|
def test_clean_update_warns_about_surviving_pre_update_serve_runtime(
|
|
monkeypatch, tmp_path, capsys
|
|
):
|
|
"""The successful update path must surface an inventoried stale serve."""
|
|
args = _update_args()
|
|
_patch_update_deps(monkeypatch, tmp_path, _make_head_moved_side_effect())
|
|
monkeypatch.setattr(
|
|
update_cmd,
|
|
"_surviving_pre_update_serve_runtimes",
|
|
lambda _plan: [
|
|
{
|
|
"pid": 5555,
|
|
"kind": "serve",
|
|
"profile": "default",
|
|
"supervisor": "manual-serve",
|
|
}
|
|
],
|
|
)
|
|
|
|
hermes_main.cmd_update(args)
|
|
|
|
out = capsys.readouterr().out
|
|
assert "pid 5555" in out
|
|
assert "serve" in out
|
|
assert "pre-update code" in out
|
|
|
|
|
|
def test_clean_update_escalates_surviving_serve_as_unaccounted(
|
|
monkeypatch, tmp_path, capsys
|
|
):
|
|
"""#100479 end to end: the plan inventoried a gateway (restarted through
|
|
``hermes-gateway.service``) and an unmanaged ``serve`` on the same
|
|
default profile. The serve survives the update as the SAME process, so
|
|
the update must (1) warn, (2) reconcile it as ``unaccounted`` instead of
|
|
borrowing the gateway's restart, and (3) exit 1 with a ``partial``
|
|
receipt — not print a clean success."""
|
|
from hermes_cli.update_inventory import (
|
|
RuntimeRecord, UpdatePlan, _restart_mechanism,
|
|
)
|
|
import hermes_cli.update_inventory as ui
|
|
|
|
args = _update_args()
|
|
_patch_update_deps(monkeypatch, tmp_path, _make_head_moved_side_effect())
|
|
|
|
plan = UpdatePlan()
|
|
plan.runtimes = [
|
|
RuntimeRecord(kind="gateway", profile="default", pid=4444,
|
|
supervisor="systemd",
|
|
restart_via=_restart_mechanism("systemd", "default")),
|
|
RuntimeRecord(kind="serve", profile="default", pid=5555,
|
|
supervisor="manual-serve",
|
|
restart_via=_restart_mechanism("manual-serve", "default"),
|
|
detail={"create_time": 1000.0}),
|
|
]
|
|
monkeypatch.setattr(ui, "collect_runtime_inventory", lambda: plan)
|
|
# The restart phase's own bookkeeping says the gateway unit restarted
|
|
# (systemd branch is stubbed off in _patch_update_deps, so feed it here).
|
|
real_match = ui.match_runtime_outcomes
|
|
|
|
def _match(p, **kw):
|
|
kw["restarted_services"] = list(kw.get("restarted_services") or []) + [
|
|
"hermes-gateway.service"
|
|
]
|
|
return real_match(p, **kw)
|
|
|
|
monkeypatch.setattr(ui, "match_runtime_outcomes", _match)
|
|
# Real survivor probe semantics against a fake ledger: pid 5555 is still
|
|
# the same incarnation the plan recorded.
|
|
import hermes_cli.process_identity as pi
|
|
|
|
monkeypatch.setattr(
|
|
pi, "ledger_entries",
|
|
lambda **_k: [{"pid": 5555, "purpose": "serve", "create_time": 1000.0}],
|
|
)
|
|
|
|
with pytest.raises(SystemExit) as excinfo:
|
|
hermes_main.cmd_update(args)
|
|
assert excinfo.value.code == 1
|
|
|
|
out = capsys.readouterr().out
|
|
assert "pid 5555" in out and "pre-update code" in out
|
|
assert "Planned runtimes the restart phase never touched" in out
|
|
assert "serve [default] pid 5555" in out
|
|
|
|
latest = get_hermes_home() / "logs" / "update_receipts" / "latest.json"
|
|
receipt = json.loads(latest.read_text(encoding="utf-8"))
|
|
assert receipt["outcome"] == "partial"
|
|
by_pid = {o["pid"]: o["outcome"] for o in receipt["runtime_outcomes"]}
|
|
assert by_pid == {4444: "restarted", 5555: "unaccounted"}
|
|
|
|
|
|
def test_interrupt_between_pull_and_restart_leaves_marker(
|
|
monkeypatch, tmp_path
|
|
):
|
|
args = _update_args()
|
|
_patch_update_deps(monkeypatch, tmp_path, _make_head_moved_side_effect())
|
|
|
|
def _interrupt(*_a, **_k):
|
|
raise KeyboardInterrupt()
|
|
|
|
monkeypatch.setattr(hermes_main, "_clear_bytecode_cache", _interrupt)
|
|
|
|
with pytest.raises(KeyboardInterrupt):
|
|
hermes_main.cmd_update(args)
|
|
|
|
marker = update_cmd._fleet_restart_pending_marker_path()
|
|
assert marker.is_file()
|
|
assert "expected_sha=def456" in marker.read_text(encoding="utf-8")
|
|
|
|
|
|
def test_already_up_to_date_runs_pending_restart_when_marker_present(
|
|
monkeypatch, tmp_path, capsys
|
|
):
|
|
args = _update_args()
|
|
_patch_update_deps(monkeypatch, tmp_path, _make_up_to_date_side_effect())
|
|
update_cmd._write_fleet_restart_pending_marker(expected_sha="def456")
|
|
|
|
seen = {"ran": False}
|
|
|
|
def _restart():
|
|
seen["ran"] = True
|
|
return True
|
|
|
|
monkeypatch.setattr(update_cmd, "_run_pending_fleet_restart", _restart)
|
|
|
|
hermes_main.cmd_update(args)
|
|
|
|
assert seen["ran"] is True
|
|
assert not update_cmd._fleet_restart_pending_marker_path().exists()
|
|
out = capsys.readouterr().out
|
|
assert "did not restart running gateways" in out
|
|
|
|
|
|
def test_already_up_to_date_runs_pending_restart_when_receipt_skewed(
|
|
monkeypatch, tmp_path, capsys
|
|
):
|
|
args = _update_args()
|
|
_patch_update_deps(monkeypatch, tmp_path, _make_up_to_date_side_effect())
|
|
|
|
disk_sha = "e" * 40
|
|
monkeypatch.setattr(update_cmd, "_current_checkout_sha", lambda: disk_sha)
|
|
receipt_dir = get_hermes_home() / "logs" / "update_receipts"
|
|
receipt_dir.mkdir(parents=True)
|
|
(receipt_dir / "latest.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"exit_code": 1,
|
|
"stop_reason": "KeyboardInterrupt: ",
|
|
"outcome": "failed",
|
|
"plan": {
|
|
"expected_sha": disk_sha,
|
|
"runtimes": [
|
|
{
|
|
"kind": "gateway",
|
|
"profile": "default",
|
|
"pid": 42,
|
|
"code_sha": "7" * 40,
|
|
}
|
|
],
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
seen = {"ran": False}
|
|
monkeypatch.setattr(
|
|
update_cmd,
|
|
"_run_pending_fleet_restart",
|
|
lambda: seen.__setitem__("ran", True) or True,
|
|
)
|
|
|
|
hermes_main.cmd_update(args)
|
|
|
|
assert seen["ran"] is True
|
|
out = capsys.readouterr().out
|
|
assert "did not restart running gateways" in out
|
|
|
|
|
|
def test_already_up_to_date_skips_restart_when_nothing_pending(
|
|
monkeypatch, tmp_path, capsys
|
|
):
|
|
args = _update_args()
|
|
_patch_update_deps(monkeypatch, tmp_path, _make_up_to_date_side_effect())
|
|
|
|
seen = {"ran": False}
|
|
monkeypatch.setattr(
|
|
update_cmd,
|
|
"_run_pending_fleet_restart",
|
|
lambda: seen.__setitem__("ran", True) or True,
|
|
)
|
|
|
|
hermes_main.cmd_update(args)
|
|
|
|
assert seen["ran"] is False
|
|
assert "did not restart running gateways" not in capsys.readouterr().out
|
|
|
|
|
|
def test_startup_warn_prints_when_marker_present(capsys):
|
|
update_cmd._write_fleet_restart_pending_marker()
|
|
update_cmd._warn_pending_fleet_restart_on_startup()
|
|
err = capsys.readouterr().err
|
|
assert "did not restart running gateways" in err
|
|
assert "hermes gateway restart" in err
|
|
|
|
|
|
def test_startup_warn_silent_when_nothing_pending(capsys):
|
|
update_cmd._warn_pending_fleet_restart_on_startup()
|
|
captured = capsys.readouterr()
|
|
assert captured.err == ""
|
|
assert captured.out == ""
|