328 lines
10 KiB
Python
328 lines
10 KiB
Python
"""Behavioral regression tests for ACP background MCP discovery + late-refresh.
|
|
|
|
These replace the previous AST-based test that only inspected source text.
|
|
They verify the *behavior*: (1) a blocked discovery doesn't block startup, and
|
|
(2) a delayed-but-reachable MCP server's tools land in the agent's snapshot
|
|
via the automatic late-refresh, cache-safely (pre-first-turn only).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import threading
|
|
import time
|
|
import types
|
|
from contextlib import nullcontext
|
|
from types import ModuleType, SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from acp_adapter.server import HermesACPAgent
|
|
from acp_adapter.session import SessionManager, SessionState
|
|
from hermes_cli import mcp_startup
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Shared helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class FakeAgent:
|
|
"""Minimal stand-in for AIAgent with the attributes late-refresh touches."""
|
|
|
|
def __init__(self):
|
|
self.model = "fake-model"
|
|
self.provider = "fake-provider"
|
|
self.enabled_toolsets = ["hermes-acp"]
|
|
self.disabled_toolsets = []
|
|
self.tools = []
|
|
self.valid_tool_names = set()
|
|
self._user_turn_count = 0
|
|
self._api_call_count = 0
|
|
|
|
|
|
class NoopDb:
|
|
def get_session(self, *_a, **_k):
|
|
return None
|
|
|
|
def create_session(self, *_a, **_k):
|
|
return None
|
|
|
|
def update_session(self, *_a, **_k):
|
|
return None
|
|
|
|
|
|
def _mod(name: str, **attrs) -> ModuleType:
|
|
module = ModuleType(name)
|
|
for key, value in attrs.items():
|
|
setattr(module, key, value)
|
|
return module
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_mcp_startup_state():
|
|
"""Ensure each test starts with a clean discovery thread state."""
|
|
saved_started = mcp_startup._mcp_discovery_started
|
|
saved_thread = mcp_startup._mcp_discovery_thread
|
|
mcp_startup._mcp_discovery_started = False
|
|
mcp_startup._mcp_discovery_thread = None
|
|
yield
|
|
thread = mcp_startup._mcp_discovery_thread
|
|
if thread is not None and thread.is_alive():
|
|
thread.join(timeout=2.0)
|
|
mcp_startup._mcp_discovery_started = saved_started
|
|
mcp_startup._mcp_discovery_thread = saved_thread
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 1 — blocked discovery does not block startup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_acp_background_discovery_does_not_block_startup(monkeypatch):
|
|
"""start_background_mcp_discovery must return immediately even if discovery hangs."""
|
|
block = threading.Event()
|
|
|
|
def _blocking_discover():
|
|
block.wait(timeout=5.0)
|
|
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"hermes_cli.config",
|
|
_mod(
|
|
"hermes_cli.config",
|
|
read_raw_config=lambda: {"mcp_servers": {"slow": {"url": "https://mcp.example.test"}}},
|
|
),
|
|
)
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"tools.mcp_oauth",
|
|
_mod("tools.mcp_oauth", suppress_interactive_oauth=lambda: nullcontext()),
|
|
)
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"tools.mcp_tool",
|
|
_mod("tools.mcp_tool", discover_mcp_tools=_blocking_discover),
|
|
)
|
|
|
|
start = time.monotonic()
|
|
mcp_startup.start_background_mcp_discovery(
|
|
logger=SimpleNamespace(debug=lambda *_a, **_k: None),
|
|
thread_name="test-acp-discovery",
|
|
)
|
|
elapsed = time.monotonic() - start
|
|
|
|
assert elapsed < 0.2, "start_background_mcp_discovery blocked for {:.3f}s".format(elapsed)
|
|
assert mcp_startup._mcp_discovery_thread is not None
|
|
assert mcp_startup._mcp_discovery_thread.is_alive()
|
|
block.set()
|
|
mcp_startup._mcp_discovery_thread.join(timeout=2.0)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 2 — delayed discovery lands tools via late-refresh (pre-first-turn)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_acp_late_refresh_adds_tools_when_discovery_lands_after_build(monkeypatch):
|
|
"""A slow MCP server that finishes after agent build must still appear in tools."""
|
|
|
|
discovery_block = threading.Event()
|
|
discovery_done = threading.Event()
|
|
|
|
def _slow_discover():
|
|
discovery_block.wait(timeout=5.0)
|
|
discovery_done.set()
|
|
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"hermes_cli.config",
|
|
_mod(
|
|
"hermes_cli.config",
|
|
read_raw_config=lambda: {"mcp_servers": {"slow": {"url": "https://mcp.example.test"}}},
|
|
),
|
|
)
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"tools.mcp_oauth",
|
|
_mod("tools.mcp_oauth", suppress_interactive_oauth=lambda: nullcontext()),
|
|
)
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"tools.mcp_tool",
|
|
_mod("tools.mcp_tool", discover_mcp_tools=_slow_discover),
|
|
)
|
|
|
|
mcp_startup.start_background_mcp_discovery(
|
|
logger=SimpleNamespace(debug=lambda *_a, **_k: None),
|
|
thread_name="test-acp-late",
|
|
)
|
|
|
|
# Build the session immediately — discovery is still in flight.
|
|
fake = FakeAgent()
|
|
manager = SessionManager(agent_factory=lambda **_k: fake, db=NoopDb())
|
|
acp_agent = HermesACPAgent(session_manager=manager)
|
|
state = manager.create_session(cwd=".")
|
|
|
|
# Discovery is blocked, so it must still be in flight.
|
|
assert not discovery_done.is_set(), "discovery finished too early for this test"
|
|
|
|
# Track refresh_agent_mcp_tools calls.
|
|
refreshed = []
|
|
|
|
def _fake_refresh(agent, **_kw):
|
|
agent.tools = [{"function": {"name": "mcp_slow_tool"}}]
|
|
agent.valid_tool_names = {"mcp_slow_tool"}
|
|
refreshed.append(agent)
|
|
return {"mcp_slow_tool"}
|
|
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"tools.mcp_tool",
|
|
_mod("tools.mcp_tool", refresh_agent_mcp_tools=_fake_refresh),
|
|
)
|
|
|
|
# Trigger late-refresh.
|
|
acp_agent._schedule_mcp_late_refresh(state)
|
|
|
|
# Release discovery so the late-refresh daemon can proceed.
|
|
discovery_block.set()
|
|
|
|
# Wait for the late-refresh daemon to finish.
|
|
deadline = time.monotonic() + 5.0
|
|
while not refreshed and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
|
|
assert refreshed, "late-refresh daemon did not call refresh_agent_mcp_tools"
|
|
assert refreshed[0] is fake
|
|
assert "mcp_slow_tool" in fake.valid_tool_names
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 3 — late-refresh is cache-safe: skips after first turn
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_acp_late_refresh_skips_after_first_turn(monkeypatch):
|
|
"""Once the user has sent a message, late-refresh must NOT rebuild tools."""
|
|
|
|
discovery_block = threading.Event()
|
|
|
|
def _slow_discover():
|
|
discovery_block.wait(timeout=5.0)
|
|
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"hermes_cli.config",
|
|
_mod(
|
|
"hermes_cli.config",
|
|
read_raw_config=lambda: {"mcp_servers": {"slow": {"url": "https://mcp.example.test"}}},
|
|
),
|
|
)
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"tools.mcp_oauth",
|
|
_mod("tools.mcp_oauth", suppress_interactive_oauth=lambda: nullcontext()),
|
|
)
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"tools.mcp_tool",
|
|
_mod("tools.mcp_tool", discover_mcp_tools=_slow_discover),
|
|
)
|
|
|
|
mcp_startup.start_background_mcp_discovery(
|
|
logger=SimpleNamespace(debug=lambda *_a, **_k: None),
|
|
thread_name="test-acp-cache",
|
|
)
|
|
|
|
fake = FakeAgent()
|
|
fake._api_call_count = 1 # simulate: user already sent a message
|
|
manager = SessionManager(agent_factory=lambda **_k: fake, db=NoopDb())
|
|
acp_agent = HermesACPAgent(session_manager=manager)
|
|
state = manager.create_session(cwd=".")
|
|
|
|
refreshed = []
|
|
|
|
def _fake_refresh(agent, **_kw):
|
|
refreshed.append(agent)
|
|
return set()
|
|
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"tools.mcp_tool",
|
|
_mod("tools.mcp_tool", refresh_agent_mcp_tools=_fake_refresh),
|
|
)
|
|
|
|
acp_agent._schedule_mcp_late_refresh(state)
|
|
|
|
# Release discovery so the daemon can proceed (if it were going to).
|
|
discovery_block.set()
|
|
|
|
# Give the daemon time to run (if it were going to).
|
|
time.sleep(0.5)
|
|
|
|
assert not refreshed, "late-refresh rebuilt tools after the first turn — cache broken!"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 4 — late-refresh is serialized with turn start: skips while running
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_acp_late_refresh_skips_while_turn_running(monkeypatch):
|
|
"""A turn in flight (state.is_running) must block the rebuild even when
|
|
the agent's counters still read zero — closes the guard/turn-start race."""
|
|
|
|
discovery_block = threading.Event()
|
|
|
|
def _slow_discover():
|
|
discovery_block.wait(timeout=5.0)
|
|
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"hermes_cli.config",
|
|
_mod(
|
|
"hermes_cli.config",
|
|
read_raw_config=lambda: {"mcp_servers": {"slow": {"url": "https://mcp.example.test"}}},
|
|
),
|
|
)
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"tools.mcp_oauth",
|
|
_mod("tools.mcp_oauth", suppress_interactive_oauth=lambda: nullcontext()),
|
|
)
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"tools.mcp_tool",
|
|
_mod("tools.mcp_tool", discover_mcp_tools=_slow_discover),
|
|
)
|
|
|
|
mcp_startup.start_background_mcp_discovery(
|
|
logger=SimpleNamespace(debug=lambda *_a, **_k: None),
|
|
thread_name="test-acp-running",
|
|
)
|
|
|
|
fake = FakeAgent() # counters are 0 — only is_running blocks the refresh
|
|
manager = SessionManager(agent_factory=lambda **_k: fake, db=NoopDb())
|
|
acp_agent = HermesACPAgent(session_manager=manager)
|
|
state = manager.create_session(cwd=".")
|
|
state.is_running = True # simulate: first prompt dispatched concurrently
|
|
|
|
refreshed = []
|
|
|
|
def _fake_refresh(agent, **_kw):
|
|
refreshed.append(agent)
|
|
return set()
|
|
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"tools.mcp_tool",
|
|
_mod("tools.mcp_tool", refresh_agent_mcp_tools=_fake_refresh),
|
|
)
|
|
|
|
acp_agent._schedule_mcp_late_refresh(state)
|
|
discovery_block.set()
|
|
time.sleep(0.5)
|
|
|
|
assert not refreshed, "late-refresh rebuilt tools while a turn was running!"
|