902 lines
35 KiB
Python
902 lines
35 KiB
Python
"""Behavior contracts for incremental tool-call persistence (#49045).
|
|
|
|
A destructive or process-terminating tool that runs during tool execution
|
|
must not lose the just-executed assistant(tool_calls) block or the tool
|
|
results that were produced before it fired. These tests pin the contract:
|
|
|
|
1. run_conversation flushes the assistant tool-call turn to the session
|
|
DB BEFORE handing control to _execute_tool_calls (so a tool that
|
|
restarts/kills the process never orphans the tool-call block).
|
|
2. The SEQUENTIAL tool path flushes each tool result to the session DB
|
|
immediately after appending it — BEFORE the next tool dispatches.
|
|
3. The CONCURRENT tool path flushes each tool result in append order.
|
|
|
|
These exercise the REAL production dispatch surfaces:
|
|
|
|
* sequential -> ``run_agent.handle_function_call`` (tool_executor ~1256/1298)
|
|
* concurrent -> ``agent._invoke_tool`` (tool_executor ~539)
|
|
|
|
Mocking the genuine dispatch surface keeps the tests deterministic (no real
|
|
``web_search`` / network) AND mutation-survivable: the ordering assertions
|
|
read snapshots captured at flush time, so removing any production flush call
|
|
makes the corresponding assertion fail.
|
|
"""
|
|
|
|
import copy
|
|
from types import SimpleNamespace
|
|
from pathlib import Path
|
|
import tempfile
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from agent.tool_dispatch_helpers import make_tool_result_message
|
|
from agent.agent_runtime_helpers import sanitize_api_messages
|
|
from agent.tool_executor import execute_tool_calls_segmented
|
|
from hermes_state import SessionDB
|
|
from run_agent import AIAgent
|
|
|
|
|
|
def _make_tool_defs(*names: str) -> list:
|
|
return [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": name,
|
|
"description": f"{name} tool",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
}
|
|
for name in names
|
|
]
|
|
|
|
|
|
def _make_agent():
|
|
hermes_home = Path(tempfile.mkdtemp(prefix="hermes-test-home-"))
|
|
(hermes_home / "logs").mkdir(parents=True, exist_ok=True)
|
|
with (
|
|
patch(
|
|
"run_agent.get_tool_definitions",
|
|
return_value=_make_tool_defs("web_search"),
|
|
),
|
|
patch("run_agent.check_toolset_requirements", return_value={}),
|
|
patch("run_agent.OpenAI"),
|
|
patch("run_agent._hermes_home", hermes_home),
|
|
patch("agent.model_metadata.fetch_model_metadata", return_value={}),
|
|
):
|
|
agent = AIAgent(
|
|
api_key="test-key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
quiet_mode=True,
|
|
skip_context_files=True,
|
|
skip_memory=True,
|
|
)
|
|
agent.client = MagicMock()
|
|
agent._cached_system_prompt = "You are helpful."
|
|
agent._use_prompt_caching = False
|
|
agent.compression_enabled = False
|
|
agent.save_trajectories = False
|
|
return agent
|
|
|
|
|
|
def _attach_real_session_db(agent, db_path: Path, session_id: str) -> SessionDB:
|
|
db = SessionDB(db_path=db_path)
|
|
db.create_session(session_id=session_id, source="tui", model="test/model")
|
|
agent._session_db = db
|
|
agent._session_db_created = True
|
|
agent.session_id = session_id
|
|
agent._last_flushed_db_idx = 0
|
|
agent._flushed_db_message_ids = set()
|
|
agent._flushed_db_message_session_id = None
|
|
agent._persist_disabled = False
|
|
return db
|
|
|
|
|
|
def _durable_messages(db_path: Path, session_id: str) -> list[dict]:
|
|
restarted_db = SessionDB(db_path=db_path)
|
|
try:
|
|
return restarted_db.get_messages_as_conversation(session_id)
|
|
finally:
|
|
restarted_db.close()
|
|
|
|
|
|
def _durable_roles(db_path: Path, session_id: str) -> list[str]:
|
|
return [message["role"] for message in _durable_messages(db_path, session_id)]
|
|
|
|
|
|
def _mock_tool_call(name="web_search", arguments="{}", call_id="call_1"):
|
|
return SimpleNamespace(
|
|
id=call_id,
|
|
type="function",
|
|
function=SimpleNamespace(name=name, arguments=arguments),
|
|
)
|
|
|
|
|
|
def _mock_response(content="Hello", finish_reason="stop", tool_calls=None):
|
|
msg = SimpleNamespace(content=content, tool_calls=tool_calls)
|
|
choice = SimpleNamespace(message=msg, finish_reason=finish_reason)
|
|
return SimpleNamespace(choices=[choice], model="test/model", usage=None)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Contract 1: run_conversation persists the assistant tool-call block BEFORE
|
|
# tool execution begins.
|
|
# ---------------------------------------------------------------------------
|
|
def test_run_conversation_flushes_assistant_tool_call_before_execution():
|
|
agent = _make_agent()
|
|
tool_call = _mock_tool_call(call_id="c1")
|
|
agent.client.chat.completions.create.side_effect = [
|
|
_mock_response(content="", finish_reason="tool_calls", tool_calls=[tool_call]),
|
|
_mock_response(content="done", finish_reason="stop"),
|
|
]
|
|
|
|
# Record a deep snapshot of the message list at every flush so the
|
|
# assertion does not depend on later mutations.
|
|
flush_snapshots: list[list] = []
|
|
|
|
def _record_flush(messages, conversation_history=None):
|
|
flush_snapshots.append(copy.deepcopy(messages))
|
|
|
|
agent._flush_messages_to_session_db = MagicMock(side_effect=_record_flush)
|
|
|
|
# Capture observations at execute time into module-level lists rather than
|
|
# asserting inside _execute_tool_calls — run_conversation's outer loop
|
|
# swallows exceptions, so an in-callback assertion would never surface.
|
|
executed = {"count": 0}
|
|
snapshot_at_execute: list = []
|
|
|
|
def _fake_execute(assistant_message, messages, effective_task_id, api_call_count=0):
|
|
executed["count"] += 1
|
|
# Record the DB state observed at the moment tool execution begins.
|
|
snapshot_at_execute.append(
|
|
copy.deepcopy(flush_snapshots[-1]) if flush_snapshots else None
|
|
)
|
|
# Simulate the tool producing a result (as the real path would).
|
|
messages.append(make_tool_result_message("web_search", "search result", "c1"))
|
|
|
|
with (
|
|
patch.object(agent, "_persist_session"),
|
|
patch.object(agent, "_save_trajectory"),
|
|
patch.object(agent, "_cleanup_task_resources"),
|
|
patch.object(agent, "_execute_tool_calls", side_effect=_fake_execute),
|
|
):
|
|
result = agent.run_conversation("search something")
|
|
|
|
assert executed["count"] == 1, "_execute_tool_calls was never reached"
|
|
# The assistant tool-call block MUST have been flushed before execution.
|
|
last = snapshot_at_execute[0]
|
|
assert last is not None, "no flush occurred before tool execution"
|
|
assert last[-1]["role"] == "assistant"
|
|
assert last[-1]["tool_calls"][0]["id"] == "c1"
|
|
assert result["final_response"] == "done"
|
|
|
|
|
|
def test_interim_assistant_is_durable_before_ui_projection_on_abnormal_exit(tmp_path):
|
|
"""A visible interim assistant row must survive an immediate process exit.
|
|
|
|
``GeneratorExit`` models an uncatchable turn interruption at the UI bridge:
|
|
no turn finalizer or graceful shutdown persistence is allowed to rescue the
|
|
row after the callback observes it.
|
|
"""
|
|
agent = _make_agent()
|
|
db_path = tmp_path / "state.db"
|
|
session_id = "interim-abnormal-exit"
|
|
db = _attach_real_session_db(agent, db_path, session_id)
|
|
tool_call = _mock_tool_call(call_id="visible-call")
|
|
agent.client.chat.completions.create.return_value = _mock_response(
|
|
content="I'll inspect the repository now.",
|
|
finish_reason="tool_calls",
|
|
tool_calls=[tool_call],
|
|
)
|
|
|
|
roles_seen_by_ui: list[str] = []
|
|
|
|
def _ui_projection(_text, *, already_streamed=False):
|
|
roles_seen_by_ui.extend(_durable_roles(db_path, session_id))
|
|
raise GeneratorExit("simulated process termination after UI projection")
|
|
|
|
agent.interim_assistant_callback = _ui_projection
|
|
try:
|
|
with pytest.raises(GeneratorExit, match="simulated process termination"):
|
|
agent.run_conversation("inspect the repository")
|
|
finally:
|
|
db.close()
|
|
|
|
assert roles_seen_by_ui == ["user", "assistant"]
|
|
durable = _durable_messages(db_path, session_id)
|
|
assert [message["role"] for message in durable] == ["user", "assistant"]
|
|
assert durable[1]["content"] == "I'll inspect the repository now."
|
|
assert durable[1]["tool_calls"][0]["id"] == "visible-call"
|
|
|
|
# Cold-resume reconciliation closes the interrupted call in the provider
|
|
# payload without mutating or duplicating the canonical transcript.
|
|
resumed = sanitize_api_messages(durable)
|
|
assert [message["role"] for message in resumed] == [
|
|
"user",
|
|
"assistant",
|
|
"tool",
|
|
]
|
|
assert resumed[2]["tool_call_id"] == "visible-call"
|
|
assert len(_durable_messages(db_path, session_id)) == 2
|
|
|
|
|
|
def test_failed_assistant_persist_blocks_ui_projection_and_tool_side_effects():
|
|
agent = _make_agent()
|
|
tool_call = _mock_tool_call(call_id="must-not-run")
|
|
agent.client.chat.completions.create.return_value = _mock_response(
|
|
content="I'll inspect the repository now.",
|
|
finish_reason="tool_calls",
|
|
tool_calls=[tool_call],
|
|
)
|
|
agent._flush_messages_to_session_db = MagicMock(return_value=False)
|
|
agent.interim_assistant_callback = MagicMock()
|
|
agent._execute_tool_calls = MagicMock()
|
|
|
|
result = agent.run_conversation("inspect the repository")
|
|
|
|
agent.interim_assistant_callback.assert_not_called()
|
|
agent._execute_tool_calls.assert_not_called()
|
|
assert agent.client is not None
|
|
assert agent.client.chat.completions.create.call_count == 1
|
|
assert result["failed"] is True
|
|
assert result["completed"] is False
|
|
assert result["turn_exit_reason"] == "session_persistence_failed"
|
|
# No exception was visible (flush returned False), so the cause is
|
|
# unknown — but the machine-readable contract fields must still be set.
|
|
assert result["failure_reason"] == "session_persistence_failed:unknown"
|
|
assert isinstance(result.get("error"), str) and result["error"].strip() != ""
|
|
|
|
|
|
def test_locked_flush_exception_surfaces_locked_cause_in_result_contract():
|
|
"""SQLite write-lock contention must surface as a 'locked' cause.
|
|
|
|
Gateway contract: result['failure_reason'] is exactly
|
|
'session_persistence_failed:locked' and result['error'] is a non-empty
|
|
string whose wording talks about busy storage, NOT disk space.
|
|
"""
|
|
import sqlite3
|
|
|
|
agent = _make_agent()
|
|
tool_call = _mock_tool_call(call_id="must-not-run")
|
|
agent.client.chat.completions.create.return_value = _mock_response(
|
|
content="I'll inspect the repository now.",
|
|
finish_reason="tool_calls",
|
|
tool_calls=[tool_call],
|
|
)
|
|
agent._flush_messages_to_session_db = MagicMock(
|
|
side_effect=sqlite3.OperationalError("database is locked")
|
|
)
|
|
agent.interim_assistant_callback = MagicMock()
|
|
agent._execute_tool_calls = MagicMock()
|
|
|
|
with (
|
|
patch.object(agent, "_persist_session"),
|
|
patch.object(agent, "_save_trajectory"),
|
|
patch.object(agent, "_cleanup_task_resources"),
|
|
):
|
|
result = agent.run_conversation("inspect the repository")
|
|
|
|
agent.interim_assistant_callback.assert_not_called()
|
|
agent._execute_tool_calls.assert_not_called()
|
|
assert result["failed"] is True
|
|
assert result["turn_exit_reason"] == "session_persistence_failed"
|
|
assert result["failure_reason"] == "session_persistence_failed:locked"
|
|
assert isinstance(result.get("error"), str) and result["error"].strip() != ""
|
|
assert "busy" in result["error"].lower()
|
|
assert "disk" not in result["error"].lower()
|
|
|
|
|
|
def test_persistence_cause_resets_between_turns():
|
|
"""A locked failure on turn 1 must not leak its cause into turn 2."""
|
|
import sqlite3
|
|
|
|
agent = _make_agent()
|
|
tool_call = _mock_tool_call(call_id="must-not-run")
|
|
agent.client.chat.completions.create.return_value = _mock_response(
|
|
content="I'll inspect the repository now.",
|
|
finish_reason="tool_calls",
|
|
tool_calls=[tool_call],
|
|
)
|
|
agent._flush_messages_to_session_db = MagicMock(
|
|
side_effect=sqlite3.OperationalError("database is locked")
|
|
)
|
|
agent._execute_tool_calls = MagicMock()
|
|
|
|
with (
|
|
patch.object(agent, "_persist_session"),
|
|
patch.object(agent, "_save_trajectory"),
|
|
patch.object(agent, "_cleanup_task_resources"),
|
|
):
|
|
first = agent.run_conversation("inspect the repository")
|
|
assert first["failure_reason"] == "session_persistence_failed:locked"
|
|
|
|
# Storage recovered but the flush function now reports a bare False
|
|
# (no exception): the stale 'locked' cause must not be reused.
|
|
agent.client.chat.completions.create.side_effect = None
|
|
agent.client.chat.completions.create.return_value = _mock_response(
|
|
content="I'll inspect the repository now.",
|
|
finish_reason="tool_calls",
|
|
tool_calls=[_mock_tool_call(call_id="must-not-run-2")],
|
|
)
|
|
agent._flush_messages_to_session_db = MagicMock(return_value=False)
|
|
second = agent.run_conversation("inspect the repository again")
|
|
|
|
assert second["turn_exit_reason"] == "session_persistence_failed"
|
|
assert second["failure_reason"] == "session_persistence_failed:unknown"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Contract 2: the SEQUENTIAL path flushes each tool result immediately, BEFORE
|
|
# the next tool dispatches. Dispatch goes through run_agent.handle_function_call
|
|
# (the real production surface), which we mock for determinism.
|
|
# ---------------------------------------------------------------------------
|
|
def test_execute_tool_calls_sequential_flushes_each_tool_result_before_next_dispatch():
|
|
agent = _make_agent()
|
|
tool_calls = [
|
|
_mock_tool_call(name="web_search", call_id="c1"),
|
|
_mock_tool_call(name="web_search", call_id="c2"),
|
|
]
|
|
messages: list = []
|
|
assistant_message = SimpleNamespace(content="", tool_calls=tool_calls)
|
|
|
|
# Ordered event log interleaving real dispatches and DB flushes.
|
|
events: list = []
|
|
|
|
def _fake_dispatch(function_name, function_args, effective_task_id, **kwargs):
|
|
# The result for call N must have been flushed before call N+1 fires.
|
|
events.append(("dispatch", kwargs.get("tool_call_id")))
|
|
return f"result-{kwargs.get('tool_call_id')}"
|
|
|
|
def _record_flush(flush_messages, conversation_history=None):
|
|
# Snapshot the tail tool result that triggered this flush.
|
|
tail = flush_messages[-1]
|
|
events.append(("flush", tail.get("role"), tail.get("tool_call_id")))
|
|
|
|
agent._flush_messages_to_session_db = MagicMock(side_effect=_record_flush)
|
|
|
|
with (
|
|
patch("run_agent.handle_function_call", side_effect=_fake_dispatch) as disp,
|
|
patch(
|
|
"agent.tool_executor.maybe_persist_tool_result",
|
|
side_effect=lambda **kwargs: kwargs["content"],
|
|
),
|
|
):
|
|
agent._execute_tool_calls_sequential(assistant_message, messages, "task-1")
|
|
|
|
# The mock proves we exercised the REAL sequential dispatch surface.
|
|
assert disp.call_count == 2, "sequential path did not dispatch via handle_function_call"
|
|
|
|
# Both tool results landed, in order.
|
|
assert [m["role"] for m in messages] == ["tool", "tool"]
|
|
assert [m["tool_call_id"] for m in messages] == ["c1", "c2"]
|
|
|
|
# Ordering contract: each tool result is flushed AFTER its own dispatch
|
|
# and BEFORE the next dispatch. Expected interleaving:
|
|
# dispatch c1 -> flush c1 -> dispatch c2 -> flush c2
|
|
assert events == [
|
|
("dispatch", "c1"),
|
|
("flush", "tool", "c1"),
|
|
("dispatch", "c2"),
|
|
("flush", "tool", "c2"),
|
|
]
|
|
|
|
|
|
def test_sequential_keyboard_interrupt_emits_results_for_all_calls():
|
|
"""A KeyboardInterrupt mid-batch must not leave dangling tool_calls.
|
|
|
|
When a tool handler raises KeyboardInterrupt, the sequential executor
|
|
re-raises to abort the turn — but it must first append a tool result for
|
|
the interrupted call AND every remaining call, or the assistant tool-call
|
|
turn is left without matching tool results (a message-role alternation
|
|
violation that malforms the next provider request). Mirrors the
|
|
cooperative-interrupt and concurrent paths, which already do this.
|
|
"""
|
|
agent = _make_agent()
|
|
tool_calls = [
|
|
_mock_tool_call(name="web_search", call_id="c1"),
|
|
_mock_tool_call(name="web_search", call_id="c2"),
|
|
_mock_tool_call(name="web_search", call_id="c3"),
|
|
]
|
|
messages: list = []
|
|
assistant_message = SimpleNamespace(content="", tool_calls=tool_calls)
|
|
|
|
def _interrupt_dispatch(function_name, function_args, effective_task_id, **kwargs):
|
|
# First tool raises a hard interrupt mid-batch.
|
|
raise KeyboardInterrupt()
|
|
|
|
agent._flush_messages_to_session_db = MagicMock()
|
|
|
|
with (
|
|
patch("run_agent.handle_function_call", side_effect=_interrupt_dispatch),
|
|
patch(
|
|
"agent.tool_executor.maybe_persist_tool_result",
|
|
side_effect=lambda **kwargs: kwargs["content"],
|
|
),
|
|
pytest.raises(KeyboardInterrupt),
|
|
):
|
|
agent._execute_tool_calls_sequential(assistant_message, messages, "task-1")
|
|
|
|
# Every call_id has a matching tool result — alternation preserved.
|
|
tool_results = [m for m in messages if m.get("role") == "tool"]
|
|
assert [m["tool_call_id"] for m in tool_results] == ["c1", "c2", "c3"]
|
|
# The results are marked as cancelled, not fabricated successes.
|
|
assert all("cancelled" in m["content"].lower() for m in tool_results)
|
|
|
|
|
|
@pytest.mark.parametrize("executor_mode", ["sequential", "concurrent"])
|
|
def test_tool_result_is_durable_before_ui_completion_on_abnormal_exit(
|
|
tmp_path,
|
|
executor_mode,
|
|
):
|
|
"""A visible tool completion must already exist in the canonical DB."""
|
|
agent = _make_agent()
|
|
db_path = tmp_path / "state.db"
|
|
session_id = f"tool-result-abnormal-exit-{executor_mode}"
|
|
db = _attach_real_session_db(agent, db_path, session_id)
|
|
tool_call = _mock_tool_call(call_id="visible-call")
|
|
messages = [
|
|
{"role": "user", "content": "inspect the repository"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "I'll inspect the repository now.",
|
|
"tool_calls": [
|
|
{
|
|
"id": "visible-call",
|
|
"type": "function",
|
|
"function": {"name": "web_search", "arguments": "{}"},
|
|
}
|
|
],
|
|
},
|
|
]
|
|
agent._flush_messages_to_session_db(messages)
|
|
|
|
roles_seen_by_ui: list[str] = []
|
|
|
|
def _ui_completion(*_args):
|
|
roles_seen_by_ui.extend(_durable_roles(db_path, session_id))
|
|
raise GeneratorExit("simulated process termination after tool completion")
|
|
|
|
agent.tool_complete_callback = _ui_completion
|
|
assistant_message = SimpleNamespace(content="", tool_calls=[tool_call])
|
|
dispatch_patch = (
|
|
patch("run_agent.handle_function_call", return_value="repository result")
|
|
if executor_mode == "sequential"
|
|
else patch.object(agent, "_invoke_tool", return_value="repository result")
|
|
)
|
|
try:
|
|
with (
|
|
dispatch_patch,
|
|
patch(
|
|
"agent.tool_executor.maybe_persist_tool_result",
|
|
side_effect=lambda **kwargs: kwargs["content"],
|
|
),
|
|
pytest.raises(GeneratorExit, match="simulated process termination"),
|
|
):
|
|
if executor_mode == "sequential":
|
|
agent._execute_tool_calls_sequential(
|
|
assistant_message,
|
|
messages,
|
|
"task-1",
|
|
)
|
|
else:
|
|
agent._execute_tool_calls_concurrent(
|
|
assistant_message,
|
|
messages,
|
|
"task-1",
|
|
)
|
|
finally:
|
|
db.close()
|
|
|
|
expected_roles = ["user", "assistant", "tool"]
|
|
assert roles_seen_by_ui == expected_roles
|
|
durable = _durable_messages(db_path, session_id)
|
|
assert [message["role"] for message in durable] == expected_roles
|
|
assert durable[2]["tool_call_id"] == "visible-call"
|
|
assert durable[2]["content"] == "repository result"
|
|
|
|
|
|
@pytest.mark.parametrize("executor_mode", ["sequential", "concurrent"])
|
|
def test_failed_tool_result_persist_blocks_completion_projection(executor_mode):
|
|
agent = _make_agent()
|
|
tool_call = _mock_tool_call(call_id="failed-persist")
|
|
assistant_message = SimpleNamespace(content="", tool_calls=[tool_call])
|
|
messages: list = []
|
|
agent._flush_messages_to_session_db = MagicMock(return_value=False)
|
|
agent.tool_complete_callback = MagicMock()
|
|
dispatch_patch = (
|
|
patch("run_agent.handle_function_call", return_value="repository result")
|
|
if executor_mode == "sequential"
|
|
else patch.object(agent, "_invoke_tool", return_value="repository result")
|
|
)
|
|
|
|
with (
|
|
dispatch_patch,
|
|
patch(
|
|
"agent.tool_executor.maybe_persist_tool_result",
|
|
side_effect=lambda **kwargs: kwargs["content"],
|
|
),
|
|
):
|
|
if executor_mode == "sequential":
|
|
agent._execute_tool_calls_sequential(
|
|
assistant_message,
|
|
messages,
|
|
"task-1",
|
|
)
|
|
else:
|
|
agent._execute_tool_calls_concurrent(
|
|
assistant_message,
|
|
messages,
|
|
"task-1",
|
|
)
|
|
|
|
agent.tool_complete_callback.assert_not_called()
|
|
assert getattr(agent, "_incremental_persistence_failed", False) is True
|
|
|
|
|
|
def test_segmented_batch_stops_before_later_segment_after_persist_failure():
|
|
agent = _make_agent()
|
|
first = _mock_tool_call(call_id="first")
|
|
second = _mock_tool_call(call_id="second")
|
|
assistant_message = SimpleNamespace(tool_calls=[first, second])
|
|
messages: list = []
|
|
agent._flush_messages_to_session_db = MagicMock(return_value=False)
|
|
|
|
with (
|
|
patch.object(agent, "_invoke_tool", return_value="first result") as invoke,
|
|
patch("run_agent.handle_function_call", return_value="second result") as dispatch,
|
|
patch(
|
|
"agent.tool_executor.maybe_persist_tool_result",
|
|
side_effect=lambda **kwargs: kwargs["content"],
|
|
),
|
|
):
|
|
execute_tool_calls_segmented(
|
|
agent,
|
|
assistant_message,
|
|
messages,
|
|
"task-1",
|
|
segments=[("parallel", [first]), ("sequential", [second])],
|
|
)
|
|
|
|
invoke.assert_called_once()
|
|
dispatch.assert_not_called()
|
|
assert getattr(agent, "_incremental_persistence_failed", False) is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Contract 3: the CONCURRENT path flushes each collected tool result in append
|
|
# order. Dispatch goes through agent._invoke_tool (the real concurrent
|
|
# surface), which we mock for determinism.
|
|
# ---------------------------------------------------------------------------
|
|
def test_execute_tool_calls_concurrent_flushes_each_tool_result_in_order():
|
|
agent = _make_agent()
|
|
tool_calls = [
|
|
_mock_tool_call(name="web_search", call_id="c1"),
|
|
_mock_tool_call(name="web_search", call_id="c2"),
|
|
]
|
|
messages: list = []
|
|
assistant_message = SimpleNamespace(content="", tool_calls=tool_calls)
|
|
|
|
invoked_ids: list = []
|
|
|
|
def _fake_invoke(function_name, function_args, effective_task_id, tool_call_id, **kwargs):
|
|
invoked_ids.append(tool_call_id)
|
|
return f"result-{tool_call_id}"
|
|
|
|
# Each flush must observe exactly one more tool result than the previous
|
|
# flush, in append order — i.e. the tail tool_call_id sequence is c1, c2.
|
|
flushed_tool_ids: list = []
|
|
flush_lengths: list = []
|
|
|
|
def _record_flush(flush_messages, conversation_history=None):
|
|
flushed_tool_ids.append(flush_messages[-1]["tool_call_id"])
|
|
flush_lengths.append(len([m for m in flush_messages if m.get("role") == "tool"]))
|
|
|
|
agent._flush_messages_to_session_db = MagicMock(side_effect=_record_flush)
|
|
|
|
with (
|
|
patch.object(agent, "_invoke_tool", side_effect=_fake_invoke) as inv,
|
|
patch(
|
|
"agent.tool_executor.maybe_persist_tool_result",
|
|
side_effect=lambda **kwargs: kwargs["content"],
|
|
),
|
|
):
|
|
agent._execute_tool_calls_concurrent(assistant_message, messages, "task-1")
|
|
|
|
# Proves the real concurrent dispatch surface was exercised.
|
|
assert inv.call_count == 2, "concurrent path did not dispatch via _invoke_tool"
|
|
assert sorted(invoked_ids) == ["c1", "c2"]
|
|
|
|
# Results appended in deterministic order.
|
|
assert [m["tool_call_id"] for m in messages] == ["c1", "c2"]
|
|
|
|
# Each tool result was flushed exactly once, in append order, with the
|
|
# running tool count growing by one each time (1 then 2). Removing either
|
|
# production flush call breaks one of these assertions.
|
|
assert flushed_tool_ids == ["c1", "c2"]
|
|
assert flush_lengths == [1, 2]
|
|
|
|
|
|
def test_empty_final_response_updates_already_flushed_blank_assistant_row(tmp_path):
|
|
"""#95514: popping _db_persisted must UPDATE the flushed row, not INSERT.
|
|
|
|
Incremental persist already wrote assistant(content=''). finalize_turn
|
|
recovers the stream buffer onto that live dict. Production flush is
|
|
append-only, so a re-insert would leave the empty row and add a second
|
|
assistant (assistant→assistant on reload).
|
|
"""
|
|
from agent.turn_finalizer import finalize_turn
|
|
|
|
agent = _make_agent()
|
|
db_path = tmp_path / "state.db"
|
|
session_id = "sess-empty-final-flush"
|
|
_attach_real_session_db(agent, db_path, session_id)
|
|
agent._current_streamed_assistant_text = "Already streamed to the user."
|
|
|
|
messages = [
|
|
{"role": "user", "content": "summarize"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{"id": "t1", "type": "function",
|
|
"function": {"name": "terminal", "arguments": "{}"}}
|
|
],
|
|
},
|
|
{"role": "tool", "tool_call_id": "t1", "name": "terminal", "content": "ok"},
|
|
{"role": "assistant", "content": ""},
|
|
]
|
|
agent._flush_messages_to_session_db(messages)
|
|
pre = _durable_messages(db_path, session_id)
|
|
assert pre[-1]["role"] == "assistant"
|
|
assert (pre[-1].get("content") or "") == ""
|
|
|
|
finalize_turn(
|
|
agent,
|
|
final_response="",
|
|
api_call_count=2,
|
|
interrupted=False,
|
|
failed=False,
|
|
messages=messages,
|
|
conversation_history=[],
|
|
effective_task_id="task",
|
|
turn_id="turn",
|
|
user_message="summarize",
|
|
original_user_message="summarize",
|
|
_should_review_memory=False,
|
|
_turn_exit_reason="text_response(final)",
|
|
)
|
|
|
|
reloaded = _durable_messages(db_path, session_id)
|
|
assistants = [m for m in reloaded if m.get("role") == "assistant"]
|
|
assert assistants[-1]["content"] == "Already streamed to the user."
|
|
assert len(assistants) == 2
|
|
assert assistants[0].get("tool_calls")
|
|
assert sum(
|
|
1 for m in assistants
|
|
if not (m.get("content") or "").strip() and not m.get("tool_calls")
|
|
) == 0
|
|
|
|
|
|
def test_flush_stale_row_id_from_other_session_still_inserts(tmp_path):
|
|
"""Copied dicts that keep a parent _row_id must still INSERT in a child session."""
|
|
parent = _make_agent()
|
|
child = _make_agent()
|
|
db_path = tmp_path / "state.db"
|
|
_attach_real_session_db(parent, db_path, "sess-parent")
|
|
_attach_real_session_db(child, db_path, "sess-child")
|
|
parent_msgs = [
|
|
{"role": "user", "content": "p-user"},
|
|
{"role": "assistant", "content": "p-answer"},
|
|
]
|
|
parent._flush_messages_to_session_db(parent_msgs)
|
|
copies = [{k: v for k, v in m.items() if k != "_db_persisted"} for m in parent_msgs]
|
|
copies.extend(
|
|
[
|
|
{"role": "user", "content": "c-user"},
|
|
{"role": "assistant", "content": "c-answer"},
|
|
]
|
|
)
|
|
child._flush_messages_to_session_db(copies)
|
|
reloaded = _durable_messages(db_path, "sess-child")
|
|
assert [m.get("content") for m in reloaded] == [
|
|
"p-user",
|
|
"p-answer",
|
|
"c-user",
|
|
"c-answer",
|
|
]
|
|
|
|
|
|
def test_flush_stale_row_id_from_other_session_does_not_fill_child_blank(tmp_path):
|
|
"""A parent `_row_id` must INSERT in the child, not steal a blank tail."""
|
|
parent = _make_agent()
|
|
child = _make_agent()
|
|
db_path = tmp_path / "state.db"
|
|
_attach_real_session_db(parent, db_path, "sess-parent")
|
|
_attach_real_session_db(child, db_path, "sess-child")
|
|
parent_msgs = [
|
|
{"role": "user", "content": "p-user"},
|
|
{"role": "assistant", "content": "p-answer"},
|
|
]
|
|
parent._flush_messages_to_session_db(parent_msgs)
|
|
child_msgs = [
|
|
{"role": "user", "content": "c-keep"},
|
|
{"role": "assistant", "content": ""},
|
|
]
|
|
child._flush_messages_to_session_db(child_msgs)
|
|
copies = [{k: v for k, v in m.items() if k != "_db_persisted"} for m in parent_msgs]
|
|
copies.extend(
|
|
[
|
|
{"role": "user", "content": "c-user"},
|
|
{"role": "assistant", "content": "c-answer"},
|
|
]
|
|
)
|
|
child._flush_messages_to_session_db(copies)
|
|
reloaded = _durable_messages(db_path, "sess-child")
|
|
assert [m.get("content") for m in reloaded] == [
|
|
"c-keep",
|
|
"",
|
|
"p-user",
|
|
"p-answer",
|
|
"c-user",
|
|
"c-answer",
|
|
]
|
|
|
|
|
|
def test_flush_archived_same_session_row_id_fills_active_clone(tmp_path):
|
|
"""Watermark compaction clones the tail; stale `_row_id` must not win.
|
|
|
|
``archive_and_compact(..., watermark=...)`` archives the original
|
|
concurrent-tail row and inserts a fresh active clone with a new id.
|
|
The live dict still holds the archived id. A rewrite keyed only on
|
|
``id + session_id`` updates the inactive row, stamps persistence, and
|
|
skips INSERT — reload then still sees the blank clone (#95514 P0).
|
|
"""
|
|
from agent.turn_finalizer import finalize_turn
|
|
|
|
agent = _make_agent()
|
|
db_path = tmp_path / "state.db"
|
|
session_id = "sess-archived-row-id"
|
|
db = _attach_real_session_db(agent, db_path, session_id)
|
|
recovered = "Already streamed to the user."
|
|
agent._current_streamed_assistant_text = recovered
|
|
messages = [
|
|
{"role": "user", "content": "summarize"},
|
|
{"role": "assistant", "content": ""},
|
|
]
|
|
agent._flush_messages_to_session_db(messages)
|
|
stale_id = messages[-1]["_row_id"]
|
|
user_id = messages[0]["_row_id"]
|
|
assert isinstance(stale_id, int)
|
|
assert stale_id > user_id
|
|
|
|
db.archive_and_compact(
|
|
session_id,
|
|
compacted_messages=[{"role": "user", "content": "prior turns summarized"}],
|
|
watermark=user_id,
|
|
)
|
|
|
|
finalize_turn(
|
|
agent,
|
|
final_response="",
|
|
api_call_count=1,
|
|
interrupted=False,
|
|
failed=False,
|
|
messages=messages,
|
|
conversation_history=[],
|
|
effective_task_id="task",
|
|
turn_id="turn",
|
|
user_message="summarize",
|
|
original_user_message="summarize",
|
|
_should_review_memory=False,
|
|
_turn_exit_reason="text_response(final)",
|
|
)
|
|
|
|
reloaded = _durable_messages(db_path, session_id)
|
|
active_assistants = [m for m in reloaded if m.get("role") == "assistant"]
|
|
assert active_assistants, "compaction clone should keep an active assistant"
|
|
assert any(
|
|
(m.get("content") or "").strip() == recovered for m in active_assistants
|
|
)
|
|
assert not any(
|
|
not (m.get("content") or "").strip() for m in active_assistants
|
|
)
|
|
inactive = db.get_messages(session_id, include_inactive=True)
|
|
archived = next(m for m in inactive if m.get("id") == stale_id)
|
|
assert archived.get("active") in (0, False)
|
|
assert messages[-1].get("_row_id") != stale_id
|
|
|
|
|
|
def test_flush_atomic_mixed_repair_and_append_rollback_on_failure(tmp_path, monkeypatch):
|
|
"""Mixed rewrite + append must be atomic: failure rolls back repair and stamps no markers.
|
|
|
|
If a turn contains both a blank assistant repair and a subsequent new message,
|
|
failure during the batch insert must roll back the assistant update and stamp
|
|
no in-memory markers.
|
|
"""
|
|
from hermes_state import SessionDB
|
|
|
|
agent = _make_agent()
|
|
db_path = tmp_path / "state.db"
|
|
session_id = "sess-atomic-rollback"
|
|
db = _attach_real_session_db(agent, db_path, session_id)
|
|
|
|
messages = [
|
|
{"role": "user", "content": "summarize"},
|
|
{"role": "assistant", "content": ""},
|
|
]
|
|
agent._flush_messages_to_session_db(messages)
|
|
orig_row_id = messages[-1]["_row_id"]
|
|
assert isinstance(orig_row_id, int)
|
|
|
|
# Now modify assistant message in memory and add a new message to the batch
|
|
messages[-1]["content"] = "Recovered stream answer"
|
|
messages[-1].pop("_db_persisted", None)
|
|
new_tool_msg = {"role": "tool", "tool_call_id": "t1", "name": "terminal", "content": "ok"}
|
|
messages.append(new_tool_msg)
|
|
|
|
# Force a failure inside the batch write transaction on the second message
|
|
orig_insert = SessionDB._insert_message_rows
|
|
|
|
def _failing_insert(self_db, conn, sid, msgs):
|
|
for msg in msgs:
|
|
if msg.get("role") == "tool":
|
|
raise RuntimeError("Simulated mid-batch persistence crash")
|
|
return orig_insert(self_db, conn, sid, msgs)
|
|
|
|
monkeypatch.setattr(SessionDB, "_insert_message_rows", _failing_insert)
|
|
|
|
success = agent._flush_messages_to_session_db(messages)
|
|
assert success is False or getattr(agent, "_incremental_persistence_failed", False) is True
|
|
|
|
# Check that in-memory markers were NOT stamped
|
|
assert messages[-1].get("_db_persisted") is not True
|
|
assert new_tool_msg.get("_db_persisted") is not True
|
|
|
|
# Check that the durable SQLite row was NOT updated (rolled back)
|
|
durable = _durable_messages(db_path, session_id)
|
|
assert len(durable) == 2
|
|
assert durable[-1]["role"] == "assistant"
|
|
assert (durable[-1].get("content") or "") == ""
|
|
|
|
|
|
def test_flush_concurrent_nonblank_winner_adopts_canonical_content(tmp_path):
|
|
"""A concurrent non-blank winner must be adopted without overwrite and synced to live dict."""
|
|
agent = _make_agent()
|
|
db_path = tmp_path / "state.db"
|
|
session_id = "sess-concurrent-winner"
|
|
db = _attach_real_session_db(agent, db_path, session_id)
|
|
|
|
messages = [
|
|
{"role": "user", "content": "hello"},
|
|
{"role": "assistant", "content": ""},
|
|
]
|
|
agent._flush_messages_to_session_db(messages)
|
|
row_id = messages[-1]["_row_id"]
|
|
assert isinstance(row_id, int)
|
|
|
|
# Concurrent writer updates the row in SQLite to non-blank canonical content
|
|
def _concurrent_write(conn):
|
|
conn.execute(
|
|
"UPDATE messages SET content = ? WHERE id = ?",
|
|
(db._encode_content("Canonical winner answer from sibling"), row_id),
|
|
)
|
|
|
|
db._execute_write(_concurrent_write)
|
|
|
|
# Live agent attempts to flush divergent content
|
|
messages[-1]["content"] = "Divergent live agent answer"
|
|
messages[-1].pop("_db_persisted", None)
|
|
|
|
success = agent._flush_messages_to_session_db(messages)
|
|
assert success is True
|
|
|
|
# SQLite must still have canonical winner content
|
|
durable = _durable_messages(db_path, session_id)
|
|
assert durable[-1]["content"] == "Canonical winner answer from sibling"
|
|
|
|
# Live dict must have adopted the canonical content and be marked persisted
|
|
assert messages[-1]["content"] == "Canonical winner answer from sibling"
|
|
assert messages[-1]["_db_persisted"] is True
|
|
assert messages[-1]["_row_id"] == row_id
|
|
|