"""Tool-call execution — sequential and concurrent dispatch. Both AIAgent methods (``_execute_tool_calls_sequential`` and ``_execute_tool_calls_concurrent``) live here as module-level functions that take the parent ``AIAgent`` as their first argument. ``run_agent`` keeps thin wrappers so existing call sites work; tests that patch ``run_agent._set_interrupt`` are honored because the extracted functions reach back through the ``run_agent`` module via ``_ra()`` for that symbol. """ from __future__ import annotations import concurrent.futures import json from pathlib import Path import logging import os import random import threading import time from dataclasses import dataclass from typing import Any, Optional from agent.display import ( KawaiiSpinner, build_tool_preview as _build_tool_preview, build_tool_label as _build_tool_label, get_cute_tool_message as _get_cute_tool_message_impl, get_tool_emoji as _get_tool_emoji, redact_tool_args_for_display as _redact_tool_args_for_display, _detect_tool_failure, ) from agent.message_sanitization import coalesce_tool_call_id from agent.tool_dispatch_helpers import ( _NEVER_PARALLEL_TOOLS, _is_destructive_command, _is_multimodal_tool_result, _multimodal_text_summary, _append_subdir_hint_to_multimodal, _plan_tool_batch_segments, make_tool_result_message, ) from tools.terminal_tool import ( get_active_env, ) from tools.thread_context import propagate_context_to_thread from tools.tool_result_storage import ( maybe_persist_tool_result, enforce_turn_budget, extract_persisted_path, ) from tools.budget_config import BudgetConfig, DEFAULT_BUDGET, budget_for_context_window logger = logging.getLogger(__name__) def _pairing_tool_call_id(tool_call: Any) -> str: """Return the canonical id used by the persisted assistant message.""" return coalesce_tool_call_id(tool_call) def _record_persisted_path_for_stub(agent, tool_call_id: str, function_result) -> None: """Tell the stall guards where a persisted result's full content lives. When a large result is spilled to disk ( preview), a later result-reference stub pointing at that first occurrence must carry the spillover file path so the reference can't dangle. Best-effort: never lets bookkeeping break tool execution. """ try: if not isinstance(function_result, str): return path = extract_persisted_path(function_result) if path: agent._tool_guardrails.record_persisted_result(tool_call_id, path) except Exception as exc: logger.debug("persisted-path record for result stub failed: %s", exc) def _ensure_file_checkpoint( agent, function_name: str, function_args: dict, effective_task_id: str, ) -> None: """Checkpoint the same workspace path that the file tool will mutate.""" file_path = function_args.get("path", "") if not file_path: return # File tools resolve relative paths against the task's live/session cwd, # which can differ from the Hermes process cwd (notably in Docker). Resolve # through that same path pipeline before asking the checkpoint manager to # discover the project root. from tools.file_tools import _resolve_path_for_task resolved_path = _resolve_path_for_task(file_path, effective_task_id or "default") work_dir = agent._checkpoint_mgr.get_working_dir_for_path(str(resolved_path)) agent._checkpoint_mgr.ensure_checkpoint(work_dir, f"before {function_name}") def _budget_for_agent(agent) -> BudgetConfig: """Resolve a tool-result BudgetConfig scaled to the agent's context window. Large-context models keep the historical 100K/200K char defaults; small models (e.g. a 65K-token local model switched into mid-session) get a budget proportional to their window so a single large tool result can't push the request past the model's limit (#23767). Falls back to the default budget when the context length isn't resolvable. """ try: ctx = getattr(getattr(agent, "context_compressor", None), "context_length", None) # budget_for_context_window(None) (rather than DEFAULT_BUDGET) so the # config-driven MCP threshold override still applies when the context # length isn't resolvable. return budget_for_context_window(int(ctx) if ctx else None) except Exception: return DEFAULT_BUDGET # Maximum number of concurrent worker threads for parallel tool execution. # Mirrors the constant in ``run_agent`` for tests/imports that look here. _MAX_TOOL_WORKERS = 8 _DEFAULT_IMAGE_PARALLEL_REQUESTS = 4 # Generous ceiling for slow-but-valid tool work (large page fetches, slow # remote backends) so the batch guard does not preempt a legitimate attempt. _DEFAULT_CONCURRENT_TOOL_TIMEOUT_S = 420.0 # Upper bound a concurrent worker will wait at the start-order gate for all # earlier-ordered tools to advance before proceeding out of order. Long enough # to cover slow-but-legitimate authorization (e.g. an approval round-trip), # short enough that one wedged dispatch cannot starve the batch forever. _START_ORDER_GATE_TIMEOUT_S = 120.0 # Fallback bound a concurrent worker will wait for the authorization gate's # serialization lock before running its prompt unserialized. The effective # bound is derived from ``approvals.timeout`` plus a margin (see # _authorization_gate_lock_timeout): a legitimate holder is at worst a human # answering an approval prompt, which self-terminates at approvals.timeout — # so a holder that overstays it is wedged and must not starve the batch. _AUTHORIZATION_GATE_LOCK_TIMEOUT_S = 360.0 def _authorization_gate_lock_timeout() -> float: """Bound for the authorization serialization lock: approval timeout + margin. Delegates to ``tools.approval.human_wait_ceiling`` — the same bound that clamps a human-wait window's deadline contribution — so the two can't drift. Long enough that serialization is never broken while a legitimate approval prompt is still answerable; short enough that a wedged holder (hanging ``pre_tool_call`` plugin, dead approval client) cannot park other workers forever (#79719). Resolved once per gate (per batch), so a mid-process ``approvals.timeout`` change applies from the next batch. """ try: from tools.approval import human_wait_ceiling # human_wait_ceiling is platform-safety-capped (agent/deadline.py # MAX_SAFE_TIMEOUT_S): a huge approvals.timeout can no longer overflow # Lock.acquire's time_t on macOS (#83220). Deliberately NOT min()'d # with _AUTHORIZATION_GATE_LOCK_TIMEOUT_S — the gate must never give # up while a legitimate approval prompt is still answerable (#79719), # so a configured approvals.timeout above 360s must extend the gate. return human_wait_ceiling() except Exception: return _AUTHORIZATION_GATE_LOCK_TIMEOUT_S class _BatchAbandoned(BaseException): """Raised inside a worker when the batch was abandoned before dispatch. Derives from BaseException so intermediate ``except Exception`` handlers in the middleware chain cannot swallow it and dispatch the tool anyway. """ def _parse_tool_arguments(raw_arguments: Any) -> tuple[dict, Optional[str]]: """Parse model-emitted arguments without repairing or coercing them.""" try: arguments = json.loads(raw_arguments) except (json.JSONDecodeError, TypeError): arguments = None if isinstance(arguments, dict): return arguments, None return {}, json.dumps( { "error": "Invalid tool arguments", "message": ( "Tool arguments must be a valid JSON object; tool was not executed." ), }, ensure_ascii=False, ) def _resolve_concurrent_tool_timeout() -> float | None: """Resolve the per-batch concurrent tool deadline. Delegates to the unified resolver (#85125): ``timeouts.tools.concurrent_batch`` in config.yaml wins, the legacy ``HERMES_CONCURRENT_TOOL_TIMEOUT_S`` env var remains the back-compat bridge, and ``0``/negative still disables the bound. """ from agent.deadline import resolve_timeout return resolve_timeout( "tools.concurrent_batch", default=_DEFAULT_CONCURRENT_TOOL_TIMEOUT_S, env_var="HERMES_CONCURRENT_TOOL_TIMEOUT_S", ) def _flush_session_db_after_tool_progress( agent, messages: list, *, stage: str, ) -> bool: """Flush tool-call progress before projecting it to any UI surface. Tool execution can perform side effects that terminate or restart the current Hermes process before the normal turn-end persistence path runs. Flush the already-appended assistant/tool messages immediately so the transcript survives destructive-but-valid tool calls. """ try: persisted = agent._flush_messages_to_session_db(messages) is not False if not persisted: agent._incremental_persistence_failed = True # The flush caught its own exception and returned False; the # classified cause (if any) was captured at the catch site. Only # fall back to 'unknown' when nothing more specific is recorded. if getattr(agent, "_last_persistence_error_cause", None) is None: agent._last_persistence_error_cause = "unknown" return persisted except Exception as exc: agent._incremental_persistence_failed = True from hermes_state import classify_persistence_error agent._last_persistence_error_cause = classify_persistence_error(exc) logger.warning("Incremental tool-call persistence failed after %s: %s", stage, exc) return False def _image_generate_parallel_limit() -> int: """Return the configured image-generation parallelism cap. Image-generation calls are slow enough that concurrent execution is useful, but backend bursts can hit TTFB or rate-limit failures. Keep the default intentionally conservative while allowing users to tune it per install. """ try: from hermes_cli.config import load_config cfg = load_config() or {} image_gen = cfg.get("image_gen") if isinstance(cfg, dict) else None value = ( image_gen.get("max_parallel_requests") if isinstance(image_gen, dict) else None ) except Exception: value = None try: limit = int(value) except (TypeError, ValueError): limit = _DEFAULT_IMAGE_PARALLEL_REQUESTS return max(1, min(limit, _MAX_TOOL_WORKERS)) def _max_workers_for_tool_batch(runnable_calls) -> int: """Return the worker cap for a concurrent tool batch.""" if not runnable_calls: return 0 max_workers = _MAX_TOOL_WORKERS if any( (call[2] if len(call) >= 3 else None) == "image_generate" for call in runnable_calls ): max_workers = min(max_workers, _image_generate_parallel_limit()) return min(len(runnable_calls), max_workers) def _ra(): """Lazy reference to ``run_agent`` so patches like ``run_agent._set_interrupt`` work.""" import run_agent return run_agent def _is_interpreter_shutdown_submit_error(exc: RuntimeError) -> bool: """Shutdown-race predicate — shared home in ``tools.interpreter_shutdown``. Delegates so all sites (cron delivery, conversation-loop retry, tool submission) recognize both CPython shutdown-message variants instead of each matching its own substring (the bug class behind #55924/#58720). """ from tools.interpreter_shutdown import interpreter_shutting_down return interpreter_shutting_down(exc) def _emit_terminal_post_tool_call( agent, *, function_name: str, function_args: dict, result: Any, effective_task_id: str, tool_call_id: str, duration_ms: int = 0, status: str | None = None, error_type: str | None = None, error_message: str | None = None, middleware_trace: Optional[list[dict[str, Any]]] = None, ) -> None: try: from model_tools import _emit_post_tool_call_hook _emit_post_tool_call_hook( function_name=function_name, function_args=function_args, result=result, task_id=effective_task_id or "", session_id=getattr(agent, "session_id", "") or "", tool_call_id=tool_call_id or "", turn_id=getattr(agent, "_current_turn_id", "") or "", api_request_id=getattr(agent, "_current_api_request_id", "") or "", duration_ms=duration_ms, status=status, error_type=error_type, error_message=error_message, middleware_trace=list(middleware_trace or []), ) except Exception: pass def _cancelled_tool_result(reason: str = "user interrupt") -> str: return json.dumps( { "error": f"Tool execution cancelled by {reason}", "status": "cancelled", }, ensure_ascii=False, ) def _emit_cancelled_terminal_post_tool_call( agent, *, function_name: str, function_args: dict, effective_task_id: str, tool_call_id: str, start_time: float, reason: str = "user interrupt", error_type: str = "keyboard_interrupt", middleware_trace: Optional[list[dict[str, Any]]] = None, ) -> str: result = _cancelled_tool_result(reason) _emit_terminal_post_tool_call( agent, function_name=function_name, function_args=function_args, result=result, effective_task_id=effective_task_id, tool_call_id=tool_call_id, duration_ms=int((time.time() - start_time) * 1000), status="cancelled", error_type=error_type, error_message=f"Tool execution cancelled by {reason}", middleware_trace=list(middleware_trace or []), ) return result def _tool_search_scoped_names(agent) -> frozenset: """Return the deferrable tool names the session may invoke via tool_call. The Tool Search unwrap dispatches the underlying tool directly, bypassing the bridge branch (and its scope check) in ``model_tools.handle_function_call``. To keep a restricted-toolset session (subagent, kanban worker, curated gateway session) from reaching tools it was never granted, the unwrap validates the underlying name against this set: the deferrable subset of the session's own enabled/disabled toolset scope. Result is cached on the agent and refreshed when the tool registry's generation changes (e.g. an MCP server reconnects), so the common case is a dict lookup, not a full tool-defs rebuild on every tool call. """ try: import model_tools from tools import tool_search as _ts from tools.registry import registry as _registry except Exception: return frozenset() enabled = getattr(agent, "enabled_toolsets", None) disabled = getattr(agent, "disabled_toolsets", None) cache_key = ( _registry.current_scope_key(), getattr(_registry, "_generation", 0), frozenset(enabled) if enabled is not None else None, frozenset(disabled) if disabled is not None else None, ) cached = getattr(agent, "_tool_search_scope_cache", None) if cached is not None and cached[0] == cache_key: return cached[1] try: scoped_defs = model_tools.get_tool_definitions( enabled_toolsets=enabled, disabled_toolsets=disabled, quiet_mode=True, skip_tool_search_assembly=True, ) or [] names = _ts.scoped_deferrable_names(scoped_defs) except Exception: names = frozenset() try: agent._tool_search_scope_cache = (cache_key, names) except Exception: pass return names @dataclass class _ManagedToolResult: result: Any args: dict[str, Any] middleware_trace: list[dict[str, Any]] blocked: bool dispatched: bool class _ToolTimeoutResult(str): """Marker for a synthesized sequential-tool timeout result.""" class _ToolCancelledResult(str): """Marker for a synthesized sequential-tool user-interrupt result. Like ``_ToolTimeoutResult``, the executor already emitted the terminal post_tool_call event for this call (status="cancelled"), so downstream emission must be suppressed — an abandoned worker finishing late must not report success for a call the user already cancelled. """ class _ConcurrentToolAuthorizationGate: """Serialize policy prompts and exclude human approval waits from batch deadlines. Serialization keeps concurrent approval prompts from interleaving on the user's screen. The acquire is BOUNDED: a worker wedged inside the gate (a hanging ``pre_tool_call`` plugin, or an approval round-trip to a client that went away) must not park every other worker forever. On expiry the worker runs its prompt unserialized — worst case is interleaved prompts, strictly better than permanent starvation (same tradeoff as the start-order gate, #79705). Deadline exclusion is measured at the SOURCE of the human wait (``tools.approval.human_wait_seconds``: the CLI prompt and the gateway approval poll loop mark their own blocking windows), NOT as residency in this gate. Gate residency is arbitrary code — using it as the exclusion signal let a wedged plugin grow the exclusion 1:1 with wall clock, keeping the batch deadline's ``remaining`` constant so it never fired and the turn hung forever (#79719). A wedged plugin now contributes nothing to the exclusion and the batch times out normally, while a genuine approval wait (which can legitimately exceed any fixed bound) is still excluded in full. """ def __init__( self, *, lock_timeout: float | None = None, session_key: str | None = None, ) -> None: self._serialization_lock = threading.Lock() self._lock_timeout = ( _authorization_gate_lock_timeout() if lock_timeout is None else lock_timeout ) self._session_key = session_key if self._session_key is None: try: from tools.approval import get_current_session_key # Snapshot the batch's session identity on the SUBMITTING # thread: excluded_seconds() is polled from the batch wait # loop, whose context may differ from the workers'. self._session_key = get_current_session_key() except Exception: logger.debug( "authorization gate could not snapshot the session key; " "human-wait exclusion will re-resolve it at poll time", exc_info=True, ) self._baseline_wait_seconds = self._human_wait_seconds() def _human_wait_seconds(self) -> float: try: from tools.approval import human_wait_seconds return human_wait_seconds(self._session_key) except Exception: return 0.0 def run(self, callback): acquired = self._serialization_lock.acquire(timeout=self._lock_timeout) if not acquired: logger.warning( "authorization gate lock not acquired after %.1fs " "(holder wedged in a pre_tool_call plugin or approval " "round-trip?); running prompt unserialized", self._lock_timeout, ) return callback() try: return callback() finally: self._serialization_lock.release() def excluded_seconds(self) -> float: """Return human-approval wait seconds accrued since the batch started.""" return max(0.0, self._human_wait_seconds() - self._baseline_wait_seconds) def _managed_values( outcome: _ManagedToolResult, ) -> tuple[Any, dict[str, Any], list[dict[str, Any]], bool, bool]: return ( outcome.result, outcome.args, outcome.middleware_trace, outcome.blocked, outcome.dispatched, ) # Cadence for the in-flight tool activity heartbeat. Must stay far below the # gateway turn-inactivity timeout (default 1800s) so a silent-but-healthy # tool call never looks idle to the watchdog. _TOOL_ACTIVITY_HEARTBEAT_INTERVAL_S = 30.0 def _run_tool_activity_heartbeat( agent, stop_event: threading.Event, label: str, interval: float = _TOOL_ACTIVITY_HEARTBEAT_INTERVAL_S, ) -> None: """Refresh the agent's activity clock while a tool call is in flight. The gateway's turn-inactivity watchdog (``gateway/run.py::_watch_gateway_turn_inactivity``) abandons a turn once ``seconds_since_activity`` exceeds the inactivity timeout (default 30 min). Activity is stamped when a tool *starts* and when it *completes*, but a tool call that runs silently for 30+ minutes (quiet builds, long pytest suites, large downloads, network waits that emit no output) previously froze the clock at "executing tool: " and the watchdog hard-abandoned a turn that was still making progress, reaping the tool's processes mid-execution. This daemon thread touches ``agent._touch_activity`` every ``interval`` seconds until ``stop_event`` is set (the tool call returned), so the gateway keeps seeing a live turn for the whole duration of the call. A tool that truly hangs is still bounded by the tool layer's own timeouts (terminal ``timeout`` default 180s, the concurrent batch deadline ~420s), so the heartbeat only extends the turn's life for as long as the tool call is legitimately executing — it does not unbind wedged tools. The 30-min gateway backstop remains for turns whose agent loop itself stalls (no API call, no tool call in flight). """ try: while not stop_event.wait(interval): agent._touch_activity(label) except Exception: # A heartbeat must never break the agent loop. pass def _run_agent_tool_execution_middleware( agent, *, function_name: str, function_args: dict, effective_task_id: str, tool_call_id: str, execute, scope_block: str | None = None, display_index: int | None = None, middleware_trace: list[dict[str, Any]] | None = None, begin_execution=None, authorization_gate: _ConcurrentToolAuthorizationGate | None = None, ) -> _ManagedToolResult: """Run Relay rewrites before Hermes policy and dispatch exactly once.""" from agent import relay_tools from hermes_cli.middleware import ( apply_tool_request_middleware, run_tool_execution_middleware, ) trace = middleware_trace if middleware_trace is not None else [] state = { "args": function_args, "middleware_trace": trace, "blocked": False, "dispatched": False, } dispatch_lock = threading.Lock() def _authorized_dispatch(final_args: dict[str, Any]) -> Any: with dispatch_lock: if state["dispatched"]: raise RuntimeError( "Hermes tool execution callback invoked more than once" ) state["dispatched"] = True state["blocked"] = False state["args"] = final_args def _begin() -> None: _begin_tool_execution( agent, function_name=function_name, function_args=final_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, display_index=display_index, ) def _advance_start_order(callback=None) -> None: if begin_execution is None: if callback is not None: callback() return begin_execution(callback) block_message = scope_block block_error_type = "tool_scope_block" if block_message is None: block_error_type = "plugin_block" def _resolve_pre_tool_block(): nonlocal final_args try: from hermes_cli.plugins import _dispatch_pre_tool_call_hooks block_msg, modified_args = _dispatch_pre_tool_call_hooks( function_name, final_args, task_id=effective_task_id or "", session_id=getattr(agent, "session_id", "") or "", tool_call_id=tool_call_id or "", turn_id=getattr(agent, "_current_turn_id", "") or "", api_request_id=getattr(agent, "_current_api_request_id", "") or "", middleware_trace=list(state["middleware_trace"]), ) if modified_args is not None: final_args = modified_args state["args"] = modified_args return block_msg except Exception: return None block_message = ( _resolve_pre_tool_block() if authorization_gate is None else authorization_gate.run(_resolve_pre_tool_block) ) guardrail_decision = None if block_message is None: guardrail_decision = agent._tool_guardrails.before_call( function_name, final_args ) if guardrail_decision.allows_execution: guardrail_decision = None if block_message is not None or guardrail_decision is not None: _advance_start_order() state["blocked"] = True if block_message is not None: result = json.dumps({"error": block_message}, ensure_ascii=False) error_type = block_error_type error_message = block_message else: result = agent._guardrail_block_result(guardrail_decision) error_type = "guardrail_block" error_message = ( getattr(guardrail_decision, "message", None) or "Tool blocked by guardrail policy" ) _emit_terminal_post_tool_call( agent, function_name=function_name, function_args=final_args, result=result, effective_task_id=effective_task_id, tool_call_id=tool_call_id, status="blocked", error_type=error_type, error_message=error_message, middleware_trace=list(state["middleware_trace"]), ) return result if function_name == "memory": agent._turns_since_memory = 0 elif function_name == "skill_manage": agent._iters_since_skill = 0 _advance_start_order(_begin) # Keep the gateway turn-inactivity watchdog from abandoning a turn # whose tool call runs silently for longer than the inactivity # timeout (#84491): stamp activity periodically while the tool is # in flight, not just at start/completion. Both the sequential and # the concurrent paths funnel through here, so a single heartbeat # covers every tool. _hb_stop = threading.Event() _hb_thread = threading.Thread( target=_run_tool_activity_heartbeat, args=(agent, _hb_stop, f"tool running: {function_name}"), kwargs={"interval": _TOOL_ACTIVITY_HEARTBEAT_INTERVAL_S}, daemon=True, name=f"tool-activity-hb-{function_name[:24]}", ) _hb_thread.start() try: return execute(final_args) finally: _hb_stop.set() _hb_thread.join(timeout=2.0) def _hermes_pipeline(relay_args: dict[str, Any]) -> Any: request_result = apply_tool_request_middleware( function_name, relay_args, skip_relay=True, task_id=effective_task_id or "", session_id=getattr(agent, "session_id", "") or "", tool_call_id=tool_call_id or "", turn_id=getattr(agent, "_current_turn_id", "") or "", api_request_id=getattr(agent, "_current_api_request_id", "") or "", ) request_args = ( request_result.payload if isinstance(request_result.payload, dict) else relay_args ) trace.clear() trace.extend(request_result.trace) return run_tool_execution_middleware( function_name, request_args, lambda next_args: _authorized_dispatch( next_args if isinstance(next_args, dict) else request_args ), original_args=function_args, task_id=effective_task_id or "", session_id=getattr(agent, "session_id", "") or "", tool_call_id=tool_call_id or "", turn_id=getattr(agent, "_current_turn_id", "") or "", api_request_id=getattr(agent, "_current_api_request_id", "") or "", ) result, _relay_args = relay_tools.execute( function_name, function_args, _hermes_pipeline, session_id=str(getattr(agent, "session_id", "") or ""), tool_call_id=tool_call_id or None, metadata={ "task_id": effective_task_id or "", "turn_id": getattr(agent, "_current_turn_id", "") or "", "api_request_id": getattr(agent, "_current_api_request_id", "") or "", "tool_call_id": tool_call_id or "", }, ) return _ManagedToolResult( result=result, args=state["args"], middleware_trace=state["middleware_trace"], blocked=bool(state["blocked"]), dispatched=bool(state["dispatched"]), ) # How often the sequential-tool wait loop wakes to check for a user # interrupt while the worker runs. Short enough that /stop or a redirect # lands within ~1s even when the tool itself never polls is_interrupted(). _SEQUENTIAL_INTERRUPT_POLL_SECONDS = 1.0 def _resolve_sequential_tool_timeout() -> float | None: """Deadline for one sequential tool call (#85125 Phase 2a). ``timeouts.tools.sequential_call`` in config.yaml wins; when unset, the sequential path inherits the concurrent batch deadline (same value, same ``HERMES_CONCURRENT_TOOL_TIMEOUT_S`` legacy bridge) so the two executor paths cannot drift apart by default. ``0``/negative disables the bound. NOTE: this path deliberately does NOT use ``agent.deadline.run_bounded_sync``. The sequential/concurrent executors extend their deadline dynamically while a human approval prompt is open (``_ConcurrentToolAuthorizationGate`` excluded seconds — a MUST-preserve invariant) and touch agent activity mid-wait; the shared primitive is fixed-deadline by design. Simpler call sites migrate onto the primitive; these two stay symmetric with each other. """ from agent.deadline import resolve_timeout return resolve_timeout( "tools.sequential_call", default=_resolve_concurrent_tool_timeout(), ) def _run_sequential_tool_execution_middleware( agent, *, function_name: str, function_args: dict, effective_task_id: str, tool_call_id: str, execute, scope_block: str | None = None, display_index: int | None = None, middleware_trace: list[dict[str, Any]] | None = None, ) -> _ManagedToolResult: """Run one sequential call with the concurrent executor's deadline. Interactive input tools such as ``clarify`` wait on a human. Their own timeout (``agent.clarify_timeout``: default 3600s, or unlimited when ``<= 0``) owns that wait. Applying the generic tool deadline here would return ``tool_timeout`` while the prompt and worker stay active. """ timeout_s = _resolve_sequential_tool_timeout() kwargs = { "function_name": function_name, "function_args": function_args, "effective_task_id": effective_task_id, "tool_call_id": tool_call_id, "execute": execute, "scope_block": scope_block, "display_index": display_index, "middleware_trace": middleware_trace, } if function_name in _NEVER_PARALLEL_TOOLS: return _run_agent_tool_execution_middleware(agent, **kwargs) from tools.daemon_pool import DaemonThreadPoolExecutor authorization_gate = _ConcurrentToolAuthorizationGate() worker_tid: list[int] = [] def _run() -> _ManagedToolResult: tid = threading.current_thread().ident worker_tid.append(tid) with agent._tool_worker_threads_lock: agent._tool_worker_threads.add(tid) try: return _run_agent_tool_execution_middleware( agent, authorization_gate=authorization_gate, **kwargs ) finally: with agent._tool_worker_threads_lock: agent._tool_worker_threads.discard(tid) try: _ra()._set_interrupt(False, tid) except Exception: pass executor = DaemonThreadPoolExecutor(max_workers=1) future = executor.submit(propagate_context_to_thread(_run)) # ``timeout_s`` disabled (None) still runs on the worker: the wait loop # below is what makes a non-cooperative tool interruptible at all, so # "no deadline" must not mean "no interrupt checks" (#86xxx class fix — # sequential path previously blocked until the tool returned). deadline = time.monotonic() + timeout_s if timeout_s is not None else None started = time.monotonic() timed_out = False interrupted = False _last_heartbeat = 0 try: while True: wait_slice = _SEQUENTIAL_INTERRUPT_POLL_SECONDS if deadline is not None: remaining = ( deadline + authorization_gate.excluded_seconds() - time.monotonic() ) if remaining <= 0: timed_out = True break wait_slice = min(wait_slice, remaining) try: return future.result(timeout=wait_slice) except concurrent.futures.TimeoutError: if agent._interrupt_requested: interrupted = True break elapsed = int(time.monotonic() - started) if elapsed - _last_heartbeat >= 30: _last_heartbeat = elapsed agent._touch_activity( f"sequential tool running ({elapsed}s): {function_name}" ) if interrupted: # Belt-and-braces: interrupt() already fans out to tracked worker # tids, but the worker may have registered after the fan-out ran. for tid in worker_tid: try: _ra()._set_interrupt( True, tid, reason=getattr(agent, "_tool_interrupt_reason", None), ) except Exception: pass # Give a cooperative tool a moment to notice its per-thread # interrupt bit and return a real result (mirrors the concurrent # path's 3s grace). concurrent.futures.wait([future], timeout=3.0) if future.done() and not future.cancelled(): return future.result() timed_out = True # reuse the abandon-shutdown path in finally future.cancel() interrupt_reason = ( getattr(agent, "_tool_interrupt_reason", None) or "interrupt requested" ) message = ( f"[Tool execution cancelled — {function_name} was abandoned: " f"{interrupt_reason}]" ) logger.info( "sequential tool %s abandoned due to %s (%.1fs elapsed)", function_name, interrupt_reason, time.monotonic() - started, ) trace = middleware_trace if middleware_trace is not None else [] _emit_terminal_post_tool_call( agent, function_name=function_name, function_args=function_args, result=message, effective_task_id=effective_task_id, tool_call_id=tool_call_id, duration_ms=int((time.monotonic() - started) * 1000), status="cancelled", error_type="tool_interrupted", error_message=f"Tool execution cancelled: {interrupt_reason}", middleware_trace=list(trace), ) return _ManagedToolResult( result=_ToolCancelledResult(message), args=function_args, middleware_trace=trace, blocked=False, dispatched=True, ) # Only reachable when a deadline exists (interrupted returns above). assert timeout_s is not None message = ( f"Error executing tool '{function_name}': " f"timed out after {timeout_s:.1f}s" ) logger.warning( "sequential tool %s timed out after %.1fs", function_name, timeout_s ) future.cancel() for tid in worker_tid: try: _ra()._set_interrupt(True, tid) except Exception: pass trace = middleware_trace if middleware_trace is not None else [] _emit_terminal_post_tool_call( agent, function_name=function_name, function_args=function_args, result=message, effective_task_id=effective_task_id, tool_call_id=tool_call_id, duration_ms=int(timeout_s * 1000), status="timeout", error_type="tool_timeout", error_message=message, middleware_trace=list(trace), ) return _ManagedToolResult( result=_ToolTimeoutResult(message), args=function_args, middleware_trace=trace, blocked=False, dispatched=True, ) finally: # Never join a wedged worker. DaemonThreadPoolExecutor also keeps it out # of the stdlib atexit join, matching the concurrent timeout path. executor.shutdown(wait=not timed_out, cancel_futures=timed_out) def _begin_tool_execution( agent, *, function_name: str, function_args: dict[str, Any], effective_task_id: str, tool_call_id: str, display_index: int | None, ) -> None: """Run user-visible and checkpoint preflight on final tool arguments.""" if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": display_args = ( _redact_tool_args_for_display(function_name, function_args) or function_args ) args_str = json.dumps(display_args, ensure_ascii=False) prefix = f"Tool {display_index}" if display_index is not None else "Tool" if agent.verbose_logging: print(f" 📞 {prefix}: {function_name}({list(display_args.keys())})") print( agent._wrap_verbose( "Args: ", json.dumps(display_args, indent=2, ensure_ascii=False) ) ) else: args_preview = ( args_str[: agent.log_prefix_chars] + "..." if len(args_str) > agent.log_prefix_chars else args_str ) print( f" 📞 {prefix}: {function_name}({list(function_args.keys())}) - " f"{args_preview}" ) agent._current_tool = function_name agent._touch_activity(f"executing tool: {function_name}") try: from tools.environments.base import set_activity_callback set_activity_callback(agent._touch_activity) except Exception: pass if agent.tool_progress_callback: try: display_args = ( _redact_tool_args_for_display(function_name, function_args) or function_args ) preview = _build_tool_preview(function_name, display_args) agent.tool_progress_callback( "tool.started", function_name, preview, display_args ) except Exception as callback_error: logging.debug("Tool progress callback error: %s", callback_error) if agent.tool_start_callback: try: display_args = ( _redact_tool_args_for_display(function_name, function_args) or function_args ) agent.tool_start_callback( tool_call_id, function_name, display_args ) except Exception as callback_error: logging.debug("Tool start callback error: %s", callback_error) if function_name in {"write_file", "patch"} and agent._checkpoint_mgr.enabled: try: _ensure_file_checkpoint( agent, function_name, function_args, effective_task_id, ) except Exception: pass if function_name == "terminal" and agent._checkpoint_mgr.enabled: try: command = function_args.get("command", "") if _is_destructive_command(command): cwd = function_args.get("workdir") or os.getenv( "TERMINAL_CWD", os.getcwd() ) agent._checkpoint_mgr.ensure_checkpoint( cwd, f"before terminal: {command[:60]}" ) except Exception: pass def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, *, finalize: bool = True) -> None: """Execute multiple tool calls concurrently using a thread pool. Results are collected in the original tool-call order and appended to messages so the API sees them in the expected sequence. ``finalize=False`` skips the end-of-batch aggregate budget enforcement and /steer injection — used when this call is one segment of a larger mixed batch and the segmented dispatcher owns the turn-end work. """ tool_calls = assistant_message.tool_calls num_tools = len(tool_calls) # Resolve the context-scaled tool-output budget once per turn (cheap, but # avoids rebuilding it per result inside the loop below). _tool_budget = _budget_for_agent(agent) # ── Pre-flight: interrupt check ────────────────────────────────── if agent._interrupt_requested: print(f"{agent.log_prefix}⚡ Interrupt: skipping {num_tools} tool call(s)") for tc in tool_calls: cancelled_result = ( f"[Tool execution cancelled — {tc.function.name} was skipped " "due to user interrupt]" ) tool_call_id = _pairing_tool_call_id(tc) messages.append(make_tool_result_message( tc.function.name, cancelled_result, tool_call_id, effect_disposition="none", )) _emit_terminal_post_tool_call( agent, function_name=tc.function.name, function_args={}, result=cancelled_result, effective_task_id=effective_task_id, tool_call_id=tool_call_id, status="cancelled", error_type="user_interrupt", error_message="Tool execution skipped due to user interrupt", ) _flush_session_db_after_tool_progress( agent, messages, stage=f"cancelled tool result {tc.function.name}", ) return # ── Parse args + pre-execution bookkeeping ─────────────────────── # (tool call, resolved name, parsed args, middleware trace, parse error, # tool-search scope block) parsed_calls = [] for tool_call in tool_calls: function_name = tool_call.function.name # Legacy tool-name aliases (2026-08 renames) — map BEFORE the # agent-loop branches (todo_list etc. dispatch above the registry). from model_tools import _LEGACY_TOOL_ALIASES as _lta function_name = _lta.get(function_name, function_name) function_args, malformed_args_result = _parse_tool_arguments( tool_call.function.arguments ) if malformed_args_result is not None: parsed_calls.append( ( tool_call, function_name, function_args, [], malformed_args_result, None, ) ) continue # ── Tool Search unwrap ──────────────────────────────────────── # When the model invokes the tool_call bridge, peel it open so # every downstream check (checkpointing, guardrails, plugin # pre-tool-call hooks, the display/activity feed, the post-call # callback) sees the underlying tool — not the bridge. This is # the OpenClaw lesson: hooks must observe the real tool name. # # The original tool_call entry on ``tool_call.function`` is left # untouched so the conversation transcript and the matching # tool_call_id are preserved exactly as the model emitted them. # # Scope gate: the unwrap dispatches the underlying tool directly # (bypassing the bridge branch in handle_function_call and its # scope check), so we enforce session toolset scope HERE. A tool # the session was not granted is rejected before any checkpoint, # hook, or dispatch fires. _ts_scope_block = None try: from tools import tool_search as _ts if function_name == _ts.TOOL_CALL_NAME: _underlying, _underlying_args, _err = _ts.resolve_underlying_call(function_args) if not _err and _underlying: if _underlying in _tool_search_scoped_names(agent): # Validate before unwrapping: the generic bridge hides # the concrete parameter schema from provider-native # tool-call validation. _probe_err = _ts.validate_deferred_call_args(_underlying, _underlying_args) if _probe_err is not None: _ts_scope_block = _probe_err else: function_name = _underlying function_args = _underlying_args else: _ts_scope_block = ( f"'{_underlying}' is not available in this session. " "Use tool_search to find tools you can call." ) except Exception: pass parsed_calls.append( (tool_call, function_name, function_args, [], None, _ts_scope_block) ) # ── Logging / callbacks ────────────────────────────────────────── tool_names_str = ", ".join(name for _, name, _, _, _, _ in parsed_calls) if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": print(f" ⚡ Concurrent: {num_tools} tool calls — {tool_names_str}") # ── Concurrent execution ───────────────────────────────────────── # Each slot holds (function_name, function_args, function_result, duration, error_flag, blocked_flag, middleware_trace) results = [None] * num_tools for i, (tc, name, args, middleware_trace, block_result, _scope_block) in enumerate(parsed_calls): if block_result is not None: results[i] = (name, args, block_result, 0.0, True, True, middleware_trace) start_condition = threading.Condition() next_start_order = 0 # Set once the batch is abandoned (deadline or interrupt) so a worker parked # at the start-order gate exits immediately instead of waking up minutes # later and dispatching a tool the turn has already reported as timed out. batch_abandoned = threading.Event() authorization_gate = _ConcurrentToolAuthorizationGate() def _abandon_batch() -> None: """Release every gate-parked worker so none dispatches post-abandon.""" batch_abandoned.set() with start_condition: start_condition.notify_all() # The gate bound must sit UNDER the batch deadline, otherwise the deadline # fires first and the parked workers are still falsely reported as timed # out without ever starting — the very bug this gate timeout fixes. A # disabled deadline (None) keeps the stock bound rather than waiting forever. def _start_order_gate_timeout(batch_timeout: float | None) -> float: if batch_timeout is None: return _START_ORDER_GATE_TIMEOUT_S return min(_START_ORDER_GATE_TIMEOUT_S, batch_timeout / 2) def _begin_in_order( order: int, callback=None, *, tool_name: str = "", gate_timeout: float | None = None ) -> bool: """Serialize dispatch by submit order. Returns False if abandoned.""" nonlocal next_start_order with start_condition: # Bounded wait: a tool that wedges during its dispatch must not # park every later-ordered worker forever. Without the timeout, # one blocking dispatch starves the whole batch (the parked tools # then get falsely reported as "timed out" by the batch deadline # despite never having started) and the parked threads leak # permanently after the batch is abandoned — f.cancel() cannot # cancel running threads and nothing ever notifies the condition # again. On expiry, proceed out of order: the worst case is # interleaved approval prompts, strictly better than permanent # starvation. The >= predicate (rather than ==) lets one worker's # timeout-jump release every skipped worker immediately instead # of each burning its own full timeout; max() keeps the counter # monotonic when workers advance out of order. batch_abandoned # short-circuits the wait so an abandoned batch releases its # parked workers in milliseconds instead of one gate timeout. in_order = start_condition.wait_for( lambda: next_start_order >= order or batch_abandoned.is_set(), timeout=( _START_ORDER_GATE_TIMEOUT_S if gate_timeout is None else gate_timeout ), ) if batch_abandoned.is_set(): # Do not run the callback or advance the counter: the turn has # already synthesized this tool's result and moved on. return False if not in_order: logger.warning( "start-order gate timed out for %s (order=%d next=%d); " "proceeding out of order", tool_name or "tool", order, next_start_order, ) try: if callback is not None: callback() finally: next_start_order = max(next_start_order, order + 1) start_condition.notify_all() return True # Resolved before the workers are defined so the start-order gate can clamp # its own bound against the batch deadline it must stay under. timeout_s = _resolve_concurrent_tool_timeout() gate_timeout_s = _start_order_gate_timeout(timeout_s) # Touch activity before launching workers so the gateway knows # we're executing tools (not stuck). agent._current_tool = tool_names_str agent._touch_activity(f"executing {num_tools} tools concurrently: {tool_names_str}") def _run_tool( index, tool_call, function_name, function_args, middleware_trace, scope_block, start_order, ): """Worker function executed in a thread.""" # Register this worker tid so the agent can fan out an interrupt # to it — see AIAgent.interrupt(). Must happen first thing, and # must be paired with discard + clear in the finally block. _worker_tid = threading.current_thread().ident with agent._tool_worker_threads_lock: agent._tool_worker_threads.add(_worker_tid) # Race: if the agent was interrupted between fan-out (which # snapshotted an empty/earlier set) and our registration, apply # the interrupt to our own tid now so is_interrupted() inside # the tool returns True on the next poll. if agent._interrupt_requested: try: _ra()._set_interrupt( True, _worker_tid, reason=getattr(agent, "_tool_interrupt_reason", None), ) except Exception: pass # Set the activity callback on THIS worker thread so # _wait_for_process (terminal commands) can fire heartbeats. # The callback is thread-local; the main thread's callback # is invisible to worker threads. try: from tools.environments.base import set_activity_callback set_activity_callback(agent._touch_activity) except Exception: pass # Approval/sudo callbacks (thread-local) and the agent turn's # ContextVars are propagated by propagate_context_to_thread() at the # submit site below (GHSA-qg5c-hvr5-hjgr, #13617). start = time.time() tool_call_id = _pairing_tool_call_id(tool_call) blocked = False dispatched = False start_advanced = False def _advance_start(callback=None) -> None: nonlocal start_advanced if start_advanced: return try: proceed = _begin_in_order( start_order, callback, tool_name=function_name, gate_timeout=gate_timeout_s, ) finally: start_advanced = True if not proceed: # Batch already abandoned: the turn synthesized this tool's # result and moved on. Abort instead of dispatching late. raise _BatchAbandoned(function_name) try: try: def _execute(next_args: dict[str, Any]) -> Any: return agent._invoke_tool( function_name, next_args, effective_task_id, tool_call_id, messages=messages, pre_tool_block_checked=True, skip_tool_request_middleware=True, skip_tool_execution_middleware=True, tool_request_middleware_trace=list(middleware_trace), ) managed = _run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=scope_block, display_index=index + 1, middleware_trace=middleware_trace, begin_execution=_advance_start, authorization_gate=authorization_gate, ) result = managed.result function_args = managed.args middleware_trace = managed.middleware_trace blocked = managed.blocked dispatched = managed.dispatched except _BatchAbandoned: # The batch was abandoned while we were parked at the start-order # gate. The main thread already synthesized this tool's result # (timeout/cancelled) and moved on, so write nothing: a late # results[index] write, post_tool_call emit, or progress print # would double-report a tool_call_id the turn already closed. logger.info( "tool %s abandoned at start-order gate; skipping dispatch", function_name, ) return except KeyboardInterrupt: try: agent.interrupt("keyboard interrupt") except Exception: pass result = _emit_cancelled_terminal_post_tool_call( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, start_time=start, middleware_trace=list(middleware_trace), ) duration = time.time() - start logger.info("tool %s cancelled (%.2fs)", function_name, duration) results[index] = ( function_name, function_args, result, duration, True, False, middleware_trace, ) return except Exception as tool_error: result = f"Error executing tool '{function_name}': {tool_error}" logger.error("_invoke_tool raised for %s: %s", function_name, tool_error, exc_info=True) duration = time.time() - start if not blocked and not dispatched: _emit_terminal_post_tool_call( agent, function_name=function_name, function_args=function_args, result=result, effective_task_id=effective_task_id, tool_call_id=tool_call_id, duration_ms=int(duration * 1000), middleware_trace=list(middleware_trace), ) is_error, _ = _detect_tool_failure(function_name, result) if is_error: logger.info("tool %s failed (%.2fs): %s", function_name, duration, result[:200]) else: logger.info("tool %s completed (%.2fs, %d chars)", function_name, duration, len(result)) results[index] = ( function_name, function_args, result, duration, is_error, blocked, middleware_trace, ) finally: # Teardown advance: keep the counter moving for any later-ordered # worker. Never let the abandonment signal escape from here — the # worker is already unwinding and the turn owns the result. try: _advance_start() except _BatchAbandoned: pass # Tear down worker-tid tracking. Clear any interrupt bit we may # have set so the next task scheduled onto this recycled tid # starts with a clean slate. This MUST be in a finally block # because BaseException subclasses (CancelledError, KeyboardInterrupt) # bypass ``except Exception`` and would otherwise leak the tid # into _interrupted_threads, poisoning the recycled thread. with agent._tool_worker_threads_lock: agent._tool_worker_threads.discard(_worker_tid) try: _ra()._set_interrupt(False, _worker_tid) except Exception: pass # Start spinner for CLI mode (skip when TUI handles tool progress) spinner = None if agent._should_emit_quiet_tool_messages() and agent._should_start_quiet_spinner(): face = random.choice(KawaiiSpinner.get_waiting_faces()) spinner = KawaiiSpinner(f"{face} ⚡ running {num_tools} tools concurrently", spinner_type='dots', print_fn=agent._print_fn) spinner.start() try: runnable_calls = [ (i, tc, name, args, scope_block) for i, (tc, name, args, _trace, parse_error, scope_block) in enumerate( parsed_calls ) if parse_error is None ] futures = [] future_to_index = {} timed_out_indices: set[int] = set() deadline = time.monotonic() + timeout_s if timeout_s is not None else None if runnable_calls: max_workers = _max_workers_for_tool_batch(runnable_calls) # Daemon workers: an interrupted/timed-out batch is abandoned with # shutdown(wait=False), but stdlib ThreadPoolExecutor workers are # non-daemon and registered in concurrent.futures' atexit hook, # which joins them unconditionally — so one wedged tool thread # would block interpreter exit forever (multi-minute CLI exits). from tools.daemon_pool import DaemonThreadPoolExecutor executor = DaemonThreadPoolExecutor(max_workers=max_workers) abandon_executor = False try: for submit_index, (i, tc, name, args, scope_block) in enumerate( runnable_calls ): # Propagate the agent turn's ContextVars (e.g. # _approval_session_key) AND thread-local approval/sudo # callbacks into the worker thread; clears callbacks on exit. try: f = executor.submit( propagate_context_to_thread(_run_tool), i, tc, name, args, parsed_calls[i][3], scope_block, submit_index, ) except RuntimeError as submit_error: if not _is_interpreter_shutdown_submit_error(submit_error): raise skipped_calls = runnable_calls[submit_index:] logger.warning( "interpreter shutdown while scheduling concurrent tools; " "skipping %d unsubmitted tool(s)", len(skipped_calls), ) for ( skipped_i, _tc, skipped_name, skipped_args, _scope_block, ) in skipped_calls: if results[skipped_i] is None: middleware_trace = parsed_calls[skipped_i][3] result = ( f"Error executing tool '{skipped_name}': " "Python interpreter is shutting down; tool was not started" ) results[skipped_i] = ( skipped_name, skipped_args, result, 0.0, True, False, middleware_trace, ) break futures.append(f) future_to_index[f] = i # Wait for all to complete with periodic heartbeats so the # gateway's inactivity monitor doesn't kill us during long # concurrent tool batches. Also check for user interrupts # so we don't block indefinitely when the user sends /stop # or a new message during concurrent tool execution. _conc_start = time.time() _interrupt_logged = False while True: wait_timeout = 5.0 if deadline is not None: effective_deadline = ( deadline + authorization_gate.excluded_seconds() ) remaining = effective_deadline - time.monotonic() if remaining <= 0: done, not_done = set(), { f for f in futures if not f.done() } else: wait_timeout = min(wait_timeout, remaining) done, not_done = concurrent.futures.wait( futures, timeout=wait_timeout, ) else: done, not_done = concurrent.futures.wait( futures, timeout=wait_timeout, ) if not not_done: break if ( deadline is not None and time.monotonic() >= deadline + authorization_gate.excluded_seconds() ): abandon_executor = True timed_out_indices = { future_to_index[f] for f in not_done if f in future_to_index } _still_running = [ parsed_calls[i][1] for i in timed_out_indices ] logger.warning( "concurrent tool batch timed out after %.1fs; " "%d tool(s) still running: %s", timeout_s, len(timed_out_indices), ", ".join(_still_running[:5]), ) for f in not_done: f.cancel() # Release gate-parked workers before the interrupt # fan-out so none of them wakes up later and dispatches # a tool this loop just reported as timed out. _abandon_batch() with agent._tool_worker_threads_lock: worker_tids = list(agent._tool_worker_threads) for tid in worker_tids: try: _ra()._set_interrupt(True, tid) except Exception: pass break # Check for interrupt — the per-thread interrupt signal # already causes individual tools (terminal, execute_code) # to abort, but tools without interrupt checks (web_search, # read_file) will run to completion. Cancel any futures # that haven't started yet so we don't block on them. if agent._interrupt_requested: abandon_executor = True if not _interrupt_logged: _interrupt_logged = True agent._vprint( f"{agent.log_prefix}⚡ Interrupt: cancelling " f"{len(not_done)} pending concurrent tool(s)", force=True, ) for f in not_done: f.cancel() # Release gate-parked workers so they abort instead of # dispatching after the turn was already interrupted. _abandon_batch() # Give already-running tools a moment to notice the # per-thread interrupt signal and exit gracefully. concurrent.futures.wait(not_done, timeout=3.0) break _conc_elapsed = int(time.time() - _conc_start) # Heartbeat every ~30s (6 × 5s poll intervals) if _conc_elapsed > 0 and _conc_elapsed % 30 < 6: _still_running = [ parsed_calls[future_to_index[f]][1] for f in not_done if f in future_to_index ] agent._touch_activity( f"concurrent tools running ({_conc_elapsed}s, " f"{len(not_done)} remaining: {', '.join(_still_running[:3])})" ) finally: # Belt-and-braces: any exit from the wait loop that abandoned # the batch must release gate-parked workers, including the # exception path that never reaches the branches above. if abandon_executor: _abandon_batch() # On abandon (interrupt or deadline) we intentionally do NOT # join hung workers: wait=False returns immediately and # cancel_futures drops queued-but-unstarted work. A wedged tool # thread is left running detached — the deliberate tradeoff vs. # deadlocking the whole batch. Normal completion joins (wait=True). executor.shutdown( wait=not abandon_executor, cancel_futures=abandon_executor, ) finally: if spinner: # Build a summary message for the spinner stop completed = sum(1 for r in results if r is not None) total_dur = sum(r[3] for r in results if r is not None) spinner.stop(f"⚡ {completed}/{num_tools} tools completed in {total_dur:.1f}s total") # ── Post-execution: display per-tool results ───────────────────── for i, (tc, name, args, middleware_trace, _parse_error, _scope_block) in enumerate( parsed_calls ): r = results[i] tool_call_id = _pairing_tool_call_id(tc) blocked = False is_error = True progress_function_name = name # A worker can finish and write results[i] in the window between the # deadline snapshot (timed_out_indices, taken from not_done) and this # loop. Prefer that real result over a fabricated timeout message — the # tool genuinely succeeded, just slightly late. effect_disposition = None if i in timed_out_indices and r is None: suffix = f"{timeout_s:.1f}s" if timeout_s is not None else "the configured timeout" function_result = f"Error executing tool '{name}': timed out after {suffix}" effect_disposition = "unknown" _emit_terminal_post_tool_call( agent, function_name=name, function_args=args, result=function_result, effective_task_id=effective_task_id, tool_call_id=tool_call_id, duration_ms=int((timeout_s or 0.0) * 1000), status="timeout", error_type="tool_timeout", error_message=function_result, middleware_trace=list(middleware_trace), ) tool_duration = float(timeout_s or 0.0) elif r is None: # Tool was cancelled (interrupt) or thread didn't return if agent._interrupt_requested: function_result = f"[Tool execution cancelled — {name} was skipped due to user interrupt]" _emit_terminal_post_tool_call( agent, function_name=name, function_args=args, result=function_result, effective_task_id=effective_task_id, tool_call_id=tool_call_id, status="cancelled", error_type="keyboard_interrupt", error_message="Tool execution cancelled by user interrupt", middleware_trace=list(middleware_trace), ) else: function_result = f"Error executing tool '{name}': thread did not return a result" _emit_terminal_post_tool_call( agent, function_name=name, function_args=args, result=function_result, effective_task_id=effective_task_id, tool_call_id=tool_call_id, status="error", error_type="thread_missing_result", error_message=function_result, middleware_trace=list(middleware_trace), ) tool_duration = 0.0 else: function_name, function_args, function_result, tool_duration, is_error, blocked, middleware_trace = r name = function_name args = function_args progress_function_name = function_name if _parse_error is not None: _emit_terminal_post_tool_call( agent, function_name=function_name, function_args=function_args, result=function_result, effective_task_id=effective_task_id, tool_call_id=tool_call_id, status="error", error_type="invalid_tool_arguments", error_message="Tool arguments must be a valid JSON object", middleware_trace=list(middleware_trace), ) if blocked: effect_disposition = "none" if not blocked: function_result = agent._append_guardrail_observation( function_name, function_args, function_result, failed=is_error, tool_call_id=tool_call_id, ) if is_error: _err_text = _multimodal_text_summary(function_result) result_preview = _err_text[:200] if len(_err_text) > 200 else _err_text logger.warning("Tool %s returned error (%.2fs): %s", function_name, tool_duration, result_preview) # Track file-mutation outcome for the turn-end verifier. # `blocked` calls never actually ran — don't let a guardrail # block count as either a failure or a success. if not blocked: try: agent._record_file_mutation_result( function_name, function_args, function_result, is_error, ) except Exception as _ver_err: logging.debug("file-mutation verifier record failed: %s", _ver_err) if agent.verbose_logging: logging.debug("Tool %s completed in %.2fs", function_name, tool_duration) logging.debug("Tool result (%d chars): %s", len(function_result), function_result) agent._current_tool = None _status_suffix = " (error)" if is_error else "" agent._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s){_status_suffix}") display_function_result = function_result function_result = maybe_persist_tool_result( content=function_result, tool_name=name, tool_use_id=tool_call_id, env=get_active_env(effective_task_id), config=_tool_budget, ) if not _is_multimodal_tool_result(function_result) else function_result _record_persisted_path_for_stub(agent, tool_call_id, function_result) subdir_hints = agent._subdirectory_hints.check_tool_call(name, args) if subdir_hints: if _is_multimodal_tool_result(function_result): # Append the hint to the text summary part so the model # still sees it; don't touch the image blocks. _append_subdir_hint_to_multimodal(function_result, subdir_hints) else: function_result += subdir_hints # Unwrap _multimodal dicts to an OpenAI-style content list so any # vision-capable provider receives [{type:text},{type:image_url}] # rather than a raw Python dict. The Anthropic adapter already # accepts content lists; vision-capable OpenAI-compatible servers # (mlx-vlm, GPT-4o, …) accept image_url in tool messages natively. # Text-only servers get a string-safe fallback here so a rejected # image tool result never poisons canonical session history. # String results pass through unchanged. _tool_content = agent._tool_result_content_for_active_model(name, function_result) tool_message = make_tool_result_message( name, _tool_content, tool_call_id, effect_disposition=effect_disposition, ) messages.append(tool_message) risk_metadata = tool_message.get("_tool_output_risk") if not _flush_session_db_after_tool_progress( agent, messages, stage=f"tool result {name}", ): return # Every completion surface is downstream of the canonical append. If # the UI bridge or process dies while projecting one of these events, # resume can reconstruct the tool result that was already visible. if not blocked and agent.tool_progress_callback: try: agent.tool_progress_callback( "tool.completed", progress_function_name, None, None, duration=tool_duration, is_error=is_error, result=display_function_result, ) except Exception as cb_err: logging.debug("Tool progress callback error: %s", cb_err) # Print cute message per tool if agent._should_emit_quiet_tool_messages(): cute_msg = _get_cute_tool_message_impl( name, args, tool_duration, result=display_function_result, ) agent._safe_print(f" {cute_msg}") elif not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": _preview_str = _multimodal_text_summary(display_function_result) if agent.verbose_logging: print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s") print(agent._wrap_verbose("Result: ", _preview_str)) else: response_preview = _preview_str[:agent.log_prefix_chars] + "..." if len(_preview_str) > agent.log_prefix_chars else _preview_str print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s - {response_preview}") if not blocked and agent.tool_complete_callback: try: display_args = _redact_tool_args_for_display(name, args) or args agent.tool_complete_callback( tool_call_id, name, display_args, display_function_result, ) except Exception as cb_err: logging.debug("Tool complete callback error: %s", cb_err) if ( risk_metadata is not None and risk_metadata.get("risk") != "low" and agent.tool_progress_callback ): try: agent.tool_progress_callback( "tool.output_risk", name, None, None, tool_call_id=tool_call_id, risk_metadata=risk_metadata, ) except Exception as cb_err: logging.debug("Tool output risk callback error: %s", cb_err) # ── Per-turn aggregate budget enforcement ───────────────────────── # Keep /steer pending until the final post-budget drain below. The model # cannot observe a partial batch, while an early drain can be discarded # when aggregate budget enforcement replaces that tool result. num_tools = len(parsed_calls) if finalize and num_tools > 0: turn_tool_msgs = messages[-num_tools:] enforce_turn_budget(turn_tool_msgs, env=get_active_env(effective_task_id), config=_tool_budget) # ── /steer injection ────────────────────────────────────────────── # Append any pending user steer text to the last tool result so the # agent sees it on its next iteration. Runs AFTER budget enforcement # so the steer marker is never truncated. See steer() for details. if finalize and num_tools > 0: agent._apply_pending_steer_to_tool_results(messages, num_tools) def _append_cancelled_tool_results(messages: list, tool_calls, *, reason: str) -> None: """Append a cancelled ``tool`` result for each call in ``tool_calls``. Used when a hard interrupt (KeyboardInterrupt / BaseException) aborts the sequential executor mid-batch. Without this, the loop re-raises leaving the assistant tool-call turn with no matching tool results — a message-role alternation violation that malforms the next provider request. Mirrors the cooperative-interrupt skip block and the concurrent path, both of which already emit a result for every call_id. """ for tc in tool_calls: name = getattr(getattr(tc, "function", None), "name", "") or "tool" messages.append(make_tool_result_message( name, f"[Tool execution cancelled — {name} was skipped due to {reason}]", _pairing_tool_call_id(tc), effect_disposition="none", )) def execute_tool_calls_sequential(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, *, finalize: bool = True) -> None: """Execute tool calls sequentially (original behavior). Used for single calls or interactive tools. ``finalize=False`` skips the end-of-batch aggregate budget enforcement and /steer injection — used when this call is one segment of a larger mixed batch and the segmented dispatcher owns the turn-end work. """ # Resolve the context-scaled tool-output budget once per turn. _tool_budget = _budget_for_agent(agent) # Keep every runtime-tool branch on one bounded execution funnel without # duplicating timeout policy across the branch-specific callbacks below. def _run_agent_tool_execution_middleware(agent, **kwargs): return _run_sequential_tool_execution_middleware(agent, **kwargs) for i, tool_call in enumerate(assistant_message.tool_calls, 1): tool_call_id = _pairing_tool_call_id(tool_call) if getattr(agent, "_incremental_persistence_failed", False): return # SAFETY: check interrupt BEFORE starting each tool. # If the user sent "stop" during a previous tool's execution, # do NOT start any more tools -- skip them all immediately. if agent._interrupt_requested: remaining_calls = assistant_message.tool_calls[i-1:] if remaining_calls: agent._vprint(f"{agent.log_prefix}⚡ Interrupt: skipping {len(remaining_calls)} tool call(s)", force=True) for skipped_tc in remaining_calls: skipped_name = skipped_tc.function.name cancelled_result = ( f"[Tool execution cancelled — {skipped_name} was skipped " "due to user interrupt]" ) messages.append(make_tool_result_message( skipped_name, cancelled_result, _pairing_tool_call_id(skipped_tc), effect_disposition="none", )) _emit_terminal_post_tool_call( agent, function_name=skipped_name, function_args={}, result=cancelled_result, effective_task_id=effective_task_id, tool_call_id=getattr(skipped_tc, "id", "") or "", status="cancelled", error_type="user_interrupt", error_message="Tool execution skipped due to user interrupt", ) if not _flush_session_db_after_tool_progress( agent, messages, stage=f"cancelled tool result {skipped_name}", ): return break function_name = tool_call.function.name # Legacy tool-name aliases (2026-08 renames) — map BEFORE the # agent-loop branches (todo_list etc. dispatch above the registry). from model_tools import _LEGACY_TOOL_ALIASES as _lta function_name = _lta.get(function_name, function_name) function_args, malformed_args_result = _parse_tool_arguments( tool_call.function.arguments ) if malformed_args_result is not None: _emit_terminal_post_tool_call( agent, function_name=function_name, function_args=function_args, result=malformed_args_result, effective_task_id=effective_task_id, tool_call_id=tool_call_id, status="error", error_type="invalid_tool_arguments", error_message="Tool arguments must be a valid JSON object", ) messages.append( make_tool_result_message( function_name, malformed_args_result, tool_call_id, ) ) if not _flush_session_db_after_tool_progress( agent, messages, stage=f"invalid tool arguments {function_name}", ): return continue # Tool Search unwrap — see execute_tool_calls_concurrent for full # rationale, including the scope gate (the unwrap dispatches the # underlying tool directly, so session toolset scope is enforced here). _ts_scope_block: Optional[str] = None try: from tools import tool_search as _ts if function_name == _ts.TOOL_CALL_NAME: _underlying, _underlying_args, _err = _ts.resolve_underlying_call(function_args) if not _err and _underlying: if _underlying in _tool_search_scoped_names(agent): # Validate before unwrapping: the generic bridge hides # the concrete parameter schema from provider-native # tool-call validation. _probe_err = _ts.validate_deferred_call_args(_underlying, _underlying_args) if _probe_err is not None: # This path wraps _block_msg in {"error": ...} — # flatten the probe payload to one plain string. try: _probe = json.loads(_probe_err) _ts_scope_block = ( f"{_probe.get('error', '')} Parameters schema: " f"{json.dumps(_probe.get('parameters', {}), ensure_ascii=False)}. " f"{_probe.get('hint', '')}" ).strip() except Exception: _ts_scope_block = _probe_err else: function_name = _underlying function_args = _underlying_args else: _ts_scope_block = ( f"'{_underlying}' is not available in this session. " "Use tool_search to find tools you can call." ) except Exception: pass middleware_trace: list[dict[str, Any]] = [] _execution_blocked = False _execution_dispatched = False tool_start_time = time.time() if function_name == "todo_list": def _execute(next_args: dict) -> Any: from tools.todo_tool import todo_tool as _todo_tool return _todo_tool( todos=next_args.get("todos"), merge=next_args.get("merge", False), store=agent._todo_store, ) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('todo_list', function_args, tool_duration, result=function_result)}") elif function_name == "message_agent": # Bot Mode teammate DM (tools/bot_mode_dm.py) — injected, not # registered: only a canonical Bot Chat session carries the # schema, and the tool re-gates on the session title itself. def _execute(next_args: dict) -> Any: from tools.bot_mode_dm import message_agent_tool as _message_agent_tool return _message_agent_tool( target=next_args.get("target", ""), message=next_args.get("message", ""), task_id=effective_task_id, agent=agent, ) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('message_agent', function_args, tool_duration, result=function_result)}") elif function_name == "session_search": def _execute(next_args: dict) -> Any: session_db = agent._get_session_db_for_recall() if not session_db: from hermes_state import format_session_db_unavailable return json.dumps({"success": False, "error": format_session_db_unavailable()}) from tools.session_search_tool import session_search as _session_search return _session_search( query=next_args.get("query", ""), role_filter=next_args.get("role_filter"), limit=next_args.get("limit", 3), session_id=next_args.get("session_id"), around_message_id=next_args.get("around_message_id"), window=next_args.get("window", 5), sort=next_args.get("sort"), detail=next_args.get("detail", "adaptive"), db=session_db, current_session_id=agent.session_id, ) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('session_search', function_args, tool_duration, result=function_result)}") elif function_name == "memory": def _execute(next_args: dict) -> Any: target = next_args.get("target", "memory") operations = next_args.get("operations") from tools.memory_tool import memory_tool as _memory_tool result = _memory_tool( action=next_args.get("action"), target=target, content=next_args.get("content"), old_text=next_args.get("old_text"), operations=operations, store=agent._memory_store, ) # Mirror successful built-in memory writes to external # providers. All gating/op-expansion lives behind the manager # interface (MemoryManager.notify_memory_tool_write). if agent._memory_manager: agent._memory_manager.notify_memory_tool_write( result, next_args, build_metadata=lambda: agent._build_memory_write_metadata( task_id=effective_task_id, tool_call_id=tool_call_id, ), ) return result function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('memory', function_args, tool_duration, result=function_result)}") elif function_name == "clarify": def _execute(next_args: dict) -> Any: from tools.clarify_tool import clarify_tool as _clarify_tool return _clarify_tool( question=next_args.get("question", ""), choices=next_args.get("choices"), multi_select=next_args.get("multi_select", False), questions=next_args.get("questions"), callback=agent.clarify_callback, ) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('clarify', function_args, tool_duration, result=function_result)}") elif function_name == "read_terminal": def _execute(next_args: dict) -> Any: from tools.read_terminal_tool import read_terminal_tool as _read_terminal_tool return _read_terminal_tool( start_line=next_args.get("start_line"), count=next_args.get("count"), callback=getattr(agent, "read_terminal_callback", None), ) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('read_terminal', function_args, tool_duration, result=function_result)}") elif function_name == "desktop_preview": def _execute(next_args: dict) -> Any: if (next_args.get("action") or "").strip() == "read": from tools.read_preview_tool import read_preview_tool as _read_preview_tool return _read_preview_tool( start=next_args.get("start"), count=next_args.get("count"), callback=getattr(agent, "read_preview_callback", None), ) from tools.preview_tool import _handle_preview return _handle_preview(next_args) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('desktop_preview', function_args, tool_duration, result=function_result)}") elif function_name == "drive_preview": def _execute(next_args: dict) -> Any: from tools.drive_preview_tool import drive_preview_tool as _drive_preview_tool return _drive_preview_tool( action=next_args.get("action", ""), ref=next_args.get("ref"), selector=next_args.get("selector"), text=next_args.get("text"), key=next_args.get("key"), submit=next_args.get("submit"), amount=next_args.get("amount"), to=next_args.get("to"), limit=next_args.get("max"), callback=getattr(agent, "drive_preview_callback", None), ) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('drive_preview', function_args, tool_duration, result=function_result)}") elif function_name == "annotate_preview": def _execute(next_args: dict) -> Any: from tools.annotate_preview_tool import annotate_preview_tool as _annotate_preview_tool return _annotate_preview_tool( action=next_args.get("action", "add"), ref=next_args.get("ref"), selector=next_args.get("selector"), label=next_args.get("label"), callback=getattr(agent, "drive_preview_callback", None), ) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('annotate_preview', function_args, tool_duration, result=function_result)}") elif function_name == "read_window_below": def _execute(next_args: dict) -> Any: from tools.read_window_tool import read_window_below_tool as _read_window_below_tool return _read_window_below_tool( callback=getattr(agent, "read_window_below_callback", None), ) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('read_window_below', function_args, tool_duration, result=function_result)}") elif function_name == "gui_tour": def _execute(next_args: dict) -> Any: from tools.tour_tool import tour_tool as _tour_tool return _tour_tool( action=next_args.get("action", ""), surface=next_args.get("surface"), selector=next_args.get("selector"), title=next_args.get("title"), text=next_args.get("text"), side=next_args.get("side"), steps=next_args.get("steps"), step_index=next_args.get("step_index"), callback=getattr(agent, "tour_callback", None), ) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('gui_tour', function_args, tool_duration, result=function_result)}") elif function_name == "setup_mcp": def _execute(next_args: dict) -> Any: from tools.setup_mcp_tool import setup_mcp_tool as _setup_mcp_tool return _setup_mcp_tool( server=next_args.get("server", ""), action=next_args.get("action", "install"), reason=next_args.get("reason", ""), callback=getattr(agent, "setup_mcp_callback", None), ) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, scope_block=_ts_scope_block, display_index=i, )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('setup_mcp', function_args, tool_duration, result=function_result)}") elif function_name == "delegate_task": _action_arg = str(function_args.get("action") or "").strip().lower() tasks_arg = function_args.get("tasks") if _action_arg in ("list", "steer", "stop"): spinner_label = f"🔀 subagent {_action_arg}" elif tasks_arg and isinstance(tasks_arg, list): spinner_label = f"🔀 delegating {len(tasks_arg)} tasks · (/agents to monitor)" else: goal_preview = (function_args.get("goal") or "")[:30] spinner_label = ( f"🔀 {goal_preview} · (/agents to monitor)" if goal_preview else "🔀 delegating · (/agents to monitor)" ) spinner = None if agent._should_emit_quiet_tool_messages() and agent._should_start_quiet_spinner(): face = random.choice(KawaiiSpinner.get_waiting_faces()) spinner = KawaiiSpinner(f"{face} {spinner_label}", spinner_type='dots', print_fn=agent._print_fn) spinner.start() agent._delegate_spinner = spinner _delegate_result = None try: def _execute(next_args: dict) -> Any: return agent._dispatch_delegate_task(next_args) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) _delegate_result = function_result finally: agent._delegate_spinner = None tool_duration = time.time() - tool_start_time cute_msg = _get_cute_tool_message_impl('delegate_task', function_args, tool_duration, result=_delegate_result) if spinner: spinner.stop(cute_msg) elif agent._should_emit_quiet_tool_messages(): agent._vprint(f" {cute_msg}") elif agent._context_engine_tool_names and function_name in agent._context_engine_tool_names: # Context engine tools (lcm_grep, lcm_describe, lcm_expand, etc.) spinner = None if agent._should_emit_quiet_tool_messages(): face = random.choice(KawaiiSpinner.get_waiting_faces()) emoji = _get_tool_emoji(function_name) display_args = _redact_tool_args_for_display(function_name, function_args) or function_args preview = _build_tool_label(function_name, display_args) or function_name spinner = KawaiiSpinner(f"{face} {emoji} {preview}", spinner_type='dots', print_fn=agent._print_fn) spinner.start() _ce_result = None try: def _execute(next_args: dict) -> Any: return agent.context_compressor.handle_tool_call(function_name, next_args, messages=messages) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) _ce_result = function_result except Exception as tool_error: function_result = json.dumps({"error": f"Context engine tool '{function_name}' failed: {tool_error}"}) logger.error("context_engine.handle_tool_call raised for %s: %s", function_name, tool_error, exc_info=True) finally: tool_duration = time.time() - tool_start_time cute_msg = _get_cute_tool_message_impl(function_name, function_args, tool_duration, result=_ce_result) if spinner: spinner.stop(cute_msg) elif agent._should_emit_quiet_tool_messages(): agent._vprint(f" {cute_msg}") elif agent._memory_manager and agent._memory_manager.has_tool(function_name): # Memory provider tools (hindsight_retain, honcho_search, etc.) # These are not in the tool registry — route through MemoryManager. spinner = None if agent._should_emit_quiet_tool_messages() and agent._should_start_quiet_spinner(): face = random.choice(KawaiiSpinner.get_waiting_faces()) emoji = _get_tool_emoji(function_name) display_args = _redact_tool_args_for_display(function_name, function_args) or function_args preview = _build_tool_label(function_name, display_args) or function_name spinner = KawaiiSpinner(f"{face} {emoji} {preview}", spinner_type='dots', print_fn=agent._print_fn) spinner.start() _mem_result = None try: def _execute(next_args: dict) -> Any: return agent._memory_manager.handle_tool_call(function_name, next_args) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, )) _mem_result = function_result except Exception as tool_error: function_result = json.dumps({"error": f"Memory tool '{function_name}' failed: {tool_error}"}) logger.error("memory_manager.handle_tool_call raised for %s: %s", function_name, tool_error, exc_info=True) finally: tool_duration = time.time() - tool_start_time cute_msg = _get_cute_tool_message_impl(function_name, function_args, tool_duration, result=_mem_result) if spinner: spinner.stop(cute_msg) elif agent._should_emit_quiet_tool_messages(): agent._vprint(f" {cute_msg}") elif agent.quiet_mode: spinner = None if agent._should_emit_quiet_tool_messages() and agent._should_start_quiet_spinner(): face = random.choice(KawaiiSpinner.get_waiting_faces()) emoji = _get_tool_emoji(function_name) display_args = _redact_tool_args_for_display(function_name, function_args) or function_args preview = _build_tool_label(function_name, display_args) or function_name spinner = KawaiiSpinner(f"{face} {emoji} {preview}", spinner_type='dots', print_fn=agent._print_fn) spinner.start() _spinner_result = None try: def _execute(next_args: dict) -> Any: from model_tools import suppress_post_tool_call_hook with suppress_post_tool_call_hook(): return _ra().handle_function_call( function_name, next_args, effective_task_id, tool_call_id=tool_call_id, session_id=agent.session_id or "", turn_id=getattr(agent, "_current_turn_id", "") or "", api_request_id=getattr(agent, "_current_api_request_id", "") or "", enabled_tools=( list(agent.valid_tool_names) if agent.valid_tool_names else None ), skip_pre_tool_call_hook=True, skip_tool_request_middleware=True, skip_tool_execution_middleware=True, tool_request_middleware_trace=list(middleware_trace), enabled_toolsets=getattr(agent, "enabled_toolsets", None), disabled_toolsets=getattr(agent, "disabled_toolsets", None), ) ( function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched, ) = _managed_values( _run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, middleware_trace=middleware_trace, ) ) _spinner_result = function_result except KeyboardInterrupt: function_result = _emit_cancelled_terminal_post_tool_call( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, start_time=tool_start_time, middleware_trace=list(middleware_trace), ) _spinner_result = function_result try: agent.interrupt("keyboard interrupt") except Exception: pass # Emit a tool result for THIS call and every remaining call in # the batch before re-raising, so the assistant tool-call turn # is never left without matching tool results (alternation). _append_cancelled_tool_results( messages, assistant_message.tool_calls[i - 1:], reason="keyboard interrupt", ) raise except Exception as tool_error: function_result = f"Error executing tool '{function_name}': {tool_error}" logger.error("handle_function_call raised for %s: %s", function_name, tool_error, exc_info=True) finally: tool_duration = time.time() - tool_start_time cute_msg = _get_cute_tool_message_impl(function_name, function_args, tool_duration, result=_spinner_result) if spinner: spinner.stop(cute_msg) elif agent._should_emit_quiet_tool_messages(): agent._vprint(f" {cute_msg}") else: try: def _execute(next_args: dict) -> Any: from model_tools import suppress_post_tool_call_hook with suppress_post_tool_call_hook(): return _ra().handle_function_call( function_name, next_args, effective_task_id, tool_call_id=tool_call_id, session_id=agent.session_id or "", turn_id=getattr(agent, "_current_turn_id", "") or "", api_request_id=getattr(agent, "_current_api_request_id", "") or "", enabled_tools=( list(agent.valid_tool_names) if agent.valid_tool_names else None ), skip_pre_tool_call_hook=True, skip_tool_request_middleware=True, skip_tool_execution_middleware=True, tool_request_middleware_trace=list(middleware_trace), enabled_toolsets=getattr(agent, "enabled_toolsets", None), disabled_toolsets=getattr(agent, "disabled_toolsets", None), ) ( function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched, ) = _managed_values( _run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, execute=_execute, scope_block=_ts_scope_block, display_index=i, middleware_trace=middleware_trace, ) ) except KeyboardInterrupt: _emit_cancelled_terminal_post_tool_call( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=tool_call_id, start_time=tool_start_time, middleware_trace=list(middleware_trace), ) try: agent.interrupt("keyboard interrupt") except Exception: pass # Emit a tool result for THIS call and every remaining call in # the batch before re-raising (see interactive branch above). _append_cancelled_tool_results( messages, assistant_message.tool_calls[i - 1:], reason="keyboard interrupt", ) raise except Exception as tool_error: function_result = f"Error executing tool '{function_name}': {tool_error}" logger.error("handle_function_call raised for %s: %s", function_name, tool_error, exc_info=True) tool_duration = time.time() - tool_start_time _execution_timed_out = isinstance( function_result, (_ToolTimeoutResult, _ToolCancelledResult) ) if isinstance(function_result, str): result_preview = function_result if agent.verbose_logging else ( function_result[:200] if len(function_result) > 200 else function_result ) _result_len = len(function_result) else: # Multimodal dict result (_multimodal=True) — not sliceable as string result_preview = function_result _result_len = len(str(function_result)) # Log tool errors to the persistent error log so [error] tags # in the UI always have a corresponding detailed entry on disk. _is_error_result, _ = _detect_tool_failure(function_name, function_result) # The agent-runtime tools above (todo, session_search, memory, # context-engine, memory-manager, clarify, delegate_task) are # dispatched inline — they never reach handle_function_call, so the # executor is the one that has to fire post_tool_call. For # Every dispatch suppresses the inner handle_function_call observer so # the executor owns one terminal event for this tool_call_id. This also # prevents an abandoned timeout worker from reporting late success. _executor_must_emit_post_hook = ( not _execution_blocked and not _execution_timed_out ) if _executor_must_emit_post_hook: _emit_terminal_post_tool_call( agent, function_name=function_name, function_args=function_args, result=function_result, effective_task_id=effective_task_id, tool_call_id=tool_call_id, duration_ms=int(tool_duration * 1000), middleware_trace=list(middleware_trace), ) if not _execution_blocked: function_result = agent._append_guardrail_observation( function_name, function_args, function_result, failed=_is_error_result, tool_call_id=tool_call_id, ) result_preview = function_result if agent.verbose_logging else ( function_result[:200] if len(function_result) > 200 else function_result ) if _is_error_result: logger.warning("Tool %s returned error (%.2fs): %s", function_name, tool_duration, result_preview) else: logger.info("tool %s completed (%.2fs, %d chars)", function_name, tool_duration, _result_len) # Track file-mutation outcome for the turn-end verifier. See # the concurrent path for the rationale; both paths must feed # the same state so the footer reflects every tool call in the # turn, not just the parallel ones. if not _execution_blocked: try: agent._record_file_mutation_result( function_name, function_args, function_result, _is_error_result, ) except Exception as _ver_err: logging.debug("file-mutation verifier record failed: %s", _ver_err) agent._current_tool = None _status_suffix = " (error)" if _is_error_result else "" agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s){_status_suffix}") if agent.verbose_logging: logging.debug("Tool %s completed in %.2fs", function_name, tool_duration) _log_result = _multimodal_text_summary(function_result) logging.debug("Tool result (%d chars): %s", len(_log_result), _log_result) display_function_result = function_result function_result = maybe_persist_tool_result( content=function_result, tool_name=function_name, tool_use_id=tool_call_id, env=get_active_env(effective_task_id), config=_tool_budget, ) if not _is_multimodal_tool_result(function_result) else function_result _record_persisted_path_for_stub(agent, tool_call_id, function_result) # Discover subdirectory context files from tool arguments subdir_hints = agent._subdirectory_hints.check_tool_call(function_name, function_args) if subdir_hints: if _is_multimodal_tool_result(function_result): _append_subdir_hint_to_multimodal(function_result, subdir_hints) else: function_result += subdir_hints # Unwrap _multimodal dicts to an OpenAI-style content list # (see parallel path for rationale). String results pass through. _tool_content = agent._tool_result_content_for_active_model(function_name, function_result) tool_message = make_tool_result_message( function_name, _tool_content, tool_call_id, effect_disposition="unknown" if _execution_timed_out else None, ) messages.append(tool_message) risk_metadata = tool_message.get("_tool_output_risk") if not _flush_session_db_after_tool_progress( agent, messages, stage=f"tool result {function_name}", ): return # UI completion/progress events are projections of the canonical tool # row, never a competing in-memory authority. if not _execution_blocked and agent.tool_progress_callback: try: agent.tool_progress_callback( "tool.completed", function_name, None, None, duration=tool_duration, is_error=_is_error_result, result=display_function_result, ) except Exception as cb_err: logging.debug("Tool progress callback error: %s", cb_err) if not _execution_blocked and agent.tool_complete_callback: try: display_args = ( _redact_tool_args_for_display(function_name, function_args) or function_args ) agent.tool_complete_callback( tool_call_id, function_name, display_args, display_function_result, ) except Exception as cb_err: logging.debug("Tool complete callback error: %s", cb_err) if ( risk_metadata is not None and risk_metadata.get("risk") != "low" and agent.tool_progress_callback ): try: agent.tool_progress_callback( "tool.output_risk", function_name, None, None, tool_call_id=tool_call_id, risk_metadata=risk_metadata, ) except Exception as cb_err: logging.debug("Tool output risk callback error: %s", cb_err) if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": if agent.verbose_logging: print(f" ✅ Tool {i} completed in {tool_duration:.2f}s") print(agent._wrap_verbose("Result: ", function_result)) else: _fr_str = function_result if isinstance(function_result, str) else str(function_result) response_preview = _fr_str[:agent.log_prefix_chars] + "..." if len(_fr_str) > agent.log_prefix_chars else _fr_str print(f" ✅ Tool {i} completed in {tool_duration:.2f}s - {response_preview}") if agent._interrupt_requested and i < len(assistant_message.tool_calls): remaining = len(assistant_message.tool_calls) - i agent._vprint(f"{agent.log_prefix}⚡ Interrupt: skipping {remaining} remaining tool call(s)", force=True) for skipped_tc in assistant_message.tool_calls[i:]: skipped_name = skipped_tc.function.name messages.append(make_tool_result_message( skipped_name, f"[Tool execution skipped — {skipped_name} was not started. User sent a new message]", _pairing_tool_call_id(skipped_tc), effect_disposition="none", )) if not _flush_session_db_after_tool_progress( agent, messages, stage=f"skipped tool result {skipped_name}", ): return break # ── Per-turn aggregate budget enforcement ───────────────────────── # Keep /steer pending until the final post-budget drain below. The model # only receives this batch after all calls finish, and an early drain can # be discarded when aggregate budget enforcement replaces a tool result. num_tools_seq = len(assistant_message.tool_calls) if finalize and num_tools_seq > 0: enforce_turn_budget(messages[-num_tools_seq:], env=get_active_env(effective_task_id), config=_tool_budget) # ── /steer injection ────────────────────────────────────────────── # See _execute_tool_calls_parallel for the rationale. Same hook, # applied to sequential execution as well. if finalize and num_tools_seq > 0: agent._apply_pending_steer_to_tool_results(messages, num_tools_seq) def execute_tool_calls_segmented(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, segments=None) -> None: """Execute a mixed tool-call batch as ordered parallel/sequential segments. ``segments`` is the ``(kind, calls)`` plan from ``_plan_tool_batch_segments``: maximal contiguous runs of parallel-safe calls execute on the concurrent path, barrier calls on the sequential path, strictly in the model's original call order. Because segments are contiguous, every tool result is still appended one-per-call in emission order and no call ever starts before an earlier barrier finishes — identical ordering and side-effect boundaries to fully-sequential execution, with I/O parallelism recovered inside the safe runs. Turn-end work (aggregate budget enforcement + /steer injection) is done once here for the WHOLE batch; the per-segment executor calls run with ``finalize=False`` so a multi-segment turn cannot multiply the budget or truncate a steer marker. Interrupt semantics: each segment executor already checks ``agent._interrupt_requested`` up front and appends a cancelled/skipped result per call, so an interrupt during segment *k* drains segments *k+1..n* without executing them while preserving one result per tool_call_id. """ from types import SimpleNamespace if segments is None: _active_env = get_active_env(effective_task_id) _exec_cwd = Path(_active_env.cwd) if _active_env is not None and _active_env.cwd else None segments = _plan_tool_batch_segments(assistant_message.tool_calls, execution_cwd=_exec_cwd) for kind, calls in segments: if getattr(agent, "_incremental_persistence_failed", False): return segment_message = SimpleNamespace(tool_calls=list(calls)) if kind == "parallel": execute_tool_calls_concurrent( agent, segment_message, messages, effective_task_id, api_call_count, finalize=False, ) else: execute_tool_calls_sequential( agent, segment_message, messages, effective_task_id, api_call_count, finalize=False, ) if getattr(agent, "_incremental_persistence_failed", False): return # ── Whole-turn finalize (budget + /steer) ───────────────────────── total_tools = len(assistant_message.tool_calls) if total_tools > 0: _tool_budget = _budget_for_agent(agent) enforce_turn_budget( messages[-total_tools:], env=get_active_env(effective_task_id), config=_tool_budget, ) agent._apply_pending_steer_to_tool_results(messages, total_tools) __all__ = [ "execute_tool_calls_concurrent", "execute_tool_calls_sequential", "execute_tool_calls_segmented", ]