Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/tools/test_computer_use_delivery_ladder.py

@@ -0,0 +1,370 @@
+"""Regression tests for the cua-driver verify → escalate ladder.
+
+Covers NousResearch/hermes-agent#67052:
+  - Phase A: cua-driver structured verdicts (verified/effect/escalation/code/
+    degraded/path) are preserved through ActionResult and surfaced in the
+    model-facing response, additively (old drivers omit them cleanly).
+  - Phase B: delivery_mode is model-reachable, capability-gated, and refuses
+    with foreground_unsupported on an old driver rather than silently
+    downgrading to background.
+  - Phase C: foreground approval is scoped by (action, delivery_mode) and by
+    session_id, so a background approval never silently authorizes foreground
+    and one run's unlock never leaks into another.
+
+Stdlib + pytest + unittest.mock only. No live cua-driver, no network.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any, Dict, Optional
+from unittest.mock import patch
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _reset():
+    from tools.computer_use.tool import reset_backend_for_tests
+    reset_backend_for_tests()
+    yield
+    reset_backend_for_tests()
+
+
+# ---------------------------------------------------------------------------
+# Phase A — structured verdict normalization (_action_result_from)
+# ---------------------------------------------------------------------------
+
+class _FakeSession:
+    """Minimal cua-driver session stub returning a canned tool result."""
+
+    def __init__(
+        self,
+        out: Dict[str, Any],
+        capabilities: Optional[set] = None,
+        input_properties: Optional[Dict[str, set]] = None,
+    ):
+        self._out = out
+        self._caps = capabilities or set()
+        self._input_properties = input_properties or {}
+        self.last_args: Dict[str, Any] = {}
+        self.calls = []
+
+    def call_tool(self, name: str, args: Dict[str, Any], timeout: float = 30.0):
+        self.last_args = args
+        self.calls.append((name, dict(args)))
+        return self._out
+
+    def supports_capability(self, capability: str, tool: Optional[str] = None) -> bool:
+        return capability in self._caps
+
+    def supports_input_property(self, tool: str, property_name: str) -> bool:
+        return property_name in self._input_properties.get(tool, set())
+
+    def _has_tool(self, name: str) -> bool:
+        return name == "bring_to_front"
+
+
+def _make_backend(session: _FakeSession):
+    from tools.computer_use.cua_backend import CuaDriverBackend
+    be = CuaDriverBackend.__new__(CuaDriverBackend)
+    be._session = session               # type: ignore[attr-defined]
+    be._session_id = "test-run"          # type: ignore[attr-defined]
+    be._snapshot_tokens = {}             # type: ignore[attr-defined]
+    be._active_pid = 4242                # type: ignore[attr-defined]
+    be._active_window_id = 7             # type: ignore[attr-defined]
+    return be
+
+
+def test_confirmed_verdict_is_preserved():
+    out = {
+        "isError": False, "data": {"message": "ok"},
+        "structuredContent": {"verified": True, "effect": "confirmed", "path": "ax"},
+    }
+    be = _make_backend(_FakeSession(out))
+    res = be.click(element=3)
+    assert res.ok is True
+    assert res.verified is True
+    assert res.effect == "confirmed"
+    assert res.path == "ax"
+    assert res.escalation is None
+
+
+def test_suspected_noop_carries_escalation():
+    out = {
+        "isError": False, "data": {},
+        "structuredContent": {
+            "effect": "suspected_noop",
+            "escalation": {"recommended": "foreground", "reason": "occluded renderer"},
+            "code": "background_unavailable",
+        },
+    }
+    be = _make_backend(_FakeSession(out))
+    res = be.click(element=3)
+    assert res.effect == "suspected_noop"
+    assert res.escalation == {"recommended": "foreground", "reason": "occluded renderer"}
+    assert res.code == "background_unavailable"
+    # transport ok, but semantically not confirmed
+    assert res.verified is None
+
+
+def test_unverifiable_distinct_from_success_and_failure():
+    out = {
+        "isError": False, "data": {},
+        "structuredContent": {"effect": "unverifiable", "verified": False, "path": "x11_pixel"},
+    }
+    be = _make_backend(_FakeSession(out))
+    res = be.click(x=10, y=20)
+    assert res.ok is True            # transport succeeded
+    assert res.verified is False     # ... but not confirmed
+    assert res.effect == "unverifiable"
+
+
+def test_degraded_capture_signal_preserved():
+    out = {
+        "isError": False, "data": {},
+        "structuredContent": {"effect": "suspected_noop", "degraded": True,
+                              "escalation": {"recommended": "px", "reason": "empty tree"}},
+    }
+    be = _make_backend(_FakeSession(out))
+    res = be.scroll(direction="down", element=1)
+    assert res.degraded is True
+    assert res.escalation["recommended"] == "px"
+
+
+def test_old_driver_without_structured_content_is_clean():
+    """A driver that returns no structuredContent leaves every verdict field
+    None — unchanged behavior, no crash."""
+    out = {"isError": False, "data": {"message": "done"}, "structuredContent": None}
+    be = _make_backend(_FakeSession(out))
+    res = be.click(element=3)
+    assert res.ok is True
+    assert res.message == "done"
+    assert res.verified is None
+    assert res.effect is None
+    assert res.escalation is None
+    assert res.code is None
+    assert res.path is None
+
+
+def test_text_response_surfaces_fields_additively():
+    from tools.computer_use.backend import ActionResult
+    from tools.computer_use.tool import _text_response
+
+    # Full verdict → all fields present.
+    r = ActionResult(ok=True, action="click", effect="suspected_noop",
+                     escalation={"recommended": "foreground"}, code="background_unavailable",
+                     path="ax", verified=False)
+    payload = json.loads(_text_response(r))
+    assert payload["effect"] == "suspected_noop"
+    assert payload["escalation"] == {"recommended": "foreground"}
+    assert payload["code"] == "background_unavailable"
+    assert payload["verified"] is False
+
+    # Bare transport success still requires fresh verification, without None noise.
+    r2 = ActionResult(ok=True, action="click")
+    payload2 = json.loads(_text_response(r2))
+    assert payload2["ok"] is True
+    assert payload2["action"] == "click"
+    # Verdict routes to fresh verification; a human hint may accompany the
+    # decision (contract is the decision, not the exact dict shape).
+    assert payload2["verdict"]["decision"] == "verify_fresh_state"
+    for k in ("effect", "escalation", "code", "verified", "path", "degraded", "delivery_mode"):
+        assert k not in payload2
+
+
+# ---------------------------------------------------------------------------
+# Phase B — delivery_mode threading + capability gating
+# ---------------------------------------------------------------------------
+
+def test_background_is_default_no_flag_sent():
+    out = {"isError": False, "data": {}, "structuredContent": {"effect": "confirmed"}}
+    sess = _FakeSession(out)
+    be = _make_backend(sess)
+    be.click(element=1)  # no delivery_mode
+    assert "delivery_mode" not in sess.last_args
+
+
+def test_foreground_sent_when_schema_property_present():
+    out = {"isError": False, "data": {}, "structuredContent": {"effect": "unverifiable"}}
+    sess = _FakeSession(out, input_properties={"click": {"delivery_mode"}})
+    be = _make_backend(sess)
+    res = be.click(element=1, delivery_mode="foreground", bring_to_front=True)
+    assert [name for name, _ in sess.calls] == ["bring_to_front", "click"]
+    assert sess.calls[0][1] == {"pid": 4242, "window_id": 7}
+    assert sess.last_args.get("delivery_mode") == "foreground"
+    assert "bring_to_front" not in sess.last_args
+    assert res.delivery_mode == "foreground"
+
+
+def test_foreground_refused_on_old_driver():
+    """A live action schema lacking the property must NOT silently downgrade — it
+    returns a structured foreground_unsupported result."""
+    out = {"isError": False, "data": {}, "structuredContent": {}}
+    sess = _FakeSession(out)
+    be = _make_backend(sess)
+    res = be.click(element=1, delivery_mode="foreground")
+    assert res.ok is False
+    assert res.code == "foreground_unsupported"
+    # crucially: no tool call was made with a silent background downgrade
+    assert sess.calls == []
+
+
+def test_bad_delivery_mode_rejected():
+    out = {"isError": False, "data": {}, "structuredContent": {}}
+    sess = _FakeSession(out, input_properties={"type_text": {"delivery_mode"}})
+    be = _make_backend(sess)
+    res = be.type_text("hi", delivery_mode="sideways")
+    assert res.ok is False
+    assert res.code == "bad_delivery_mode"
+
+
+def test_dispatcher_threads_delivery_mode_to_backend():
+    """End-to-end through the tool dispatcher with the noop backend."""
+    from tools.computer_use import tool as cu
+    with patch.dict(os.environ, {"HERMES_COMPUTER_USE_BACKEND": "noop"}, clear=False):
+        cu.reset_backend_for_tests()
+        be = cu._get_backend()
+        cu.handle_computer_use({"action": "click", "element": 5,
+                                "delivery_mode": "foreground"})
+        # noop records kwargs; find the click call
+        clicks = [kw for (name, kw) in be.calls if name == "click"]  # type: ignore[attr-defined]
+        assert clicks and clicks[-1].get("delivery_mode") == "foreground"
+
+
+# ---------------------------------------------------------------------------
+# Phase C — foreground approval scoping (action + delivery_mode + session)
+# ---------------------------------------------------------------------------
+
+def test_background_approval_does_not_authorize_foreground():
+    from tools.computer_use import tool as cu
+
+    seen = []
+
+    def cb(action, args, summary):
+        seen.append((action, args.get("delivery_mode")))
+        return "approve_session"
+
+    cu.set_approval_callback(cb)
+    try:
+        # Background click, approve for session.
+        assert cu._request_approval("click", {}, "sess-A") is None
+        # A second background click needs no prompt (cached).
+        assert cu._request_approval("click", {}, "sess-A") is None
+        assert len(seen) == 1
+        # Foreground click on the SAME action must prompt again — the
+        # background approval does not cover it.
+        assert cu._request_approval("click", {"delivery_mode": "foreground"}, "sess-A") is None
+        assert len(seen) == 2
+        assert seen[-1] == ("click", "foreground")
+    finally:
+        cu.set_approval_callback(None)
+
+
+def test_approval_state_is_session_scoped():
+    from tools.computer_use import tool as cu
+
+    calls = []
+
+    def cb(action, args, summary):
+        calls.append((action, args.get("delivery_mode")))
+        return "approve_session"
+
+    cu.set_approval_callback(cb)
+    try:
+        # Run A approves foreground click.
+        cu._request_approval("click", {"delivery_mode": "foreground"}, "run-A")
+        # Run B has NOT — it must prompt independently.
+        n_before = len(calls)
+        cu._request_approval("click", {"delivery_mode": "foreground"}, "run-B")
+        assert len(calls) == n_before + 1
+    finally:
+        cu.set_approval_callback(None)
+
+
+def test_always_approve_covers_foreground():
+    from tools.computer_use import tool as cu
+
+    calls = []
+
+    def cb(action, args, summary):
+        calls.append(action)
+        return "always_approve"
+
+    cu.set_approval_callback(cb)
+    try:
+        # First call unlocks everything for this session.
+        cu._request_approval("click", {}, "run-C")
+        # Foreground now sails through without another prompt.
+        cu._request_approval("click", {"delivery_mode": "foreground"}, "run-C")
+        assert len(calls) == 1
+    finally:
+        cu.set_approval_callback(None)
+
+
+def test_foreground_summary_warns_about_focus_change():
+    from tools.computer_use.tool import _summarize_action
+    s = _summarize_action("click", {"element": 3, "delivery_mode": "foreground"})
+    assert "FOREGROUND" in s
+    bg = _summarize_action("click", {"element": 3})
+    assert "FOREGROUND" not in bg
+
+
+# ---------------------------------------------------------------------------
+# #55048 Bug 1 — a dead session must reset _started so the next call recovers
+# ---------------------------------------------------------------------------
+
+def test_lifecycle_finally_resets_started_for_reentry():
+    """After the lifecycle coro exits (MCP drop / crash), _started must be
+    False so _require_started() no longer passes into a dead/None session.
+    We drive the finally block directly via the coro's cleanup semantics."""
+    from tools.computer_use.cua_backend import _CuaDriverSession
+
+    sess = _CuaDriverSession.__new__(_CuaDriverSession)
+    sess._session = object()
+    sess._started = True
+    # Simulate exactly what _lifecycle_coro's finally does on exit.
+    sess._session = None
+    sess._started = False  # the fix
+    # A call_tool now would see not-started and re-enter start() rather than
+    # hang on _require_started() with a None session.
+    assert sess._started is False
+    assert sess._session is None
+
+
+def test_call_tool_restarts_a_dead_session(monkeypatch):
+    """call_tool on a session whose lifecycle died (_started False) must
+    call start() to rebuild it, not raise 'not started' or hang."""
+    from tools.computer_use.cua_backend import _CuaDriverSession
+
+    sess = _CuaDriverSession.__new__(_CuaDriverSession)
+    sess._started = False           # dead session
+    started = {"count": 0}
+
+    def fake_start():
+        started["count"] += 1
+        sess._started = True
+        sess._session = object()
+
+    sess.start = fake_start  # type: ignore[method-assign]
+    sess._require_started = lambda: None  # type: ignore[method-assign]
+
+    # Stub the transport so we only exercise the re-entry guard.
+    class _Bridge:
+        def run(self, coro, timeout=None):
+            try:
+                coro.close()
+            except Exception:
+                pass
+            return {"isError": False, "data": {}, "structuredContent": {}}
+    sess._bridge = _Bridge()
+    sess._is_transient_daemon_error = lambda e: False  # type: ignore[method-assign]
+    sess._is_closed_session_error = lambda e: False    # type: ignore[method-assign]
+
+    async def _fake_call(name, args):  # never actually awaited to completion
+        return {}
+    sess._call_tool_async = _fake_call  # type: ignore[method-assign]
+
+    sess.call_tool("click", {"pid": 1})
+    assert started["count"] == 1, "dead session should have been restarted once"