Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,471 @@
|
||||
"""Opt-in macOS smoke test for the installed cua-driver live MCP contract.
|
||||
|
||||
This script never installs, updates, or grants an existing browser profile. Start
|
||||
an isolated daemon separately, then point this script at its socket:
|
||||
|
||||
cua-driver serve --embedded --socket /tmp/hermes-cua-0-9-live.sock \
|
||||
--no-permissions-gate --no-overlay
|
||||
CUA_DRIVER_LIVE_SOCKET=/tmp/hermes-cua-0-9-live.sock \
|
||||
.venv/bin/python tests/computer_use/live_cua_0_9_smoke.py
|
||||
|
||||
The output deliberately excludes process IDs, window IDs, socket paths, and
|
||||
driver payloads. Each cell is classified as pass, structured_refusal,
|
||||
environment_unavailable, or unproven.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from mcp import ClientSession, StdioServerParameters
|
||||
from mcp.client.stdio import stdio_client
|
||||
|
||||
|
||||
def structured(result: Any) -> dict[str, Any]:
|
||||
value = getattr(result, "structuredContent", None)
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
dumped = result.model_dump(by_alias=True) if hasattr(result, "model_dump") else {}
|
||||
for key in ("structuredContent", "structured_content"):
|
||||
value = dumped.get(key)
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
for block in getattr(result, "content", []) or []:
|
||||
text = getattr(block, "text", None)
|
||||
if not isinstance(text, str):
|
||||
continue
|
||||
try:
|
||||
value = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
return {}
|
||||
|
||||
|
||||
def refusal_code(payload: dict[str, Any]) -> str | None:
|
||||
refusal = payload.get("refusal")
|
||||
return payload.get("code") or (
|
||||
refusal.get("code") if isinstance(refusal, dict) else None
|
||||
)
|
||||
|
||||
|
||||
def textedit_process_contains(pid: int, marker: str) -> bool:
|
||||
"""Read the exact throwaway process through the native AX script bridge."""
|
||||
script = """
|
||||
on run argv
|
||||
set targetPid to item 1 of argv as integer
|
||||
set markerText to item 2 of argv
|
||||
tell application "System Events"
|
||||
tell first application process whose unix id is targetPid
|
||||
set documentText to value of text area 1 of scroll area 1 of window 1
|
||||
end tell
|
||||
end tell
|
||||
return (documentText contains markerText) as text
|
||||
end run
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["osascript", "-e", script, "--", str(pid), marker],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
check=False,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return False
|
||||
return result.returncode == 0 and result.stdout.strip().lower() == "true"
|
||||
|
||||
|
||||
async def run_smoke(socket_path: str) -> dict[str, dict[str, Any]]:
|
||||
session_id = f"hermes-cua-live-{uuid.uuid4().hex[:8]}"
|
||||
params = StdioServerParameters(
|
||||
command="cua-driver",
|
||||
args=["mcp", "--embedded", "--socket", socket_path],
|
||||
)
|
||||
report: dict[str, dict[str, Any]] = {
|
||||
"foreground": {"classification": "unproven"},
|
||||
"typed_browser": {"classification": "unproven"},
|
||||
}
|
||||
launched_pid: int | None = None
|
||||
isolated_browser_pid: int | None = None
|
||||
browser_pid: int | None = None
|
||||
prior_foreground_pids: set[int] = set()
|
||||
file_descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix="hermes-cua-live-", suffix=".txt"
|
||||
)
|
||||
os.close(file_descriptor)
|
||||
smoke_path = Path(temporary_name)
|
||||
|
||||
try:
|
||||
async with stdio_client(params) as (read, write):
|
||||
async with ClientSession(read, write) as client:
|
||||
await client.initialize()
|
||||
await client.call_tool("start_session", {"session": session_id})
|
||||
try:
|
||||
before_windows = structured(
|
||||
await client.call_tool(
|
||||
"list_windows",
|
||||
{"on_screen_only": True, "session": session_id},
|
||||
)
|
||||
)
|
||||
prior_foreground_pids = {
|
||||
pid
|
||||
for row in before_windows.get("windows") or []
|
||||
if "textedit" in str(row.get("app_name") or "").lower()
|
||||
and isinstance((pid := row.get("pid")), int)
|
||||
}
|
||||
launched = structured(
|
||||
await client.call_tool(
|
||||
"launch_app",
|
||||
{
|
||||
"name": "TextEdit",
|
||||
"urls": [smoke_path.as_uri()],
|
||||
"creates_new_application_instance": True,
|
||||
"session": session_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
launched_pid = launched.get("pid")
|
||||
windows = launched.get("windows") or []
|
||||
if isinstance(launched_pid, int) and not windows:
|
||||
await client.call_tool(
|
||||
"wait", {"seconds": 1, "session": session_id}
|
||||
)
|
||||
refreshed = structured(
|
||||
await client.call_tool(
|
||||
"list_windows",
|
||||
{"on_screen_only": True, "session": session_id},
|
||||
)
|
||||
)
|
||||
windows = [
|
||||
row
|
||||
for row in refreshed.get("windows") or []
|
||||
if row.get("pid") == launched_pid
|
||||
]
|
||||
window_id = windows[0].get("window_id") if windows else None
|
||||
if (
|
||||
not isinstance(launched_pid, int)
|
||||
or launched_pid in prior_foreground_pids
|
||||
or not isinstance(window_id, int)
|
||||
):
|
||||
report["foreground"] = {
|
||||
"classification": "environment_unavailable",
|
||||
"stage": "throwaway_target",
|
||||
}
|
||||
else:
|
||||
focus = await client.call_tool(
|
||||
"bring_to_front",
|
||||
{"pid": launched_pid, "window_id": window_id},
|
||||
)
|
||||
before = structured(
|
||||
await client.call_tool(
|
||||
"get_window_state",
|
||||
{
|
||||
"pid": launched_pid,
|
||||
"window_id": window_id,
|
||||
"session": session_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
editor = next(
|
||||
(
|
||||
element
|
||||
for element in before.get("elements") or []
|
||||
if str(element.get("role") or "").lower()
|
||||
in {"axtextarea", "axtextfield"}
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not isinstance(editor, dict):
|
||||
report["foreground"] = {
|
||||
"classification": "unproven",
|
||||
"stage": "editor_discovery",
|
||||
}
|
||||
else:
|
||||
marker = "hermes foreground smoke"
|
||||
type_args = {
|
||||
"pid": launched_pid,
|
||||
"window_id": window_id,
|
||||
"element_index": editor.get("index"),
|
||||
"text": marker,
|
||||
"delivery_mode": "foreground",
|
||||
"session": session_id,
|
||||
}
|
||||
token = editor.get("element_token")
|
||||
if isinstance(token, str) and token:
|
||||
type_args["element_token"] = token
|
||||
typed = structured(
|
||||
await client.call_tool("type_text", type_args)
|
||||
)
|
||||
saved = structured(
|
||||
await client.call_tool(
|
||||
"hotkey",
|
||||
{
|
||||
"pid": launched_pid,
|
||||
"window_id": window_id,
|
||||
"keys": ["cmd", "s"],
|
||||
"delivery_mode": "foreground",
|
||||
"session": session_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
await client.call_tool(
|
||||
"wait", {"seconds": 0.5, "session": session_id}
|
||||
)
|
||||
after = structured(
|
||||
await client.call_tool(
|
||||
"get_window_state",
|
||||
{
|
||||
"pid": launched_pid,
|
||||
"window_id": window_id,
|
||||
"session": session_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
fresh_contains_marker = marker in json.dumps(
|
||||
after.get("elements") or []
|
||||
)
|
||||
native_document_confirmed = textedit_process_contains(
|
||||
launched_pid, marker
|
||||
)
|
||||
file_contains_marker = marker in smoke_path.read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
report["foreground"] = {
|
||||
"classification": (
|
||||
"pass"
|
||||
if not focus.isError
|
||||
and not refusal_code(typed)
|
||||
and not refusal_code(saved)
|
||||
and (
|
||||
typed.get("verified") is True
|
||||
or fresh_contains_marker
|
||||
or native_document_confirmed
|
||||
or file_contains_marker
|
||||
)
|
||||
else "unproven"
|
||||
),
|
||||
"focus_transport_ok": not focus.isError,
|
||||
"effect": typed.get("effect"),
|
||||
"verified": typed.get("verified"),
|
||||
"fresh_state": bool(after.get("elements")),
|
||||
"fresh_state_confirmed": fresh_contains_marker,
|
||||
"native_document_confirmed": (
|
||||
native_document_confirmed
|
||||
),
|
||||
"saved_file_confirmed": file_contains_marker,
|
||||
"action_schema_omitted_bring_to_front": (
|
||||
"bring_to_front" not in type_args
|
||||
),
|
||||
}
|
||||
|
||||
# Use only a driver-owned isolated profile. Never request,
|
||||
# mint, print, or persist an existing-profile grant token.
|
||||
listed = structured(
|
||||
await client.call_tool(
|
||||
"list_windows",
|
||||
{"on_screen_only": True, "session": session_id},
|
||||
)
|
||||
)
|
||||
browser_row = next(
|
||||
(
|
||||
row
|
||||
for row in listed.get("windows") or []
|
||||
if "chrome" in str(row.get("app_name") or "").lower()
|
||||
),
|
||||
None,
|
||||
)
|
||||
browser_pid = browser_row.get("pid") if browser_row else None
|
||||
browser_window = (
|
||||
browser_row.get("window_id") if browser_row else None
|
||||
)
|
||||
if not isinstance(browser_pid, int) or not isinstance(
|
||||
browser_window, int
|
||||
):
|
||||
report["typed_browser"] = {
|
||||
"classification": "environment_unavailable",
|
||||
"stage": "browser_target",
|
||||
}
|
||||
else:
|
||||
prepared = structured(
|
||||
await client.call_tool(
|
||||
"browser_prepare",
|
||||
{
|
||||
"pid": browser_pid,
|
||||
"window_id": browser_window,
|
||||
"allow_launch": True,
|
||||
"profile": {"mode": "isolated_new"},
|
||||
"session": session_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
isolated_browser_pid = prepared.get("prepared_pid")
|
||||
code = refusal_code(prepared)
|
||||
if prepared.get("status") == "refused" or code:
|
||||
report["typed_browser"] = {
|
||||
"classification": "structured_refusal",
|
||||
"code": code,
|
||||
}
|
||||
else:
|
||||
prepared_pid = prepared.get("prepared_pid") or browser_pid
|
||||
await client.call_tool(
|
||||
"wait", {"seconds": 1, "session": session_id}
|
||||
)
|
||||
prepared_windows = structured(
|
||||
await client.call_tool(
|
||||
"list_windows",
|
||||
{
|
||||
"on_screen_only": True,
|
||||
"session": session_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
prepared_row = next(
|
||||
(
|
||||
row
|
||||
for row in prepared_windows.get("windows") or []
|
||||
if row.get("pid") == prepared_pid
|
||||
),
|
||||
None,
|
||||
)
|
||||
prepared_window = (
|
||||
prepared_row.get("window_id")
|
||||
if prepared_row
|
||||
else browser_window
|
||||
)
|
||||
bound = structured(
|
||||
await client.call_tool(
|
||||
"get_browser_state",
|
||||
{
|
||||
"pid": prepared_pid,
|
||||
"window_id": prepared_window,
|
||||
"session": session_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
tabs = bound.get("tabs") or []
|
||||
tab_id = tabs[0].get("tab_id") if tabs else None
|
||||
target_id = bound.get("target_id")
|
||||
if (
|
||||
bound.get("status") == "ok"
|
||||
and bound.get("binding_quality") == "exact"
|
||||
and bound.get("mutation_allowed") is True
|
||||
and isinstance(tab_id, str)
|
||||
and isinstance(target_id, str)
|
||||
):
|
||||
snapshot = structured(
|
||||
await client.call_tool(
|
||||
"get_browser_state",
|
||||
{
|
||||
"target_id": target_id,
|
||||
"tab_id": tab_id,
|
||||
"snapshot_format": "semantic_v2",
|
||||
"session": session_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
navigated = structured(
|
||||
await client.call_tool(
|
||||
"browser_navigate",
|
||||
{
|
||||
"target_id": target_id,
|
||||
"tab_id": tab_id,
|
||||
"url": "about:blank",
|
||||
"session": session_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
fresh = structured(
|
||||
await client.call_tool(
|
||||
"get_browser_state",
|
||||
{
|
||||
"target_id": target_id,
|
||||
"tab_id": tab_id,
|
||||
"snapshot_format": "semantic_v2",
|
||||
"session": session_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
report["typed_browser"] = {
|
||||
"classification": "pass",
|
||||
"exact_binding": True,
|
||||
"mutation_allowed": True,
|
||||
"initial_snapshot": snapshot.get("status")
|
||||
in (None, "ok"),
|
||||
"mutation_transport": navigated.get("status")
|
||||
in (None, "ok"),
|
||||
"fresh_verification": fresh.get("status")
|
||||
in (None, "ok"),
|
||||
}
|
||||
else:
|
||||
report["typed_browser"] = {
|
||||
"classification": "unproven",
|
||||
"stage": "exact_binding",
|
||||
"code": refusal_code(bound),
|
||||
}
|
||||
finally:
|
||||
if (
|
||||
isinstance(launched_pid, int)
|
||||
and launched_pid not in prior_foreground_pids
|
||||
):
|
||||
await client.call_tool(
|
||||
"kill_app", {"pid": launched_pid, "session": session_id}
|
||||
)
|
||||
if (
|
||||
isinstance(isolated_browser_pid, int)
|
||||
and isolated_browser_pid != browser_pid
|
||||
):
|
||||
await client.call_tool(
|
||||
"kill_app",
|
||||
{"pid": isolated_browser_pid, "session": session_id},
|
||||
)
|
||||
await client.call_tool("end_session", {"session": session_id})
|
||||
finally:
|
||||
smoke_path.unlink(missing_ok=True)
|
||||
return report
|
||||
|
||||
|
||||
def main() -> int:
|
||||
report: dict[str, dict[str, Any]] = {
|
||||
"foreground": {"classification": "environment_unavailable"},
|
||||
"typed_browser": {"classification": "environment_unavailable"},
|
||||
}
|
||||
if sys.platform != "darwin":
|
||||
for cell in report.values():
|
||||
cell["stage"] = "macos_host_required"
|
||||
else:
|
||||
socket_path = os.environ.get(
|
||||
"CUA_DRIVER_LIVE_SOCKET", "/tmp/hermes-cua-0-9-live.sock"
|
||||
)
|
||||
if not Path(socket_path).is_socket():
|
||||
for cell in report.values():
|
||||
cell["stage"] = "isolated_daemon_required"
|
||||
else:
|
||||
try:
|
||||
report = asyncio.run(run_smoke(socket_path))
|
||||
except Exception as exc: # pragma: no cover - host/driver boundary
|
||||
report = {
|
||||
"foreground": {
|
||||
"classification": "environment_unavailable",
|
||||
"stage": "driver_connection",
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
"typed_browser": {
|
||||
"classification": "environment_unavailable",
|
||||
"stage": "driver_connection",
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
}
|
||||
print(json.dumps(report, indent=2, sort_keys=True))
|
||||
return int(any(cell.get("classification") != "pass" for cell in report.values()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user