220 lines
8.1 KiB
Python
220 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Interact with the in-app browser / preview pane in the Hermes desktop GUI.
|
|
|
|
``open_preview`` shows a page and ``read_preview`` reads it; this tool is the
|
|
third leg — clicking, typing, scrolling, and history — so the agent can drive
|
|
the same page the user is looking at instead of narrating from the outside.
|
|
|
|
Elements are addressed by refs from ``action="elements"`` that say what they
|
|
are: ``btn-sign-in``, ``inp-email``. A ref lasts as long as the page is open,
|
|
including across a re-render that destroys and rebuilds the element, and only a
|
|
navigation retires it — the renderer says so rather than acting on whatever now
|
|
occupies the spot.
|
|
|
|
Because the refs hold, the renderer answers with a *delta* — what appeared,
|
|
what went, what changed, and what was rebound — instead of re-sending the whole
|
|
inventory after every click. That is the cheap half of the arrangement, and it
|
|
only works because the refs are legible enough to read on their own three turns
|
|
later.
|
|
|
|
Round-trips through the gateway's blocking-prompt bridge like ``read_preview``:
|
|
tui_gateway emits ``preview.act.request``, the renderer injects the interaction
|
|
engine into the pane's webview and answers ``preview.act.respond`` with the
|
|
outcome plus whatever moved. This module is just schema + a thin dispatcher
|
|
over the platform-injected callback.
|
|
|
|
Lives in the ``desktop_ui`` toolset, which the GUI gateway enables only for
|
|
desktop-sourced sessions.
|
|
"""
|
|
|
|
import json
|
|
from typing import Callable, Optional
|
|
|
|
from tools.registry import registry, tool_error
|
|
|
|
ACTIONS = (
|
|
"elements",
|
|
"click",
|
|
"hover",
|
|
"type",
|
|
"scroll",
|
|
"press",
|
|
"strobe",
|
|
"back",
|
|
"forward",
|
|
"reload",
|
|
)
|
|
SCROLL_TO = ("top", "bottom")
|
|
|
|
# Verbs that need something to act on — a ref from the last inventory, or a
|
|
# raw CSS selector. `scroll` is deliberately absent: bare, it scrolls the page.
|
|
NEEDS_TARGET = ("click", "hover", "type", "press")
|
|
|
|
|
|
def drive_preview_tool(
|
|
action: str = "",
|
|
ref: Optional[str] = None,
|
|
selector: Optional[str] = None,
|
|
text: Optional[str] = None,
|
|
key: Optional[str] = None,
|
|
submit: Optional[bool] = None,
|
|
amount: Optional[int] = None,
|
|
to: Optional[str] = None,
|
|
limit: Optional[int] = None,
|
|
full: Optional[bool] = None,
|
|
callback: Optional[Callable] = None,
|
|
) -> str:
|
|
"""Dispatch one interaction to the desktop renderer and return its outcome."""
|
|
if callback is None:
|
|
return tool_error("drive_preview is only available in the Hermes desktop app.")
|
|
|
|
verb = (action or "").strip().lower()
|
|
if verb not in ACTIONS:
|
|
return tool_error(f"action must be one of: {', '.join(ACTIONS)}.")
|
|
|
|
if verb in NEEDS_TARGET and not (ref or selector):
|
|
return tool_error(
|
|
f"{verb} needs a ref from action='elements' (e.g. 'btn-sign-in') or a CSS selector."
|
|
)
|
|
|
|
if verb == "type" and text is None:
|
|
return tool_error("type needs the text to enter.")
|
|
|
|
if verb == "press" and not key:
|
|
return tool_error("press needs a key, e.g. 'Enter' or 'Escape'.")
|
|
|
|
if to is not None and to not in SCROLL_TO:
|
|
return tool_error(f"to must be one of: {', '.join(SCROLL_TO)}.")
|
|
|
|
try:
|
|
payload = {
|
|
name: val
|
|
for name, val in (
|
|
("action", verb),
|
|
("ref", ref),
|
|
("selector", selector),
|
|
("text", text),
|
|
("key", key),
|
|
("submit", submit),
|
|
("full", full),
|
|
("to", to),
|
|
("amount", None if amount is None else int(amount)),
|
|
("max", None if limit is None else int(limit)),
|
|
)
|
|
if val is not None
|
|
}
|
|
except (TypeError, ValueError):
|
|
return tool_error("amount and max must be integers.")
|
|
|
|
try:
|
|
raw = callback(payload)
|
|
except Exception as exc:
|
|
return tool_error(f"Failed to act on the in-app browser: {exc}")
|
|
|
|
if not raw:
|
|
return tool_error(
|
|
"The action timed out, or no GUI window answered. "
|
|
"Open a page with open_preview first."
|
|
)
|
|
|
|
# The renderer answers with a JSON object; pass it through, else wrap it.
|
|
try:
|
|
return json.dumps(json.loads(raw), ensure_ascii=False)
|
|
except (TypeError, ValueError):
|
|
return json.dumps({"text": str(raw)}, ensure_ascii=False)
|
|
|
|
|
|
ACT_PREVIEW_SCHEMA = {
|
|
"name": "drive_preview",
|
|
# Dieted (#95681): world-building compressed; response-shape teaching
|
|
# kept only where skipping it causes wasted calls (delta semantics,
|
|
# rebound refs, strobe's burst) — those are pre-effect: a model that
|
|
# doesn't know them re-reads pages or loops strobe.
|
|
"description": (
|
|
"Use the web page open in the desktop preview pane (the one "
|
|
"`desktop_preview` opens): log in, fill forms, click through flows. ALWAYS "
|
|
"start with action='elements' — it inventories clickable/typable "
|
|
"things as refs ('btn-sign-in') with role/label/value; act by ref, "
|
|
"not guessed selectors. Refs survive re-renders and only die on "
|
|
"navigation (you'll be told they're stale — call elements again). "
|
|
"After the first full inventory, actions answer with a DELTA: "
|
|
"'added' in full, 'changed' as ref + moved fields, 'removed'/"
|
|
"'rebound' as ref lists ('rebound' needs nothing from you — the ref "
|
|
"already follows the rebuilt element). Anything unmentioned is "
|
|
"unchanged; do not re-read to check. Input is real (pointer travels, "
|
|
"hover menus open). Actions: elements, click, hover (park the "
|
|
"pointer — opens dropdowns before clicking in), type (submit=true "
|
|
"also presses Enter), scroll, press, strobe (visual flourish only — "
|
|
"one call runs a multi-second burst; never loop it), back/forward/"
|
|
"reload. Moves draw live and fade; annotate_preview leaves a lasting "
|
|
"mark. Page text only: desktop_preview action=read. Separate automated "
|
|
"browser: browser_* tools."
|
|
),
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"action": {
|
|
"type": "string",
|
|
"enum": list(ACTIONS),
|
|
"description": "Start with 'elements'.",
|
|
},
|
|
"ref": {
|
|
"type": "string",
|
|
"description": "Element ref from an earlier elements call.",
|
|
},
|
|
"selector": {
|
|
"type": "string",
|
|
"description": "CSS selector fallback. Prefer ref.",
|
|
},
|
|
"text": {"type": "string", "description": "type: the text."},
|
|
"submit": {
|
|
"type": "boolean",
|
|
"description": "type: press Enter + submit the form after.",
|
|
},
|
|
"key": {
|
|
"type": "string",
|
|
"description": "press: key name ('Enter', 'Escape', 'ArrowDown').",
|
|
},
|
|
"amount": {
|
|
"type": "integer",
|
|
"description": "scroll: pixels (negative = up; default ~one screen).",
|
|
},
|
|
"to": {
|
|
"type": "string",
|
|
"enum": list(SCROLL_TO),
|
|
"description": "scroll: jump to top/bottom instead.",
|
|
},
|
|
"max": {
|
|
"type": "integer",
|
|
"description": "elements: cap the inventory.",
|
|
},
|
|
"full": {
|
|
"type": "boolean",
|
|
"description": "elements: full re-read instead of a delta. Rarely needed.",
|
|
},
|
|
},
|
|
"required": ["action"],
|
|
},
|
|
}
|
|
|
|
|
|
registry.register(
|
|
name="drive_preview",
|
|
toolset="desktop_ui",
|
|
schema=ACT_PREVIEW_SCHEMA,
|
|
handler=lambda args, **kw: drive_preview_tool(
|
|
action=args.get("action", ""),
|
|
ref=args.get("ref"),
|
|
selector=args.get("selector"),
|
|
text=args.get("text"),
|
|
key=args.get("key"),
|
|
submit=args.get("submit"),
|
|
amount=args.get("amount"),
|
|
to=args.get("to"),
|
|
limit=args.get("max"),
|
|
full=args.get("full"),
|
|
callback=kw.get("callback"),
|
|
),
|
|
emoji="🖱️",
|
|
)
|