Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Interact with the in-app browser / preview pane in the Hermes desktop GUI.
|
||||
|
||||
``open_preview`` shows a page and ``read_preview`` reads it; this tool is the
|
||||
third leg — clicking, typing, scrolling, and history — so the agent can drive
|
||||
the same page the user is looking at instead of narrating from the outside.
|
||||
|
||||
Elements are addressed by refs from ``action="elements"`` that say what they
|
||||
are: ``btn-sign-in``, ``inp-email``. A ref lasts as long as the page is open,
|
||||
including across a re-render that destroys and rebuilds the element, and only a
|
||||
navigation retires it — the renderer says so rather than acting on whatever now
|
||||
occupies the spot.
|
||||
|
||||
Because the refs hold, the renderer answers with a *delta* — what appeared,
|
||||
what went, what changed, and what was rebound — instead of re-sending the whole
|
||||
inventory after every click. That is the cheap half of the arrangement, and it
|
||||
only works because the refs are legible enough to read on their own three turns
|
||||
later.
|
||||
|
||||
Round-trips through the gateway's blocking-prompt bridge like ``read_preview``:
|
||||
tui_gateway emits ``preview.act.request``, the renderer injects the interaction
|
||||
engine into the pane's webview and answers ``preview.act.respond`` with the
|
||||
outcome plus whatever moved. This module is just schema + a thin dispatcher
|
||||
over the platform-injected callback.
|
||||
|
||||
Lives in the ``desktop_ui`` toolset, which the GUI gateway enables only for
|
||||
desktop-sourced sessions.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Callable, Optional
|
||||
|
||||
from tools.registry import registry, tool_error
|
||||
|
||||
ACTIONS = (
|
||||
"elements",
|
||||
"click",
|
||||
"hover",
|
||||
"type",
|
||||
"scroll",
|
||||
"press",
|
||||
"strobe",
|
||||
"back",
|
||||
"forward",
|
||||
"reload",
|
||||
)
|
||||
SCROLL_TO = ("top", "bottom")
|
||||
|
||||
# Verbs that need something to act on — a ref from the last inventory, or a
|
||||
# raw CSS selector. `scroll` is deliberately absent: bare, it scrolls the page.
|
||||
NEEDS_TARGET = ("click", "hover", "type", "press")
|
||||
|
||||
|
||||
def drive_preview_tool(
|
||||
action: str = "",
|
||||
ref: Optional[str] = None,
|
||||
selector: Optional[str] = None,
|
||||
text: Optional[str] = None,
|
||||
key: Optional[str] = None,
|
||||
submit: Optional[bool] = None,
|
||||
amount: Optional[int] = None,
|
||||
to: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
full: Optional[bool] = None,
|
||||
callback: Optional[Callable] = None,
|
||||
) -> str:
|
||||
"""Dispatch one interaction to the desktop renderer and return its outcome."""
|
||||
if callback is None:
|
||||
return tool_error("drive_preview is only available in the Hermes desktop app.")
|
||||
|
||||
verb = (action or "").strip().lower()
|
||||
if verb not in ACTIONS:
|
||||
return tool_error(f"action must be one of: {', '.join(ACTIONS)}.")
|
||||
|
||||
if verb in NEEDS_TARGET and not (ref or selector):
|
||||
return tool_error(
|
||||
f"{verb} needs a ref from action='elements' (e.g. 'btn-sign-in') or a CSS selector."
|
||||
)
|
||||
|
||||
if verb == "type" and text is None:
|
||||
return tool_error("type needs the text to enter.")
|
||||
|
||||
if verb == "press" and not key:
|
||||
return tool_error("press needs a key, e.g. 'Enter' or 'Escape'.")
|
||||
|
||||
if to is not None and to not in SCROLL_TO:
|
||||
return tool_error(f"to must be one of: {', '.join(SCROLL_TO)}.")
|
||||
|
||||
try:
|
||||
payload = {
|
||||
name: val
|
||||
for name, val in (
|
||||
("action", verb),
|
||||
("ref", ref),
|
||||
("selector", selector),
|
||||
("text", text),
|
||||
("key", key),
|
||||
("submit", submit),
|
||||
("full", full),
|
||||
("to", to),
|
||||
("amount", None if amount is None else int(amount)),
|
||||
("max", None if limit is None else int(limit)),
|
||||
)
|
||||
if val is not None
|
||||
}
|
||||
except (TypeError, ValueError):
|
||||
return tool_error("amount and max must be integers.")
|
||||
|
||||
try:
|
||||
raw = callback(payload)
|
||||
except Exception as exc:
|
||||
return tool_error(f"Failed to act on the in-app browser: {exc}")
|
||||
|
||||
if not raw:
|
||||
return tool_error(
|
||||
"The action timed out, or no GUI window answered. "
|
||||
"Open a page with open_preview first."
|
||||
)
|
||||
|
||||
# The renderer answers with a JSON object; pass it through, else wrap it.
|
||||
try:
|
||||
return json.dumps(json.loads(raw), ensure_ascii=False)
|
||||
except (TypeError, ValueError):
|
||||
return json.dumps({"text": str(raw)}, ensure_ascii=False)
|
||||
|
||||
|
||||
ACT_PREVIEW_SCHEMA = {
|
||||
"name": "drive_preview",
|
||||
# Dieted (#95681): world-building compressed; response-shape teaching
|
||||
# kept only where skipping it causes wasted calls (delta semantics,
|
||||
# rebound refs, strobe's burst) — those are pre-effect: a model that
|
||||
# doesn't know them re-reads pages or loops strobe.
|
||||
"description": (
|
||||
"Use the web page open in the desktop preview pane (the one "
|
||||
"`desktop_preview` opens): log in, fill forms, click through flows. ALWAYS "
|
||||
"start with action='elements' — it inventories clickable/typable "
|
||||
"things as refs ('btn-sign-in') with role/label/value; act by ref, "
|
||||
"not guessed selectors. Refs survive re-renders and only die on "
|
||||
"navigation (you'll be told they're stale — call elements again). "
|
||||
"After the first full inventory, actions answer with a DELTA: "
|
||||
"'added' in full, 'changed' as ref + moved fields, 'removed'/"
|
||||
"'rebound' as ref lists ('rebound' needs nothing from you — the ref "
|
||||
"already follows the rebuilt element). Anything unmentioned is "
|
||||
"unchanged; do not re-read to check. Input is real (pointer travels, "
|
||||
"hover menus open). Actions: elements, click, hover (park the "
|
||||
"pointer — opens dropdowns before clicking in), type (submit=true "
|
||||
"also presses Enter), scroll, press, strobe (visual flourish only — "
|
||||
"one call runs a multi-second burst; never loop it), back/forward/"
|
||||
"reload. Moves draw live and fade; annotate_preview leaves a lasting "
|
||||
"mark. Page text only: desktop_preview action=read. Separate automated "
|
||||
"browser: browser_* tools."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": list(ACTIONS),
|
||||
"description": "Start with 'elements'.",
|
||||
},
|
||||
"ref": {
|
||||
"type": "string",
|
||||
"description": "Element ref from an earlier elements call.",
|
||||
},
|
||||
"selector": {
|
||||
"type": "string",
|
||||
"description": "CSS selector fallback. Prefer ref.",
|
||||
},
|
||||
"text": {"type": "string", "description": "type: the text."},
|
||||
"submit": {
|
||||
"type": "boolean",
|
||||
"description": "type: press Enter + submit the form after.",
|
||||
},
|
||||
"key": {
|
||||
"type": "string",
|
||||
"description": "press: key name ('Enter', 'Escape', 'ArrowDown').",
|
||||
},
|
||||
"amount": {
|
||||
"type": "integer",
|
||||
"description": "scroll: pixels (negative = up; default ~one screen).",
|
||||
},
|
||||
"to": {
|
||||
"type": "string",
|
||||
"enum": list(SCROLL_TO),
|
||||
"description": "scroll: jump to top/bottom instead.",
|
||||
},
|
||||
"max": {
|
||||
"type": "integer",
|
||||
"description": "elements: cap the inventory.",
|
||||
},
|
||||
"full": {
|
||||
"type": "boolean",
|
||||
"description": "elements: full re-read instead of a delta. Rarely needed.",
|
||||
},
|
||||
},
|
||||
"required": ["action"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
registry.register(
|
||||
name="drive_preview",
|
||||
toolset="desktop_ui",
|
||||
schema=ACT_PREVIEW_SCHEMA,
|
||||
handler=lambda args, **kw: drive_preview_tool(
|
||||
action=args.get("action", ""),
|
||||
ref=args.get("ref"),
|
||||
selector=args.get("selector"),
|
||||
text=args.get("text"),
|
||||
key=args.get("key"),
|
||||
submit=args.get("submit"),
|
||||
amount=args.get("amount"),
|
||||
to=args.get("to"),
|
||||
limit=args.get("max"),
|
||||
full=args.get("full"),
|
||||
callback=kw.get("callback"),
|
||||
),
|
||||
emoji="🖱️",
|
||||
)
|
||||
Reference in New Issue
Block a user