#!/usr/bin/env python3 """Run ONE (arm, model, task, rep) cell of the PR #97979 A/B in an isolated process. Usage: worker.py Env: OPENROUTER_API_KEY must be set. Exit 3 = infra/config error (do not score). """ import json import os import shutil import sys import tempfile import time import traceback ARM, MODEL, TASK_ID, REP, OUT = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]), sys.argv[5] # Arm trees: plain checkouts of the two SHAs under test (git worktree/clone — # NEVER `pip install -e .` from them). Set both env vars before running: # ABDEFER_BASE_TREE=/path/to/checkout-of-baseline-sha # ABDEFER_PR_TREE=/path/to/checkout-of-pr-sha TREE = os.environ.get(f"ABDEFER_{ARM.upper()}_TREE") or "" if not TREE or not os.path.isdir(TREE): print(f"ABORT: ABDEFER_{ARM.upper()}_TREE not set or not a directory", file=sys.stderr) sys.exit(3) HARNESS = os.path.dirname(os.path.abspath(__file__)) if not os.environ.get("OPENROUTER_API_KEY"): print("ABORT: OPENROUTER_API_KEY missing", file=sys.stderr) sys.exit(3) # --- hermetic env BEFORE any hermes import ------------------------------- for var in list(os.environ): if var.endswith(("_API_KEY", "_TOKEN")) and var != "OPENROUTER_API_KEY": os.environ.pop(var, None) os.environ.pop("FAL_KEY", None) os.environ.pop("HERMES_PROFILE", None) tmp_root = tempfile.mkdtemp(prefix=f"ab-{ARM}-{TASK_ID}-") hermes_home = os.path.join(tmp_root, ".hermes") workspace = os.path.join(tmp_root, "ws") os.makedirs(hermes_home) os.makedirs(workspace) with open(os.path.join(hermes_home, "config.yaml"), "w", encoding="utf-8") as f: f.write("model:\n provider: openrouter\n model: %s\n" % MODEL) os.environ["HERMES_HOME"] = hermes_home os.environ["TERMINAL_CWD"] = workspace os.chdir(workspace) sys.path.insert(0, HARNESS) sys.path.insert(0, TREE) import tasks as taskmod # noqa: E402 TASK = taskmod.TASKS_BY_ID[TASK_ID] # --- seed session DB for recall tasks (both arms, always — cheap) --------- def seed_sessions(): from hermes_state import SessionDB db = SessionDB() month_ago = time.time() - 30 * 86400 def sess(sid, msgs, t0): db.create_session(sid, source="cli") t = t0 for role, content in msgs: db.append_message(sid, role, content=content, timestamp=t) t += 60 sess("seed_shadow_vet", [ ("user", "Back from the vet with Shadow. They put him on carprofen for the leg inflammation."), ("assistant", "Got it — what dose did they prescribe for Shadow?"), ("user", "Shadow's carprofen dose is 12.5 mg, twice a day with food. Two week course."), ("assistant", "Noted: Shadow takes 12.5 mg carprofen twice daily with food, for two weeks."), ], month_ago) sess("seed_backup_talk", [ ("user", "I keep worrying about my repos. The sparks-data repo really needs nightly backups, it has irreplaceable training data."), ("assistant", "Agreed — sparks-data should get a nightly backup job. The toybox repo is scratch space so it can be skipped."), ("user", "Right, toybox doesn't matter. Just sparks-data."), ], month_ago + 3 * 86400) sess("seed_decoy_cat", [ ("user", "My cat Biscuit is on 5 mg cetirizine for allergies."), ("assistant", "Noted — Biscuit: 5 mg cetirizine daily."), ], month_ago + 5 * 86400) sess("seed_decoy_dose", [ ("user", "I bumped the server worker count from 8 to 25 mg— sorry, to 25 workers. Typo."), ("assistant", "25 workers, got it."), ], month_ago + 6 * 86400) db.close() seed_sessions() if TASK.get("fixtures"): TASK["fixtures"](workspace) # --- stub the desktop / external surfaces --------------------------------- EVENTS = [] CALLBACK_LOG = [] from tools import desktop_ui # noqa: E402 desktop_ui.set_emitter(lambda sid, event, payload: EVENTS.append( {"sid": sid, "event": event, "payload": payload})) FOCUSED = taskmod.FOCUSED_APP PREVIEW_TITLE = taskmod.PREVIEW_TITLE TERMINAL_TAIL = taskmod.TERMINAL_TAIL WINDOW_BELOW = taskmod.WINDOW_BELOW_TEXT IMG_URL = taskmod.IMG_URL _clarify_answers = list(TASK.get("clarify_answers") or []) def clarify_cb(question, choices, multi_select=False): CALLBACK_LOG.append({"name": "clarify", "question": question, "choices": choices}) if _clarify_answers: ans = _clarify_answers.pop(0) else: ans = "Use your best judgement." if choices: for c in choices: if ans.lower() in str(c).lower(): return str(c) return ans def tour_cb(payload): CALLBACK_LOG.append({"name": "tour", "payload": payload}) action = payload.get("action", "") if action == "targets": return json.dumps({"success": True, "targets": [ {"selector": "[data-tour='settings']", "label": "Settings button", "stable": True}, {"selector": "[data-tour='composer']", "label": "Message composer", "stable": True}, {"selector": "[data-tour='sidebar']", "label": "Session sidebar", "stable": True}, {"selector": "[data-tour='model-picker']", "label": "Model picker", "stable": True}, ]}) if action in ("start", "steps", "show"): return json.dumps({"success": True, "shown": True, "steps_total": len(payload.get("steps") or []) or 1, "completed": True}) return json.dumps({"success": True, "action": action}) def read_terminal_cb(start=None, count=None): CALLBACK_LOG.append({"name": "read_terminal", "start": start, "count": count}) lines = ["$ make build", "compiling core...", "linking...", TERMINAL_TAIL] return json.dumps({"total_lines": 4, "start": 0, "end": 3, "viewport_rows": 24, "cursor_row": 3, "text": "\n".join(lines)}) def read_preview_cb(start=None, count=None): CALLBACK_LOG.append({"name": "read_preview", "start": start, "count": count}) return json.dumps({"title": PREVIEW_TITLE, "url": "https://example.com/docs/", "text": ("Example Domain\nThis domain is for use in documents.\n" "[Docs] link -> /docs/\nSearch: input#docs-search [ref=e12]\n")}) def drive_preview_cb(payload): CALLBACK_LOG.append({"name": "drive_preview", "payload": payload}) action = payload.get("action", "") if "annotate" in json.dumps(payload) or action in ("highlight", "point", "underline", "clear", "hold"): return json.dumps({"success": True, "annotated": payload.get("selector") or payload.get("ref")}) if action in ("click", "goto", "navigate"): return json.dumps({"success": True, "title": PREVIEW_TITLE, "url": "https://example.com/docs/", "text": "Docs index. Search box: input#docs-search [ref=e12]"}) if action in ("snapshot", "read", "links"): return json.dumps({"success": True, "title": PREVIEW_TITLE, "url": "https://example.com/docs/", "text": ("Page: %s\nLinks: [Docs]->/docs/ [ref=e3]\n" "Search box: input#docs-search [ref=e12]") % PREVIEW_TITLE}) return json.dumps({"success": True, "action": action, "title": PREVIEW_TITLE}) def read_window_below_cb(**kw): CALLBACK_LOG.append({"name": "read_window_below", "kw": kw}) return json.dumps({"title": "Invoices — draft", "text": WINDOW_BELOW}) def setup_mcp_cb(name, action, reason): CALLBACK_LOG.append({"name": "setup_mcp", "server": name, "action": action}) return json.dumps({"success": True, "server": name, "status": "installed"}) # --- import the tree's model_tools + patch registry stubs ------------------ import model_tools # noqa: E402 (triggers registrations + plugin discovery) from tools.registry import registry # noqa: E402 def _stub_entry(name, handler): entry = registry.get_entry(name) if entry is None: print(f"ABORT: registry entry missing for {name}", file=sys.stderr) sys.exit(3) entry.handler = handler entry.check_fn = None entry.is_async = False def computer_use_stub(args, **kw): CALLBACK_LOG.append({"name": "computer_use", "args": args}) action = (args or {}).get("action", "screenshot") shot = os.path.join(tmp_root, "screen.png") with open(shot, "wb") as f: f.write(b"\x89PNG\r\n\x1a\nstub") return json.dumps({ "success": True, "action": action, "screenshot": shot, "analysis": ("Focused window: %s. It shows a note titled 'Shadow feeding " "schedule' with a table of meal times. No error dialogs visible." % FOCUSED), }) def image_generate_stub(args, **kw): CALLBACK_LOG.append({"name": "image_generate", "args": args}) return json.dumps({"success": True, "image": IMG_URL, "prompt_used": (args or {}).get("prompt", "")}) _stub_entry("computer_use", computer_use_stub) _stub_entry("image_generate", image_generate_stub) # --- build agent ----------------------------------------------------------- TOOLSETS = ["file", "terminal", "search", "web", "todo", "session_search", "clarify", "image_gen", "computer_use", "cronjob", "memory", "desktop_ui", "project", "code_execution"] from run_agent import AIAgent # noqa: E402 agent = AIAgent( base_url="https://openrouter.ai/api/v1", api_key=os.environ["OPENROUTER_API_KEY"], provider="openrouter", model=MODEL, quiet_mode=True, skip_context_files=True, skip_memory=True, skip_background_review=True, enabled_toolsets=TOOLSETS, max_iterations=40, clarify_callback=clarify_cb, tour_callback=tour_cb, read_terminal_callback=read_terminal_cb, read_preview_callback=read_preview_cb, drive_preview_callback=drive_preview_cb, read_window_below_callback=read_window_below_cb, setup_mcp_callback=setup_mcp_cb, ) PREAMBLE = ("You are running inside the Hermes desktop app on the user's machine. " "Your working directory (the workspace) is: %s\n\nTask: " % workspace) t0 = time.time() error = None convo = None user_roundtrips = 0 try: convo = agent.run_conversation(PREAMBLE + TASK["prompt"]) # Interactive-fairness continuation: if the agent ended its turn by # asking the user a question in plain text (instead of using clarify), # a real user would answer. Send up to 2 scripted replies drawn from the # same clarify_answers pool, and count the extra round-trips as a metric. for _ in range(2): _msgs = (convo or {}).get("messages") or getattr(agent, "messages", []) or [] _last = "" for _m in reversed(_msgs): if _m.get("role") == "assistant" and (_m.get("content") or "").strip(): _last = _m["content"].strip() break if "?" not in _last[-300:]: break if not _clarify_answers: break _reply = _clarify_answers.pop(0) user_roundtrips += 1 convo = agent.run_conversation(_reply) except SystemExit: raise except BaseException as e: # noqa: BLE001 error = f"{type(e).__name__}: {e}" traceback.print_exc() wall = time.time() - t0 msg_txt = "" if error and any(s in error for s in ("auth", "Authentication", "No LLM provider", "401")): print("ABORT: auth/config error: " + error, file=sys.stderr) sys.exit(3) messages = (convo or {}).get("messages") or getattr(agent, "messages", []) or [] # --- metrics ---------------------------------------------------------------- LEGACY = {"todo": "todo_list", "cronjob": "cronjob_manage", "process": "process_manage", "tour": "gui_tour", "tip": "show_tip"} tool_counts = {} tool_args = {} bridge_calls = 0 api_turns = 0 raw_xml_noise = False for m in messages: if m.get("role") == "assistant": api_turns += 1 if "