Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

evals/core_tool_deferral/README.md

@@ -0,0 +1,73 @@
+# core_tool_deferral — live A/B harness for tool-visibility changes
+
+Built for the PR #97979 maintainer battery (core-tool deferral behind the
+tool_search bridge). Runs REAL in-process `AIAgent`s from two pinned
+checkouts and grades task outcomes programmatically — accuracy, api turns,
+tokens, wall, bridge-call counts — across any set of models.
+
+Original verdict + full numbers: `results/SUMMARY.md` and the PR #97979 body
+(288 runs; gpt-5.6-terra / glm-5.3-flash / qwen3.8-27b).
+
+## Layout
+
+- `tasks.py` — 14-task battery: one task per deferred tool, multistep
+  (todo discipline, GUI chains), long-range (session_search → backup →
+  cron → todo), a destructive-ambiguity clarify trap, an eager-only
+  control, and a false-discovery distractor. Each task carries fixtures,
+  a programmatic grader (0–1 partial credit), and scripted user replies.
+- `worker.py` — one (arm, model, task, rep) cell in an isolated
+  subprocess: temp HERMES_HOME + workspace, hermetic env (only
+  OPENROUTER_API_KEY survives), seeded session DB (targets + decoys),
+  deterministic desktop-surface stubs (desktop_ui emitter + agent
+  callbacks), computer_use/image_generate stubbed at the registry
+  handler. Terminal/files/cron/process/session-DB are REAL.
+  Exit 3 = infra/config error (never scored).
+- `orchestrator.py` — battery runner: resume-safe, per-task wall
+  timeouts, parallel cells, errored-record retry, 3-infra-abort fuse.
+- `report.py` — per-task table both arms (score spread, turns, tok, wall,
+  bridge calls), mean-of-task-means, noise/error accounting.
+
+## Running
+
+```bash
+# 1. Two plain checkouts pinned to the SHAs under test (never pip install -e)
+git worktree add /tmp/abdefer-base <baseline-sha>
+git worktree add /tmp/abdefer-pr   <pr-sha>
+
+export ABDEFER_BASE_TREE=/tmp/abdefer-base
+export ABDEFER_PR_TREE=/tmp/abdefer-pr
+export OPENROUTER_API_KEY=...   # the only key the worker keeps
+
+# 2. Smoke one cheap cell first
+python3 worker.py base openai/gpt-5.6-terra config_grep_distractor 1 /tmp/smoke.json
+
+# 3. Battery (per model; start with the STRONGEST model to validate variance)
+python3 orchestrator.py openai/gpt-5.6-terra 3 --parallel=5
+python3 orchestrator.py z-ai/glm-5.3-flash 3 --parallel=5
+python3 orchestrator.py qwen/qwen3.8-27b   3 --parallel=5
+
+# 4. Readout
+python3 report.py
+```
+
+`ABDEFER_PYTHON` overrides the worker interpreter (defaults to the
+orchestrator's own); `ABDEFER_RESULTS` overrides the results root.
+
+## Discipline (from the readtool/session_search harness lineage)
+
+- Verify model slugs against the live OpenRouter list before launching.
+- Interactive fairness: if the agent ends its turn with a plain-text
+  question, the worker sends the scripted reply (max 2, counted as
+  `user_roundtrips`) — without this, every clarify-shaped task scores 0
+  unfairly and the battery is poisoned (the first terra run was discarded
+  for exactly this).
+- Same-denominator rule: errored runs score 0 and STAY in the accuracy
+  denominator; they are excluded from efficiency means.
+- Extend contested cells (score spread at n=3) to n=6 before concluding.
+- For discovery-rate regressions, always check base-arm usage on the same
+  tasks first — a tool models skip even when visible is not a deferral
+  regression.
+- Audit anomalous cells from `*.transcript.json` before publishing.
+
+`results/` is gitignored except SUMMARY.md — rep JSONs are rebuildable,
+verdicts are the artifact.

evals/core_tool_deferral/orchestrator.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""Orchestrate the PR #97979 A/B battery. Resume-safe; per-run wall timeout.
+
+Usage: orchestrator.py <model_slug> <reps> [--tasks id1,id2] [--arms base,pr] [--parallel N]
+Results land in results/<model_short>/<arm>__<task>__rep<r>.json (override
+the results root with ABDEFER_RESULTS).
+"""
+import json
+import os
+import subprocess
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+HARNESS = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HARNESS)
+import tasks as taskmod
+
+MODEL = sys.argv[1]
+REPS = int(sys.argv[2])
+task_ids = [t["id"] for t in taskmod.TASKS]
+arms = ["base", "pr"]
+parallel = 4
+for a in sys.argv[3:]:
+    if a.startswith("--tasks="):
+        task_ids = a.split("=", 1)[1].split(",")
+    elif a.startswith("--arms="):
+        arms = a.split("=", 1)[1].split(",")
+    elif a.startswith("--parallel="):
+        parallel = int(a.split("=", 1)[1])
+
+short = MODEL.split("/")[-1]
+RESULTS = os.path.join(os.environ.get("ABDEFER_RESULTS", os.path.join(HARNESS, "results")), short)
+os.makedirs(RESULTS, exist_ok=True)
+PY = os.environ.get("ABDEFER_PYTHON", sys.executable)
+
+cells = []
+for task_id in task_ids:
+    for arm in arms:
+        for rep in range(1, REPS + 1):
+            out = f"{RESULTS}/{arm}__{task_id}__rep{rep}.json"
+            if os.path.exists(out):
+                try:
+                    with open(out, encoding="utf-8") as f:
+                        rec = json.load(f)
+                    if rec.get("error") is None or rec.get("score", 0) > 0:
+                        continue  # keep good/attempted records
+                    # errored record -> retry
+                    os.remove(out)
+                except Exception:
+                    os.remove(out)
+            cells.append((arm, task_id, rep, out))
+
+print(f"model={MODEL} cells to run: {len(cells)} (parallel={parallel})", flush=True)
+
+def run_cell(cell):
+    arm, task_id, rep, out = cell
+    timeout = taskmod.TASKS_BY_ID[task_id].get("timeout", 600)
+    cmd = [PY, os.path.join(HARNESS, "worker.py"), arm, MODEL, task_id, str(rep), out]
+    t0 = time.time()
+    try:
+        p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout + 60,
+                           env=os.environ.copy())
+        if p.returncode == 3:
+            return (cell, "INFRA_ABORT", p.stderr[-500:])
+        if p.returncode != 0 and not os.path.exists(out):
+            rec = {"arm": arm, "model": MODEL, "task": task_id, "rep": rep,
+                   "score": 0.0, "error": f"worker exit {p.returncode}",
+                   "notes": [p.stderr[-400:]], "api_turns": None,
+                   "total_tokens": None, "wall_s": round(time.time() - t0, 1),
+                   "bridge_calls": None, "tool_calls_total": None,
+                   "tool_counts": {}, "raw_xml_noise": False}
+            with open(out, "w", encoding="utf-8") as f:
+                json.dump(rec, f, indent=1)
+            return (cell, "WORKER_ERR", p.stderr[-300:])
+        return (cell, "OK", p.stdout.strip().splitlines()[-1] if p.stdout.strip() else "")
+    except subprocess.TimeoutExpired:
+        rec = {"arm": arm, "model": MODEL, "task": task_id, "rep": rep,
+               "score": 0.0, "error": "wall timeout", "notes": ["hard wall timeout"],
+               "api_turns": None, "total_tokens": None,
+               "wall_s": round(time.time() - t0, 1), "bridge_calls": None,
+               "tool_calls_total": None, "tool_counts": {}, "raw_xml_noise": False}
+        with open(out, "w", encoding="utf-8") as f:
+            json.dump(rec, f, indent=1)
+        return (cell, "TIMEOUT", "")
+
+done = 0
+infra_aborts = 0
+with ThreadPoolExecutor(max_workers=parallel) as ex:
+    futs = {ex.submit(run_cell, c): c for c in cells}
+    for fut in as_completed(futs):
+        cell, status, info = fut.result()
+        done += 1
+        print(f"[{done}/{len(cells)}] {cell[0]}/{cell[1]}/rep{cell[2]}: {status} {info}", flush=True)
+        if status == "INFRA_ABORT":
+            infra_aborts += 1
+            if infra_aborts >= 3:
+                print("FATAL: 3 infra aborts — stopping battery", flush=True)
+                sys.exit(3)
+print("BATTERY COMPLETE", flush=True)

evals/core_tool_deferral/report.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Aggregate A/B results. Usage: report.py [model_short ...]"""
+import json
+import glob
+import os
+import statistics
+import sys
+
+BASE = os.environ.get("ABDEFER_RESULTS", os.path.join(os.path.dirname(os.path.abspath(__file__)), "results"))
+models = sys.argv[1:] or sorted(
+    d for d in os.listdir(BASE) if os.path.isdir(os.path.join(BASE, d)) and d != "smoke")
+
+def load(model):
+    recs = []
+    for p in glob.glob(f"{BASE}/{model}/*.json"):
+        if p.endswith(".transcript.json"):
+            continue
+        with open(p, encoding="utf-8") as f:
+            recs.append(json.load(f))
+    return recs
+
+def fmt(v, nd=1):
+    return "-" if v is None else (f"{v:.{nd}f}" if isinstance(v, float) else str(v))
+
+for model in models:
+    recs = load(model)
+    if not recs:
+        continue
+    tasks = sorted({r["task"] for r in recs})
+    print(f"\n{'='*100}\nMODEL: {model}   (runs: {len(recs)})\n{'='*100}")
+    hdr = f"{'task':<28} | {'arm':<4} | {'n':>1} | {'score':>10} | {'turns':>6} | {'tok(k)':>7} | {'wall':>6} | {'bridge':>6} | {'err':>3}"
+    print(hdr)
+    print("-" * len(hdr))
+    agg = {"base": {"s": [], "t": [], "k": [], "w": []}, "pr": {"s": [], "t": [], "k": [], "w": []}}
+    for task in tasks:
+        for arm in ("base", "pr"):
+            rs = [r for r in recs if r["task"] == task and r["arm"] == arm]
+            if not rs:
+                continue
+            scores = [r["score"] for r in rs]
+            ok = [r for r in rs if not r.get("error")]
+            turns = [r["api_turns"] for r in ok if r.get("api_turns")]
+            toks = [r["total_tokens"] for r in ok if r.get("total_tokens")]
+            walls = [r["wall_s"] for r in ok if r.get("wall_s")]
+            bridges = [r.get("bridge_calls") or 0 for r in ok]
+            nerr = sum(1 for r in rs if r.get("error"))
+            smean = statistics.mean(scores)
+            sspread = f"{smean:.2f} [{min(scores):.1f}-{max(scores):.1f}]"
+            print(f"{task:<28} | {arm:<4} | {len(rs)} | {sspread:>10} | "
+                  f"{fmt(statistics.mean(turns) if turns else None):>6} | "
+                  f"{fmt(statistics.mean(toks)/1000 if toks else None):>7} | "
+                  f"{fmt(statistics.mean(walls) if walls else None):>6} | "
+                  f"{fmt(statistics.mean(bridges) if bridges else None):>6} | {nerr:>3}")
+            agg[arm]["s"].append(smean)
+            if turns: agg[arm]["t"].append(statistics.mean(turns))
+            if toks: agg[arm]["k"].append(statistics.mean(toks))
+            if walls: agg[arm]["w"].append(statistics.mean(walls))
+    print("-" * len(hdr))
+    for arm in ("base", "pr"):
+        a = agg[arm]
+        if a["s"]:
+            print(f"{'MEAN-OF-TASK-MEANS':<28} | {arm:<4} |   | {statistics.mean(a['s']):>10.3f} | "
+                  f"{fmt(statistics.mean(a['t']) if a['t'] else None):>6} | "
+                  f"{fmt(statistics.mean(a['k'])/1000 if a['k'] else None):>7} | "
+                  f"{fmt(statistics.mean(a['w']) if a['w'] else None):>6} |")
+    noise = [r for r in recs if r.get("raw_xml_noise")]
+    errs = [r for r in recs if r.get("error")]
+    if noise:
+        print(f"raw-XML noise runs: {len(noise)} -> " + ", ".join(f"{r['arm']}/{r['task']}/r{r['rep']}" for r in noise))
+    if errs:
+        print(f"errored runs: {len(errs)} -> " + ", ".join(f"{r['arm']}/{r['task']}/r{r['rep']}: {r['error'][:60]}" for r in errs))

evals/core_tool_deferral/results/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!SUMMARY.md

evals/core_tool_deferral/results/SUMMARY.md

@@ -0,0 +1,75 @@
+# PR #97979 A/B verdict — core-tool deferral (288 live runs)
+
+Date: 2026-08-29 · Harness: /tmp/ab97979/harness · Method: METHOD.md
+
+## Arms
+base = origin/main 3f36c87e1ebd (27 direct tools in the eval assembly, 47.4KB schema chars)
+pr   = main + #97979 e16ad33a9d24 (12 direct: 9 working set + 3 bridge; 19 deferred; 21.0KB schema chars, −56%)
+
+## Headline (mean of task means, 14 tasks × 3 reps; contested cells re-run to n=6)
+
+| model | arm | accuracy | turns | tokens(k) | wall(s) |
+|---|---|---|---|---|---|
+| gpt-5.6-terra (large) | base | 0.938 | 6.0 | 80.9 | 27.6 |
+| gpt-5.6-terra | pr | 0.879 | 6.6 | **62.5 (−23%)** | 27.1 |
+| glm-5.3-flash (medium) | base | 0.915 | 6.0 | 101.0 | 56.5 |
+| glm-5.3-flash | pr | **0.963 (+0.05)** | 8.8 | **89.6 (−11%)** | 59.5 |
+| qwen3.8-27b (small) | base | 0.915 | 7.1 | 127.4 | 53.5 |
+| qwen3.8-27b | pr | 0.907 | 9.4 | **118.4 (−7%)** | 79.2 |
+
+Grand accuracy: base 0.923 vs pr 0.916 — flat within rep noise once the two
+contested tasks were extended to n=6. Tokens down on every model. Turns up
+~1–2 (bridge discovery round-trips), wall flat on terra/glm, +48% on qwen
+(27B pays real latency for extra bridge turns).
+
+## Deferred-tool discovery (PR arm, tasks requiring the tool, all models)
+Perfect (9/9 or 18/18): session_search, todo_list, image_generate,
+desktop_project, desktop_preview, drive_preview, annotate_preview,
+apply_layout, focus_pane, read_terminal, read_window_below.
+Near-perfect: cronjob_manage 16/18, gui_tour 8/9, process_manage 8/9.
+Weak: computer_use 6/9, show_tip 6/9, clarify 7/18, setup_mcp 4/9*,
+close_terminal 4/9*.
+(*base-arm usage on the same tasks: setup_mcp 3/9, close_terminal 0/9 —
+these two are NOT deferral regressions; models skip them even when visible.)
+
+## The one real regression: clarify
+base: clarify used 18/18, score 1.00 on the ambiguous-delete trap, all models.
+pr: clarify used 7/18 → terra 0/6 (0.50), glm 3/6 (0.80), qwen 4/6 (0.87).
+Models still ask — but as plain text, ending the turn (extra user round-trip,
+no structured choices). The harness credits scripted replies; without that
+continuation the task scores 0. Exactly trade-off #1 flagged in the PR body.
+Safety note: in 0 of 288 runs was the WRONG file deleted — the failure mode
+is degraded UX, never destructive action.
+
+## screenshot_ambiguous (n=6): split, not directional
+terra base 1.00 → pr 0.67 (2 reps answered from read_window_below instead of
+discovering computer_use — catalog-stub misrouting to a cheaper adjacent tool);
+but glm 0.67→1.00 and qwen 0.50→0.83 IMPROVED under deferral (the focused
+catalog line beats 27 competing schemas for weaker models). Model-split, nets
+to ~flat across the tier ladder.
+
+## Controls
+eager_refactor_control (eager-only tools): pr arm −49% tokens at held 1.00 —
+pure schema-shrink win, no behavior change.
+config_grep_distractor: 1.00 both arms, 0 false bridge calls on terra/glm —
+no discovery-overhead tax on tasks that don't need deferred tools.
+
+## Anomalies audited
+- glm pr layout rep2 (41 turns, 514k tok): after completing the GUI task via
+  bridge it burned 30 terminal calls "verifying"; score 1.0. Model paranoia,
+  not a bridge failure.
+- qwen pr screenshot rep3: hard wall timeout, scored 0, kept in denominator.
+- 1 errored run / 288 total; raw-XML provider noise: 0.
+
+## Verdict: SHIP, with one follow-up — un-defer (or pin) `clarify`.
+The deferral mechanism works: discovery is essentially perfect for 14/19
+tools, accuracy is flat overall (large model −0.06, medium +0.05, small
+−0.01), token cost drops on every model, and the eager-surface control shows
+the −49%-token schema win with zero accuracy cost. The single consistent
+regression is clarify: structured ask-the-user collapses to plain-text
+questions when the schema is invisible (7/18 vs 18/18). PR #91125
+(always-visible deferred-tool pins) is the natural mechanism — pin clarify
+eager by default, or drop it from _DEFAULT_DEFERRED_TOOLS (~250 tok cost).
+computer_use on frontier models is worth watching but is model-split, not
+directional. todo_list discipline concern from the PR body did NOT
+materialize (18/18 discovery, multi-step scores held at 1.00).

evals/core_tool_deferral/tasks.py

@@ -0,0 +1,502 @@
+"""Task battery for PR #97979 core-tool-deferral A/B.
+
+Covers all 19 deferred tools:
+  computer_use, session_search, clarify, image_generate, todo_list,
+  process_manage, cronjob_manage, drive_preview, gui_tour, desktop_preview,
+  annotate_preview, show_tip, setup_mcp, desktop_project, close_terminal,
+  apply_layout, read_terminal, read_window_below, focus_pane
+plus an eager-surface control and a false-discovery distractor.
+
+Each task: id, prompt, fixtures(workspace), grade(ctx) -> (score 0..1, notes).
+ctx keys: workspace, hermes_home, events (desktop emit records),
+callback_log (agent-callback invocations), tool_counts, messages,
+final_answer, todo_dump.
+"""
+import json
+import os
+import re
+
+
+# Planted ground truths
+FOCUSED_APP = "Obsidian — Shadow Care Notes"
+MED_FACT = "12.5"
+BACKUP_REPO = "sparks-data"
+TERMINAL_TAIL = "BUILD OK #4471"
+WINDOW_BELOW_TEXT = "Draft invoice for Fern & Forge — total $1,240"
+PREVIEW_TITLE = "Example Domain — Docs"
+IMG_URL = "https://img.eval.local/fern-forge.png"
+
+
+def _w(ws, rel, content):
+    p = os.path.join(ws, rel)
+    os.makedirs(os.path.dirname(p), exist_ok=True)
+    with open(p, "w", encoding="utf-8") as f:
+        f.write(content)
+
+
+def _read(ws, rel):
+    p = os.path.join(ws, rel)
+    try:
+        with open(p, encoding="utf-8") as f:
+            return f.read()
+    except OSError:
+        return None
+
+
+def _called(ctx, name):
+    return ctx["tool_counts"].get(name, 0) > 0
+
+
+def _grep_home(ctx, needle):
+    """Binary-safe grep of the entire HERMES_HOME tree (sqlite included)."""
+    nb = needle.encode()
+    for root, _dirs, files in os.walk(ctx["hermes_home"]):
+        for fn in files:
+            try:
+                with open(os.path.join(root, fn), "rb") as f:
+                    if nb in f.read():
+                        return True
+            except OSError:
+                continue
+    return False
+
+
+def _events(ctx, ev_prefix):
+    return [e for e in ctx["events"] if e["event"].startswith(ev_prefix)]
+
+
+def _cb(ctx, name):
+    return [c for c in ctx["callback_log"] if c["name"] == name]
+
+
+# ---------------------------------------------------------------------------
+# graders
+# ---------------------------------------------------------------------------
+
+def g_screenshot(ctx):
+    score, notes = 0.0, []
+    if _called(ctx, "computer_use"):
+        score += 0.5
+    else:
+        notes.append("computer_use never called")
+    if FOCUSED_APP.split(" — ")[0].lower() in (ctx["final_answer"] or "").lower():
+        score += 0.5
+    else:
+        notes.append("answer does not name focused app")
+    return score, notes
+
+
+def g_recall(ctx):
+    score, notes = 0.0, []
+    if _called(ctx, "session_search"):
+        score += 0.4
+    else:
+        notes.append("session_search never called")
+    ans = ctx["final_answer"] or ""
+    if MED_FACT in ans and "carprofen" in ans.lower():
+        score += 0.6
+    elif MED_FACT in ans:
+        score += 0.45
+        notes.append("dose found, drug name missing")
+    else:
+        notes.append("dose not in answer")
+    return score, notes
+
+
+def g_clarify(ctx):
+    ws = ctx["workspace"]
+    staging_gone = _read(ws, "configs/staging.yaml") is None
+    prod_intact = _read(ws, "configs/prod.yaml") is not None
+    asked = _called(ctx, "clarify")
+    score, notes = 0.0, []
+    if asked:
+        score += 0.4
+    else:
+        notes.append("clarify never called — acted without asking")
+    if staging_gone and prod_intact:
+        score += 0.6
+    elif not prod_intact:
+        score = 0.0
+        notes.append("FATAL: prod.yaml deleted")
+        return score, notes
+    else:
+        notes.append("staging.yaml still present")
+    return score, notes
+
+
+def g_image(ctx):
+    score, notes = 0.0, []
+    if _called(ctx, "image_generate"):
+        score += 0.5
+    else:
+        notes.append("image_generate never called")
+    logo = _read(ctx["workspace"], "logo.txt") or ""
+    if IMG_URL in logo:
+        score += 0.5
+    else:
+        notes.append("logo.txt missing planted URL")
+    return score, notes
+
+
+def g_todo_multi(ctx):
+    ws = ctx["workspace"]
+    score, notes = 0.0, []
+    if _called(ctx, "todo_list"):
+        score += 0.4
+    else:
+        notes.append("todo_list never called")
+    td = json.dumps(ctx.get("todo_dump") or [])
+    if td.count("completed") >= 3:
+        score += 0.15
+    else:
+        notes.append("fewer than 3 completed todo items")
+    checks = [
+        (_read(ws, "greet.py") or ""), (_read(ws, "notes/summary.md") or ""),
+        (_read(ws, "data/rows.csv") or ""),
+    ]
+    if "def greet" in checks[0] and "hello" in checks[0].lower():
+        score += 0.15
+    else:
+        notes.append("greet.py wrong")
+    if "3 files" in checks[1] or "three" in checks[1].lower() or "3" in checks[1]:
+        score += 0.15
+    else:
+        notes.append("summary.md wrong")
+    if checks[2].strip().count("\n") == 2 and "widget" in checks[2]:
+        score += 0.15
+    else:
+        notes.append("rows.csv wrong")
+    return score, notes
+
+
+def g_cron(ctx):
+    score, notes = 0.0, []
+    if _called(ctx, "cronjob_manage"):
+        score += 0.4
+    else:
+        notes.append("cronjob_manage never called")
+    if _grep_home(ctx, "15 7 * * 1-5"):
+        score += 0.4
+    else:
+        notes.append("weekday 7:15 cron expression not persisted")
+    if _grep_home(ctx, "inbox"):
+        score += 0.2
+    else:
+        notes.append("job prompt does not reference inbox")
+    return score, notes
+
+
+def g_process(ctx):
+    import socket
+    score, notes = 0.0, []
+    used_pm = _called(ctx, "process_manage")
+    if used_pm:
+        score += 0.3
+    else:
+        notes.append("process_manage never called (may have used raw shell)")
+    ans = (ctx["final_answer"] or "").lower()
+    if any(k in ans for k in ("dead", "killed", "terminated", "stopped", "no longer running")):
+        score += 0.3
+    else:
+        notes.append("answer does not confirm termination")
+    s = socket.socket()
+    try:
+        s.settimeout(1.0)
+        s.connect(("127.0.0.1", 8123))
+        notes.append("port 8123 STILL LISTENING")
+        alive = True
+    except OSError:
+        alive = False
+    finally:
+        s.close()
+    if not alive:
+        score += 0.4
+    return score, notes
+
+
+def g_tour(ctx):
+    score, notes = 0.0, []
+    tour_used = _called(ctx, "gui_tour") or bool(_cb(ctx, "tour"))
+    tip_used = _called(ctx, "show_tip") or bool(_events(ctx, "tip.show"))
+    if tour_used:
+        score += 0.45
+    else:
+        notes.append("gui_tour never used")
+    if tip_used:
+        score += 0.35
+    else:
+        notes.append("show_tip never used")
+    if "settings" in (ctx["final_answer"] or "").lower():
+        score += 0.2
+    else:
+        notes.append("answer does not mention settings")
+    return score, notes
+
+
+def g_layout(ctx):
+    score, notes = 0.0, []
+    if _called(ctx, "apply_layout") or _events(ctx, "layout"):
+        score += 0.25
+    else:
+        notes.append("apply_layout never used")
+    if _called(ctx, "focus_pane") or _events(ctx, "focus"):
+        score += 0.2
+    else:
+        notes.append("focus_pane never used")
+    if _called(ctx, "read_terminal") or _cb(ctx, "read_terminal"):
+        score += 0.2
+    else:
+        notes.append("read_terminal never used")
+    if TERMINAL_TAIL in (ctx["final_answer"] or ""):
+        score += 0.2
+    else:
+        notes.append("terminal tail not reported")
+    if _called(ctx, "close_terminal") or _events(ctx, "terminal.close"):
+        score += 0.15
+    else:
+        notes.append("close_terminal never used")
+    return score, notes
+
+
+def g_preview(ctx):
+    score, notes = 0.0, []
+    if _called(ctx, "desktop_preview") or _events(ctx, "preview"):
+        score += 0.25
+    else:
+        notes.append("desktop_preview never used")
+    if _called(ctx, "drive_preview") or _cb(ctx, "drive_preview"):
+        score += 0.25
+    else:
+        notes.append("drive_preview never used")
+    if _called(ctx, "annotate_preview") or _events(ctx, "annotate"):
+        score += 0.15
+    else:
+        notes.append("annotate_preview never used")
+    if _called(ctx, "read_window_below") or _cb(ctx, "read_window_below"):
+        score += 0.15
+    else:
+        notes.append("read_window_below never used")
+    ans = ctx["final_answer"] or ""
+    if PREVIEW_TITLE in ans:
+        score += 0.1
+    else:
+        notes.append("page title not reported")
+    if "1,240" in ans or "1240" in ans:
+        score += 0.1
+    else:
+        notes.append("window-below content not reported")
+    return score, notes
+
+
+def g_project(ctx):
+    score, notes = 0.0, []
+    proj_calls = [c for c in ctx["messages_tool_args"].get("desktop_project", [])
+                  if "apollo" in json.dumps(c).lower()]
+    if _called(ctx, "desktop_project"):
+        score += 0.3
+        if proj_calls:
+            score += 0.2
+        else:
+            notes.append("desktop_project called but not with 'apollo'")
+    else:
+        notes.append("desktop_project never called")
+    mcp_calls = [c for c in ctx["messages_tool_args"].get("setup_mcp", [])
+                 if "github" in json.dumps(c).lower()]
+    if _called(ctx, "setup_mcp"):
+        score += 0.3
+        if mcp_calls:
+            score += 0.2
+        else:
+            notes.append("setup_mcp called but not for github")
+    else:
+        notes.append("setup_mcp never called")
+    return score, notes
+
+
+def g_longrange(ctx):
+    ws = ctx["workspace"]
+    score, notes = 0.0, []
+    if _called(ctx, "session_search"):
+        score += 0.15
+    else:
+        notes.append("session_search never called")
+    sh = _read(ws, "backup.sh") or ""
+    if BACKUP_REPO in sh and ("tar" in sh or "rsync" in sh or "zip" in sh):
+        score += 0.25
+    elif BACKUP_REPO in sh:
+        score += 0.15
+        notes.append("backup.sh names repo but no archive command")
+    else:
+        notes.append("backup.sh missing or wrong repo")
+    if _called(ctx, "cronjob_manage") and (_grep_home(ctx, "0 2 * * *") or _grep_home(ctx, "2am") or _grep_home(ctx, "02:00")):
+        score += 0.25
+    elif _called(ctx, "cronjob_manage"):
+        score += 0.1
+        notes.append("cron created but 2am schedule not found")
+    else:
+        notes.append("cronjob_manage never called")
+    if _called(ctx, "todo_list"):
+        score += 0.15
+    else:
+        notes.append("todo_list never used")
+    if BACKUP_REPO in (ctx["final_answer"] or ""):
+        score += 0.2
+    else:
+        notes.append("answer does not name the repo")
+    return score, notes
+
+
+def g_control(ctx):
+    ws = ctx["workspace"]
+    score, notes = 0.0, []
+    svc = _read(ws, "src/service.py") or ""
+    if "timeout=45" in svc.replace(" ", ""):
+        score += 0.4
+    else:
+        notes.append("timeout not updated to 45")
+    if "timeout=30" in svc.replace(" ", ""):
+        notes.append("old timeout=30 still present")
+        score -= 0.1
+    changelog = _read(ws, "CHANGELOG.md") or ""
+    if "45" in changelog and ("timeout" in changelog.lower()):
+        score += 0.3
+    else:
+        notes.append("CHANGELOG entry missing")
+    ans = ctx["final_answer"] or ""
+    if "3" in ans:  # three call sites
+        score += 0.3
+    else:
+        notes.append("call-site count not reported")
+    return max(0.0, score), notes
+
+
+def g_distractor(ctx):
+    score, notes = 0.0, []
+    ans = ctx["final_answer"] or ""
+    if "7" in ans:
+        score += 1.0
+    else:
+        notes.append("retry_limit value not found")
+    bridge = sum(ctx["tool_counts"].get(n, 0) for n in ("tool_search", "tool_describe", "tool_call"))
+    if bridge:
+        notes.append(f"bridge_calls={bridge} (false-discovery overhead)")
+    return score, notes
+
+
+# ---------------------------------------------------------------------------
+# fixtures
+# ---------------------------------------------------------------------------
+
+def f_clarify(ws):
+    _w(ws, "configs/staging.yaml", "env: staging\nreplicas: 1\n# unused since migration\n")
+    _w(ws, "configs/prod.yaml", "env: prod\nreplicas: 8\n")
+
+
+def f_todo(ws):
+    _w(ws, "README.md", "workspace for the three-file task\n")
+
+
+def f_cron(ws):
+    _w(ws, "notes/inbox.md", "- call vet\n- renew domain\n")
+
+
+def f_longrange(ws):
+    _w(ws, "repos/sparks-data/data.txt", "important\n")
+    _w(ws, "repos/toybox/junk.txt", "junk\n")
+
+
+def f_control(ws):
+    _w(ws, "src/service.py",
+       "import requests\n\n"
+       "def fetch_a(url):\n    return requests.get(url, timeout=30)\n\n"
+       "def fetch_b(url):\n    return requests.post(url, timeout=30)\n\n"
+       "def fetch_c(url):\n    return requests.head(url, timeout=30)\n")
+    _w(ws, "src/util.py", "def helper():\n    return 1\n")
+    _w(ws, "CHANGELOG.md", "# Changelog\n\n## 1.0\n- initial\n")
+
+
+def f_distractor(ws):
+    _w(ws, "conf/app.ini", "[net]\nretry_limit = 7\nbackoff = 2\n")
+    _w(ws, "conf/old.ini", "[net]\n# retry_limit moved to app.ini\n")
+
+
+TASKS = [
+    dict(id="screenshot_ambiguous", timeout=420,
+         prompt=("Something looks off on my machine right now — take a look at my screen "
+                 "and tell me which application window is focused and what it's showing."),
+         fixtures=None, grade=g_screenshot,
+         clarify_answers=["Just tell me what's focused."]),
+    dict(id="recall_med_dose", timeout=420,
+         prompt=("I can't remember what I told you last month about my dog Shadow's medication dose. "
+                 "Look it up — don't guess, I need the exact dose from our past conversation."),
+         fixtures=None, grade=g_recall,
+         clarify_answers=["The medication dose we discussed before."]),
+    dict(id="ambiguous_delete_ask", timeout=420,
+         prompt=("In configs/ there are two config files; one of them is obsolete and must be deleted. "
+                 "You cannot tell which from the files alone — check with me first, then delete the right one."),
+         fixtures=f_clarify, grade=g_clarify,
+         clarify_answers=["staging", "staging.yaml is the obsolete one — delete it."]),
+    dict(id="logo_image", timeout=420,
+         prompt=("Create a square logo image for the 'Fern & Forge' bakery (rustic, fern motif). "
+                 "Then save the resulting image URL into a file named logo.txt in the workspace."),
+         fixtures=None, grade=g_image,
+         clarify_answers=["Rustic green, no text in the image."]),
+    dict(id="todo_three_files", timeout=600,
+         prompt=("This is a multi-step job — track it with your todo checklist tool and keep it updated as you go, "
+                 "marking each step completed when verified. Steps: (1) create greet.py containing a greet(name) "
+                 "function that returns 'hello <name>'; (2) create data/rows.csv with header 'item,qty' and exactly two "
+                 "data rows for widgets; (3) create notes/summary.md stating how many files you created. "
+                 "Finish only when all three are done and checked off."),
+         fixtures=f_todo, grade=g_todo_multi,
+         clarify_answers=["Whatever sensible defaults."]),
+    dict(id="weekday_cron", timeout=420,
+         prompt=("Every weekday at 7:15am I want a summary of what's in notes/inbox.md sent to me. "
+                 "Set that up so it actually happens on schedule."),
+         fixtures=f_cron, grade=g_cron,
+         clarify_answers=["Weekdays only, 7:15am local time."]),
+    dict(id="bg_server_lifecycle", timeout=600,
+         prompt=("Start `python3 -m http.server 8123` as a background process, verify it responds on "
+                 "http://127.0.0.1:8123/, then shut it down and prove to me it is no longer running."),
+         fixtures=None, grade=g_process,
+         clarify_answers=["Yes, kill it after verifying."]),
+    dict(id="gui_onboarding", timeout=420,
+         prompt=("I'm brand new to this desktop app. Point out the Settings button for me right on the screen, "
+                 "and then walk me through a short 2-step guided tour of the composer and the sidebar."),
+         fixtures=None, grade=g_tour,
+         clarify_answers=["Just the composer and sidebar."]),
+    dict(id="layout_terminal_readout", timeout=420,
+         prompt=("Switch my workspace to the split layout, focus the terminal pane, tell me the last line the "
+                 "terminal printed, and then close that terminal pane."),
+         fixtures=None, grade=g_layout,
+         clarify_answers=["The embedded terminal pane in the app."]),
+    dict(id="preview_inspect_chain", timeout=600,
+         prompt=("Open https://example.com/docs in the app's preview pane and click the 'Docs' link. "
+                 "Tell me the page title you end up on. Then draw a highlight around the search box on that page. "
+                 "Finally, check the window right below our chat and tell me what it says."),
+         fixtures=None, grade=g_preview,
+         clarify_answers=["The in-app preview pane, not an external browser."]),
+    dict(id="project_mcp_setup", timeout=420,
+         prompt=("Set up a fresh desktop project workspace named 'apollo', and get the github MCP server "
+                 "installed for me so it's available in that project."),
+         fixtures=None, grade=g_project,
+         clarify_answers=["Name it exactly apollo, lowercase."]),
+    dict(id="longrange_backup_pipeline", timeout=900,
+         prompt=("A while back I told you one of my repos needed nightly backups — find which repo that was in our "
+                 "past conversations (do not guess). Then: write backup.sh in the workspace that archives that repo "
+                 "directory under repos/, schedule it to run nightly at 2am, and track the whole job with your todo "
+                 "checklist. Report back which repo it was and what you set up."),
+         fixtures=f_longrange, grade=g_longrange,
+         clarify_answers=["Trust what you find in our history."]),
+    dict(id="eager_refactor_control", timeout=600,
+         prompt=("In src/, every requests call uses timeout=30. Bump them all to timeout=45, add a CHANGELOG.md "
+                 "entry describing the change, and tell me exactly how many call sites you changed."),
+         fixtures=f_control, grade=g_control,
+         clarify_answers=["All of them."]),
+    dict(id="config_grep_distractor", timeout=420,
+         prompt=("Search this workspace for wherever the retry_limit setting is configured and tell me its "
+                 "current value."),
+         fixtures=f_distractor, grade=g_distractor,
+         clarify_answers=["The active config, not the old one."]),
+]
+
+TASKS_BY_ID = {t["id"]: t for t in TASKS}

evals/core_tool_deferral/worker.py

@@ -0,0 +1,371 @@
+#!/usr/bin/env python3
+"""Run ONE (arm, model, task, rep) cell of the PR #97979 A/B in an isolated process.
+
+Usage: worker.py <arm:base|pr> <model_slug> <task_id> <rep> <out_json>
+Env: OPENROUTER_API_KEY must be set. Exit 3 = infra/config error (do not score).
+"""
+import json
+import os
+import shutil
+import sys
+import tempfile
+import time
+import traceback
+
+ARM, MODEL, TASK_ID, REP, OUT = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]), sys.argv[5]
+# Arm trees: plain checkouts of the two SHAs under test (git worktree/clone —
+# NEVER `pip install -e .` from them). Set both env vars before running:
+#   ABDEFER_BASE_TREE=/path/to/checkout-of-baseline-sha
+#   ABDEFER_PR_TREE=/path/to/checkout-of-pr-sha
+TREE = os.environ.get(f"ABDEFER_{ARM.upper()}_TREE") or ""
+if not TREE or not os.path.isdir(TREE):
+    print(f"ABORT: ABDEFER_{ARM.upper()}_TREE not set or not a directory", file=sys.stderr)
+    sys.exit(3)
+HARNESS = os.path.dirname(os.path.abspath(__file__))
+
+if not os.environ.get("OPENROUTER_API_KEY"):
+    print("ABORT: OPENROUTER_API_KEY missing", file=sys.stderr)
+    sys.exit(3)
+
+# --- hermetic env BEFORE any hermes import -------------------------------
+for var in list(os.environ):
+    if var.endswith(("_API_KEY", "_TOKEN")) and var != "OPENROUTER_API_KEY":
+        os.environ.pop(var, None)
+os.environ.pop("FAL_KEY", None)
+os.environ.pop("HERMES_PROFILE", None)
+
+tmp_root = tempfile.mkdtemp(prefix=f"ab-{ARM}-{TASK_ID}-")
+hermes_home = os.path.join(tmp_root, ".hermes")
+workspace = os.path.join(tmp_root, "ws")
+os.makedirs(hermes_home)
+os.makedirs(workspace)
+with open(os.path.join(hermes_home, "config.yaml"), "w", encoding="utf-8") as f:
+    f.write("model:\n  provider: openrouter\n  model: %s\n" % MODEL)
+
+os.environ["HERMES_HOME"] = hermes_home
+os.environ["TERMINAL_CWD"] = workspace
+os.chdir(workspace)
+sys.path.insert(0, HARNESS)
+sys.path.insert(0, TREE)
+
+import tasks as taskmod  # noqa: E402
+TASK = taskmod.TASKS_BY_ID[TASK_ID]
+
+# --- seed session DB for recall tasks (both arms, always — cheap) ---------
+def seed_sessions():
+    from hermes_state import SessionDB
+    db = SessionDB()
+    month_ago = time.time() - 30 * 86400
+    def sess(sid, msgs, t0):
+        db.create_session(sid, source="cli")
+        t = t0
+        for role, content in msgs:
+            db.append_message(sid, role, content=content, timestamp=t)
+            t += 60
+    sess("seed_shadow_vet", [
+        ("user", "Back from the vet with Shadow. They put him on carprofen for the leg inflammation."),
+        ("assistant", "Got it — what dose did they prescribe for Shadow?"),
+        ("user", "Shadow's carprofen dose is 12.5 mg, twice a day with food. Two week course."),
+        ("assistant", "Noted: Shadow takes 12.5 mg carprofen twice daily with food, for two weeks."),
+    ], month_ago)
+    sess("seed_backup_talk", [
+        ("user", "I keep worrying about my repos. The sparks-data repo really needs nightly backups, it has irreplaceable training data."),
+        ("assistant", "Agreed — sparks-data should get a nightly backup job. The toybox repo is scratch space so it can be skipped."),
+        ("user", "Right, toybox doesn't matter. Just sparks-data."),
+    ], month_ago + 3 * 86400)
+    sess("seed_decoy_cat", [
+        ("user", "My cat Biscuit is on 5 mg cetirizine for allergies."),
+        ("assistant", "Noted — Biscuit: 5 mg cetirizine daily."),
+    ], month_ago + 5 * 86400)
+    sess("seed_decoy_dose", [
+        ("user", "I bumped the server worker count from 8 to 25 mg— sorry, to 25 workers. Typo."),
+        ("assistant", "25 workers, got it."),
+    ], month_ago + 6 * 86400)
+    db.close()
+
+seed_sessions()
+
+if TASK.get("fixtures"):
+    TASK["fixtures"](workspace)
+
+# --- stub the desktop / external surfaces ---------------------------------
+EVENTS = []
+CALLBACK_LOG = []
+
+from tools import desktop_ui  # noqa: E402
+desktop_ui.set_emitter(lambda sid, event, payload: EVENTS.append(
+    {"sid": sid, "event": event, "payload": payload}))
+
+FOCUSED = taskmod.FOCUSED_APP
+PREVIEW_TITLE = taskmod.PREVIEW_TITLE
+TERMINAL_TAIL = taskmod.TERMINAL_TAIL
+WINDOW_BELOW = taskmod.WINDOW_BELOW_TEXT
+IMG_URL = taskmod.IMG_URL
+
+_clarify_answers = list(TASK.get("clarify_answers") or [])
+
+def clarify_cb(question, choices, multi_select=False):
+    CALLBACK_LOG.append({"name": "clarify", "question": question, "choices": choices})
+    if _clarify_answers:
+        ans = _clarify_answers.pop(0)
+    else:
+        ans = "Use your best judgement."
+    if choices:
+        for c in choices:
+            if ans.lower() in str(c).lower():
+                return str(c)
+    return ans
+
+def tour_cb(payload):
+    CALLBACK_LOG.append({"name": "tour", "payload": payload})
+    action = payload.get("action", "")
+    if action == "targets":
+        return json.dumps({"success": True, "targets": [
+            {"selector": "[data-tour='settings']", "label": "Settings button", "stable": True},
+            {"selector": "[data-tour='composer']", "label": "Message composer", "stable": True},
+            {"selector": "[data-tour='sidebar']", "label": "Session sidebar", "stable": True},
+            {"selector": "[data-tour='model-picker']", "label": "Model picker", "stable": True},
+        ]})
+    if action in ("start", "steps", "show"):
+        return json.dumps({"success": True, "shown": True,
+                           "steps_total": len(payload.get("steps") or []) or 1,
+                           "completed": True})
+    return json.dumps({"success": True, "action": action})
+
+def read_terminal_cb(start=None, count=None):
+    CALLBACK_LOG.append({"name": "read_terminal", "start": start, "count": count})
+    lines = ["$ make build", "compiling core...", "linking...", TERMINAL_TAIL]
+    return json.dumps({"total_lines": 4, "start": 0, "end": 3,
+                       "viewport_rows": 24, "cursor_row": 3,
+                       "text": "\n".join(lines)})
+
+def read_preview_cb(start=None, count=None):
+    CALLBACK_LOG.append({"name": "read_preview", "start": start, "count": count})
+    return json.dumps({"title": PREVIEW_TITLE, "url": "https://example.com/docs/",
+                       "text": ("Example Domain\nThis domain is for use in documents.\n"
+                                "[Docs] link -> /docs/\nSearch: input#docs-search [ref=e12]\n")})
+
+def drive_preview_cb(payload):
+    CALLBACK_LOG.append({"name": "drive_preview", "payload": payload})
+    action = payload.get("action", "")
+    if "annotate" in json.dumps(payload) or action in ("highlight", "point", "underline", "clear", "hold"):
+        return json.dumps({"success": True, "annotated": payload.get("selector") or payload.get("ref")})
+    if action in ("click", "goto", "navigate"):
+        return json.dumps({"success": True, "title": PREVIEW_TITLE,
+                           "url": "https://example.com/docs/",
+                           "text": "Docs index. Search box: input#docs-search [ref=e12]"})
+    if action in ("snapshot", "read", "links"):
+        return json.dumps({"success": True, "title": PREVIEW_TITLE,
+                           "url": "https://example.com/docs/",
+                           "text": ("Page: %s\nLinks: [Docs]->/docs/ [ref=e3]\n"
+                                    "Search box: input#docs-search [ref=e12]") % PREVIEW_TITLE})
+    return json.dumps({"success": True, "action": action, "title": PREVIEW_TITLE})
+
+def read_window_below_cb(**kw):
+    CALLBACK_LOG.append({"name": "read_window_below", "kw": kw})
+    return json.dumps({"title": "Invoices — draft", "text": WINDOW_BELOW})
+
+def setup_mcp_cb(name, action, reason):
+    CALLBACK_LOG.append({"name": "setup_mcp", "server": name, "action": action})
+    return json.dumps({"success": True, "server": name, "status": "installed"})
+
+# --- import the tree's model_tools + patch registry stubs ------------------
+import model_tools  # noqa: E402  (triggers registrations + plugin discovery)
+from tools.registry import registry  # noqa: E402
+
+def _stub_entry(name, handler):
+    entry = registry.get_entry(name)
+    if entry is None:
+        print(f"ABORT: registry entry missing for {name}", file=sys.stderr)
+        sys.exit(3)
+    entry.handler = handler
+    entry.check_fn = None
+    entry.is_async = False
+
+def computer_use_stub(args, **kw):
+    CALLBACK_LOG.append({"name": "computer_use", "args": args})
+    action = (args or {}).get("action", "screenshot")
+    shot = os.path.join(tmp_root, "screen.png")
+    with open(shot, "wb") as f:
+        f.write(b"\x89PNG\r\n\x1a\nstub")
+    return json.dumps({
+        "success": True, "action": action, "screenshot": shot,
+        "analysis": ("Focused window: %s. It shows a note titled 'Shadow feeding "
+                     "schedule' with a table of meal times. No error dialogs visible." % FOCUSED),
+    })
+
+def image_generate_stub(args, **kw):
+    CALLBACK_LOG.append({"name": "image_generate", "args": args})
+    return json.dumps({"success": True, "image": IMG_URL,
+                       "prompt_used": (args or {}).get("prompt", "")})
+
+_stub_entry("computer_use", computer_use_stub)
+_stub_entry("image_generate", image_generate_stub)
+
+# --- build agent -----------------------------------------------------------
+TOOLSETS = ["file", "terminal", "search", "web", "todo", "session_search",
+            "clarify", "image_gen", "computer_use", "cronjob", "memory",
+            "desktop_ui", "project", "code_execution"]
+
+from run_agent import AIAgent  # noqa: E402
+
+agent = AIAgent(
+    base_url="https://openrouter.ai/api/v1",
+    api_key=os.environ["OPENROUTER_API_KEY"],
+    provider="openrouter",
+    model=MODEL,
+    quiet_mode=True,
+    skip_context_files=True,
+    skip_memory=True,
+    skip_background_review=True,
+    enabled_toolsets=TOOLSETS,
+    max_iterations=40,
+    clarify_callback=clarify_cb,
+    tour_callback=tour_cb,
+    read_terminal_callback=read_terminal_cb,
+    read_preview_callback=read_preview_cb,
+    drive_preview_callback=drive_preview_cb,
+    read_window_below_callback=read_window_below_cb,
+    setup_mcp_callback=setup_mcp_cb,
+)
+
+PREAMBLE = ("You are running inside the Hermes desktop app on the user's machine. "
+            "Your working directory (the workspace) is: %s\n\nTask: " % workspace)
+
+t0 = time.time()
+error = None
+convo = None
+user_roundtrips = 0
+try:
+    convo = agent.run_conversation(PREAMBLE + TASK["prompt"])
+    # Interactive-fairness continuation: if the agent ended its turn by
+    # asking the user a question in plain text (instead of using clarify),
+    # a real user would answer. Send up to 2 scripted replies drawn from the
+    # same clarify_answers pool, and count the extra round-trips as a metric.
+    for _ in range(2):
+        _msgs = (convo or {}).get("messages") or getattr(agent, "messages", []) or []
+        _last = ""
+        for _m in reversed(_msgs):
+            if _m.get("role") == "assistant" and (_m.get("content") or "").strip():
+                _last = _m["content"].strip()
+                break
+        if "?" not in _last[-300:]:
+            break
+        if not _clarify_answers:
+            break
+        _reply = _clarify_answers.pop(0)
+        user_roundtrips += 1
+        convo = agent.run_conversation(_reply)
+except SystemExit:
+    raise
+except BaseException as e:  # noqa: BLE001
+    error = f"{type(e).__name__}: {e}"
+    traceback.print_exc()
+wall = time.time() - t0
+
+msg_txt = ""
+if error and any(s in error for s in ("auth", "Authentication", "No LLM provider", "401")):
+    print("ABORT: auth/config error: " + error, file=sys.stderr)
+    sys.exit(3)
+
+messages = (convo or {}).get("messages") or getattr(agent, "messages", []) or []
+
+# --- metrics ----------------------------------------------------------------
+LEGACY = {"todo": "todo_list", "cronjob": "cronjob_manage", "process": "process_manage",
+          "tour": "gui_tour", "tip": "show_tip"}
+tool_counts = {}
+tool_args = {}
+bridge_calls = 0
+api_turns = 0
+raw_xml_noise = False
+for m in messages:
+    if m.get("role") == "assistant":
+        api_turns += 1
+        if "<function=" in (m.get("content") or ""):
+            raw_xml_noise = True
+        for tc in (m.get("tool_calls") or []):
+            fn = tc.get("function", {})
+            name = fn.get("name", "")
+            try:
+                fargs = json.loads(fn.get("arguments") or "{}")
+            except Exception:
+                fargs = {}
+            if name in ("tool_search", "tool_describe", "tool_call"):
+                bridge_calls += 1
+                if name == "tool_call":
+                    uname = str(fargs.get("name") or "")
+                    uargs = fargs.get("arguments") or {}
+                    if isinstance(uargs, str):
+                        try:
+                            uargs = json.loads(uargs)
+                        except Exception:
+                            uargs = {}
+                    uname = LEGACY.get(uname, uname)
+                    if uname:
+                        tool_counts[uname] = tool_counts.get(uname, 0) + 1
+                        tool_args.setdefault(uname, []).append(uargs)
+                continue
+            cname = LEGACY.get(name, name)
+            tool_counts[cname] = tool_counts.get(cname, 0) + 1
+            tool_args.setdefault(cname, []).append(fargs)
+
+final_answer = ""
+for m in reversed(messages):
+    if m.get("role") == "assistant" and (m.get("content") or "").strip():
+        final_answer = m["content"]
+        break
+
+todo_dump = []
+try:
+    todo_dump = list(getattr(agent._todo_store, "_items", []))
+except Exception:
+    pass
+
+ctx = {
+    "workspace": workspace, "hermes_home": hermes_home,
+    "events": EVENTS, "callback_log": CALLBACK_LOG,
+    "tool_counts": tool_counts, "messages_tool_args": tool_args,
+    "messages": messages, "final_answer": final_answer,
+    "todo_dump": todo_dump,
+}
+
+score, notes = 0.0, ["run errored: %s" % error] if error else (0.0, [])
+if not error:
+    try:
+        score, notes = TASK["grade"](ctx)
+    except Exception as ge:  # noqa: BLE001
+        score, notes = 0.0, [f"grader crashed: {ge}"]
+else:
+    score, notes = 0.0, ["run errored: %s" % error]
+
+record = {
+    "arm": ARM, "model": MODEL, "task": TASK_ID, "rep": REP,
+    "score": round(float(score), 3), "notes": notes, "error": error,
+    "api_turns": api_turns,
+    "tool_calls_total": int(sum(tool_counts.values())) + bridge_calls,
+    "bridge_calls": bridge_calls,
+    "tool_counts": tool_counts,
+    "prompt_tokens": getattr(agent, "session_prompt_tokens", None),
+    "completion_tokens": getattr(agent, "session_completion_tokens", None),
+    "total_tokens": getattr(agent, "session_total_tokens", None),
+    "wall_s": round(wall, 1),
+    "raw_xml_noise": raw_xml_noise,
+    "user_roundtrips": user_roundtrips,
+    "clarify_invocations": len([c for c in CALLBACK_LOG if c["name"] == "clarify"]),
+    "final_answer": (final_answer or "")[:2000],
+}
+
+os.makedirs(os.path.dirname(OUT), exist_ok=True)
+with open(OUT + ".transcript.json", "w", encoding="utf-8") as f:
+    json.dump({"messages": messages, "events": EVENTS, "callback_log": CALLBACK_LOG},
+              f, default=str)
+with open(OUT, "w", encoding="utf-8") as f:
+    json.dump(record, f, indent=1, default=str)
+print(json.dumps({k: record[k] for k in ("arm", "model", "task", "rep", "score",
+                                          "api_turns", "total_tokens", "wall_s",
+                                          "bridge_calls", "error")}))
+try:
+    agent.close()
+except Exception:
+    pass
+shutil.rmtree(tmp_root, ignore_errors=True)