Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

evals/readtool/runner.py

@@ -0,0 +1,206 @@
+"""Run the read-tool eval through the REAL Hermes AIAgent.
+
+For each task: fresh temp HERMES_HOME, fresh fixture workspace, real
+AIAgent with the file+terminal+search toolsets, real provider API. Collects
+accuracy plus efficiency metrics (API turns, tool calls, read_file calls,
+prompt/completion tokens, wall time).
+
+Usage:
+  python3 evals/readtool/runner.py --model anthropic/claude-opus-4.8 \\
+      --provider nous --reps 3 --label baseline
+  python3 evals/readtool/runner.py --model qwen/qwen3.8-max \\
+      --provider openrouter --reps 3 --label baseline --tasks fifo_hang
+
+Results land in evals/readtool/results/<label>/<model-slug>/rep<N>.json.
+Compare two labels with report.py.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+EVAL_DIR = Path(__file__).resolve().parent
+REPO_ROOT = EVAL_DIR.parent.parent
+sys.path.insert(0, str(EVAL_DIR))
+sys.path.insert(0, str(REPO_ROOT))
+
+from fixtures import build_workspace  # noqa: E402
+from tasks import TASKS, TASKS_BY_ID  # noqa: E402
+
+SYSTEM_SUFFIX = (
+    "You are working inside the project directory {ws}. All paths in the "
+    "task are relative to it. Work autonomously; do not ask questions. "
+    "When done, state your final answer plainly."
+)
+
+
+def _count_metrics(messages: list) -> dict:
+    api_turns = 0
+    tool_calls = 0
+    read_calls = 0
+    read_errors = 0
+    for m in messages:
+        role = m.get("role")
+        if role == "assistant":
+            api_turns += 1
+            for tc in m.get("tool_calls") or []:
+                tool_calls += 1
+                fn = (tc.get("function") or {}).get("name", "")
+                if fn == "read_file":
+                    read_calls += 1
+        elif role == "tool":
+            content = m.get("content") or ""
+            if isinstance(content, list):
+                content = " ".join(
+                    c.get("text", "") for c in content if isinstance(c, dict)
+                )
+            if '"error"' in content or "File not found" in content:
+                read_errors += 1
+    return {
+        "api_turns": api_turns,
+        "tool_calls": tool_calls,
+        "read_file_calls": read_calls,
+        "tool_error_results": read_errors,
+    }
+
+
+def run_task(task, model: str, provider: str, timeout_mult: float,
+             toolsets: list[str]) -> dict:
+    ws = Path(tempfile.mkdtemp(prefix=f"readtool-{task.task_id}-"))
+    hermes_home = Path(tempfile.mkdtemp(prefix="readtool-home-")) / ".hermes"
+    hermes_home.mkdir(parents=True)
+    build_workspace(ws)
+
+    old_env = dict(os.environ)
+    os.environ["HERMES_HOME"] = str(hermes_home)
+    os.environ["TERMINAL_CWD"] = str(ws)
+    # Keep only the API key the run needs; hide the rest so provider
+    # auto-detection can't wander (mirrors run_tests.sh hermeticity).
+    for var in list(os.environ):
+        if var.endswith("_API_KEY") and var != "OPENROUTER_API_KEY":
+            os.environ.pop(var)
+    result: dict = {"task_id": task.task_id, "capability": task.capability}
+    t0 = time.monotonic()
+    try:
+        # Import inside the env so profile-aware paths bind to the temp home.
+        from run_agent import AIAgent  # noqa: PLC0415
+
+        agent = AIAgent(
+            model=model,
+            provider=provider,
+            quiet_mode=True,
+            skip_context_files=True,
+            skip_memory=True,
+            enabled_toolsets=toolsets,
+            max_iterations=40,
+        )
+        convo = agent.run_conversation(
+            SYSTEM_SUFFIX.format(ws=ws) + "\n\nTask: " + task.prompt,
+        )
+        final = convo.get("final_response") or ""
+        messages = convo.get("messages") or []
+        result.update(_count_metrics(messages))
+        result.update(
+            {
+                "final_response": final,
+                "score": task.grade(final),
+                "prompt_tokens": getattr(agent, "session_prompt_tokens", 0),
+                "completion_tokens": getattr(agent, "session_completion_tokens", 0),
+                "total_tokens": getattr(agent, "session_total_tokens", 0),
+                "wall_s": round(time.monotonic() - t0, 1),
+                "error": None,
+            }
+        )
+    except Exception as exc:  # noqa: BLE001
+        msg = f"{type(exc).__name__}: {exc}"
+        if "No LLM provider configured" in str(exc) or "authentication" in str(exc).lower():
+            # Harness misconfiguration, not a model result. Abort the whole
+            # run rather than writing poisoned zero-score records.
+            raise SystemExit(f"ABORT (harness config error, not a result): {msg}")
+        result.update(
+            {
+                "final_response": "",
+                "score": 0.0,
+                "wall_s": round(time.monotonic() - t0, 1),
+                "error": msg,
+            }
+        )
+    finally:
+        os.environ.clear()
+        os.environ.update(old_env)
+        shutil.rmtree(ws, ignore_errors=True)
+        shutil.rmtree(hermes_home.parent, ignore_errors=True)
+    return result
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--provider", required=True)
+    ap.add_argument("--reps", type=int, default=3)
+    ap.add_argument("--label", required=True, help="e.g. baseline, feat-fifo-guard")
+    ap.add_argument("--tasks", default="", help="comma-separated task ids (default all)")
+    ap.add_argument("--timeout-mult", type=float, default=1.0)
+    ap.add_argument(
+        "--toolsets",
+        default="file,terminal,search",
+        help=(
+            "Comma-separated toolsets. Use 'file' alone for the "
+            "discriminative arm (no terminal escape hatch — the read tool "
+            "must handle the hostile file itself)."
+        ),
+    )
+    args = ap.parse_args()
+
+    if not os.environ.get("OPENROUTER_API_KEY"):
+        raise SystemExit(
+            "OPENROUTER_API_KEY not in environment. Run: set -a; "
+            "source ~/.hermes/.env; set +a  — then relaunch."
+        )
+
+    slate = (
+        [TASKS_BY_ID[t] for t in args.tasks.split(",") if t]
+        if args.tasks
+        else TASKS
+    )
+    slug = args.model.replace("/", "_")
+    out_dir = EVAL_DIR / "results" / args.label / slug
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    for rep in range(1, args.reps + 1):
+        rep_path = out_dir / f"rep{rep}.json"
+        if rep_path.exists():
+            print(f"rep{rep} exists, skipping")
+            continue
+        records = []
+        for task in slate:
+            print(f"[rep{rep}] {task.task_id} ...", flush=True)
+            rec = run_task(task, args.model, args.provider, args.timeout_mult,
+                           [t for t in args.toolsets.split(",") if t])
+            print(
+                f"[rep{rep}] {task.task_id}: score={rec['score']:.2f} "
+                f"turns={rec.get('api_turns', '?')} tok={rec.get('total_tokens', '?')} "
+                f"wall={rec['wall_s']}s err={rec.get('error')}",
+                flush=True,
+            )
+            records.append(rec)
+        rep_path.write_text(
+            json.dumps(
+                {"model": args.model, "provider": args.provider, "label": args.label,
+                 "rep": rep, "records": records},
+                indent=2,
+            )
+        )
+        print(f"wrote {rep_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())