Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,184 @@
|
||||
"""Cloud-backend battery: pr arm attached to a provisioned cloud browser per cell.
|
||||
|
||||
Backends:
|
||||
nous-cloud - Nous Portal-provisioned Browser Use cloud browser
|
||||
(plugins.browser.browser_use provider; needs gateway access)
|
||||
browserbase - fresh Browserbase session per cell
|
||||
(needs BROWSERBASE_API_KEY + BROWSERBASE_PROJECT_ID)
|
||||
|
||||
Usage:
|
||||
BUBENCH_BASE_TREE=... BUBENCH_PR_TREE=... \
|
||||
python3 orchestrate_cloud.py --backend nous-cloud [--reps 2]
|
||||
python3 orchestrate_cloud.py --backend browserbase [--reps 1]
|
||||
|
||||
Per cell: provision a session, export its CDP endpoint via BENCH_CDP_URL /
|
||||
BU_CDP_WS, run single_run.py (pr arm), close the session. Resume-safe.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
ROOT = os.environ.get("BUBENCH_ROOT", os.path.dirname(os.path.abspath(__file__)))
|
||||
PY = sys.executable
|
||||
ENV_BASE = {**os.environ}
|
||||
ENV_BASE["PATH"] = (
|
||||
os.path.expanduser("~/.local/bin") + os.pathsep + ENV_BASE.get("PATH", "")
|
||||
)
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--backend", required=True, choices=["nous-cloud", "browserbase"])
|
||||
parser.add_argument("--tasks", default=os.path.join(ROOT, "tasks", "hard.json"))
|
||||
parser.add_argument("--models", default="anthropic/claude-opus-4.8,moonshotai/kimi-k3")
|
||||
parser.add_argument("--reps", type=int, default=2)
|
||||
parser.add_argument("--results", default=None)
|
||||
parser.add_argument("--run-timeout", type=int, default=1200)
|
||||
args = parser.parse_args()
|
||||
|
||||
RESULTS = args.results or os.path.join(ROOT, "results", f"results_{args.backend}.jsonl")
|
||||
os.makedirs(os.path.dirname(RESULTS), exist_ok=True)
|
||||
MODELS = args.models.split(",")
|
||||
TASKS = list(json.load(open(args.tasks, encoding="utf-8")).keys())
|
||||
REPS = list(range(1, args.reps + 1))
|
||||
|
||||
done = set()
|
||||
if os.path.exists(RESULTS):
|
||||
for line in open(RESULTS, encoding="utf-8"):
|
||||
try:
|
||||
r = json.loads(line)
|
||||
done.add((r["task"], r["model"], r["rep"]))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class NousCloud:
|
||||
def __init__(self):
|
||||
sys.path.insert(0, os.environ["BUBENCH_PR_TREE"])
|
||||
import importlib
|
||||
|
||||
self._mod = importlib.import_module("plugins.browser.browser_use.provider")
|
||||
|
||||
def create(self, name):
|
||||
self._prov = self._mod.BrowserUseBrowserProvider()
|
||||
sess = self._prov.create_session(name)
|
||||
return sess, {"BENCH_CDP_URL": sess["cdp_url"]}
|
||||
|
||||
def close(self, sess):
|
||||
self._prov.close_session(
|
||||
sess.get("bb_session_id") or sess.get("session_name", "")
|
||||
)
|
||||
|
||||
|
||||
class Browserbase:
|
||||
def create(self, name):
|
||||
req = urllib.request.Request(
|
||||
"https://api.browserbase.com/v1/sessions",
|
||||
data=json.dumps({
|
||||
"projectId": os.environ["BROWSERBASE_PROJECT_ID"]
|
||||
}).encode(),
|
||||
headers={
|
||||
"x-bb-api-key": os.environ["BROWSERBASE_API_KEY"],
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
sess = json.load(resp)
|
||||
return sess, {
|
||||
"BU_CDP_WS": sess["connectUrl"],
|
||||
"BENCH_CDP_URL": sess["connectUrl"],
|
||||
}
|
||||
|
||||
def close(self, sess):
|
||||
req = urllib.request.Request(
|
||||
f"https://api.browserbase.com/v1/sessions/{sess['id']}",
|
||||
data=json.dumps({
|
||||
"projectId": os.environ["BROWSERBASE_PROJECT_ID"],
|
||||
"status": "REQUEST_RELEASE",
|
||||
}).encode(),
|
||||
headers={
|
||||
"x-bb-api-key": os.environ["BROWSERBASE_API_KEY"],
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
urllib.request.urlopen(req, timeout=30)
|
||||
|
||||
|
||||
provider = NousCloud() if args.backend == "nous-cloud" else Browserbase()
|
||||
|
||||
cells = [(t, m, rep) for m, t, rep in itertools.product(MODELS, TASKS, REPS)]
|
||||
total = len(cells)
|
||||
n = 0
|
||||
for task, model, rep in cells:
|
||||
n += 1
|
||||
if (task, model, rep) in done:
|
||||
continue
|
||||
print(f"[{n}/{total}] {args.backend} pr {task} {model} rep{rep}", flush=True)
|
||||
sess = None
|
||||
t0 = time.time()
|
||||
try:
|
||||
sess, extra_env = provider.create(f"bubench-{task}-{rep}")
|
||||
except Exception as e: # noqa: BLE001
|
||||
rec = {
|
||||
"arm": f"pr-{args.backend}",
|
||||
"task": task,
|
||||
"model": model,
|
||||
"rep": rep,
|
||||
"ok": False,
|
||||
"error": f"session-create: {e}",
|
||||
}
|
||||
with open(RESULTS, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(rec) + "\n")
|
||||
continue
|
||||
env = {**ENV_BASE, **extra_env, "BUBENCH_TASKS": args.tasks}
|
||||
subprocess.run(["pkill", "-f", "browser_harness"], capture_output=True)
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[PY, os.path.join(ROOT, "single_run.py"), "pr", task, model, str(rep)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=args.run_timeout,
|
||||
env=env,
|
||||
)
|
||||
rec = None
|
||||
for line in (proc.stdout or "").splitlines():
|
||||
if line.startswith("RESULT_JSON:"):
|
||||
rec = json.loads(line[len("RESULT_JSON:") :])
|
||||
if rec is None:
|
||||
rec = {
|
||||
"task": task,
|
||||
"model": model,
|
||||
"rep": rep,
|
||||
"ok": False,
|
||||
"error": "no-result",
|
||||
"stderr_tail": (proc.stderr or "")[-600:],
|
||||
}
|
||||
except subprocess.TimeoutExpired:
|
||||
rec = {
|
||||
"task": task,
|
||||
"model": model,
|
||||
"rep": rep,
|
||||
"ok": False,
|
||||
"error": f"timeout-{args.run_timeout}s",
|
||||
}
|
||||
finally:
|
||||
try:
|
||||
provider.close(sess)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" close warning: {e}", flush=True)
|
||||
rec["arm"] = f"pr-{args.backend}"
|
||||
rec["cell_wall_s"] = round(time.time() - t0, 1)
|
||||
with open(RESULTS, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
print(
|
||||
f" -> ok={rec.get('ok')} err={rec.get('error')} wall={rec.get('cell_wall_s')}s",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
print("CLOUD BATTERY COMPLETE", flush=True)
|
||||
Reference in New Issue
Block a user