yilsem/aiturk-hermes-ide
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 3 Wiki Activity

Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

Browse Source
This commit is contained in:
yilsem
2026-09-05 13:26:46 +03:00
commit 03634b1ca3
11340 changed files with 3442369 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/LIVETEST_README.md
+45
View File
@@ -0,0 +1,45 @@
# Tool Search live test harness
Runs five scenarios against a real model (Claude Haiku 4.5 via OpenRouter) to
verify that the bridge tools work end-to-end. Records transcripts in
`scripts/out/`.
## Running
```bash
cd <repo root>
python3 scripts/tool_search_livetest.py # runs all 5 scenarios x 2 modes
python3 scripts/analyze_livetest.py # side-by-side report
```
Requires `OPENROUTER_API_KEY` set or present in `~/.hermes/.env`.
## What it verifies
| Scenario | Tests |
|----------|-------|
| A obvious_single | BM25 retrieval on an obvious tool name (github_create_issue) |
| B vague_paraphrased | Retrieval when the model has to paraphrase ("schedule meeting" → evt_create) |
| C multi_tool_chain | Multi-step task chaining two deferred tools (GitHub + Slack) |
| D core_plus_deferred | Mixed: core tool (read_file) called directly, deferred tool (Slack) via bridge |
| E no_tool_needed | Pure-knowledge prompt; verify no spurious tool_search invocations |
Each scenario runs with `tool_search.enabled = on` and again with `off` for an
A/B baseline. The harness records:
- bridge_calls (the tool_search / tool_describe / tool_call sequence the model emitted)
- underlying_tool_calls (what actually ran through the registry dispatcher)
- final_response, iteration count, elapsed time, any errors
## Output structure
```
scripts/out/
<scenario>__enabled.json # tool_search ON
<scenario>__disabled.json # tool_search OFF
_summary.json # one-line summary across all runs
```
The 2026-05 baseline run is checked in for reference. Re-running may produce
slightly different transcripts (the model is non-deterministic) but the
expected_underlying_tools assertions should remain satisfied.
scripts/aa_quality_sync.py
+94
View File
@@ -0,0 +1,94 @@
"""Propose catalog quality updates from Artificial Analysis.
Authoring-time helper — NEVER called at runtime (their terms forbid
client-side keys, the fleet would burn the rate limit, and a
recommendation must not change because a third-party endpoint
hiccuped). Run it when adding a model or refreshing the ordering;
review the printed diff and edit catalog.json yourself. The script
proposes, the commit decides.
The catalog's `quality` stays OUR field: AA-informed where they cover a
model, editorially set where they don't (day-0 releases lag their evals;
some entries never appear). AA's Intelligence Index grades the
full-precision cloud model, not our Q4 build — fine for ordering, never
for display.
Usage:
export AA_API_KEY=... # from https://artificialanalysis.ai (free tier)
python scripts/aa_quality_sync.py
Attribution: scores by Artificial Analysis (https://artificialanalysis.ai).
"""
from __future__ import annotations
import json
import os
import sys
import urllib.request
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
CATALOG_PATH = REPO_ROOT / "hermes_cli" / "local_runtime" / "catalog.json"
AA_URL = "https://artificialanalysis.ai/api/v2/data/llms/models"
# Catalog entry id -> AA slug. Hand-maintained: AA's naming rarely matches
# HF repo names, and a wrong match silently mis-ranks a model. An entry
# absent here (or mapped to None) is editorial-only and never overwritten.
AA_SLUG_BY_ENTRY = {
"qwen3.8-27b": "qwen3-8-27b",
"qwen3.8-flash-next": "qwen3-8-flash-next",
"qwen3.6-35b-a3b": "qwen3-6-35b-a3b",
"deepseek-v4-flash": "deepseek-v4-flash",
}
def fetch_aa_models(api_key: str) -> dict[str, dict]:
req = urllib.request.Request(AA_URL, headers={"x-api-key": api_key})
with urllib.request.urlopen(req, timeout=30) as r:
doc = json.load(r)
return {m["slug"]: m for m in doc.get("data", [])}
def main() -> int:
api_key = os.environ.get("AA_API_KEY", "").strip()
if not api_key:
print("AA_API_KEY not set — create a free key at "
"https://artificialanalysis.ai and export it.", file=sys.stderr)
return 2
catalog = json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
aa = fetch_aa_models(api_key)
print(f"{'entry':24s} {'catalog q':>9s} {'AA index':>9s} note")
print("-" * 70)
for model in catalog["models"]:
entry_id = model["id"]
current = model.get("quality", 0)
slug = AA_SLUG_BY_ENTRY.get(entry_id)
if not slug:
print(f"{entry_id:24s} {current:>9d} {'—':>9s} editorial only (no AA mapping)")
continue
hit = aa.get(slug)
if hit is None:
print(f"{entry_id:24s} {current:>9d} {'—':>9s} not in AA data (slug {slug!r})")
continue
index = (hit.get("evaluations") or {}).get(
"artificial_analysis_intelligence_index")
if index is None:
print(f"{entry_id:24s} {current:>9d} {'—':>9s} AA row lacks the index")
continue
proposed = round(float(index))
marker = "" if proposed == current else " <-- proposes change"
print(f"{entry_id:24s} {current:>9d} {proposed:>9d}{marker}")
print("\nReview against the decision table before editing: a quality "
"change that flips cells in tests/hermes_cli/"
"test_local_recommendation.py is the actual decision being made.")
print("Attribution: scores by Artificial Analysis "
"(https://artificialanalysis.ai).")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/add_contributor.py
+138
View File
@@ -0,0 +1,138 @@
#!/usr/bin/env python3
"""Add a contributor email → GitHub login mapping.
Writes one file per email under contributors/emails/ (filename = email,
content = login). File additions never merge-conflict, unlike the legacy
AUTHOR_MAP dict in scripts/release.py, which is frozen — do not append to it.
Usage (from the repo root):
python3 scripts/add_contributor.py <email> <github-login> [comment...]
# e.g.
python3 scripts/add_contributor.py jane@example.com janedoe "PR #12345 salvage"
Idempotent: if the mapping already exists with the same login, prints
"present" and exits 0. If the email maps to a DIFFERENT login (here or in the
legacy AUTHOR_MAP), refuses with exit 1 so a typo can't silently reassign
someone's commits.
"""
import re
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
EMAILS_DIR = REPO_ROOT / "contributors" / "emails"
_EMAIL_RE = re.compile(r"^[^/\\\s]+@[^/\\\s]+$")
# GitHub's *current* signup rules forbid consecutive hyphens, but legacy
# accounts with them exist and are valid (e.g. Roger--Han, verified via the
# users API July 2026). Accept any alphanumeric/hyphen login that doesn't
# start or end with a hyphen, max 39 chars.
_LOGIN_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$")
def read_mapping_file(path: Path) -> str | None:
"""Return the login from a mapping file (first non-comment line)."""
try:
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line and not line.startswith("#"):
return line
except OSError:
pass
return None
def _legacy_login(email: str) -> str | None:
"""Look the email up in the frozen legacy AUTHOR_MAP in release.py."""
try:
sys.path.insert(0, str(REPO_ROOT / "scripts"))
from release import LEGACY_AUTHOR_MAP # noqa: PLC0415
return LEGACY_AUTHOR_MAP.get(email)
except Exception:
return None
def _case_collision(email: str) -> str | None:
"""An existing mapping whose filename differs from `email` only in case.
Returns the colliding filename, or None. Exact matches are not collisions --
that is the ordinary "already mapped" path handled by the caller.
"""
if not EMAILS_DIR.is_dir():
return None
# casefold (not lower) matches how macOS/Windows fold non-ASCII text —
# same key scripts/check-case-collisions.py uses repo-wide.
folded = email.casefold()
for entry in EMAILS_DIR.iterdir():
if entry.name != email and entry.name.casefold() == folded:
return entry.name
return None
def add_contributor(email: str, login: str, comment: str = "") -> int:
email = email.strip()
login = login.strip().lstrip("@")
if not _EMAIL_RE.match(email):
print(f"error: {email!r} does not look like a commit-author email", file=sys.stderr)
return 2
if not _LOGIN_RE.match(login):
print(f"error: {login!r} is not a valid GitHub login", file=sys.stderr)
return 2
path = EMAILS_DIR / email
# One file per email means the FILENAME is the key, and on a
# case-insensitive filesystem (Windows, default macOS) two emails differing
# only in case are the same file. Creating both makes the repo impossible to
# check out cleanly there -- `git status` reports a phantom modification
# forever, because whichever file git wrote second wins on disk. Refuse for
# the same reason a conflicting login is refused: resolve it deliberately.
collision = _case_collision(email)
if collision is not None:
print(
f"error: {email} collides with existing mapping {collision} on "
"case-insensitive filesystems (Windows/macOS) — the two are the same "
"file there. Reuse that mapping, or resolve manually.",
file=sys.stderr,
)
return 1
existing = read_mapping_file(path) if path.is_file() else None
if existing is None:
existing = _legacy_login(email)
if existing is not None:
if existing == login:
print("present")
return 0
print(
f"error: {email} already maps to {existing!r} (asked for {login!r}) — "
"resolve manually",
file=sys.stderr,
)
return 1
EMAILS_DIR.mkdir(parents=True, exist_ok=True)
body = login + "\n"
if comment:
body += f"# {comment}\n"
path.write_text(body, encoding="utf-8")
print(f"added: contributors/emails/{email} -> {login}")
return 0
def main() -> int:
if len(sys.argv) < 3:
print(__doc__, file=sys.stderr)
return 2
email, login = sys.argv[1], sys.argv[2]
comment = " ".join(sys.argv[3:])
return add_contributor(email, login, comment)
if __name__ == "__main__":
sys.exit(main())
scripts/analyze_livetest.py
+124
View File
@@ -0,0 +1,124 @@
#!/usr/bin/env python3
"""Compare enabled vs disabled runs and produce a readable report.
Reads scripts/out/_summary.json and the per-scenario JSONs, prints a side-by-
side comparison of what happened, and flags anomalies.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
OUT = HERE / "out"
def load_record(scenario_id: str, mode: str):
path = OUT / f"{scenario_id}__{mode}.json"
if not path.exists():
return None
return json.loads(path.read_text(encoding="utf-8"))
def fmt_tool_seq(calls):
if not calls:
return "(none)"
return " → ".join(c["name"] for c in calls)
def fmt_bridge_seq(calls):
if not calls:
return "(none)"
parts = []
for c in calls:
if c["name"] == "tool_call":
inner = (c.get("args") or {}).get("name", "?")
parts.append(f"tool_call→{inner}")
elif c["name"] == "tool_search":
args = c.get("args") or {}
qs = args.get("queries")
if isinstance(qs, list):
q = "; ".join(str(x) for x in qs)
else: # legacy single-query transcripts
q = str(args["query"] if "query" in args else "?")
parts.append(f"search('{q[:30]}')")
elif c["name"] == "tool_describe":
args = c.get("args") or {}
ns = args.get("names")
if isinstance(ns, list):
n = ", ".join(str(x) for x in ns)
else: # legacy single-name transcripts
n = str(args.get("name", "?"))
parts.append(f"describe({n})")
return " → ".join(parts)
def main():
if not OUT.exists():
print("No output directory at", OUT)
sys.exit(1)
summary_path = OUT / "_summary.json"
if not summary_path.exists():
print("No _summary.json yet")
sys.exit(1)
summary = json.loads(summary_path.read_text(encoding="utf-8"))
scenarios = sorted({row["scenario"] for row in summary})
print(f"{'='*78}")
print(" Live test results: tool_search ENABLED vs DISABLED")
print(f"{'='*78}\n")
fails = 0
for sid in scenarios:
en = load_record(sid, "enabled")
di = load_record(sid, "disabled")
if not en or not di:
continue
expected = set(en["expected_underlying_tools"])
print(f"┌─ {sid} ({en['scenario_description']})")
print(f"│ Prompt: {en['prompt'][:120]}")
print(f"│ Expected underlying tools: {sorted(expected) or '(none)'}")
print("│")
for label, rec in [("ENABLED ", en), ("DISABLED", di)]:
called_under = [c["name"] for c in rec["underlying_tool_calls"]]
called_set = set(called_under)
missing = expected - called_set
extra = called_set - expected - {"read_file", "search_files", "terminal", "todo", "memory"}
mark = "✓" if (expected.issubset(called_set) and not rec["error"]) else "✗"
if mark == "✗":
fails += 1
print(f"│ {label} {mark} bridges={len(rec['bridge_calls']):2} underlying={len(rec['underlying_tool_calls']):2} "
f"iters={rec['n_iterations']:2} elapsed={rec['elapsed_seconds']:5.1f}s err={bool(rec['error'])}")
print(f"│ underlying: {fmt_tool_seq(rec['underlying_tool_calls'])}")
if rec["bridge_calls"]:
print(f"│ bridges: {fmt_bridge_seq(rec['bridge_calls'])}")
if missing:
print(f"│ ⚠ MISSING expected tools: {sorted(missing)}")
if extra:
print(f"│ ⓘ extra tools called: {sorted(extra)}")
if rec["error"]:
print(f"│ 💥 error: {rec['error'][:200]}")
# Bridge-trip count vs direct (interesting comparator)
en_bridges = len(en["bridge_calls"])
di_underlying = len(di["underlying_tool_calls"])
en_underlying = len(en["underlying_tool_calls"])
overhead = en_bridges + en_underlying - di_underlying
print(f"│ Δ round-trip cost: enabled used {en_bridges + en_underlying} calls vs disabled {di_underlying} → +{overhead}")
print(f"│ Final (enabled): {(en.get('final_response') or '')[:140]}")
print(f"│ Final (disabled): {(di.get('final_response') or '')[:140]}")
print("└──")
print()
print(f"\nFails: {fails}/{2*len(scenarios)}")
if __name__ == "__main__":
main()
scripts/audit_pr_attribution.py
+147
View File
@@ -0,0 +1,147 @@
#!/usr/bin/env python3
"""Audit (and auto-fix) contributor email mappings for a PR branch.
Mirrors the CI gate in .github/workflows/contributor-check.yml so salvage
branches never bounce off the check-attribution job. Run it from the branch
you are about to push:
python3 scripts/audit_pr_attribution.py # report only
python3 scripts/audit_pr_attribution.py --fix # create mapping files
Logic (kept in sync with contributor-check.yml):
- scans ``git log $(git merge-base origin/main HEAD)..HEAD --format=%ae``
- skips teknium/bot emails and ``<id>+<login>@users.noreply.github.com``
(CI auto-resolves those)
- everything else must have ``contributors/emails/<email>`` or a legacy
AUTHOR_MAP entry in scripts/release.py
``--fix`` resolution order for an unmapped email:
1. bare ``<login>@users.noreply.github.com`` → ``<login>``, verified via
``gh api users/<login>``. A warning is printed: the local part is
*usually* the GitHub login but is user-controlled (the historical
``bryan@…`` → ``hydraxman`` case) — eyeball it against the PR author.
2. ``gh api 'search/users?q=<email>+in:email'``
3. otherwise: prints the manual ``add_contributor.py`` command and exits 1.
"""
import argparse
import json
import re
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
SKIP_SUBSTRINGS = (
"teknium",
"noreply@github.com",
"dependabot",
"github-actions",
"anthropic.com",
"cursor.com",
)
ID_NOREPLY_RE = re.compile(r"\d+\+.+@users\.noreply\.github\.com$")
BARE_NOREPLY_RE = re.compile(r"^([A-Za-z0-9](?:[A-Za-z0-9]|-(?=[A-Za-z0-9])){0,38})@users\.noreply\.github\.com$")
def run(*args: str, check: bool = True) -> str:
result = subprocess.run(
list(args), capture_output=True, text=True, encoding="utf-8",
errors="replace", cwd=str(REPO_ROOT),
)
if check and result.returncode != 0:
raise RuntimeError(f"{' '.join(args)}: {result.stderr.strip()}")
return result.stdout.strip()
def new_emails() -> list[str]:
base = run("git", "merge-base", "origin/main", "HEAD")
log = run("git", "log", f"{base}..HEAD", "--format=%ae", "--no-merges", check=False)
return sorted({e for e in log.splitlines() if e.strip()})
def is_mapped(email: str) -> bool:
if any(s in email for s in SKIP_SUBSTRINGS):
return True
if ID_NOREPLY_RE.search(email):
return True
if (REPO_ROOT / "contributors" / "emails" / email).is_file():
return True
release_py = REPO_ROOT / "scripts" / "release.py"
try:
if f'"{email}"' in release_py.read_text(encoding="utf-8", errors="replace"):
return True
except OSError:
pass
return False
def gh_json(*args: str):
try:
out = run("gh", "api", *args, check=False)
return json.loads(out) if out else None
except (RuntimeError, json.JSONDecodeError, FileNotFoundError):
return None
def resolve_login(email: str) -> tuple[str, str] | None:
"""Return (login, how) or None."""
m = BARE_NOREPLY_RE.match(email)
if m:
login = m.group(1)
user = gh_json(f"users/{login}")
if user and user.get("login"):
return user["login"], "bare-noreply local part (verified user exists)"
found = gh_json(f"search/users?q={email}+in:email")
if found and found.get("items"):
return found["items"][0]["login"], "GitHub email search"
return None
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--fix", action="store_true",
help="auto-create contributors/emails/ mapping files")
args = parser.parse_args()
unmapped = [e for e in new_emails() if not is_mapped(e)]
if not unmapped:
print("✅ All contributor emails on this branch are mapped.")
return 0
failed = []
for email in unmapped:
author = run("git", "log", f"--author={email}", "--format=%an", "-1", check=False)
if not args.fix:
print(f"⚠️ unmapped: {email} ({author})")
continue
resolved = resolve_login(email)
if resolved:
login, how = resolved
run("python3", "scripts/add_contributor.py", email, login)
print(f"✔ mapped {email} -> {login} [{how}]")
if BARE_NOREPLY_RE.match(email):
print(f" ⚠ local part is user-controlled — confirm @{login} really is "
f"the contributor (git name: {author!r}) before pushing.")
else:
failed.append((email, author))
if not args.fix:
print("\nRun with --fix to auto-create mapping files, or manually:")
for email in unmapped:
print(f" python3 scripts/add_contributor.py {email} <github-username>")
return 1
if failed:
print("\nCould not auto-resolve; map manually:")
for email, author in failed:
print(f" python3 scripts/add_contributor.py {email} <github-username> # {author}")
return 1
print("\nDone — remember to `git add contributors && git commit`.")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/benchmark_browser_eval.py
+138
View File
@@ -0,0 +1,138 @@
"""Quick benchmark: subprocess eval vs supervisor-WS eval.
Runs both paths against the same live Chrome and prints a comparison table.
Not a pytest — a script you run manually for the PR description.
Usage:
.venv/bin/python scripts/benchmark_browser_eval.py [--iterations N]
"""
from __future__ import annotations
import argparse
import shutil
import statistics
import subprocess
import sys
import tempfile
import time
import urllib.request
import json
def _find_chrome() -> str:
for c in ("google-chrome", "chromium", "chromium-browser"):
p = shutil.which(c)
if p:
return p
print("No Chrome binary found.", file=sys.stderr)
sys.exit(1)
def _start_chrome(port: int):
profile = tempfile.mkdtemp(prefix="hermes-bench-eval-")
proc = subprocess.Popen(
[
_find_chrome(),
f"--remote-debugging-port={port}",
f"--user-data-dir={profile}",
"--no-first-run",
"--no-default-browser-check",
"--headless=new",
"--disable-gpu",
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/version", timeout=1) as r:
info = json.loads(r.read().decode())
return proc, profile, info["webSocketDebuggerUrl"]
except Exception:
time.sleep(0.25)
proc.terminate()
raise RuntimeError("Chrome didn't expose CDP")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--iterations", type=int, default=50)
parser.add_argument("--port", type=int, default=9333)
args = parser.parse_args()
proc, profile, cdp_url = _start_chrome(args.port)
try:
from tools.browser_supervisor import SUPERVISOR_REGISTRY
# Warm up: start the supervisor, navigate to a page.
supervisor = SUPERVISOR_REGISTRY.get_or_start(
task_id="bench-eval", cdp_url=cdp_url
)
# Give it a moment to attach.
time.sleep(1.0)
# Sanity check: one eval over WS should succeed.
sanity = supervisor.evaluate_runtime("1 + 1")
if not sanity.get("ok") or sanity.get("result") != 2:
print(f"sanity check failed: {sanity}", file=sys.stderr)
sys.exit(2)
# ── Bench 1: supervisor WS path ──────────────────────────────────
ws_times: list[float] = []
for _ in range(args.iterations):
t0 = time.monotonic()
out = supervisor.evaluate_runtime("1 + 1")
t1 = time.monotonic()
assert out.get("ok"), out
ws_times.append((t1 - t0) * 1000)
# ── Bench 2: agent-browser subprocess path ────────────────────────
# Skip if agent-browser isn't installed — the WS bench still tells
# us what we need.
if shutil.which("agent-browser") is None and shutil.which("npx") is None:
print("agent-browser CLI not found — skipping subprocess bench.")
sub_times = []
else:
from tools.browser_tool import _run_browser_command, _last_session_key
task_id = _last_session_key("bench-eval")
sub_times = []
for _ in range(args.iterations):
t0 = time.monotonic()
_run_browser_command(task_id, "eval", ["1 + 1"])
t1 = time.monotonic()
sub_times.append((t1 - t0) * 1000)
def fmt(name: str, ts: list[float]) -> str:
if not ts:
return f" {name:<40} (skipped)"
mean = statistics.mean(ts)
median = statistics.median(ts)
mn, mx = min(ts), max(ts)
return (
f" {name:<40} mean={mean:>7.2f}ms median={median:>7.2f}ms "
f"min={mn:>7.2f}ms max={mx:>7.2f}ms"
)
print()
print(f"browser_eval benchmark — {args.iterations} iterations of `1 + 1`")
print("-" * 90)
print(fmt("supervisor WS (Runtime.evaluate)", ws_times))
print(fmt("agent-browser subprocess (eval)", sub_times))
if ws_times and sub_times:
speedup = statistics.mean(sub_times) / statistics.mean(ws_times)
print()
print(f"Speedup: {speedup:.1f}x (mean)")
finally:
SUPERVISOR_REGISTRY.stop_all()
proc.terminate()
try:
proc.wait(timeout=3)
except Exception:
proc.kill()
shutil.rmtree(profile, ignore_errors=True)
if __name__ == "__main__":
main()
scripts/build_model_catalog.py
Executable
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env python3
"""Build the Hermes Model Catalog — a centralized JSON manifest of curated models.
This script reads the in-repo hardcoded curated lists (``OPENROUTER_MODELS``,
``_PROVIDER_MODELS["nous"]``) and writes them to a JSON manifest that the
Hermes CLI fetches at runtime. Publishing the catalog through the docs site
lets maintainers update model lists without shipping a Hermes release.
The runtime fetcher falls back to the same in-repo hardcoded lists if the
manifest is unreachable, so this script is a convenience for keeping the
manifest in sync — not a source of truth.
Usage::
python scripts/build_model_catalog.py
Output: ``website/static/api/model-catalog.json``
Live URL (after ``deploy-site.yml`` runs on merge to main):
``https://hermes-agent.nousresearch.com/docs/api/model-catalog.json``
"""
from __future__ import annotations
import json
import os
import sys
from datetime import datetime, timezone
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, REPO_ROOT)
# Ensure HERMES_HOME is set for imports that touch it at module level.
os.environ.setdefault("HERMES_HOME", os.path.join(os.path.expanduser("~"), ".hermes"))
from hermes_cli.models import ( # noqa: E402
OPENROUTER_MODELS,
PREFERRED_SILENT_DEFAULT_MODEL,
_PROVIDER_MODELS,
)
OUTPUT_PATH = os.path.join(REPO_ROOT, "website", "static", "api", "model-catalog.json")
CATALOG_VERSION = 1
def _openrouter_entry(mid: str, desc: str) -> dict:
entry: dict = {"id": mid, "description": desc}
if mid == PREFERRED_SILENT_DEFAULT_MODEL:
entry["description"] = desc or "default"
entry["default"] = True
return entry
def _nous_entry(mid: str) -> dict:
entry: dict = {"id": mid}
if mid == PREFERRED_SILENT_DEFAULT_MODEL:
entry["default"] = True
return entry
def build_catalog() -> dict:
return {
"version": CATALOG_VERSION,
"updated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"metadata": {
"source": "hermes-agent repo",
"docs": "https://hermes-agent.nousresearch.com/docs/reference/model-catalog",
},
"providers": {
"openrouter": {
"metadata": {
"display_name": "OpenRouter",
"note": (
"Descriptions drive picker badges. Live /api/v1/models "
"filters curated ids by tool-calling support and free pricing. "
'The entry labeled "default": true is the model Hermes '
"silently lands on when the user never picked one."
),
},
"models": [
_openrouter_entry(mid, desc)
for mid, desc in OPENROUTER_MODELS
],
},
"nous": {
"metadata": {
"display_name": "Nous Portal",
"note": (
"Free-tier gating is determined live via Portal pricing "
"(partition_nous_models_by_tier), not this manifest. "
'The entry labeled "default": true is the model Hermes '
"silently lands on when the user never picked one."
),
},
"models": [
_nous_entry(mid)
for mid in _PROVIDER_MODELS.get("nous", [])
],
},
},
}
def main() -> int:
catalog = build_catalog()
os.makedirs(os.path.dirname(OUTPUT_PATH), exist_ok=True)
with open(OUTPUT_PATH, "w", encoding="utf-8") as fh:
json.dump(catalog, fh, indent=2)
fh.write("\n")
print(f"Wrote {OUTPUT_PATH}")
for provider, block in catalog["providers"].items():
print(f" {provider}: {len(block['models'])} models")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/build_skills_index.py
+459
View File
@@ -0,0 +1,459 @@
#!/usr/bin/env python3
"""Build the Hermes Skills Index — a centralized JSON catalog of all skills.
This script crawls every skill source (skills.sh, GitHub taps, official,
clawhub, lobehub) and writes a JSON index with resolved
GitHub paths. The index is served as a static file on the docs site so that
`hermes skills search/install` can use it without hitting the GitHub API.
Usage:
# Local (uses gh CLI or GITHUB_TOKEN for auth)
python scripts/build_skills_index.py
# CI (set GITHUB_TOKEN as secret)
GITHUB_TOKEN=ghp_... python scripts/build_skills_index.py
Output: website/static/api/skills-index.json
"""
import json
import os
import sys
import time
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
# Allow importing from repo root
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, REPO_ROOT)
# Ensure HERMES_HOME is set (needed by tools/skills_hub.py imports)
os.environ.setdefault("HERMES_HOME", os.path.join(os.path.expanduser("~"), ".hermes"))
from tools.skills_hub import (
GitHubAuth,
GitHubSource,
SkillsShSource,
OptionalSkillSource,
WellKnownSkillSource,
ClawHubSource,
LobeHubSource,
BrowseShSource,
SkillMeta,
)
import httpx
OUTPUT_PATH = os.path.join(REPO_ROOT, "website", "static", "api", "skills-index.json")
INDEX_VERSION = 1
def _meta_to_dict(meta: SkillMeta) -> dict:
"""Convert a SkillMeta to a serializable dict."""
return {
"name": meta.name,
"description": meta.description,
"source": meta.source,
"identifier": meta.identifier,
"trust_level": meta.trust_level,
"repo": meta.repo or "",
"path": meta.path or "",
"tags": meta.tags or [],
"extra": meta.extra or {},
}
def crawl_source(source, source_name: str, limit: int) -> list:
"""Crawl a single source and return skill dicts."""
print(f" Crawling {source_name}...", flush=True)
start = time.time()
try:
results = source.search("", limit=limit)
except Exception as e:
print(f" Error crawling {source_name}: {e}", file=sys.stderr)
return []
skills = [_meta_to_dict(m) for m in results]
elapsed = time.time() - start
print(f" {source_name}: {len(skills)} skills ({elapsed:.1f}s)", flush=True)
return skills
def crawl_skills_sh(source: SkillsShSource) -> list:
"""Crawl skills.sh via its sitemap to enumerate the full catalog (~20k entries).
Previously walked a hardcoded list of ~28 popular keywords (each capped at
50 results) which yielded ~850 unique skills — about 4% of the real catalog.
The SkillsShSource.search("") path now hits the sitemap directly, returning
the full 20k-entry catalog deduplicated by canonical identifier.
"""
print(" Crawling skills.sh (sitemap)...", flush=True)
start = time.time()
try:
results = source.search("", limit=0) # 0 = no cap, return the whole catalog
except Exception as e:
print(f" Warning: skills.sh sitemap walk failed: {e}", file=sys.stderr)
results = []
all_skills: dict[str, dict] = {}
for meta in results:
entry = _meta_to_dict(meta)
if entry["identifier"] not in all_skills:
all_skills[entry["identifier"]] = entry
elapsed = time.time() - start
print(f" skills.sh: {len(all_skills)} unique skills ({elapsed:.1f}s)",
flush=True)
return list(all_skills.values())
def _fetch_repo_tree(repo: str, auth: GitHubAuth) -> list:
"""Fetch the recursive tree for a repo. Returns list of tree entries."""
headers = auth.get_headers()
try:
resp = httpx.get(
f"https://api.github.com/repos/{repo}",
headers=headers, timeout=15, follow_redirects=True,
)
if resp.status_code != 200:
return []
branch = resp.json().get("default_branch", "main")
resp = httpx.get(
f"https://api.github.com/repos/{repo}/git/trees/{branch}",
params={"recursive": "1"},
headers=headers, timeout=30, follow_redirects=True,
)
if resp.status_code != 200:
return []
data = resp.json()
if data.get("truncated"):
return []
return data.get("tree", [])
except Exception:
return []
def batch_resolve_paths(skills: list, auth: GitHubAuth) -> list:
"""Resolve GitHub paths for skills.sh entries using batch tree lookups.
Instead of resolving each skill individually (N×M API calls), we:
1. Group skills by repo
2. Fetch one tree per repo (2 API calls per repo)
3. Find all SKILL.md files in the tree
4. Match skills to their resolved paths
"""
# Filter to skills.sh entries that need resolution
skills_sh = [s for s in skills if s["source"] in {"skills.sh", "skills-sh"}]
if not skills_sh:
return skills
print(f" Resolving paths for {len(skills_sh)} skills.sh entries...",
flush=True)
start = time.time()
# Group by repo
by_repo: dict[str, list] = defaultdict(list)
for s in skills_sh:
repo = s.get("repo", "")
if repo:
by_repo[repo].append(s)
print(f" {len(by_repo)} unique repos to scan", flush=True)
resolved_count = 0
# Fetch trees in parallel (up to 6 concurrent)
def _resolve_repo(repo: str, entries: list):
tree = _fetch_repo_tree(repo, auth)
if not tree:
return 0
# Find all SKILL.md paths in this repo
skill_paths = {} # skill_dir_name -> full_path
for item in tree:
if item.get("type") != "blob":
continue
path = item.get("path", "")
if path.endswith("/SKILL.md"):
skill_dir = path[: -len("/SKILL.md")]
dir_name = skill_dir.split("/")[-1]
skill_paths[dir_name.lower()] = f"{repo}/{skill_dir}"
# Also check SKILL.md frontmatter name if we can match by path
# For now, just index by directory name
elif path == "SKILL.md":
# Root-level SKILL.md
skill_paths["_root_"] = f"{repo}"
count = 0
for entry in entries:
# Try to match the skill's name/path to a tree entry
skill_name = entry.get("name", "").lower()
skill_path = entry.get("path", "").lower()
identifier = entry.get("identifier", "")
# Extract the skill token from the identifier
# e.g. "skills-sh/d4vinci/scrapling/scrapling-official" -> "scrapling-official"
parts = identifier.replace("skills-sh/", "").replace("skills.sh/", "")
skill_token = parts.split("/")[-1].lower() if "/" in parts else ""
# Try matching in order of likelihood
for candidate in [skill_token, skill_name, skill_path]:
if not candidate:
continue
matched = skill_paths.get(candidate)
if matched:
entry["resolved_github_id"] = matched
count += 1
break
else:
# Try fuzzy: skill_token with common transformations
for tree_name, tree_path in skill_paths.items():
if (skill_token and (
tree_name.replace("-", "") == skill_token.replace("-", "")
or skill_token in tree_name
or tree_name in skill_token
)):
entry["resolved_github_id"] = tree_path
count += 1
break
return count
with ThreadPoolExecutor(max_workers=6) as pool:
futures = {
pool.submit(_resolve_repo, repo, entries): repo
for repo, entries in by_repo.items()
}
for future in as_completed(futures):
try:
resolved_count += future.result()
except Exception as e:
repo = futures[future]
print(f" Warning: {repo}: {e}", file=sys.stderr)
elapsed = time.time() - start
print(f" Resolved {resolved_count}/{len(skills_sh)} paths ({elapsed:.1f}s)",
flush=True)
return skills
def main():
print("Building Hermes Skills Index...", flush=True)
overall_start = time.time()
auth = GitHubAuth()
print(f"GitHub auth: {auth.auth_method()}")
if auth.auth_method() == "anonymous":
print("WARNING: No GitHub authentication — rate limit is 60/hr. "
"Set GITHUB_TOKEN for better results.", file=sys.stderr)
skills_sh_source = SkillsShSource(auth=auth)
sources = {
"official": OptionalSkillSource(),
"well-known": WellKnownSkillSource(),
"github": GitHubSource(auth=auth),
"clawhub": ClawHubSource(),
"lobehub": LobeHubSource(),
"browse-sh": BrowseShSource(),
}
all_skills: list[dict] = []
# Crawl skills.sh
all_skills.extend(crawl_skills_sh(skills_sh_source))
# Crawl other sources in parallel.
# Per-source soft caps — sources stop returning when they run out, so these
# are ceilings, not targets. ClawHub has 20k+ skills; bumping to 100k
# (well above current catalog size) lets the full catalog land in the
# index instead of being truncated at an arbitrary build-time limit.
SOURCE_LIMITS = {
# 0 = unbounded catalog walk (max_items=0 in ClawHubSource). A positive
# limit bounds the walk and also enables the interactive 12s budget.
"clawhub": 0,
"lobehub": 100_000,
"browse-sh": 5_000,
"github": 5_000,
"well-known": 5_000,
"official": 5_000,
}
DEFAULT_SOURCE_LIMIT = 500
with ThreadPoolExecutor(max_workers=4) as pool:
futures = {}
for name, source in sources.items():
limit = SOURCE_LIMITS.get(name, DEFAULT_SOURCE_LIMIT)
futures[pool.submit(crawl_source, source, name, limit)] = name
for future in as_completed(futures):
try:
all_skills.extend(future.result())
except Exception as e:
print(f" Error: {e}", file=sys.stderr)
# Batch resolve GitHub paths for skills.sh entries
all_skills = batch_resolve_paths(all_skills, auth)
# Enrich ClawHub skills with owner handles. The listing API does not
# include owner info, so we fetch each skill's detail page concurrently.
# This is needed to build valid "View source" URLs on the Skills Hub page:
# https://clawhub.ai/{owner}/skills/{slug}. Without the owner segment the
# URL leads to a 404.
clawhub_skills = [s for s in all_skills if s["source"] == "clawhub"]
if clawhub_skills:
# Convert dicts back to SkillMeta for enrichment, then update in place.
clawhub_metas = []
for s in clawhub_skills:
meta = SkillMeta(
name=s["name"],
description=s["description"],
source=s["source"],
identifier=s["identifier"],
trust_level=s["trust_level"],
repo=s.get("repo") or None,
path=s.get("path") or None,
tags=s.get("tags") or [],
extra=dict(s.get("extra") or {}),
)
clawhub_metas.append(meta)
print(f" Enriching {len(clawhub_metas)} ClawHub skills with owner handles...",
flush=True)
enrich_start = time.time()
enriched = sources["clawhub"].enrich_owners(clawhub_metas, max_workers=30)
# Write enriched owner back into the index dicts.
meta_by_id = {m.identifier: m for m in clawhub_metas}
for s in clawhub_skills:
meta = meta_by_id.get(s["identifier"])
if meta and meta.extra.get("owner"):
s.setdefault("extra", {})["owner"] = meta.extra["owner"]
enrich_elapsed = time.time() - enrich_start
print(f" Enriched {enriched}/{len(clawhub_metas)} ClawHub owners "
f"({enrich_elapsed:.1f}s)", flush=True)
# Collect which sources hit a GitHub API rate limit during the crawl.
# github / well-known both read api.github.com, so a rate-limited token
# zeroes both at once — surfaced below so the failure message names the
# real cause instead of "source returned 0".
rate_limited_sources = {
name for name, source in sources.items()
if getattr(source, "is_rate_limited", False)
}
if rate_limited_sources:
print(
" WARNING: GitHub API rate limit hit for: "
+ ", ".join(sorted(rate_limited_sources)),
file=sys.stderr,
)
# Deduplicate by identifier
seen: dict[str, dict] = {}
for skill in all_skills:
key = skill["identifier"]
if key not in seen:
seen[key] = skill
deduped = list(seen.values())
# Sort
source_order = {"official": 0, "skills-sh": 1, "skills.sh": 1,
"github": 2, "well-known": 3, "clawhub": 4,
"browse-sh": 5, "lobehub": 6}
deduped.sort(key=lambda s: (source_order.get(s["source"], 99), s["name"]))
from collections import Counter
by_source = Counter(s["source"] for s in deduped)
print(f"\nCrawled {len(deduped)} skills in {time.time() - overall_start:.0f}s")
for src, count in sorted(by_source.items(), key=lambda x: -x[1]):
resolved = sum(1 for s in deduped
if s["source"] == src and s.get("resolved_github_id"))
extra = f" ({resolved} resolved)" if resolved else ""
print(f" {src}: {count}{extra}")
# Health check: catch silent breakage early. Every source listed below
# has historically returned at least `floor` entries; a zero (or near-
# zero) result almost certainly means a tap path moved, an API changed,
# or rate limiting kicked in. Failing here forces a human look before
# the broken index reaches the live docs.
EXPECTED_FLOORS = {
# skills.sh now uses the sitemap walker (~20k catalog as of May 2026).
# Anything under 10k means the sitemap shape changed or fetches failed
# — better to fail loudly than ship a regression to the 858-skill
# popular-queries era.
"skills.sh": 10000,
"lobehub": 100,
# ClawHub had 49,698+ skills as of May 2026 — anything under 20k means
# pagination broke or the API surface changed. Fail loudly rather
# than ship a degenerate index (we shipped 200/50000 silently for
# weeks because the floor was 50).
"clawhub": 20000,
"official": 50,
"github": 30, # collapsed across all GitHub taps
"browse-sh": 50,
}
health_errors = []
for src, floor in EXPECTED_FLOORS.items():
# 'skills-sh' and 'skills.sh' are the same source; both labels exist.
count = by_source.get(src, 0)
if src == "skills.sh":
count = by_source.get("skills.sh", 0) + by_source.get("skills-sh", 0)
if count < floor:
health_errors.append(f" {src}: {count} < expected floor {floor}")
MIN_TOTAL = 1500
if len(deduped) < MIN_TOTAL:
health_errors.append(
f" total: {len(deduped)} < expected floor {MIN_TOTAL}"
)
if health_errors:
print(
"\nERROR: skills index health check failed — refusing to ship "
"a degenerate index. Investigate the following sources:",
file=sys.stderr,
)
for line in health_errors:
print(line, file=sys.stderr)
if rate_limited_sources:
print(
"\nGitHub API rate limit was hit during this crawl for: "
+ ", ".join(sorted(rate_limited_sources))
+ ". This is the usual cause of an all-GitHub-tap collapse "
"(github / well-known dropping to zero together). "
"Re-run with a higher-quota GITHUB_TOKEN.",
file=sys.stderr,
)
print(
"\nIf the drop is expected (e.g. a hub is genuinely shutting "
"down), lower the floor in scripts/build_skills_index.py "
"EXPECTED_FLOORS in the same PR.",
file=sys.stderr,
)
# IMPORTANT: do NOT write OUTPUT_PATH on failure. The index file is
# gitignored, so a fresh deploy checkout has no copy on disk — leaving
# it absent lets website/scripts/extract-skills.py fall back to the
# legacy snapshot cache (or skip the unified index) instead of reading
# a degenerate file. Writing-then-exiting-2 was the bug that shipped an
# index with every GitHub-API source dropped to zero: deploy-site.yml
# swallows the exit code with `|| echo non-fatal`, and the partial file
# was already on disk for extract-skills to pick up.
sys.exit(2)
# Healthy — only now write the index out for the docs build to consume.
index = {
"version": INDEX_VERSION,
"generated_at": datetime.now(timezone.utc).isoformat(),
"skill_count": len(deduped),
"skills": deduped,
}
os.makedirs(os.path.dirname(OUTPUT_PATH), exist_ok=True)
with open(OUTPUT_PATH, "w", encoding="utf-8") as f:
json.dump(index, f, separators=(",", ":"), ensure_ascii=False)
file_size = os.path.getsize(OUTPUT_PATH)
print(f"\nDone! {len(deduped)} skills indexed in "
f"{time.time() - overall_start:.0f}s")
print(f"Output: {OUTPUT_PATH} ({file_size / 1024:.0f} KB)")
if __name__ == "__main__":
main()
scripts/capture-cage-terminal.sh
Executable
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -euo pipefail
out=${1:?usage: $0 OUTPUT.png [command...]}
shift
mkdir -p "$(dirname "$out")"
# This runs inside `cage --`, after Ghostty has already opened the isolated
# Wayland display. Capture before this script exits so Cage keeps the surface
# alive long enough for grim to see it.
if (($#)); then
"$@"
fi
if ! command -v grim >/dev/null 2>&1; then
echo "grim is required inside the Cage client session" >&2
exit 127
fi
grim "$out"
scripts/check-case-collisions.py
+114
View File
@@ -0,0 +1,114 @@
#!/usr/bin/env python3
"""
Blocking check for tracked files whose paths collide when case is ignored.
Linux is case-sensitive; Windows and macOS (default) are not. Two tracked
paths that differ only by case — ``README.md`` and ``readme.md``, or
``src/Foo.py`` and ``SRC/foo.py`` — coexist happily in a Linux checkout and
silently break every clone on a case-insensitive host: the filesystem can
hold only one of them, so checkout either refuses or whichever file is
written last wins and clobbers the other. Git itself won't stop the pair
from landing — it only warns at checkout time, on a case-insensitive FS,
for whichever client happens to do the checkout, and the collision is
invisible on Linux. This check is the enforcement point: scan the index,
fail the build, name the offenders.
Usage:
# Check the checkout this script lives in (CI + the common local case)
python scripts/check-case-collisions.py
# Check an arbitrary git checkout (tests, other worktrees)
python scripts/check-case-collisions.py /path/to/other/repo
Exit status:
0 — no case-colliding tracked paths
1 — at least one collision group (paths printed to stdout)
2 — not in a git repository / git failed
Comparison key: the casefolded FULL path (``str.casefold``), not the
basename — on a case-insensitive filesystem the entire path is
case-insensitive, so ``dir/Foo.txt`` and ``DIR/foo.txt`` collide just like
same-directory pairs. ``casefold`` (not ``lower``) is used because it
matches how the OSes fold case for non-ASCII text (straße vs strasse,
sigma variants); a pair it flags is a genuine collision on macOS/Windows
even when Linux disagrees.
Deliberately out of scope: Unicode NFC/NFD normalization collisions (macOS
stores NFD, Linux NFC). git already handles those at checkout via
``core.precomposeunicode``; this check is strictly about case.
"""
from __future__ import annotations
import argparse
import os
import subprocess
import sys
from collections import defaultdict
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"root",
nargs="?",
default=str(REPO_ROOT),
help="git checkout to scan (default: the repo this script lives in)",
)
args = parser.parse_args()
try:
os.chdir(args.root)
except OSError as exc:
print(f"::error::cannot enter {args.root}: {exc}")
return 2
proc = subprocess.run(["git", "ls-files", "-z"], capture_output=True)
if proc.returncode != 0:
msg = proc.stderr.decode("utf-8", errors="replace").strip()
print(f"::error::git ls-files failed in {args.root}: {msg}")
return 2
paths = [
p.decode("utf-8", errors="surrogateescape")
for p in proc.stdout.split(b"\0")
if p
]
by_casefold: dict[str, list[str]] = defaultdict(list)
for path in paths:
by_casefold[path.casefold()].append(path)
collisions = {key: group for key, group in by_casefold.items() if len(group) > 1}
if not collisions:
print(f"::notice::{len(paths)} tracked files, no case-colliding paths.")
return 0
print(
f"::error::Found {len(collisions)} case-collision group(s) among "
f"{len(paths)} tracked files."
)
print(
"Paths that differ only by case are ONE file on Windows/macOS but "
"several on Linux - the pair breaks every clone on a case-insensitive "
"host. Rename one member of each group so the paths differ beyond case."
)
print()
for key, group in sorted(collisions.items()):
for path in sorted(group):
print(f" {path}")
print()
print(
"Fix: `git mv` one path in each group to a name that doesn't collide. "
"On Windows/macOS you may need two steps (`git mv a.txt tmp && git mv "
"tmp A.txt`) because the filesystem can't hold both spellings at once."
)
return 1
if __name__ == "__main__":
sys.exit(main())
scripts/check-windows-footguns.py
+814
View File
@@ -0,0 +1,814 @@
#!/usr/bin/env python3
"""
Grep-based checker for Windows cross-platform footguns.
Flags common patterns that break silently on Windows. Run before PRs —
cheap, fast, catches regressions in a codebase that runs on three OSes.
Usage:
# Scan staged changes (default when run from a git checkout)
python scripts/check-windows-footguns.py
# Scan the full tree (full-repo audit)
python scripts/check-windows-footguns.py --all
# Scan a specific file or directory
python scripts/check-windows-footguns.py path/to/file.py path/to/dir/
# Scan only modified files vs. main
python scripts/check-windows-footguns.py --diff main
Exit status:
0 — no Windows footguns found (or all matches suppressed)
1 — at least one unsuppressed match
Suppress an intentional use (e.g. tests or platform-gated code) with:
os.kill(pid, 0) # windows-footgun: ok — only called on POSIX
"""
from __future__ import annotations
import argparse
import os
import re
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable
REPO_ROOT = Path(__file__).resolve().parent.parent
SUPPRESS_MARKER = re.compile(r"#\s*windows-footgun\s*:\s*ok\b", re.IGNORECASE)
# Line-level guard hints. If a line contains any of these tokens, we assume
# the programmer wrote the line in full awareness of the Windows pitfall —
# e.g. `if hasattr(os, 'setsid'): ... os.setsid()`, or the classic
# `getattr(signal, 'SIGKILL', signal.SIGTERM)`, or `shutil.which("wmic")`.
# False negatives are fine here — the inline `# windows-footgun: ok` marker
# is still the authoritative suppression. This is just to reduce the noise
# floor on obviously-guarded lines so the signal-to-noise stays useful.
GUARD_HINTS = (
"hasattr(os,",
"hasattr(signal,",
"getattr(os,",
"getattr(signal,",
"shutil.which(",
"if platform.system() != \"Windows\"",
"if platform.system() != 'Windows'",
"if sys.platform == \"win32\"",
"if sys.platform != \"win32\"",
"if sys.platform == 'win32'",
"if sys.platform != 'win32'",
"IS_WINDOWS",
"is_windows",
)
# Dirs we never scan.
EXCLUDED_DIRS = {
".git",
"node_modules",
"venv",
".venv",
"__pycache__",
"build",
"dist",
".tox",
".mypy_cache",
".pytest_cache",
"site-packages",
"website/build",
"optional-skills", # external skills
}
# File globs we never scan (beyond the dirs above).
EXCLUDED_SUFFIXES = {
".pyc",
".pyo",
".so",
".dll",
".exe",
".png",
".jpg",
".gif",
".ico",
".svg",
".mp4",
".mp3",
".wav",
".pdf",
".zip",
".tar",
".gz",
".whl",
".lock",
".min.js",
".min.css",
}
# Files we never scan (self-referential — this script mentions the
# patterns it detects — and the CONTRIBUTING docs that list them).
EXCLUDED_FILES = {
"scripts/check-windows-footguns.py",
"CONTRIBUTING.md",
}
@dataclass
class Footgun:
"""A Windows cross-platform footgun pattern."""
name: str
pattern: re.Pattern
message: str
fix: str
# If set, matches in files/paths containing any of these substrings are
# silently ignored (e.g. tests that legitimately exercise the footgun
# behind a platform guard). Prefer `# windows-footgun: ok` inline
# suppression over this list; only use path_allowlist for whole files
# that are inherently tests of the footgun itself.
path_allowlist: tuple[str, ...] = ()
# Optional post-match predicate. Takes the re.Match and returns True
# if the match is a REAL footgun (not a false positive). Use this when
# the regex can't fully distinguish (e.g. open() where mode may contain
# "b" for binary, or the line may have `encoding=` elsewhere).
post_filter: "callable | None" = None
FOOTGUNS: list[Footgun] = [
Footgun(
name="open() without encoding= on text mode",
# Match builtins.open() specifically — NOT os.open(), .open()
# method calls (Path.open, tarfile.open, zf.open, webbrowser.open,
# Image.open, wave.open, etc), or `async def open()` method
# definitions. The pattern requires a start-of-identifier boundary
# before `open(` so `os.open`, `.open`, `def open` are all skipped.
# Note: Path.open() is ALSO affected by the encoding default, but
# rather than flagging all `.open(` (huge noise), we require an
# explicit builtins-style open() call. Path.open() is rare in the
# codebase compared to open() and can be audited separately.
pattern=re.compile(
r"""(?:^|[\s\(,;=])(?<![.\w])open\s*\(\s*[^,)]+\s*(?:,\s*['"](?P<mode>[^'"]*)['"])?"""
),
message=(
"open() without an explicit encoding= uses the platform default "
"(UTF-8 on POSIX, cp1252/mbcs on Windows) — files round-tripped "
"between hosts get mojibake. Always pass encoding='utf-8' for "
"text files, or use open(path, 'rb')/'wb' for binary."
),
fix=(
"open(path, 'r', encoding='utf-8') # or 'utf-8-sig' if the "
"file may have a BOM"
),
# Filter: only flag if mode is missing-or-text AND the line doesn't
# already pass encoding=. Skip binary mode (contains "b").
post_filter=lambda m, line: (
"b" not in (m.group("mode") or "")
and "encoding=" not in line
and "encoding =" not in line
# Skip `def open(` and `async def open(` (method definitions)
and not line.lstrip().startswith("def ")
and not line.lstrip().startswith("async def ")
# Skip open(path, **kwargs) patterns — encoding may be in the dict.
# Too expensive to trace; require the author to set encoding in
# the dict and trust them (or they can add a # windows-footgun: ok).
and "**" not in line
),
),
Footgun(
name="os.fdopen() without encoding= on text mode",
# ruff PLW1514 covers builtins.open/Path.read_text/write_text/
# Path.open but NOT os.fdopen — a bare text-mode fdopen still
# decodes/encodes with the locale default (cp1252 on Windows).
# This is the exact hole the July 2026 encoding sweep kept
# re-fixing by hand (PRs #56033/#56940/#65565), so gate it here.
pattern=re.compile(
r"""(?:os\s*\.\s*)?\bfdopen\s*\(\s*[^,)]+\s*(?:,\s*['"](?P<mode>[^'"]*)['"])?"""
),
message=(
"os.fdopen() without an explicit encoding= uses the platform "
"default (cp1252/mbcs on Windows) in text mode — the same "
"mojibake class as bare open(). ruff PLW1514 does not cover "
"fdopen, so this checker is the only gate."
),
fix=(
"os.fdopen(fd, 'w', encoding='utf-8') # or mode 'wb' for binary"
),
post_filter=lambda m, line: (
"b" not in (m.group("mode") or "")
and "encoding=" not in line
and "encoding =" not in line
and "**" not in line
),
),
Footgun(
name="os.kill(pid, 0)",
pattern=re.compile(r"\bos\.kill\s*\(\s*[^,]+,\s*0\s*\)"),
message=(
"os.kill(pid, 0) is NOT a no-op on Windows — it sends "
"CTRL_C_EVENT to the target's console process group, "
"hard-killing the target and potentially unrelated siblings. "
"See bpo-14484."
),
fix=(
"Use psutil.pid_exists(pid) (psutil is a core dependency). "
"Or gateway.status._pid_exists(pid) for the hermes wrapper "
"with a stdlib fallback."
),
),
Footgun(
name="bare os.setsid",
pattern=re.compile(r"(?<!hasattr\()\bos\.setsid\b"),
message=(
"os.setsid does not exist on Windows and raises "
"AttributeError. Subprocesses that need detachment on "
"Windows use creationflags instead."
),
fix=(
"if platform.system() != 'Windows':\n"
" kwargs['preexec_fn'] = os.setsid\n"
"else:\n"
" kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP"
),
),
Footgun(
name="bare os.killpg",
pattern=re.compile(r"\bos\.killpg\b"),
message="os.killpg does not exist on Windows.",
fix=(
"Use psutil for cross-platform process-tree kill:\n"
" p = psutil.Process(pid)\n"
" for c in p.children(recursive=True): c.kill()\n"
" p.kill()"
),
),
Footgun(
name="bare os.getuid / os.geteuid / os.getgid",
pattern=re.compile(r"\bos\.(?:getuid|geteuid|getgid|getegid)\b"),
message=(
"os.getuid / os.geteuid / os.getgid do not exist on Windows "
"and raise AttributeError at import time if referenced."
),
fix=(
"Use getpass.getuser() for the username, or gate with "
"hasattr(os, 'getuid')."
),
),
Footgun(
name="bare os.fork",
pattern=re.compile(r"(?<!hasattr\()\bos\.fork\s*\("),
message="os.fork does not exist on Windows.",
fix=(
"Use subprocess.Popen for daemonization, or guard with "
"hasattr(os, 'fork') and a Windows fallback path."
),
),
Footgun(
name="bare signal.SIGKILL",
pattern=re.compile(r"\bsignal\.SIGKILL\b"),
message=(
"signal.SIGKILL does not exist on Windows and raises "
"AttributeError at import time."
),
fix="Use getattr(signal, 'SIGKILL', signal.SIGTERM).",
),
Footgun(
name="bare signal.SIGHUP / SIGUSR1 / SIGUSR2 / SIGALRM / SIGCHLD / SIGPIPE / SIGQUIT",
pattern=re.compile(
r"\bsignal\.(?:SIGHUP|SIGUSR1|SIGUSR2|SIGALRM|SIGCHLD|SIGPIPE|SIGQUIT)\b"
),
message=(
"These POSIX signals don't exist on Windows; referencing "
"them raises AttributeError at import time."
),
fix=(
"Use getattr(signal, 'SIGXXX', None) and check for None "
"before using, or gate the whole block behind a platform check."
),
),
Footgun(
name="subprocess shebang script invocation",
pattern=re.compile(
r"subprocess\.(?:run|Popen|call|check_output|check_call)\s*\(\s*\[\s*['\"]\./"
),
message=(
"Running a script via './scriptname' doesn't work on Windows — "
"shebang lines aren't honored. CreateProcessW can't execute "
"bash/python scripts without an explicit interpreter."
),
fix="Use [sys.executable, 'scriptname.py', ...] explicitly.",
),
Footgun(
name="wmic invocation without shutil.which guard",
# Match wmic appearing as a subprocess argument — NOT the
# shutil.which("wmic") guard pattern itself. Looks for wmic in a
# list or as first arg of subprocess.run/Popen.
pattern=re.compile(
r"""(?:subprocess\.\w+\s*\(\s*\[\s*['"]wmic['"]|['"]wmic\.exe['"])"""
),
message=(
"wmic was removed in Windows 10 21H1 and later. Always "
"gate with shutil.which('wmic') and fall back to "
"PowerShell (Get-CimInstance Win32_Process)."
),
fix=(
"if shutil.which('wmic'):\n"
" ... wmic path ...\n"
"else:\n"
" subprocess.run(['powershell', '-NoProfile', '-Command',\n"
" 'Get-CimInstance Win32_Process | ...'])"
),
),
Footgun(
name="hardcoded ~/Desktop (OneDrive trap)",
pattern=re.compile(
r"""['"](?:~|~/|[A-Z]:[/\\]Users[/\\][^/\\'"]+[/\\])Desktop\b"""
),
message=(
"When OneDrive Backup is enabled on Windows, the real Desktop "
"is at %USERPROFILE%\\OneDrive\\Desktop, not %USERPROFILE%\\"
"Desktop (which exists as an empty husk)."
),
fix=(
"On Windows, resolve via ctypes + SHGetKnownFolderPath, or "
"read the Shell Folders registry key, or run PowerShell "
"[Environment]::GetFolderPath('Desktop')."
),
),
Footgun(
name="asyncio add_signal_handler without try/except",
pattern=re.compile(r"\.add_signal_handler\s*\("),
message=(
"loop.add_signal_handler raises NotImplementedError on "
"Windows — always wrap in try/except or gate with a "
"platform check."
),
fix=(
"try:\n"
" loop.add_signal_handler(sig, handler, sig)\n"
"except NotImplementedError:\n"
" pass # Windows asyncio doesn't support signal handlers"
),
),
Footgun(
name="subprocess text=True without explicit encoding=",
# Match ``text=True`` (or ``text = True``) anywhere on a line. We
# rely on the post_filter to (a) skip lines that already pass
# ``encoding=`` on the same line, and (b) skip false positives like
# ``def text(self, ...)`` or string literals. ``text=True`` is
# overwhelmingly a subprocess kwarg, so a bare match + filter has a
# high signal-to-noise ratio and avoids the complexity of parsing
# multi-line subprocess calls (which the line-based scanner can't
# reliably attribute to a single line anyway).
pattern=re.compile(r"\btext\s*=\s*True\b"),
message=(
"subprocess text=True without explicit encoding= decodes "
"child output with locale.getpreferredencoding() — cp936 "
"(GBK) on Chinese Windows, cp1252 on Western Windows — "
"which crashes _readerthread with UnicodeDecodeError on "
"non-default-codepage bytes. Always pass encoding='utf-8' "
"(and errors='replace' for Windows-native CLIs that emit "
"non-UTF-8). See issues #47939, #53428, #57238."
),
fix=(
"subprocess.run(..., text=True, encoding='utf-8', "
"errors='replace')\n"
"Both params are required: encoding alone still crashes on "
"non-UTF-8 bytes from Windows-native CLIs (tasklist, "
"schtasks)."
),
post_filter=lambda m, line: (
# Skip if the same line already specifies encoding=.
"encoding=" not in line
and "encoding =" not in line
# Skip method definitions named ``text`` (def text(self, ...)).
and not line.lstrip().startswith("def ")
and not line.lstrip().startswith("async def ")
# Skip ``text=True`` inside string literals (heuristic: the
# substring appears between matching quotes that aren't part
# of an f-string expression). This is imperfect but catches
# the common case of docstrings mentioning text=True.
and not _looks_like_string_literal(line, m)
# Skip lines that are obviously not subprocess calls — e.g.
# DataFrame.rename(text=True) or similar. We can't know for
# sure without parsing, so we accept some false negatives by
# only flagging when ``subprocess`` or a known subprocess-
# shaped call (run/Popen/call/check_output/check_call/
# check_output) appears on the same line. This keeps the
# rule focused on the actual footgun.
and _is_likely_subprocess_call(line)
),
),
Footgun(
name="bare Path.read_text()/write_text() without encoding=",
# Match ``.read_text(`` / ``.write_text(`` when the same line does
# not pass ``encoding=``. Multi-line calls where encoding= sits on
# a later line are handled by the post_filter's lookahead-free
# heuristic accepting a small false-negative rate — the AST guard
# test in tests/gateway/test_gateway_utf8_encoding.py catches the
# gateway/adapters exactly, and this rule catches the common
# single-line form everywhere else.
pattern=re.compile(r"\.(read_text|write_text)\s*\("),
message=(
"Path.read_text()/write_text() without encoding= uses "
"locale.getpreferredencoding() — cp936/cp1252 on Windows — "
"so UTF-8 content (config JSON, session state, skills) "
"crashes with UnicodeDecodeError or writes mojibake. "
"See issue #37423 and the #71014 / read_text campaign."
),
fix='path.read_text(encoding="utf-8") / path.write_text(data, encoding="utf-8")',
post_filter=lambda m, line: (
"encoding=" not in line
and "encoding =" not in line
and not _looks_like_string_literal(line, m)
# Skip calls that continue onto the next line — if the call's
# own closing paren isn't on this line, encoding= may follow
# on a later line. Balance parens from the call opener instead
# of requiring the line to END with ``)`` so chained forms like
# ``read_text()[:4000]`` / ``read_text().splitlines()`` are
# still caught. AST-level enforcement for multi-line calls
# lives in the gateway guard test.
and _call_closes_on_line(line, m.end())
),
),
]
def should_scan_file(path: Path) -> bool:
"""Return True if this file is in scope for the checker."""
# Skip the excluded dirs
parts = set(path.parts)
if parts & EXCLUDED_DIRS:
return False
# Skip excluded suffixes
for suffix in EXCLUDED_SUFFIXES:
if str(path).endswith(suffix):
return False
# Skip self and docs that intentionally mention the patterns
rel = path.relative_to(REPO_ROOT).as_posix()
if rel in EXCLUDED_FILES:
return False
# Only scan text files (rough heuristic — .py, .md, .sh, .ps1, .yaml, etc.)
if path.suffix in {".py", ".pyw", ".pyi"}:
return True
# Other file types are read but only Python-specific patterns would match;
# that's fine and cheap to skip.
return False
def iter_files(paths: Iterable[Path]) -> Iterable[Path]:
for p in paths:
if p.is_file():
if should_scan_file(p):
yield p
elif p.is_dir():
for root, dirs, files in os.walk(p):
# prune excluded dirs in-place for speed
dirs[:] = [d for d in dirs if d not in EXCLUDED_DIRS]
for fname in files:
fpath = Path(root) / fname
if should_scan_file(fpath):
yield fpath
def _strip_code(line: str) -> str:
"""Return just the code portion of a line — strip trailing comments and
skip lines that are entirely inside a string literal or comment.
Heuristic only (we don't parse Python); good enough to avoid flagging
our own `# ``os.kill(pid, 0)`` is NOT a no-op` docstring-style comments.
"""
stripped = line.lstrip()
# Line starts with # — entirely a comment.
if stripped.startswith("#"):
return ""
# Remove trailing "# ..." inline comment. Naive — doesn't handle `#`
# inside strings — but on balance reduces noise far more than it adds.
hash_idx = _find_unquoted_hash(line)
if hash_idx is not None:
return line[:hash_idx]
return line
def _find_unquoted_hash(line: str) -> int | None:
"""Index of the first `#` not inside a single/double/triple-quoted string.
Simple state machine — good enough for the 99% case of "code, then
optional trailing comment."
"""
i = 0
n = len(line)
in_s = False # single-quote string
in_d = False # double-quote string
while i < n:
c = line[i]
if c == "\\" and (in_s or in_d) and i + 1 < n:
i += 2
continue
if not in_d and c == "'":
in_s = not in_s
elif not in_s and c == '"':
in_d = not in_d
elif c == "#" and not in_s and not in_d:
return i
i += 1
return None
# Subprocess method names that accept ``text=`` and are affected by the
# encoding-default footgun. Used by ``_is_likely_subprocess_call`` below to
# keep the ``text=True`` rule focused on subprocess calls (and avoid flagging
# unrelated APIs that happen to accept a ``text`` kwarg).
_SUBPROCESS_METHODS = (
"subprocess.run",
"subprocess.Popen",
"subprocess.call",
"subprocess.check_output",
"subprocess.check_call",
"_sp.run", # common alias
"_sp.Popen",
"_sp.check_output",
"_sp.check_call",
"_sp.call",
".run(", # bare .run( — usually subprocess.run
".Popen(",
".check_output(",
".check_call(",
".call(",
)
def _is_likely_subprocess_call(line: str) -> bool:
"""Heuristic: does this line look like a subprocess invocation?
The ``text=True`` footgun rule only fires when the matched line also
contains a subprocess-shaped call site. This avoids false positives on
unrelated APIs that accept a ``text`` kwarg (e.g. DataFrame.rename,
custom library calls). Multi-line calls where the ``subprocess.X(``
prefix is on a previous line won't be flagged — that's an acceptable
false negative for a line-based scanner.
"""
return any(token in line for token in _SUBPROCESS_METHODS)
def _call_closes_on_line(line: str, open_paren_end: int) -> bool:
"""True when the call whose ``(`` sits at ``open_paren_end - 1`` closes
on this same line (paren-balance walk). Multi-line calls return False —
the missing ``encoding=`` may sit on a continuation line, so the caller
should skip them rather than false-positive."""
depth = 1
for ch in line[open_paren_end:]:
if ch == "(":
depth += 1
elif ch == ")":
depth -= 1
if depth == 0:
return True
return False
def _looks_like_string_literal(line: str, match: "re.Match") -> bool:
"""Heuristic: is the ``text=True`` match inside a string literal?
Catches the common case of docstrings/comments that mention ``text=True``
as prose. Walks the line tracking single/double quote state and returns
True if the match start index falls inside a quoted region.
"""
start = match.start()
in_s = False
in_d = False
i = 0
while i < start and i < len(line):
c = line[i]
if c == "\\" and (in_s or in_d) and i + 1 < len(line):
i += 2
continue
if not in_d and c == "'":
in_s = not in_s
elif not in_s and c == '"':
in_d = not in_d
i += 1
return in_s or in_d
def scan_file(path: Path, footguns: list[Footgun]) -> list[tuple[int, str, Footgun]]:
"""Return a list of (line_number, line, footgun) for unsuppressed matches."""
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError:
return []
matches: list[tuple[int, str, Footgun]] = []
# Track whether we're inside a triple-quoted string (docstring/raw block).
# Simple state machine — handles both ''' and """, toggled by the FIRST
# triple-quote we see; we don't try to handle nested or f-string cases.
in_triple: str | None = None # None, "'''", or '"""'
for i, line in enumerate(text.splitlines(), start=1):
# Update triple-quote state based on this line's occurrences.
code_for_scan = line
if in_triple:
# We're inside a docstring — skip the whole line's scan.
# Check if it closes here.
if in_triple in line:
# Find the closing delimiter; anything after it is real code.
after = line.split(in_triple, 1)[1]
in_triple = None
code_for_scan = after
else:
continue
# Now check for docstring-open in the (possibly after-triple) portion.
# Scan for the first unescaped '''/""" in the current code_for_scan.
stripped = code_for_scan.strip()
for delim in ('"""', "'''"):
if delim in code_for_scan:
# Count occurrences — even count means single-line docstring,
# odd means we've entered a multi-line one.
count = code_for_scan.count(delim)
if count % 2 == 1:
# Odd — we're now inside the triple-quoted block.
# Scan only the part BEFORE the opening delimiter.
before = code_for_scan.split(delim, 1)[0]
code_for_scan = before
in_triple = delim
break
else:
# Even — entire docstring fits on one line. Strip it
# from the scan text to avoid matching on prose.
parts = code_for_scan.split(delim)
# Keep the "outside" parts (every other chunk, starting
# with index 0) as code, drop the "inside" parts.
code_for_scan = "".join(parts[::2])
break
if SUPPRESS_MARKER.search(line):
continue
# Skip if the line has an obvious guard — e.g. hasattr/getattr/
# shutil.which or a platform check. False negatives are acceptable;
# the inline suppression marker is the authoritative override.
if any(hint in line for hint in GUARD_HINTS):
continue
code = _strip_code(code_for_scan)
if not code.strip():
continue
for fg in footguns:
if fg.path_allowlist and any(s in str(path) for s in fg.path_allowlist):
continue
match = fg.pattern.search(code)
if not match:
continue
if fg.post_filter is not None:
try:
if not fg.post_filter(match, line):
continue
except (IndexError, AttributeError):
# Post-filter assumed a named group that isn't there — skip.
continue
matches.append((i, line.rstrip(), fg))
return matches
def get_staged_files() -> list[Path]:
"""Return paths staged in the current git index. Empty on non-git trees."""
try:
out = subprocess.check_output(
["git", "diff", "--cached", "--name-only", "--diff-filter=ACMR"],
cwd=REPO_ROOT,
stderr=subprocess.DEVNULL,
text=True, encoding='utf-8', errors='replace',
)
except (subprocess.CalledProcessError, FileNotFoundError):
return []
return [REPO_ROOT / f for f in out.splitlines() if f.strip()]
def get_diff_files(ref: str) -> list[Path]:
"""Return paths modified vs. the given git ref."""
try:
out = subprocess.check_output(
["git", "diff", f"{ref}...HEAD", "--name-only", "--diff-filter=ACMR"],
cwd=REPO_ROOT,
stderr=subprocess.DEVNULL,
text=True, encoding='utf-8', errors='replace',
)
except (subprocess.CalledProcessError, FileNotFoundError):
return []
return [REPO_ROOT / f for f in out.splitlines() if f.strip()]
def parse_args(argv: list[str]) -> argparse.Namespace:
p = argparse.ArgumentParser(
description="Flag Windows cross-platform footguns in Python code."
)
p.add_argument(
"paths",
nargs="*",
type=Path,
help="Specific files/dirs to scan (default: staged changes).",
)
p.add_argument(
"--all",
action="store_true",
help="Scan the full repository (hermes_cli/, gateway/, tools/, cron/, etc.).",
)
p.add_argument(
"--diff",
metavar="REF",
help="Scan files changed vs. the given git ref (e.g. --diff main).",
)
p.add_argument(
"--list",
action="store_true",
help="List all known footgun rules and exit.",
)
return p.parse_args(argv)
def print_rules() -> None:
print("Known Windows footguns checked by this script:\n")
for i, fg in enumerate(FOOTGUNS, start=1):
print(f"{i:2}. {fg.name}")
print(f" {fg.message}")
print(f" Fix: {fg.fix}")
print()
def main(argv: list[str]) -> int:
# Windows terminals default to cp1252, which can't encode the ✓/✗
# characters used in the output. Reconfigure streams to UTF-8 so the
# script works correctly on the very platform it is designed to help.
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8")
if hasattr(sys.stderr, "reconfigure"):
sys.stderr.reconfigure(encoding="utf-8")
args = parse_args(argv)
if args.list:
print_rules()
return 0
if args.all:
# Scan main Python packages + scripts
roots = [
REPO_ROOT / "hermes_cli",
REPO_ROOT / "gateway",
REPO_ROOT / "tools",
REPO_ROOT / "cron",
REPO_ROOT / "agent",
REPO_ROOT / "plugins",
REPO_ROOT / "scripts",
REPO_ROOT / "acp_adapter",
]
roots = [r for r in roots if r.exists()]
elif args.diff:
roots = get_diff_files(args.diff)
elif args.paths:
roots = [p.resolve() for p in args.paths]
else:
# Default: staged changes
roots = get_staged_files()
if not roots:
print(
"No staged files to scan. Pass --all for a full-repo scan, "
"--diff <ref> for a range diff, or paths explicitly.",
file=sys.stderr,
)
return 0
total_matches = 0
files_scanned = 0
for path in iter_files(roots):
files_scanned += 1
matches = scan_file(path, FOOTGUNS)
for lineno, line, fg in matches:
rel = path.relative_to(REPO_ROOT).as_posix()
print(f"{rel}:{lineno}: [{fg.name}]")
print(f" {line.strip()}")
print(f" — {fg.message}")
print(f" Fix: {fg.fix.splitlines()[0]}")
print()
total_matches += 1
if total_matches:
print(
f"\n✗ {total_matches} Windows footgun(s) found across "
f"{files_scanned} file(s) scanned.",
file=sys.stderr,
)
print(
" If an individual match is a false positive or intentionally "
"platform-gated, suppress it with `# windows-footgun: ok` on "
"the same line.\n Run with --list to see all rules.",
file=sys.stderr,
)
return 1
print(
f"✓ No Windows footguns found ({files_scanned} file(s) scanned)."
)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
scripts/check_subprocess_stdin.py
+227
View File
@@ -0,0 +1,227 @@
#!/usr/bin/env python3
"""Check that subprocess calls in TUI-context code specify stdin=.
When Hermes runs in TUI mode, the gateway child process communicates with
the Node.js parent over a JSON-RPC protocol on stdin. Subprocess calls that
inherit this fd can cause the gateway to exit with stdin EOF during tool
execution (issue #14036, PR #39257).
This script checks that all subprocess.run() and subprocess.Popen() calls
in TUI-context files (agent/, tools/, plugins/, tui_gateway/) explicitly
set stdin= to prevent fd inheritance.
Exit codes:
0 — all calls are safe
1 — violations found
2 — script error
Usage:
python scripts/check_subprocess_stdin.py [--fix]
With --fix, prints the commands to add stdin=subprocess.DEVNULL to each
violation (does not modify files).
"""
from __future__ import annotations
import os
import re
import sys
from pathlib import Path
# Directories that run inside the TUI gateway child process.
TUI_CONTEXT_DIRS = [
"agent/",
"tools/",
"plugins/",
"tui_gateway/",
]
# User plugin roots — scanned at runtime if they exist. Plugins load from
# ``get_hermes_home() / "plugins"`` (user) and ``./.hermes/plugins/`` (project,
# gated behind ``HERMES_ENABLE_PROJECT_PLUGINS``) — see
# ``hermes_cli/plugins.py:10-12``. The guard only checked the bundled
# ``plugins/`` dir, missing user-installed code that spawns subprocesses
# (gap reported in #67639).
#
# Import is deferred to ``main()`` (after ``os.chdir(repo_root)``) because
# this script runs as a standalone subprocess — ``hermes_constants`` isn't
# on ``sys.path`` until the repo root is added.
# subprocess and os APIs that inherit stdin by default when called without
# an explicit stdin= argument. The original regex only covered run/Popen
# (gap #1 in #67639); call, check_output, check_call, os.system, and
# asyncio.create_subprocess_* all inherit fd 0 equally.
_SUBPROCESS_PATTERNS = [
r"subprocess\.(run|Popen|call|check_output|check_call)\s*\([\"'a-zA-Z_\[\(]",
r"os\.system\s*\([\"'a-zA-Z_\[\(]",
r"asyncio\.create_subprocess_(exec|shell)\s*\([\"'a-zA-Z_\[\(]",
]
# Files with intentional stdin= override (e.g. input= creates a pipe).
# Format: "filepath:line" or just "filepath" to skip the whole file.
KNOWN_SAFE = {
"agent/shell_hooks.py", # uses input=stdin_json, creates a pipe
"plugins/security-guidance/patterns.py", # subprocess mentions are in reminder strings, not calls
}
# Inline marker that exempts a single subprocess call from this check.
# Put it in a comment on (or within) the call when the process MUST inherit
# stdin — e.g. an interactive login the user explicitly invokes. Travels with
# the line, so it survives edits that shift line numbers (unlike a pinned
# file:line entry).
EXEMPT_MARKER = "noqa: subprocess-stdin"
# Directories to skip entirely.
SKIP_DIRS = {
"tests/",
"scripts/",
"skills/",
"optional-skills/",
"hermes_cli/",
"gateway/",
"cron/",
}
def find_subprocess_calls(content: str, filepath: str) -> list[dict]:
"""Find all subprocess/os/asyncio calls missing stdin= in content."""
violations = []
lines = content.split("\n")
# Match only actual function calls — not comments, docstrings, or prose.
# Multiple patterns cover subprocess.run/Popen/call/check_output/check_call,
# os.system, and asyncio.create_subprocess_exec/shell.
patterns = [re.compile(p) for p in _SUBPROCESS_PATTERNS]
for i, line in enumerate(lines):
# Skip comments.
stripped = line.lstrip()
if stripped.startswith("#"):
continue
# Skip lines where the match is inside backticks (docstring references).
if "``subprocess" in line:
continue
if not any(p.search(line) for p in patterns):
continue
# Collect the full call (may span multiple lines).
call_start = i
paren_depth = 0
found_open = False
call_lines = []
for j in range(i, min(i + 30, len(lines))):
call_lines.append(lines[j])
for ch in lines[j]:
if ch == "(":
paren_depth += 1
found_open = True
elif ch == ")":
paren_depth -= 1
if found_open and paren_depth == 0:
call_text = "\n".join(call_lines)
# Already has stdin= → safe.
if "stdin=" in call_text:
break
# Has input= → creates a pipe, safe.
if "input=" in call_text:
break
# Inline exemption marker on the call itself or within
# the few comment lines immediately above it → the call
# intentionally inherits stdin.
window_start = max(0, i - 4)
preceding = "\n".join(lines[window_start:i])
if EXEMPT_MARKER in call_text or EXEMPT_MARKER in preceding:
break
violations.append({
"file": filepath,
"line": i + 1,
"snippet": line.strip()[:120],
})
break
else:
continue
break
return violations
def main() -> int:
fix_mode = "--fix" in sys.argv
repo_root = Path(__file__).resolve().parent.parent
os.chdir(repo_root)
# Add repo root to sys.path so we can import hermes_constants (this script
# runs as a standalone subprocess, not as a module).
sys.path.insert(0, str(repo_root))
from hermes_constants import get_hermes_home
all_violations = []
for tui_dir in TUI_CONTEXT_DIRS:
dirpath = repo_root / tui_dir
if not dirpath.exists():
continue
for py_file in dirpath.rglob("*.py"):
rel = str(py_file.relative_to(repo_root))
# Skip known-safe files.
if rel in KNOWN_SAFE:
continue
# Skip test files inside tools/ etc.
parts = py_file.parts
if any(skip.rstrip("/") in parts for skip in SKIP_DIRS):
continue
content = py_file.read_text(encoding="utf-8")
violations = find_subprocess_calls(content, rel)
all_violations.extend(violations)
# Scan user plugin directories (Gap 1: guard missed user-installed
# plugins in get_hermes_home()/plugins/ and project plugins in
# ./.hermes/plugins/, where code like ori/hooks.py can spawn
# subprocesses with inherited stdin — #67639).
plugin_roots: list[Path] = [get_hermes_home() / "plugins"]
if os.environ.get("HERMES_ENABLE_PROJECT_PLUGINS"):
plugin_roots.append(Path.cwd() / ".hermes" / "plugins")
seen_roots: set[Path] = set()
for plugin_root in plugin_roots:
resolved = plugin_root.resolve()
if resolved in seen_roots or not resolved.is_dir():
continue
seen_roots.add(resolved)
for py_file in resolved.rglob("*.py"):
rel = str(py_file)
if py_file.name in ("conftest.py",) or "/tests/" in rel:
continue
try:
content = py_file.read_text(encoding="utf-8")
except Exception:
continue
violations = find_subprocess_calls(content, rel)
all_violations.extend(violations)
if all_violations:
print(f"❌ {len(all_violations)} subprocess calls missing stdin=:")
for v in all_violations:
print(f" {v['file']}:{v['line']}: {v['snippet']}")
if fix_mode:
print("\nAdd stdin=subprocess.DEVNULL to each call above.")
return 1
else:
print("✅ All TUI-context subprocess calls have explicit stdin=")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/ci/assemble_review_comment.py
+444
View File
@@ -0,0 +1,444 @@
#!/usr/bin/env python3
"""Assemble the unified CI review comment for a pull request.
Every CI job that wants to appear in the review comment emits a
``review_status`` output: a JSON array of objects, each with a ``source``
(the workflow name, used for dedup) and a ``results`` array of typed
result objects::
[
{
"source": "review-label-gate",
"results": [
{"kind": "action_required", "title": "...", "summary": "...",
"how_to_fix": "..."},
{"kind": "info", "title": "...", "summary": "..."}
]
},
{
"source": "ci-timings",
"results": [
{"kind": "warning", "title": "CI timings", "summary": "...",
"detail": "...", "link": "..."}
]
}
]
Each result object has:
kind: "error" | "action_required" | "warning" | "info" | "debug"
title: section heading
summary: one-line description
detail: markdown detail (optional)
how_to_fix: markdown checklist (optional)
link: URL (optional)
link_label: label for the link (optional, default "View logs")
The assembler flattens all results into a flat list of ReviewItems,
grouped by severity in the comment. Jobs that failed (from the
``needs`` context) but didn't emit any status get synthesized ❌ Error
items. Jobs that DID emit a status are excluded from the synthesized
error list — their own output is the authority for their classification.
Exits 0 always — comment posting is best-effort (fork PRs are read-only).
"""
from __future__ import annotations
import argparse
import json
import sys
from dataclasses import dataclass
from pathlib import Path
# Hidden marker the comment system uses to find-and-edit its
# previous comment instead of stacking new ones on each run.
MARKER = "<!-- hermes-ci-review-bot -->"
# Severity ordering for display.
_SEVERITY_ORDER = ["error", "action_required", "warning", "info", "debug"]
# Severities that trigger the "blocking issues" layout (vs. the
# "looks good!" banner).
_BLOCKING_SEVERITIES = ("error", "action_required", "warning")
_SEVERITY_GROUP_HEADER = {
"error": "## ❌ Job failures",
"action_required": "## ⚠️ Action required",
"warning": "## ⚠️ Warnings",
"info": "## ℹ️ Details",
}
@dataclass
class ReviewItem:
"""A single piece of review information with a severity tag."""
severity: str # "error" | "action_required" | "warning" | "info" | "debug"
title: str # short section title, e.g. "package-lock.json"
summary: str # one-line summary
detail: str = "" # optional markdown detail (tables, bullet lists, etc.)
link: str = "" # optional URL emitted by the job (e.g. report URL)
link_label: str = "View report" # label for the emitted link
how_to_fix: str = "" # optional markdown checklist for action_required items
source: str = "" # workflow that declared this status (for dedup)
job_url: str = "" # auto-attached per-job log link (from the live poller)
# ---------------------------------------------------------------------------
# Collectors — each returns a list of ReviewItems (possibly empty)
# ---------------------------------------------------------------------------
def collect_from_statuses(review_statuses_json: str) -> tuple[list[ReviewItem], set[str]]:
"""Parse the nested review_status JSON into flat ReviewItems.
The input is a JSON array of ``{source, results: [...]}`` objects.
Each entry in ``results`` becomes one ReviewItem, tagged with the
parent's ``source``.
Returns ``(items, sources)`` where ``sources`` is the set of source
values — used by :func:`collect_failed_jobs` to exclude jobs that
already declared their own status (so a failing job that emitted an
``action_required`` status doesn't also show as a synthesized ❌ Error).
"""
if not review_statuses_json:
return [], set()
try:
data = json.loads(review_statuses_json)
except (json.JSONDecodeError, TypeError):
return [], set()
if not isinstance(data, list):
return [], set()
items: list[ReviewItem] = []
sources: set[str] = set()
for entry in data:
if not isinstance(entry, dict):
continue
source = entry.get("source", "")
if source:
sources.add(source)
for r in entry.get("results", []):
if not isinstance(r, dict):
continue
kind = r.get("kind", "info")
if kind not in _SEVERITY_ORDER:
kind = "info"
items.append(ReviewItem(
severity=kind,
title=r.get("title", "Unknown"),
summary=r.get("summary", ""),
detail=r.get("detail", ""),
link=r.get("link", ""),
link_label=r.get("link_label", "View logs"),
how_to_fix=r.get("how_to_fix", ""),
source=source,
))
return items, sources
def collect_failed_jobs(
needs_json: str,
run_url: str,
exclude_sources: set[str] | None = None,
job_urls: dict[str, str] | None = None,
) -> list[ReviewItem]:
"""Build error items for failed CI jobs from the ``needs`` context.
``needs_json`` is the JSON string emitted by ``all-checks-pass`` — a
``{job_name: result}`` dict where result is ``success`` / ``failure``
/ ``skipped``. Only ``failure`` entries become error items.
``exclude_sources`` is a set of ``source`` values from status objects
declared by workflow_call jobs. Job names containing any of these
source strings are excluded — their failure is already covered by their
own status output.
``job_urls`` is an optional ``{job_name: html_url}`` dict from the
live poller. When a job's name is in this dict, the ❌ Error link
points directly to that job's logs page instead of the whole run.
Falls back to ``run_url`` when no per-job URL is available.
"""
if not needs_json:
return []
try:
needs = json.loads(needs_json)
except (json.JSONDecodeError, TypeError):
return []
# Pre-normalize exclude sources once: lowercase + hyphens→spaces, so
# "review-label-gate" matches "Review label gate / Review label gate".
norm_sources = {
src.lower().replace("-", " ") for src in (exclude_sources or set())
}
items: list[ReviewItem] = []
for name, result in sorted(needs.items()):
if result != "failure":
continue
if norm_sources:
norm = name.lower().replace("-", " ")
if any(src in norm for src in norm_sources):
continue
job_url = (job_urls or {}).get(name, run_url)
items.append(ReviewItem(
severity="error",
title=name,
summary=f"Job **{name}** failed.",
job_url=job_url,
))
return items
# ---------------------------------------------------------------------------
# Rendering
# ---------------------------------------------------------------------------
def _render_item(item: ReviewItem) -> str:
"""Render a single ReviewItem as a markdown block.
The group header (``## ❌ Job failures`` etc.) carries the severity
emoji, so items don't repeat it. Links are shown inline next to the
title. Layout per item::
### {title} · [View report](url) · [View job](url)
{summary}
{detail}
**How to fix:**
{how_to_fix}
"""
title = f"### {item.title}"
# Build inline links next to the title.
links: list[str] = []
if item.link:
links.append(f"[{item.link_label}]({item.link})")
if item.job_url:
links.append(f"[View job]({item.job_url})")
if links:
title += " · " + " · ".join(links)
parts = [title, "", item.summary]
if item.detail:
parts += ["", item.detail]
if item.how_to_fix:
parts += ["", "**How to fix:**", "", item.how_to_fix]
return "\n".join(parts)
def _render_group(header: str, items: list[ReviewItem]) -> str:
"""Render a severity group: ``##`` header + items separated by ``---``."""
blocks = [_render_item(i) for i in items]
return f"{header}\n\n" + "\n\n---\n\n".join(blocks)
def _render_debug_details(items: list[ReviewItem]) -> str:
"""Render each debug item as its own collapsible ``<details>`` block."""
blocks = []
for item in items:
inner = _render_item(item)
blocks.append(
f"<details>\n<summary>{item.title}</summary>\n\n{inner}\n\n</details>"
)
return "### debug info\n\n" + "\n\n".join(blocks)
def _render_pending_items(pending_jobs: list[str]) -> str:
"""Render the dimmed ``<sub>`` items for jobs still running."""
job_list = ", ".join(f"`{j}`" for j in sorted(pending_jobs))
return f"\n\n---\n\n<sub>Still running {len(pending_jobs)} job{'s' if len(pending_jobs) != 1 else ''}: {job_list}</sub>\n"
def render_comment(
items: list[ReviewItem],
pending_jobs: list[str] | None = None,
commit_info: str = "",
waiting: bool = False,
) -> str:
"""Render the full comment body from a list of review items.
Items are grouped by severity under ``##`` group headers, separated
by ``---``. Errors and action_required items are always visible.
Warnings are shown only when present. Info items are visible; debug items
are in a collapsible ``<details>`` block. If ``pending_jobs`` is non-empty, a dimmed
``<sub>`` footer is appended listing jobs still running.
When there are no errors, action_required, or warnings, an "all good!"
banner is shown at the top. Info items remain visible and debug items
follow in collapsible ``<details>`` blocks.
``waiting`` means a workflow run is still queued or in progress even
though no individual job is visibly pending — GitHub has not spawned
the jobs yet. The comment must not look final in that state, so the
"all good!" banner is replaced by a waiting note and a dimmed footer
marks the comment as still live.
"""
pending = pending_jobs or []
# Group by severity
by_severity: dict[str, list[ReviewItem]] = {s: [] for s in _SEVERITY_ORDER}
for item in items:
by_severity.setdefault(item.severity, []).append(item)
info = by_severity.get("info", [])
debug = by_severity.get("debug", [])
has_blocking = any(by_severity.get(s) for s in _BLOCKING_SEVERITIES)
body = f"{MARKER}\n# ૮ >ﻌ< ა ci review\n\n"
if commit_info:
body += f"{commit_info}\n\n"
if not items and not pending:
if waiting:
return f"{body}<sub>waiting for jobs to start…</sub>"
return f"{body}all good!"
sections: list[str] = []
for sev in _BLOCKING_SEVERITIES:
group = by_severity.get(sev, [])
if group:
sections.append(_render_group(_SEVERITY_GROUP_HEADER[sev], group))
if info:
sections.append(_render_group("## ℹ️ Info", info))
# Debug: collapsible <details>
if debug:
sections.append(_render_debug_details(debug))
if pending:
body += _render_pending_items(pending)
elif waiting:
body += "\n\n---\n\n<sub>waiting for more jobs to start…</sub>\n"
if sections:
body += "\n\n---\n\n".join(sections)
return body
# ---------------------------------------------------------------------------
# Assembly
# ---------------------------------------------------------------------------
def _attach_job_urls(items: list[ReviewItem], job_urls: dict[str, str], run_url: str) -> None:
"""Fill in per-job log links for all items.
Uses the same case-insensitive, hyphen-normalized matching as
:func:`collect_failed_jobs`: the item's ``source`` is matched against
job names in ``job_urls``. Sets ``job_url`` on the item — this is
separate from ``link`` (the job-emitted URL, e.g. a report artifact),
so both can appear in the rendered comment.
"""
if not job_urls and not run_url:
return
# Pre-normalize job_url keys once.
norm_urls: dict[str, str] = {}
for name, url in job_urls.items():
norm_urls[name.lower().replace("-", " ")] = url
for item in items:
if item.job_url:
continue
src = item.source.lower().replace("-", " ")
# Try exact match first, then substring match.
if src in norm_urls:
item.job_url = norm_urls[src]
continue
for norm_name, url in norm_urls.items():
if src and src in norm_name:
item.job_url = url
break
# If no per-job URL found, fall back to run_url for items with a source.
if not item.job_url and item.source and run_url:
item.job_url = run_url
def assemble(
needs_json: str = "",
run_url: str = "",
job_urls: dict[str, str] | None = None,
review_statuses_json: str = "",
pending_jobs: list[str] | None = None,
commit_info: str = "",
waiting: bool = False,
) -> str:
"""Assemble the full comment body from all available inputs."""
items: list[ReviewItem] = []
# 1. Structured statuses from workflow_call jobs (review-labels, etc.)
status_items, sources = collect_from_statuses(review_statuses_json)
items.extend(status_items)
# 2. Synthesized error items for failed jobs not covered by statuses
items.extend(collect_failed_jobs(needs_json, run_url, exclude_sources=sources, job_urls=job_urls))
# 3. Attach per-job log links to all items (not just synthesized errors)
_attach_job_urls(items, job_urls or {}, run_url)
return render_comment(items, pending_jobs, commit_info, waiting=waiting)
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--needs-json",
default="",
help="JSON string of {job_name: result} from the all-checks-pass job.",
)
parser.add_argument(
"--run-url",
default="",
help="URL to the CI run summary page (for failed job links).",
)
parser.add_argument(
"--review-statuses-json",
default="",
help="JSON array of {source, results: [...]} objects from workflow_call jobs.",
)
parser.add_argument(
"--pending-jobs",
default="",
help="Comma-separated list of job names still running (shown in a dimmed footer).",
)
parser.add_argument(
"--output",
type=Path,
required=True,
help="Output file for the assembled comment body.",
)
args = parser.parse_args()
pending = [j.strip() for j in args.pending_jobs.split(",") if j.strip()] if args.pending_jobs else None
body = assemble(
needs_json=args.needs_json,
run_url=args.run_url,
review_statuses_json=args.review_statuses_json,
pending_jobs=pending,
)
args.output.write_text(body, encoding="utf-8")
print(f"Wrote {len(body)} chars to {args.output}")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/ci/check_profile_archive_boundary.py
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env python3
"""Reject profile export archives before publication.
``.gitignore`` and ``.dockerignore`` are useful first-line filters, but both
can be bypassed (for example with ``git add -f`` or a non-standard build
context). This check is the blocking, executable policy at the CI and image
publication boundaries. It intentionally checks the filesystem rather than
Git's index so a generated archive cannot enter a build after checkout.
"""
from __future__ import annotations
import argparse
import os
from pathlib import Path
_PROFILE_ARCHIVE_SUFFIXES = (".tar.gz", ".tgz")
def find_forbidden_profile_archives(root: Path) -> list[Path]:
"""Return profile archive paths anywhere in the checkout."""
root = root.resolve()
if not root.is_dir():
raise ValueError(f"repository root is not a directory: {root}")
offenders: list[Path] = []
for directory, dirnames, filenames in os.walk(root, followlinks=False):
dirnames[:] = [
name
for name in dirnames
if name not in {".git", ".venv", "venv", "node_modules", "__pycache__"}
]
for name in (*dirnames, *filenames):
if name.casefold().endswith(_PROFILE_ARCHIVE_SUFFIXES):
offenders.append((Path(directory) / name).relative_to(root))
return sorted(offenders, key=lambda path: path.as_posix().casefold())
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
description="Reject profile export archives in the checkout."
)
parser.add_argument(
"--root",
type=Path,
default=Path.cwd(),
help="repository root to inspect (default: current directory)",
)
args = parser.parse_args(argv)
try:
offenders = find_forbidden_profile_archives(args.root)
except ValueError as exc:
parser.error(str(exc))
if not offenders:
print("No profile export archives detected in the checkout.")
return 0
print(
"::error::profile export archives are forbidden "
"in source and Docker build contexts"
)
for path in offenders:
print(f" {path.as_posix()}")
print(
"Move the archive outside the checkout or pass an explicit external "
"output path to the profile export command."
)
return 1
if __name__ == "__main__":
raise SystemExit(main())
scripts/ci/classify_changes.py
+340
View File
@@ -0,0 +1,340 @@
#!/usr/bin/env python3
"""Classify a PR's changed files into CI work lanes.
Reads newline-separated changed paths on stdin and writes ``key=value``
booleans (one per lane) to ``$GITHUB_OUTPUT`` and stdout. The
``detect-changes`` composite action consumes them so steps gate on
``if: steps.changes.outputs.<lane> == 'true'``.
Lanes:
* ``python`` — pytest / ruff / ty / footguns.
* ``python_prod`` — Python changes OUTSIDE tests/ — gates jobs that ship or
run the product (Desktop E2E backend, Docker image) but never import the
test suite. A tests-only PR keeps ``python`` (pytest must run) while
skipping those product jobs.
* ``docker_meta`` — Dockerfiles etc.
* ``docker`` — any product change + docker meta
* ``nix`` — ``nix flake check``: the flake inputs and any product change.
* ``frontend`` — TS typecheck matrix + desktop build.
* ``site`` — Docusaurus + generated skill docs.
* ``scan`` — supply-chain scan (Python files, .pth, setup hooks).
* ``deps`` — pyproject.toml dependency bounds check.
* ``uv_lock`` — ``uv lock --check``. Re-resolves the whole graph against
PyPI, so a diff that touches neither ``pyproject.toml`` nor ``uv.lock``
must not run it.
* ``npm_lock`` — semantic package-lock.json diff PR comment.
* ``installer`` — PowerShell installer tests (Windows runner).
* ``desktop_updater`` — the Windows desktop-update hand-off script and the
tests that drive the REAL ``windows.ps1`` (``-SelfTestUi`` / pipe drain /
retry policy). These are integration tests of a PowerShell process on a
shared runner; running them on every Python PR made their timing noise
everyone's problem. They still run on push (fail-open) and whenever the
script, its siblings, or their tests change.
* ``rust`` — ``cargo test`` for the Tauri bootstrap installer. ``.rs``
lives under ``apps/``, so without this lane a Rust change matched ``frontend``
and only the TypeScript matrix ran.
* ``mcp_catalog`` — bundled MCP catalog / installer review.
Docker is not a lane — it builds on push-to-main and release only,
never per-PR.
Contract — *fail open, never closed*. We may run a lane we didn't need, but
must never skip one a change could break:
* An empty diff, or any ``.github/`` change, runs everything.
* ``python`` is a denylist: skipped only when *every* file is provably prose
or a frontend-only package; an unrecognized path keeps it on.
* ``skills/`` (incl. ``SKILL.md``) is python-relevant — the skill-doc tests
read that tree, so a doc-looking edit can still break Python.
* ``nix/``, ``flake.nix`` and ``flake.lock`` are the exception the other way:
only the flake reads them, so they skip the Python lanes and run ``nix``
alone. ``pyproject.toml`` and ``uv.lock`` are flake inputs too, but the
packaging tests read them, so they keep every Python lane.
* ``website/static/oauth/`` is python-relevant too: it publishes the OAuth
Client ID Metadata Document that ``tests/tools/test_mcp_cimd.py`` checks
against the pinned callback ports in ``tools/mcp_oauth.py``.
* ``website/docs/`` and ``website/scripts/`` are python-relevant for the same
reason: the docs tree generates ``llms.txt``, and
``tests/website/test_generate_llms_txt.py`` asserts every page reaches it.
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
_FRONTEND = ("ui-tui/", "web/", "apps/") # TS typecheck-matrix packages
_ROOT_NPM = {"package.json", "package-lock.json"} # shifts every package's tree
_DOCKER_META = ("docker/", ".hadolint.yml", "Dockerfile") # docker setup
_NIX_PATHS = ("nix/",) # nix files
_NIX_FILES = {"flake.nix", "flake.lock"} # base nix files
_SITE = ("website/", "skills/", "optional-skills/") # docs site + skill pages
# Prose/frontend trees that can't touch Python. skills/ is excluded on purpose.
_PY_SKIP = ("docs/", "website/") + _FRONTEND
# Published artifacts that live under website/ but that Python asserts about.
# The OAuth Client ID Metadata Document is cross-checked against the pinned
# callback ports in tools/mcp_oauth.py, so editing it alone must still run the
# Python lane — otherwise dropping a redirect URI goes green here and breaks
# every CIMD login on main.
# website/docs/ and website/scripts/ are asserted about the same way. The docs
# tree generates llms.txt — the index every LLM (Hermes included, via the
# hermes-agent skill) reads to learn what Hermes can do — and
# tests/website/test_generate_llms_txt.py holds every page to appearing in it.
# Skipping Python on a docs-only PR is how the index drifted to 53% coverage.
_PY_RELEVANT_SITE = (
"website/static/oauth/",
"website/docs/",
"website/scripts/",
)
# CI-sensitive files: eslint config, workflow files, composite actions.
# Changes here can influence what code the autofix job executes and pushes to
# main, so they require explicit maintainer review (ci-reviewed label).
#
# package.json is deliberately NOT listed here: npm scripts only execute on the
# unprivileged generate-patch runner (contents: read), never on the privileged
# apply-patch job. The two-job split means a malicious package.json script
# can't get push access — it runs on an ephemeral runner with zero write perms.
_CI_REVIEW_FILES = {
".prettierrc",
}
_CI_REVIEW_PATHS = (".github/workflows/", ".github/actions/")
# Supply-chain scan: files that can execute code at install/import time.
_SCAN_EXTS = (".py", ".pth")
_SCAN_FILES = {"setup.cfg", "pyproject.toml"}
# MCP catalog files that require explicit security review.
_MCP_CATALOG_PATHS = ("optional-mcps/",)
_MCP_CATALOG_FILES = {"hermes_cli/mcp_catalog.py"}
# Windows installer + its PowerShell tests. These only run on a Windows runner,
# so they get their own lane rather than riding along with ``python``.
_INSTALLER_PATHS = ("scripts/tests/",)
_INSTALLER_FILES = {"scripts/install.ps1", "scripts/install.cmd"}
# Windows desktop-update hand-off (scripts/desktop-update/windows.ps1 + the
# Electron side that launches it) and the pytest files that spawn it.
_DESKTOP_UPDATER_PATHS = ("scripts/desktop-update/",)
_DESKTOP_UPDATER_TEST_PREFIX = "tests/test_desktop_update_"
_DESKTOP_UPDATER_FILES = {
"apps/desktop/electron/updater-process.ts",
"apps/desktop/electron/managed-ssh-update.ts",
"tests/conftest.py",
"pyproject.toml",
}
# Rust crates — currently just the Tauri bootstrap installer (Hermes-Setup).
# These live under ``apps/``, so before this lane existed a ``.rs`` edit matched
# ``frontend`` and nothing more: the TypeScript matrix built, cargo never ran,
# and the crate's unit tests had never executed in CI at all.
_RUST_PATHS = ("apps/bootstrap-installer/src-tauri/",)
_RUST_FILENAMES = {"Cargo.toml", "Cargo.lock"}
def _is_docs(p: str) -> bool:
if p.startswith(("skills/", "optional-skills/")):
return False
return p.endswith((".md", ".mdx")) or p.startswith("docs/") or p.startswith("LICENSE")
def _is_nix(p: str) -> bool:
return p.startswith(_NIX_PATHS) or p in _NIX_FILES
def _py_irrelevant(p: str) -> bool:
if p.startswith(_PY_RELEVANT_SITE):
return False
return (
_is_docs(p)
or p in _ROOT_NPM
or p.startswith(_PY_SKIP)
or p.startswith(_DOCKER_META)
or _is_nix(p)
)
def _py_test_only(p: str) -> bool:
"""Is ``p`` inside the test suite (never shipped / imported by the product)?
Product jobs (Desktop E2E's ``hermes serve`` backend, the Docker image)
run installed code — nothing under ``tests/`` is packaged or importable
there. scripts/run_tests.sh and run_tests_parallel.py are deliberately
NOT test-only: they are runner infrastructure, and a bad edit there can
mask real failures, so they stay conservative (python_prod=true).
"""
return p.startswith("tests/")
def _is_scan(p: str) -> bool:
return p.endswith(_SCAN_EXTS) or p in _SCAN_FILES
def _is_mcp_catalog(p: str) -> bool:
return p.startswith(_MCP_CATALOG_PATHS) or p in _MCP_CATALOG_FILES
def _is_installer(p: str) -> bool:
return p.startswith(_INSTALLER_PATHS) or p in _INSTALLER_FILES
def _is_desktop_updater(p: str) -> bool:
return (
p.startswith(_DESKTOP_UPDATER_PATHS)
or p.startswith(_DESKTOP_UPDATER_TEST_PREFIX)
or p in _DESKTOP_UPDATER_FILES
)
def _is_rust(p: str) -> bool:
return (
p.endswith(".rs")
or p.startswith(_RUST_PATHS)
or os.path.basename(p) in _RUST_FILENAMES
)
def _is_ci_review(p: str) -> bool:
if p in _CI_REVIEW_FILES or p.startswith(_CI_REVIEW_PATHS):
return True
# Any eslint config file at any path — eslint configs can define custom
# fix functions that execute arbitrary code, so they all require review.
return os.path.basename(p).startswith("eslint.config.")
def ci_review_files(files: list[str]) -> list[str]:
"""Return the CI-sensitive paths that need maintainer review."""
return sorted({f.strip() for f in files if f.strip() and _is_ci_review(f.strip())})
def classify(files: list[str]) -> dict[str, bool]:
"""Map changed paths to ``{lane: should_run}``."""
files = [f.strip() for f in files if f.strip()]
python = any(not _py_irrelevant(f) for f in files)
python_prod = any(not _py_irrelevant(f) and not _py_test_only(f) for f in files)
frontend = any(f.startswith(_FRONTEND) or f in _ROOT_NPM for f in files)
deps = any(f == "pyproject.toml" for f in files)
npm_lock = any(f.split("/")[-1] == "package-lock.json" for f in files)
docker_meta = any(f.startswith(_DOCKER_META) for f in files)
ret = {
"python": python,
"python_prod": python_prod,
"docker": docker_meta or python_prod or frontend,
"docker_meta": docker_meta,
"frontend": frontend,
"site": any(f.startswith(_SITE) for f in files),
"scan": any(_is_scan(f) for f in files),
"deps": deps,
"uv_lock": any(f in ("pyproject.toml", "uv.lock") for f in files),
"npm_lock": npm_lock,
"installer": any(_is_installer(f) for f in files),
"desktop_updater": any(_is_desktop_updater(f) for f in files),
"rust": any(_is_rust(f) for f in files),
"mcp_catalog": any(_is_mcp_catalog(f) for f in files),
"ci_review": any(_is_ci_review(f) for f in files),
"nix": python_prod or frontend or any(_is_nix(f) for f in files)
}
if not files or any(f.startswith(".github/") for f in files):
ret["python"] = True
ret["python_prod"] = True
ret["docker"] = True
ret["docker_meta"] = True
ret["frontend"] = True
ret["site"] = True
ret["scan"] = True
ret["deps"] = True
ret["uv_lock"] = True
ret["npm_lock"] = True
ret["installer"] = True
ret["desktop_updater"] = True
ret["rust"] = True
ret["nix"] = True
ret["ci_review"] = True
# explicitly skip mcp catalog here. it's not needed unless those files are modified.
return ret
def _pull_request_number() -> str | None:
"""Read the PR number from the Actions event payload, if present."""
event_path = os.environ.get("GITHUB_EVENT_PATH")
if not event_path:
return None
try:
with open(event_path, encoding="utf-8") as fh:
payload = json.load(fh)
except (OSError, json.JSONDecodeError):
return None
number = (payload.get("pull_request") or {}).get("number")
return str(number) if number else None
def pull_request_changed_files() -> list[str]:
"""Recover the PR file list when the compare API returned nothing.
``detect-changes`` calls ``repos/.../compare/base...head`` with raw SHAs.
A fork force-push can 404 for ~30s until GitHub attaches the new head SHA
to the base repo, so the action fails open with an empty file list. That
forces ``ci_review=true`` and blocks the PR on a ``ci-reviewed`` label
even when no CI-sensitive file changed.
The pull-request files endpoint already knows the PR's files (it is how
this action used to classify), so use it as a fallback on pull_request
events only. Push/dispatch keep the empty-diff fail-open.
"""
if os.environ.get("EVENT_NAME") != "pull_request":
return []
repo = os.environ.get("REPO") or os.environ.get("GITHUB_REPOSITORY") or ""
pr = _pull_request_number()
if not repo or not pr:
return []
try:
completed = subprocess.run(
[
"gh",
"api",
"--paginate",
f"repos/{repo}/pulls/{pr}/files",
"--jq",
".[].filename",
],
check=False,
capture_output=True,
text=True,
timeout=30,
)
except (OSError, subprocess.TimeoutExpired):
return []
if completed.returncode != 0:
return []
return [line.strip() for line in completed.stdout.splitlines() if line.strip()]
def main() -> int:
files = sys.stdin.read().splitlines()
if not any(f.strip() for f in files):
recovered = pull_request_changed_files()
if recovered:
print(
f"compare API returned no files; recovered {len(recovered)} "
"path(s) from the pull request files endpoint",
file=sys.stderr,
)
files = recovered
lanes = classify(files)
out = "\n".join([
*(f"{key}={str(value).lower()}" for key, value in lanes.items()),
f"ci_review_files={json.dumps(ci_review_files(files))}",
])
if dest := os.environ.get("GITHUB_OUTPUT"):
with open(dest, "a", encoding="utf-8") as fh:
fh.write(out + "\n")
print(out) # echo for local runs + CI step logs
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/ci/e2e_screenshot_status.py
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""Select Desktop E2E visual evidence and build its CI review status."""
from __future__ import annotations
import argparse
import hashlib
import json
import shutil
from pathlib import Path
SOURCE = "playwright e2e"
EVIDENCE_START = "<!-- hermes-e2e-evidence:start -->"
EVIDENCE_END = "<!-- hermes-e2e-evidence:end -->"
def _files(root: Path, pattern: str) -> list[Path]:
return sorted(path for path in root.rglob(pattern) if path.is_file()) if root.exists() else []
def _is_explicit_screenshot(path: Path) -> bool:
"""Exclude Playwright's automatic and visual-comparator PNG outputs."""
return not (
path.name.startswith(("test-finished-", "test-failed-"))
or path.name.endswith(("-actual.png", "-expected.png", "-diff.png"))
)
def build_manifest(results_dir: Path) -> dict:
"""Record stable screenshot names from one E2E run for main/PR comparison."""
screenshots = [path for path in _files(results_dir, "*.png") if _is_explicit_screenshot(path)]
return {"version": 1, "screenshot_names": sorted({path.name for path in screenshots})}
def _base_screenshot_names(path: Path | None) -> set[str] | None:
"""Return ``None`` when main evidence is unavailable (never guess newness)."""
if path is None or not path.is_file():
return None
try:
data = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return None
names = data.get("screenshot_names", []) if isinstance(data, dict) else []
if not isinstance(data, dict) or not isinstance(names, list):
return None
return {name for name in names if isinstance(name, str)}
def _stage_name(kind: str, path: Path, results_dir: Path) -> str:
relative = path.relative_to(results_dir).as_posix()
digest = hashlib.sha256(relative.encode("utf-8")).hexdigest()[:12]
return f"{kind}-{digest}-{path.name}"
def select_evidence(results_dir: Path, base_manifest: Path | None = None) -> dict:
"""Select only screenshots new to main, plus every generated visual diff."""
base_names = _base_screenshot_names(base_manifest)
screenshots = [] if base_names is None else [
path for path in _files(results_dir, "*.png")
if _is_explicit_screenshot(path) and path.name not in base_names
]
diffs: list[dict[str, Path]] = []
for diff in _files(results_dir, "*-diff.png"):
stem = diff.with_name(diff.name.removesuffix("-diff.png"))
entry = {"diff": diff}
for kind in ("actual", "expected"):
candidate = stem.with_name(f"{stem.name}-{kind}.png")
if candidate.is_file():
entry[kind] = candidate
diffs.append(entry)
return {"screenshots": screenshots, "diffs": diffs}
def stage_evidence(results_dir: Path, evidence_dir: Path, selection: dict) -> dict:
"""Copy selected PNGs into a flat, path-safe evidence artifact."""
evidence_dir.mkdir(parents=True, exist_ok=True)
staged: dict[Path, str] = {}
def stage(kind: str, path: Path) -> str:
if path in staged:
return staged[path]
name = _stage_name(kind, path, results_dir)
shutil.copyfile(path, evidence_dir / name)
staged[path] = name
return name
manifest = {"version": 1, "screenshots": [], "diffs": []}
for screenshot in selection["screenshots"]:
manifest["screenshots"].append({
"name": screenshot.name,
"file": stage("screenshot", screenshot),
})
for diff in selection["diffs"]:
entry = {"name": diff["diff"].name.removesuffix("-diff.png"), "diff": stage("diff", diff["diff"])}
for kind in ("actual", "expected"):
if kind in diff:
entry[kind] = stage(kind, diff[kind])
manifest["diffs"].append(entry)
(evidence_dir / "e2e-evidence.json").write_text(
json.dumps(manifest, sort_keys=True) + "\n", encoding="utf-8"
)
return manifest
def build_status(selection: dict, artifact_url: str = "") -> list[dict]:
"""Return the review status. The trusted publisher replaces its marker."""
screenshots = selection["screenshots"]
diffs = selection["diffs"]
if not screenshots and not diffs:
return []
summary_parts = []
if screenshots:
summary_parts.append(
f"{len(screenshots)} new screenshot{'s' if len(screenshots) != 1 else ''} vs main"
)
if diffs:
summary_parts.append(f"{len(diffs)} visual diff{'s' if len(diffs) != 1 else ''}")
result: dict[str, str] = {
"kind": "info",
"title": "Desktop E2E visual evidence",
"summary": "; ".join(summary_parts) + ".",
"detail": "\n".join((EVIDENCE_START, "<sub>inline evidence is publishing...</sub>", EVIDENCE_END)),
}
if artifact_url:
result["link"] = artifact_url
result["link_label"] = "View test artifacts"
return [{"source": SOURCE, "results": [result]}]
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--results-dir", type=Path, required=True)
parser.add_argument("--base-manifest", type=Path)
parser.add_argument("--manifest-output", type=Path, required=True)
parser.add_argument("--evidence-dir", type=Path, required=True)
parser.add_argument("--artifact-url", default="")
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
args.manifest_output.write_text(
json.dumps(build_manifest(args.results_dir), sort_keys=True) + "\n", encoding="utf-8"
)
selection = select_evidence(args.results_dir, args.base_manifest)
stage_evidence(args.results_dir, args.evidence_dir, selection)
args.output.write_text(
json.dumps(build_status(selection, args.artifact_url)) + "\n", encoding="utf-8"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/ci/emit_review_status.py
+214
View File
@@ -0,0 +1,214 @@
#!/usr/bin/env python3
"""Emit review_status JSON for the review-labels workflow.
Builds a JSON array with one entry::
[
{
"source": "review-label-gate",
"results": [
{"kind": "action_required", "title": "...", "summary": "...",
"how_to_fix": "..."},
{"kind": "info", "title": "...", "summary": "..."}
]
}
]
The ``source`` field is the workflow name that declared the status; the
assembler uses it to exclude the corresponding job from the synthesized
❌ Error list (the job already has its own status section).
The array can contain 0 to 3 results — one per lane that ran
(``ci_review``, ``mcp_catalog``, ``supply_chain``). When the ``ci-reviewed`` label is
present, the kind is ``info``; when missing, it's ``action_required``
with the verification checklist.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import sys
from urllib.parse import quote
# The source identifier used for error-synthesis exclusion. This must
# match (as a normalized substring) the job name as it appears in the
# GitHub Actions API. The ci.yml job key is ``review-labels`` with
# ``name: Review label gate``, and the reusable workflow's job is also
# ``name: Review label gate``, so the API shows the job as
# "Review label gate / Review label gate". Normalizing "review-label-gate"
# (lowercase, hyphens→spaces) gives "review label gate", which is a
# substring of "review label gate / review label gate".
SOURCE = "review-label-gate"
def _ci_review_detail(
files_json: str, repo_url: str, base_sha: str, head_sha: str,
) -> str:
"""Render links to the changed CI-sensitive files that triggered review."""
try:
files = json.loads(files_json)
except (json.JSONDecodeError, TypeError):
return ""
if not isinstance(files, list) or not repo_url or not base_sha or not head_sha:
return ""
links = []
for path in files:
if not isinstance(path, str) or not path:
continue
label = path.replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]")
path_hash = hashlib.sha256(path.encode()).hexdigest()
url = (
f"{repo_url}/compare/{quote(base_sha, safe='')}...{quote(head_sha, safe='')}"
f"#diff-{path_hash}"
)
links.append(f"- [`{label}`]({url})")
return "**Sensitive files changed:**\n" + "\n".join(links) if links else ""
def build_results(
ci_review: bool,
mcp_catalog: bool,
supply_chain: bool,
label_present: bool,
ci_review_files: str = "[]",
repo_url: str = "",
base_sha: str = "",
head_sha: str = "",
) -> list[dict]:
"""Build the list of result objects for this source."""
results: list[dict] = []
if ci_review:
detail = _ci_review_detail(ci_review_files, repo_url, base_sha, head_sha)
if label_present:
result = {
"kind": "info",
"title": "CI-sensitive file review",
"summary": (
"PR touches sensitive files, but the `ci-reviewed` label has been "
"added, approving them."
),
}
else:
result = {
"kind": "action_required",
"title": "CI-sensitive file review",
"summary": (
"This PR changes CI-sensitive files (eslint config, "
"workflow YAMLs, or composite actions). These influence "
"what the js-autofix job executes and pushes to main."
),
"how_to_fix": (
"Add the `ci-reviewed` label after verifying:\n"
"- no new eslint rules with custom `fix` functions that write outside linted paths,\n"
"- no workflow changes that widen permissions or remove guards,\n"
"- no composite action changes that alter what gets executed."
),
}
if detail:
result["detail"] = detail
results.append(result)
if mcp_catalog:
if label_present:
results.append({
"kind": "debug",
"title": "MCP catalog security review",
"summary": "`ci-reviewed` label is present.",
})
else:
results.append({
"kind": "action_required",
"title": "MCP catalog security review",
"summary": (
"This PR changes the bundled MCP catalog or MCP catalog "
"installer code. MCP entries can define local commands "
"that users later install into `mcp_servers`, so this "
"needs explicit maintainer review before merge."
),
"how_to_fix": (
"Add the `ci-reviewed` label after verifying:\n"
"- any new/changed `optional-mcps/**/manifest.yaml` command and args are expected,\n"
"- stdio transports do not use shell+egress/exfiltration payloads,\n"
"- git install refs are pinned and bootstrap commands are minimal,\n"
"- requested env vars/secrets match the upstream MCP's documented needs."
),
})
if supply_chain and not label_present:
results.append({
"kind": "action_required",
"title": "Critical supply chain risk",
"summary": "Critical supply chain risk patterns were detected in this PR.",
"how_to_fix": (
"Review the flagged code carefully. If it is intentional, add the "
"`ci-reviewed` label to confirm maintainer review."
),
})
return results
def build_statuses(
ci_review: bool,
mcp_catalog: bool,
supply_chain: bool,
label_present: bool,
ci_review_files: str = "[]",
repo_url: str = "",
base_sha: str = "",
head_sha: str = "",
) -> list[dict]:
"""Build the full review_status array (one entry with a results list)."""
results = build_results(
ci_review, mcp_catalog, supply_chain, label_present,
ci_review_files, repo_url, base_sha, head_sha,
)
if not results:
return []
return [{"source": SOURCE, "results": results}]
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--ci-review", action="store_true",
help="Whether CI-sensitive files changed.")
parser.add_argument("--ci-review-files", default="[]",
help="JSON list of CI-sensitive files changed.")
parser.add_argument("--mcp-catalog", action="store_true",
help="Whether the MCP catalog / installer changed.")
parser.add_argument("--supply-chain", action="store_true",
help="Whether the critical supply-chain scanner found a risk.")
parser.add_argument("--label-present", action="store_true",
help="Whether the ci-reviewed label is present.")
parser.add_argument("--repo-url", default="",
help="Repository URL used for changed-file links.")
parser.add_argument("--base-sha", default="",
help="Pull request base SHA used for changed-file links.")
parser.add_argument("--head-sha", default="",
help="Pull request head SHA used for changed-file links.")
parser.add_argument("--output", default="-",
help="Output file ('-' for stdout, or a GITHUB_OUTPUT path).")
args = parser.parse_args()
statuses = build_statuses(
args.ci_review, args.mcp_catalog, args.supply_chain, args.label_present,
args.ci_review_files, args.repo_url, args.base_sha, args.head_sha,
)
json_str = json.dumps(statuses)
if args.output == "-":
print(json_str)
else:
# GITHUB_OUTPUT format: key=value\n
with open(args.output, "a", encoding="utf-8") as f:
f.write(f"review_status={json_str}\n")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/ci/list_os_marked_tests.py
+115
View File
@@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""List the test files that carry a given OS marker.
Used by ``.github/workflows/tests-os.yml`` to scope what the macOS and
Windows lanes import.
Why scope at all, when ``pytest -m macos_only`` already selects correctly?
Because ``-m`` filters AFTER collection, and collection IMPORTS every test
module under ``tests/``. On the Linux lane that is fine (it runs them all
anyway), but on the macOS/Windows lanes it would drag ~900 unrelated modules
through import on a host they were never expected to import on — one
unrelated ImportError would fail a job whose actual subject passed. Narrowing
the paths keeps each lane's failure signal about its own tests.
``-m`` is still passed by the workflow and remains the authoritative
selector: this script only decides which files get imported, never which
tests run. Over-selecting here is harmless (``-m`` drops the extras); the
failure mode to care about is UNDER-selecting, which is why the workflow
fails the job when zero tests end up selected.
Usage:
python scripts/ci/list_os_marked_tests.py macos_only [tests_root]
Prints one path per line (POSIX separators, repo-relative), sorted.
"""
from __future__ import annotations
import re
import sys
from pathlib import Path
_VALID_MARKERS = ("linux_only", "macos_only", "windows_only")
def find_marked_files(marker: str, root: Path) -> list[Path]:
"""Return every ``test_*.py`` under *root* that references *marker*.
Matches the marker as a whole word so ``macos_only`` doesn't pick up a
hypothetical ``macos_only_extra``. Catches both the decorator form
(``@pytest.mark.macos_only``, on a function or a class) and the
module-level ``pytestmark`` form.
"""
pattern = re.compile(rf"\b{re.escape(marker)}\b")
hits: list[Path] = []
for path in sorted(root.rglob("test_*.py")):
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
if pattern.search(text):
hits.append(path)
return hits
def main(argv: list[str]) -> int:
if len(argv) < 2:
print(__doc__, file=sys.stderr)
return 2
marker = argv[1]
if marker not in _VALID_MARKERS:
print(
f"error: unknown marker {marker!r} (expected one of "
f"{', '.join(_VALID_MARKERS)})",
file=sys.stderr,
)
return 2
repo_root = Path(__file__).resolve().parents[2]
root = Path(argv[2]) if len(argv) > 2 else repo_root / "tests"
if not root.exists():
print(f"error: no such directory: {root}", file=sys.stderr)
return 2
files = find_marked_files(marker, root)
if not files:
print(
f"error: no test file references @pytest.mark.{marker} — the marker "
"was probably renamed or dropped. Refusing to emit an empty list, "
"which would let the OS lane pass without running anything.",
file=sys.stderr,
)
return 1
lines: list[str] = []
for path in files:
# POSIX separators so the output is safe to paste into a bash
# command line on the Windows runner (Git Bash accepts them).
#
# Relative to the repo root when the path is inside it (the CI case —
# pytest is invoked from the repo root). A root outside the repo is a
# test/manual invocation; emit it as-is rather than raising, since
# ``relative_to`` refuses non-descendant paths.
try:
rel = path.resolve().relative_to(repo_root)
except ValueError:
lines.append(path.as_posix())
else:
lines.append(rel.as_posix())
# Write bytes with explicit LF rather than print(), which on Windows
# translates "\n" to "\r\n" in text mode. The consumer reads this list with
# ``$(cat ...)`` in bash, and word splitting uses IFS (space/tab/newline) —
# a CR is NOT a separator, so it stays glued to each path and pytest then
# fails with "file or directory not found: tests/...py" for a path that
# looks correct in the log because the CR is invisible. Emitting bytes makes
# the output identical on every host instead of depending on the platform's
# newline translation.
sys.stdout.buffer.write(b"".join(line.encode("utf-8") + b"\n" for line in lines))
sys.stdout.buffer.flush()
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
scripts/ci/live_comment.py
+794
View File
@@ -0,0 +1,794 @@
#!/usr/bin/env python3
"""Live-updating CI review comment.
Polls the GitHub Actions API for job statuses in the CI run, assembles
the review comment from whatever results are available, and upserts it as a
PR comment. Repeats every ``--interval`` seconds until all jobs are
completed (or ``--timeout`` is reached), so the comment updates in real time
as each job finishes.
The comment is identified by the ``<!-- hermes-ci-review-bot -->`` marker
— the same one ``assemble_review_comment.py`` uses — so it replaces any
previous comment from an earlier run.
This runs from ``.github/workflows/ci-review-comment.yml``, a separate
``workflow_run`` workflow. Thus ``CI_RUN_ID`` names the CI run to report
on, not the run that contains this script. (The variable cannot be
called ``GITHUB_RUN_ID``: the Actions runner sets the ``GITHUB_*``
defaults itself and ignores an ``env:`` override, so that name would
silently resolve to the poller's own run — which stays ``in_progress``
for as long as the poller runs, deadlocking it against itself.)
The poller reports on runs that
it does not belong to. This is also how it covers a workflow that CI does
not contain: ``WATCH_WORKFLOWS`` names sibling workflows that the same
commit triggered (the Docker image build). Their jobs join the comment.
Architecture:
- :func:`classify_jobs` (pure, testable) — takes a list of raw API job
dicts and returns ``(completed, pending, job_urls)`` where ``completed``
is a ``{name: result}`` dict (for :func:`assemble_review_comment.assemble`)
and ``pending`` is a list of job names still running.
- :func:`select_watched_runs` (pure, testable) — picks the sibling runs
to merge in, newest attempt per workflow.
- :func:`find_comment_id` / :func:`upsert_comment` — thin API wrappers.
- :func:`fetch_all_review_statuses` — lists all ``review-status-*``
artifacts on the CI run (GitHub attaches reusable-workflow
artifacts to the caller run), downloads each, parses the
``review_status=`` line from ``review-status.json``, and merges into
one array. Recomputed from source every poll cycle, so statuses
appear as soon as each job uploads its artifact.
- :func:`run` — the polling loop. Calls the API, classifies,
fetches artifacts, assembles, upserts, sleeps, repeats. Before
its final exit, it gives downstream jobs a short grace period
to appear.
The orchestrator job names (detect, all-checks-pass, comment-live, etc.)
are excluded from the comment — they're infrastructure, not review signal.
"""
from __future__ import annotations
import argparse
import json
import os
import shutil
import sys
import time
import urllib.error
import urllib.request
import zipfile
from pathlib import Path
API_BASE = "https://api.github.com"
# Job names that are infrastructure (this script, the gate, the detector)
# and should never appear in the review comment.
_INFRA_JOBS = frozenset({
"detect",
"all-checks-pass",
"comment-pending",
"comment-results",
"comment-live",
"CI review comment (pending)",
"CI review comment (results)",
"CI review comment (live)",
"All required checks pass",
"Detect affected areas",
})
# Map GitHub API conclusion values to our result strings.
_CONCLUSION_MAP = {
"success": "success",
"failure": "failure",
"skipped": "skipped",
"cancelled": "skipped",
"neutral": "skipped",
"timed_out": "failure",
"action_required": "skipped",
}
def classify_jobs(api_jobs: list[dict]) -> tuple[dict[str, str], list[str], dict[str, str]]:
"""Classify raw API job dicts into completed + pending + job_urls.
Returns ``(completed, pending, job_urls)``:
- ``completed``: ``{job_name: result}`` where result is
``"success"`` / ``"failure"`` / ``"skipped"``. Only non-infra jobs
that have finished.
- ``pending``: list of job names still running (in_progress / queued
/ waiting). Excludes infra jobs.
- ``job_urls``: ``{job_name: html_url}`` — direct links to each
job's logs page, for the assembler to use in ❌ Error links.
The API returns orchestrator-level jobs and sub-workflow jobs
(workflow_call) in separate runs — :func:`collect_run_jobs` merges
them. Each sub-workflow job has a ``_workflow_name`` prefix so the
display name is ``"Workflow / job"``.
"""
completed: dict[str, str] = {}
pending: list[str] = []
job_urls: dict[str, str] = {}
for job in api_jobs:
name = job.get("name", "unknown")
if job.get("_workflow_name"):
name = f"{job['_workflow_name']} / {name}"
if name in _INFRA_JOBS:
continue
status = job.get("status", "")
conclusion = job.get("conclusion", "")
html_url = job.get("html_url", "")
if html_url:
job_urls[name] = html_url
if status in ("in_progress", "queued", "waiting"):
pending.append(name)
elif status == "completed":
result = _CONCLUSION_MAP.get(conclusion, "skipped")
completed[name] = result
# else: unknown status → skip
return completed, pending, job_urls
# ---------------------------------------------------------------------------
# API helpers
# ---------------------------------------------------------------------------
def _api_request(url: str, token: str) -> dict:
"""Authenticated GitHub API GET (single page)."""
req = urllib.request.Request(url, headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
"User-Agent": "ci-live-comment",
})
with urllib.request.urlopen(req) as resp:
data: dict = json.loads(resp.read())
return data
def _api_get_paginated(url: str, token: str, list_key: str | None = None) -> list:
"""Authenticated GitHub API GET with pagination."""
results: list = []
while url:
req = urllib.request.Request(url, headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
"User-Agent": "ci-live-comment",
})
with urllib.request.urlopen(req) as resp:
data = json.loads(resp.read())
link_header = resp.headers.get("Link", "")
if list_key:
results.extend(data.get(list_key, []))
elif isinstance(data, list):
results.extend(data)
else:
return data
next_url = None
for part in link_header.split(","):
part = part.strip()
if 'rel="next"' in part:
next_url = part[part.find("<") + 1:part.find(">")]
break
url = next_url
return results
def select_watched_runs(
runs: list[dict], watch_names: list[str], exclude_run_id: str = "",
) -> list[dict]:
"""Pick the sibling runs whose jobs belong in the comment.
``runs`` is the API's run list for one commit. ``watch_names`` holds
workflow names from ``WATCH_WORKFLOWS``. One commit can have more than
one run of the same workflow, after a rerun or a new push. Thus this
keeps only the newest run for each workflow name. An older attempt
reports results that a rerun replaced.
``exclude_run_id`` removes the CI run itself when its name is also in
``watch_names``.
"""
newest: dict[str, dict] = {}
wanted = {n.strip() for n in watch_names if n.strip()}
for candidate in runs:
name = str(candidate.get("name", ""))
if name not in wanted:
continue
if exclude_run_id and str(candidate.get("id", "")) == str(exclude_run_id):
continue
current = newest.get(name)
if current is None or str(candidate.get("created_at", "")) > str(current.get("created_at", "")):
newest[name] = candidate
return list(newest.values())
def runs_all_completed(runs: list[dict]) -> bool:
"""True only when every run in the list reports ``status: completed``.
The job list alone cannot answer "is CI done": a run that GitHub just
created has no jobs yet, and a mid-run poll can catch the moment where
every visible job finished but a downstream sub-workflow has not
spawned its jobs. Both look identical to "all done" at the job level.
The run's own ``status`` is the authoritative signal, so the poller
must not exit while any relevant run is still ``queued`` or
``in_progress``. An empty list is not done — it means the poller has
no run information at all.
"""
return bool(runs) and all(str(r.get("status", "")) == "completed" for r in runs)
def collect_run_jobs(
token: str, repo: str, run_id: str, watch_workflows: list[str] | None = None,
) -> tuple[list[dict], bool]:
"""Collect all jobs in the CI run + any watched sibling runs.
Returns ``(jobs, runs_completed)``: a flat list of job dicts (same
shape as the API returns, plus ``_workflow_name`` on jobs from a
watched run), and whether the CI run and every selected watched run
report ``status: completed`` (see :func:`runs_all_completed`).
Reusable-workflow (``workflow_call``) jobs need no special handling:
GitHub flattens them into the caller run's job list, already named
``\"Workflow / job\"``. Watched runs are separate top-level runs
(the Docker image build), so their jobs are fetched per run and
prefixed here.
"""
owner, repo_name = repo.split("/")
run_info = _api_request(f"{API_BASE}/repos/{owner}/{repo_name}/actions/runs/{run_id}", token)
head_sha = run_info.get("head_sha", "")
# CI run jobs (includes every reusable-workflow job).
all_jobs: list[dict] = []
orch_jobs = _api_get_paginated(
f"{API_BASE}/repos/{owner}/{repo_name}/actions/runs/{run_id}/jobs",
token, list_key="jobs",
)
# Skip workflow-call placeholder steps (they're sub-workflow triggers,
# not review signal), but KEEP in_progress / queued jobs so the poller
# knows they're still running.
for job in orch_jobs:
steps = job.get("steps") or []
if any(s.get("name", "").startswith("Run ./.github/workflows/") for s in steps):
continue
all_jobs.append(job)
if not watch_workflows or not head_sha:
return all_jobs, runs_all_completed([run_info])
# Watched sibling runs for the same commit. A run can be absent on the
# first polls. Then classify_jobs() shows nothing for it.
sibling_runs = _api_get_paginated(
f"{API_BASE}/repos/{owner}/{repo_name}/actions/runs?head_sha={head_sha}&per_page=100",
token, list_key="workflow_runs",
)
relevant_runs = [run_info]
for watched in select_watched_runs(sibling_runs, watch_workflows, exclude_run_id=run_id):
relevant_runs.append(watched)
watched_jobs = _api_get_paginated(
f"{API_BASE}/repos/{owner}/{repo_name}/actions/runs/{watched['id']}/jobs",
token, list_key="jobs",
)
for job in watched_jobs:
job["_workflow_name"] = watched.get("name", "")
all_jobs.append(job)
return all_jobs, runs_all_completed(relevant_runs)
def find_comment_id(token: str, repo: str, pr_number: str) -> int | None:
"""Find our existing review comment by marker prefix."""
owner, repo_name = repo.split("/")
comments = _api_get_paginated(
f"{API_BASE}/repos/{owner}/{repo_name}/issues/{pr_number}/comments",
token,
)
for c in comments:
body = c.get("body", "") if isinstance(c, dict) else ""
if body.startswith("<!-- hermes-ci-review-bot -->"):
return c.get("id") if isinstance(c, dict) else None
return None
def upsert_comment(
token: str, repo: str, pr_number: str, body: str, comment_id: int | None = None
) -> int | None:
"""Create or update the review comment. Returns the comment ID."""
owner, repo_name = repo.split("/")
if comment_id is None:
comment_id = find_comment_id(token, repo, pr_number)
if comment_id:
url = f"{API_BASE}/repos/{owner}/{repo_name}/issues/comments/{comment_id}"
method = "PATCH"
else:
url = f"{API_BASE}/repos/{owner}/{repo_name}/issues/{pr_number}/comments"
method = "POST"
data = json.dumps({"body": body}).encode("utf-8")
req = urllib.request.Request(url, data=data, method=method, headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
"Content-Type": "application/json",
"User-Agent": "ci-live-comment",
})
try:
with urllib.request.urlopen(req) as resp:
result = json.loads(resp.read())
return result.get("id")
except urllib.error.HTTPError as e:
print(f" API error {e.code}: {e.reason}", file=sys.stderr)
return None
# ---------------------------------------------------------------------------
# Artifact fetching (dynamic review-status artifacts)
# ---------------------------------------------------------------------------
# Prefix for all review-status artifacts uploaded by status-producing jobs.
# Each job uploads a ``review-status-<name>`` artifact containing a
# ``review-status.json`` file in GITHUB_OUTPUT format:
# review_status=<json array of {source, results: [...]} objects>
_REVIEW_STATUS_ARTIFACT_PREFIX = "review-status-"
def _list_artifacts(token: str, repo: str, run_id: str) -> list[dict]:
"""List artifacts for a given run (paginated)."""
owner, repo_name = repo.split("/")
return _api_get_paginated(
f"{API_BASE}/repos/{owner}/{repo_name}/actions/runs/{run_id}/artifacts",
token, list_key="artifacts",
)
class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
"""Redirect handler that never follows — used to capture the Location."""
def redirect_request(self, *args, **kwargs):
return None
def _download_artifact(
token: str, repo: str, artifact: dict, dest_dir: Path,
) -> Path | None:
"""Download a single artifact zip via the API and extract it.
Returns the path to ``review-status.json`` inside the extracted dir,
or ``None`` if the download or extraction failed.
"""
owner, repo_name = repo.split("/")
archive_download_url = artifact.get("archive_download_url", "")
if not archive_download_url:
return None
# The archive_download_url is an API URL that 302s to a signed blob
# URL. Hop 1 authenticates to the API; hop 2 follows the redirect
# WITHOUT the Authorization header — the blob rejects a request that
# carries both a SAS token and an Authorization header (401).
opener = urllib.request.build_opener(_NoRedirectHandler)
location = ""
try:
opener.open(urllib.request.Request(archive_download_url, headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
"User-Agent": "ci-live-comment",
}), timeout=30)
except urllib.error.HTTPError as e:
location = e.headers.get("Location", "") if e.code == 302 else ""
except Exception:
location = ""
if not location:
return None
zip_path = dest_dir / f"{artifact['name']}.zip"
try:
# No auth headers here; further redirects are safe to follow.
with urllib.request.urlopen(
urllib.request.Request(location, headers={"User-Agent": "ci-live-comment"}),
timeout=60,
) as resp:
zip_path.write_bytes(resp.read())
except Exception:
return None
extract_dir = dest_dir / artifact["name"]
extract_dir.mkdir(parents=True, exist_ok=True)
try:
with zipfile.ZipFile(zip_path) as zf:
if any(".." in name or name.startswith("/") for name in zf.namelist()):
return None
zf.extractall(extract_dir)
except Exception:
return None
status_file = extract_dir / "review-status.json"
return status_file if status_file.exists() else None
def _parse_status_file(status_file: Path) -> list[dict]:
"""Parse a review-status.json file in GITHUB_OUTPUT format."""
try:
content = status_file.read_text(encoding="utf-8").strip()
if content.startswith("review_status="):
content = content[len("review_status="):]
statuses = json.loads(content)
if isinstance(statuses, list):
return statuses
except (json.JSONDecodeError, OSError):
pass
return []
def fetch_all_review_statuses(
token: str, repo: str, run_id: str,
) -> list[dict]:
"""Fetch and merge all review-status artifacts from the run.
Lists artifacts with the ``review-status-`` prefix on the orchestrator
run, downloads each, parses the ``review-status.json`` inside, and
merges into a single flat array. GitHub attaches artifacts uploaded by
reusable workflow jobs to the caller run, so one listing covers every
status-producing job.
Returns the merged list of ``{source, results: [...]}`` objects.
Artifacts that don't exist yet or fail to parse are silently skipped.
"""
all_statuses: list[dict] = []
temp_base = Path("/tmp/review-status-artifacts")
try:
artifacts = _list_artifacts(token, repo, run_id)
except Exception:
return all_statuses
rs_artifacts = [
a for a in artifacts
if a.get("name", "").startswith(_REVIEW_STATUS_ARTIFACT_PREFIX)
]
if not rs_artifacts:
return all_statuses
# Clean temp dir for this run's artifacts.
run_dl_dir = temp_base / str(run_id)
if run_dl_dir.exists():
shutil.rmtree(run_dl_dir)
run_dl_dir.mkdir(parents=True, exist_ok=True)
for artifact in rs_artifacts:
status_file = _download_artifact(token, repo, artifact, run_dl_dir)
if status_file is None:
continue
statuses = _parse_status_file(status_file)
all_statuses.extend(statuses)
# A re-run can leave several non-expired artifacts with the same name,
# each carrying the same source — dedupe by source so the comment
# doesn't render duplicate sections.
seen: set[str] = set()
deduped: list[dict] = []
for status in all_statuses:
src = status.get("source", "")
if src in seen:
continue
if src:
seen.add(src)
deduped.append(status)
return deduped
# ---------------------------------------------------------------------------
# Comment assembly
# ---------------------------------------------------------------------------
def _import_assembler():
"""Import assemble_review_comment.py from the same directory."""
here = Path(__file__).resolve().parent
sys.path.insert(0, str(here))
import assemble_review_comment as asm
return asm
def build_comment_body(
asm_mod,
completed: dict[str, str],
pending: list[str],
run_url: str,
job_urls: dict[str, str],
review_statuses_json: str,
commit_info: str = "",
waiting: bool = False,
) -> str:
"""Assemble the comment body from current job states + static inputs."""
needs_json = json.dumps(completed) if completed else ""
return asm_mod.assemble(
needs_json=needs_json,
run_url=run_url,
job_urls=job_urls,
review_statuses_json=review_statuses_json,
pending_jobs=pending if pending else None,
commit_info=commit_info,
waiting=waiting,
)
def _commit_info_for_state(commit_info: str, pending: bool) -> str:
"""Use past tense in the final comment after every CI job completes."""
if pending:
return commit_info
return commit_info.replace("<sub>running on ", "<sub>ran on ", 1)
# ---------------------------------------------------------------------------
# Polling loop
# ---------------------------------------------------------------------------
def run(
token: str,
repo: str,
run_id: str,
pr_number: str,
run_url: str,
commit_info: str = "",
interval: int = 15,
timeout: int = 1800,
dry_run: bool = False,
watch_workflows: list[str] | None = None,
) -> int:
"""Poll for job statuses and update the PR comment until all done.
Always returns 0. The poller reports on the CI run from a different run.
Thus a failed CI job is not a failure of this job. The CI run has its
own gate, which reports that. Comment posting is best-effort.
"""
asm = _import_assembler()
start = time.time()
last_body = ""
quiet_grace_used = False
prev_completed: dict[str, str] = {}
prev_pending: list[str] = []
prev_artifact_count = 0
while True:
elapsed = time.time() - start
if elapsed > timeout:
print(f"Timeout ({timeout}s) reached — stopping poll.", file=sys.stderr)
break
try:
jobs, runs_completed = collect_run_jobs(token, repo, run_id, watch_workflows)
except Exception as e:
print(f" API error collecting jobs: {e}", file=sys.stderr)
time.sleep(interval)
continue
completed, pending, job_urls = classify_jobs(jobs)
total = len(completed) + len(pending)
infra_count = len(jobs) - total
print(f" [{elapsed:.0f}s] fetched {len(jobs)} jobs from API "
f"({infra_count} infra filtered) → {len(completed)} completed, "
f"{len(pending)} pending ({total} review jobs)")
# Log transitions since last poll.
new_completed = {k: v for k, v in completed.items() if k not in prev_completed}
new_pending = [j for j in pending if j not in prev_pending]
gone_pending = [j for j in prev_pending if j not in pending and j not in completed]
if new_completed:
parts = [f"{name}={result}" for name, result in new_completed.items()]
print(f" → {len(new_completed)} job(s) newly completed: {', '.join(parts)}")
if new_pending:
print(f" → {len(new_pending)} job(s) newly appeared: {', '.join(new_pending)}")
if gone_pending:
print(f" → {len(gone_pending)} job(s) disappeared from pending: {', '.join(gone_pending)}")
# Dynamically fetch all review-status artifacts from the run.
artifact_statuses = fetch_all_review_statuses(token, repo, run_id)
artifact_count_changed = len(artifact_statuses) != prev_artifact_count
if artifact_count_changed:
print(f" Found {len(artifact_statuses)} review status entries from artifacts "
f"(was {prev_artifact_count} last poll)")
prev_artifact_count = len(artifact_statuses)
merged_json = json.dumps(artifact_statuses) if artifact_statuses else ""
# The run status is authoritative for "done": an empty job list on
# a run that is still queued/in_progress means GitHub has not
# spawned the jobs yet, not that everything passed.
all_done = not pending and runs_completed
current_commit_info = _commit_info_for_state(commit_info, pending=not all_done)
body = build_comment_body(
asm, completed, pending, run_url, job_urls,
merged_json,
current_commit_info,
waiting=not runs_completed,
)
if body != last_body:
change_reasons = []
if new_completed:
change_reasons.append(f"{len(new_completed)} new completion(s)")
if new_pending:
change_reasons.append(f"{len(new_pending)} new pending job(s)")
if gone_pending:
change_reasons.append(f"{len(gone_pending)} job(s) left pending")
if artifact_count_changed:
change_reasons.append("artifact statuses updated")
if not change_reasons:
change_reasons.append("initial post")
reason = "; ".join(change_reasons)
if dry_run:
print(f" Comment body changed ({reason}) — DRY RUN:")
print("--- DRY RUN — comment body ---")
print(body)
print("--- END ---")
else:
cid = upsert_comment(token, repo, pr_number, body)
if cid:
print(f" Updated comment {cid} ({reason})")
else:
print(f" Failed to update comment ({reason}, will retry)", file=sys.stderr)
last_body = body
else:
if pending:
print(f" No change since last poll. Still waiting on: {', '.join(pending)}")
else:
print(" No change since last poll.")
prev_completed = completed
prev_pending = pending
if all_done and not quiet_grace_used:
quiet_grace_used = True
print(" No jobs pending and runs report completed — "
"waiting 10s for downstream jobs to appear.")
time.sleep(10)
continue
if all_done:
failed = [name for name, result in completed.items() if result == "failure"]
if failed:
print(f" All jobs done, {len(failed)} failed: {', '.join(failed)}")
else:
print(" All jobs completed — done.")
break
if not pending:
print(" No visible jobs pending, but a run is still queued or "
"in progress — waiting for its jobs to appear.")
quiet_grace_used = False
time.sleep(interval)
return 0
def parse_watch_workflows(raw: str) -> list[str]:
"""Parse the ``WATCH_WORKFLOWS`` value into workflow names.
One name per line. Not comma-separated: a workflow name can contain a
comma ("Docker Build, Test, and Publish").
"""
return [name.strip() for name in raw.splitlines() if name.strip()]
def resolve_pr_number(token: str, repo: str, head_sha: str) -> str:
"""Find the PR number for a commit when the event payload has none.
``workflow_run.pull_requests`` is empty for some runs. The poller has no
comment to post without a number.
"""
if not head_sha:
return ""
owner, repo_name = repo.split("/")
try:
results = _api_get_paginated(
f"{API_BASE}/repos/{owner}/{repo_name}/commits/{head_sha}/pulls",
token,
)
except Exception as e:
print(f" API error resolving PR number: {e}", file=sys.stderr)
return ""
for item in results:
if isinstance(item, dict) and item.get("state") == "open":
return str(item.get("number", ""))
return ""
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--interval", type=int, default=15,
help="Seconds between polls (default: 15).")
parser.add_argument("--timeout", type=int, default=1800,
help="Max seconds to poll before giving up (default: 1800).")
parser.add_argument("--dry-run", action="store_true",
help="Print comment body instead of posting to PR.")
args = parser.parse_args()
token = os.environ.get("GITHUB_TOKEN", "")
repo = os.environ.get("GITHUB_REPOSITORY", "")
run_id = os.environ.get("CI_RUN_ID", "")
pr_number = os.environ.get("PR_NUMBER", "")
run_url = os.environ.get("RUN_URL", "")
# Sibling workflows to merge into the comment, one name per line. Their
# runs are separate from the CI run, so the poller resolves them by name.
watch_workflows = parse_watch_workflows(os.environ.get("WATCH_WORKFLOWS", ""))
if not args.dry_run:
if not token:
print("GITHUB_TOKEN is required", file=sys.stderr)
return 1
if not repo:
print("GITHUB_REPOSITORY is required", file=sys.stderr)
return 1
if not run_id:
print("CI_RUN_ID is required", file=sys.stderr)
return 1
# Build commit info line from env vars (set by ci-review-comment.yml).
commit_sha = os.environ.get("COMMIT_SHA", "")
commit_msg = os.environ.get("COMMIT_MESSAGE", "")
if not pr_number and not args.dry_run:
pr_number = resolve_pr_number(token, repo, commit_sha)
if not pr_number:
print("No PR number found — nothing to comment on.", file=sys.stderr)
return 0
print(f"Resolved PR #{pr_number} from commit {commit_sha[:7]}")
commit_url = os.environ.get("COMMIT_URL", "")
if not commit_url and commit_sha and pr_number:
server = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
commit_url = f"{server}/{repo}/pull/{pr_number}/commits/{commit_sha}"
commit_info = ""
if commit_sha:
short_sha = commit_sha[:7]
if commit_msg:
# Truncate commit message to first line, max 60 chars.
first_line = commit_msg.split("\n")[0][:60]
if commit_url:
commit_info = f"<sub>running on [{short_sha}]({commit_url}) — {first_line}</sub>"
else:
commit_info = f"<sub>running on {short_sha} — {first_line}</sub>"
elif commit_url:
commit_info = f"<sub>running on [{short_sha}]({commit_url})</sub>"
else:
commit_info = f"<sub>running on {short_sha}</sub>"
return run(
token=token,
repo=repo,
run_id=run_id,
pr_number=pr_number,
run_url=run_url,
commit_info=commit_info,
interval=args.interval,
timeout=args.timeout,
dry_run=args.dry_run,
watch_workflows=watch_workflows,
)
if __name__ == "__main__":
sys.exit(main())
scripts/ci/lockfile_diff.py
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env python3
"""Semantic diff of npm ``package-lock.json`` files for PR comments.
``git diff`` on a lockfile is unreadable: npm reorders entries, rewrites
integrity hashes, and moves packages between nesting levels, so a one-line
``package.json`` bump can produce a thousand-line textual diff. This script
ignores the text entirely — it parses the ``packages`` map out of both
versions of each lockfile (lockfileVersion 2/3), reduces each to
``{install path: version}``, and set-diffs the two dicts. Reordering and
hash churn vanish; what's left is the actual dependency change.
Usage (from a checkout that still has the base ref available):
python scripts/ci/lockfile_diff.py --base <ref> --head <ref> \
--output diff.md [--repo-root .]
Reads every ``package-lock.json`` tracked at either ref (top-level and
nested — the repo has several), diffs each, and writes a Markdown fragment
to ``--output``. Exits 0 always; an empty output file means "no version
changes" (the caller uses that to decide whether to include the section).
The fragment is consumed by ``scripts/ci/assemble_review_comment.py``,
which wraps it in a section with a header and action note.
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
def parse_lockfile(text: str) -> dict[str, str]:
"""Reduce lockfile JSON to ``{install path: version}``.
Keys are the ``packages`` map's keys (e.g. ``node_modules/react`` or
``node_modules/foo/node_modules/react``), so the same package deduped
at two versions shows up as two distinct entries. The root entry
(``""``, the workspace itself) is skipped, as are versionless link
entries.
"""
data = json.loads(text)
out: dict[str, str] = {}
for path, meta in data.get("packages", {}).items():
if not path:
continue # root project entry, not a dependency
version = meta.get("version")
if version:
out[path] = version
return out
def diff_locks(base: dict[str, str], head: dict[str, str]) -> dict[str, list]:
"""Set-diff two ``{path: version}`` maps.
Returns ``added`` / ``removed`` as ``[(path, version)]`` and
``updated`` as ``[(path, base_version, head_version)]``, each sorted
by path.
"""
added = sorted((p, v) for p, v in head.items() if p not in base)
removed = sorted((p, v) for p, v in base.items() if p not in head)
updated = sorted(
(p, base[p], head[p]) for p in base.keys() & head.keys() if base[p] != head[p]
)
return {"added": added, "removed": removed, "updated": updated}
def _display_name(path: str) -> str:
"""``node_modules/foo/node_modules/@scope/bar`` → ``@scope/bar (nested under foo)``."""
parts = path.split("node_modules/")
name = parts[-1].rstrip("/")
if len(parts) > 2:
parents = " → ".join(p.rstrip("/") for p in parts[1:-1])
return f"{name} *(nested under {parents})*"
return name
def render_markdown(diffs: dict[str, dict[str, list]]) -> str:
"""Render per-lockfile diffs as a Markdown fragment.
``diffs`` maps lockfile repo-path → the output of :func:`diff_locks`.
Lockfiles with no version changes are omitted. Returns ``""`` when
nothing changed anywhere (caller skips the section entirely).
The output is a fragment — per-lockfile ``####`` subsections with
tables — not a standalone comment. The ``assemble_review_comment``
script wraps this in a section with its own header and action note,
so no top-level header or comment marker is emitted here.
"""
sections = []
for lockfile, d in sorted(diffs.items()):
added, removed, updated = d["added"], d["removed"], d["updated"]
n = len(added) + len(removed) + len(updated)
if n == 0:
continue
lines = [f"#### `{lockfile}`", ""]
lines.append("| Package | Before | After |")
lines.append("| --- | --- | --- |")
for path, old, new in updated:
lines.append(f"| {_display_name(path)} | `{old}` | `{new}` |")
for path, version in added:
lines.append(f"| ➕ {_display_name(path)} | — | `{version}` |")
for path, version in removed:
lines.append(f"| ➖ {_display_name(path)} | `{version}` | — |")
sections.append("\n".join(lines))
if not sections:
return ""
return "\n\n".join(sections) + "\n"
def _git_show(ref: str, path: str, repo_root: str) -> str | None:
"""Contents of ``path`` at ``ref``, or None if it doesn't exist there."""
proc = subprocess.run(
["git", "show", f"{ref}:{path}"],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
cwd=repo_root,
)
return proc.stdout if proc.returncode == 0 else None
def _tracked_lockfiles(ref: str, repo_root: str) -> set[str]:
proc = subprocess.run(
["git", "ls-tree", "-r", "--name-only", ref],
capture_output=True,
text=True, encoding="utf-8", errors="replace",
cwd=repo_root,
check=True,
)
return {
line
for line in proc.stdout.splitlines()
if line.split("/")[-1] == "package-lock.json"
}
def diff_refs(base: str, head: str, repo_root: str = ".") -> dict[str, dict[str, list]]:
"""Diff every package-lock.json tracked at either ref."""
lockfiles = _tracked_lockfiles(base, repo_root) | _tracked_lockfiles(head, repo_root)
diffs = {}
for path in sorted(lockfiles):
base_text = _git_show(base, path, repo_root)
head_text = _git_show(head, path, repo_root)
base_map = parse_lockfile(base_text) if base_text else {}
head_map = parse_lockfile(head_text) if head_text else {}
diffs[path] = diff_locks(base_map, head_map)
return diffs
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--base", required=True, help="base git ref (merge base)")
ap.add_argument("--head", required=True, help="head git ref")
ap.add_argument("--output", required=True, help="markdown output path")
ap.add_argument("--repo-root", default=".", help="repository root")
args = ap.parse_args()
diffs = diff_refs(args.base, args.head, args.repo_root)
markdown = render_markdown(diffs)
with open(args.output, "w", encoding="utf-8") as fh:
fh.write(markdown)
if markdown:
changed = sum(len(v) for d in diffs.values() for v in d.values())
print(f"{changed} package version change(s) — report written to {args.output}")
else:
print("No package version changes.")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/ci/publish_e2e_evidence.py
+337
View File
@@ -0,0 +1,337 @@
#!/usr/bin/env python3
"""Publish validated E2E evidence as GitHub attachments and update its PR comment.
This script only runs from the trusted ``workflow_run`` publisher. It never
checks out PR code: it accepts the small evidence artifact produced by the
untrusted E2E workflow, validates its manifest and PNG bytes, uploads the
approved files as GitHub attachments, and replaces the placeholder in the
source PR's CI review comment with those attachment URLs.
"""
from __future__ import annotations
import argparse
import html
import json
import os
import re
import subprocess
import sys
import time
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from typing import Any
API_BASE = "https://api.github.com"
EVIDENCE_START = "<!-- hermes-e2e-evidence:start -->"
EVIDENCE_END = "<!-- hermes-e2e-evidence:end -->"
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
MAX_FILES = 20
MAX_FILE_BYTES = 5 * 1024 * 1024
MAX_TOTAL_BYTES = 20 * 1024 * 1024
MAX_DIMENSION = 8_000
COMMENT_LOOKUP_ATTEMPTS = 6
COMMENT_LOOKUP_DELAY_SECONDS = 2
_SAFE_FILE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*\.png$")
_ATTACHMENT_URL = re.compile(r"^!\[[^\]\r\n]*\]\((https://github\.com/user-attachments/assets/[0-9a-fA-F-]+)\)$")
@dataclass(frozen=True)
class EvidenceFile:
"""One validated PNG and the label used when rendering the PR comment."""
filename: str
label: str
def _api_request(
url: str,
token: str,
method: str = "GET",
payload: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Send one authenticated GitHub API request and return its JSON object."""
data = json.dumps(payload).encode("utf-8") if payload is not None else None
request = urllib.request.Request(
url,
data=data,
method=method,
headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"Content-Type": "application/json",
"X-GitHub-Api-Version": "2022-11-28",
"User-Agent": "hermes-e2e-evidence-publisher",
},
)
with urllib.request.urlopen(request) as response:
parsed = json.loads(response.read())
if not isinstance(parsed, dict):
raise ValueError(f"Expected an object from {url}")
return parsed
def _read_png(path: Path) -> bytes:
"""Read a bounded PNG, rejecting corrupt and unexpectedly large images."""
if not path.is_file() or path.is_symlink():
raise ValueError(f"Evidence file is not a regular file: {path.name}")
size = path.stat().st_size
if size == 0 or size > MAX_FILE_BYTES:
raise ValueError(f"Evidence file has invalid size: {path.name}")
data = path.read_bytes()
if not data.startswith(PNG_SIGNATURE) or len(data) < 24 or data[12:16] != b"IHDR":
raise ValueError(f"Evidence file is not a PNG: {path.name}")
width = int.from_bytes(data[16:20], "big")
height = int.from_bytes(data[20:24], "big")
if not 0 < width <= MAX_DIMENSION or not 0 < height <= MAX_DIMENSION:
raise ValueError(f"Evidence image has invalid dimensions: {path.name}")
return data
def _manifest_files(manifest: dict[str, Any]) -> list[EvidenceFile]:
"""Flatten a version-one manifest into ordered, reviewer-facing images."""
if manifest.get("version") != 1:
raise ValueError("Unsupported E2E evidence manifest version")
files: list[EvidenceFile] = []
screenshots = manifest.get("screenshots", [])
diffs = manifest.get("diffs", [])
if not isinstance(screenshots, list) or not isinstance(diffs, list):
raise ValueError("Evidence manifest lists are malformed")
for entry in screenshots:
if not isinstance(entry, dict) or not isinstance(entry.get("name"), str) or not isinstance(entry.get("file"), str):
raise ValueError("Evidence screenshot entry is malformed")
files.append(EvidenceFile(entry["file"], f"new screenshot: {entry['name']}"))
for entry in diffs:
if not isinstance(entry, dict) or not isinstance(entry.get("name"), str) or not isinstance(entry.get("diff"), str):
raise ValueError("Evidence visual-diff entry is malformed")
files.append(EvidenceFile(entry["diff"], f"visual diff: {entry['name']}"))
for kind in ("actual", "expected"):
value = entry.get(kind)
if value is not None:
if not isinstance(value, str):
raise ValueError("Evidence visual-diff companion is malformed")
files.append(EvidenceFile(value, f"visual {kind}: {entry['name']}"))
names = [item.filename for item in files]
if len(files) > MAX_FILES or len(set(names)) != len(names):
raise ValueError("Evidence manifest has too many or duplicate files")
if any(not _SAFE_FILE.fullmatch(name) for name in names):
raise ValueError("Evidence manifest contains an unsafe filename")
return files
def load_evidence(evidence_dir: Path) -> tuple[list[EvidenceFile], dict[str, bytes]]:
"""Load the manifest and return only the validated files it declares."""
manifest_path = evidence_dir / "e2e-evidence.json"
if not manifest_path.is_file() or manifest_path.is_symlink():
raise ValueError("E2E evidence manifest is missing")
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise ValueError("E2E evidence manifest is not JSON") from exc
if not isinstance(manifest, dict):
raise ValueError("E2E evidence manifest is not an object")
files = _manifest_files(manifest)
payloads: dict[str, bytes] = {}
total = 0
for item in files:
path = evidence_dir / item.filename
if path.parent != evidence_dir:
raise ValueError("Evidence file escaped its artifact directory")
payload = _read_png(path)
total += len(payload)
if total > MAX_TOTAL_BYTES:
raise ValueError("E2E evidence exceeds the total size limit")
payloads[item.filename] = payload
return files, payloads
def render_evidence(files: list[EvidenceFile], attachment_urls: dict[str, str]) -> str:
"""Render validated GitHub attachment URLs inside the review-comment marker."""
blocks = [EVIDENCE_START]
for item in files:
url = attachment_urls.get(item.filename)
if url is None:
raise ValueError(f"Missing attachment URL for {item.filename}")
blocks.extend((
"<details>",
f"<summary>{item.label}</summary>",
"",
f"![{item.label}]({url})",
"",
"</details>",
))
blocks.append(EVIDENCE_END)
return "\n".join(blocks)
def render_upload_failure(error: Exception) -> str:
"""Render an escaped upload error inside the review-comment marker."""
return "\n".join((
EVIDENCE_START,
"<sub>inline evidence upload failed.</sub>",
"",
f"<pre>{html.escape(str(error))}</pre>",
EVIDENCE_END,
))
def replace_evidence_marker(comment: str, evidence: str) -> str:
"""Replace exactly the pending-evidence region in a CI review comment."""
pattern = re.compile(f"{re.escape(EVIDENCE_START)}.*?{re.escape(EVIDENCE_END)}", re.DOTALL)
result, count = pattern.subn(evidence, comment, count=1)
if count != 1:
raise ValueError("CI review comment does not contain one evidence marker")
return result
def _find_review_comment(comments: object) -> dict[str, Any] | None:
"""Find a live CI review comment only after it contains this marker."""
if not isinstance(comments, list):
raise ValueError("GitHub comments response is malformed")
for item in comments:
if not isinstance(item, dict):
continue
body = str(item.get("body", ""))
if body.startswith("<!-- hermes-ci-review-bot -->") and EVIDENCE_START in body and EVIDENCE_END in body:
return item
return None
def _wait_for_review_comment(token: str, source_repo: str, pr_number: str) -> dict[str, Any] | None:
"""Wait briefly for GitHub's comment API to expose the completed marker."""
request = urllib.request.Request(
f"{API_BASE}/repos/{source_repo}/issues/{pr_number}/comments?per_page=100",
headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
"User-Agent": "hermes-e2e-evidence-publisher",
},
)
for attempt in range(COMMENT_LOOKUP_ATTEMPTS):
with urllib.request.urlopen(request) as response:
comment = _find_review_comment(json.loads(response.read()))
if comment is not None:
return comment
if attempt + 1 < COMMENT_LOOKUP_ATTEMPTS:
time.sleep(COMMENT_LOOKUP_DELAY_SECONDS)
return None
def upload_evidence(
files: list[EvidenceFile],
evidence_dir: Path,
source_repo: str,
session_token: str,
) -> dict[str, str]:
"""Upload validated files through gh-image and accept only attachment URLs."""
environment = os.environ.copy()
environment["GH_SESSION_TOKEN"] = session_token
attachment_urls: dict[str, str] = {}
for item in files:
try:
result = subprocess.run(
[
"gh",
"image",
"--repo",
source_repo,
str(evidence_dir / item.filename),
],
check=True,
capture_output=True,
text=True, encoding="utf-8", errors="replace",
env=environment,
)
except subprocess.CalledProcessError as exc:
output = "; ".join(
value.strip()
for value in (exc.stdout, exc.stderr)
if value and value.strip()
)
message = f"Failed to upload {item.filename} with gh image (exit code {exc.returncode})"
if output:
message = f"{message}: {output}"
print(message, file=sys.stderr)
raise RuntimeError(message) from exc
match = _ATTACHMENT_URL.fullmatch(result.stdout.strip())
if match is None:
raise ValueError(f"gh-image returned an invalid attachment reference for {item.filename}")
attachment_urls[item.filename] = match.group(1)
return attachment_urls
def publish(
token: str,
source_repo: str,
evidence_dir: Path,
pr_number: str,
session_token: str,
) -> bool:
"""Publish evidence and patch its source PR comment; false means nothing to show."""
files, _ = load_evidence(evidence_dir)
if not files:
print("No inline E2E evidence to publish.")
return False
comment = _wait_for_review_comment(token, source_repo, pr_number)
if comment is None:
# A fork PR gets no CI review comment (the live poller needs a
# write token there), so there is no marker to patch. The
# evidence stays available in the workflow artifact.
print(
f"PR #{pr_number} has no CI review comment with an E2E evidence "
"marker; the evidence stays in the workflow artifact."
)
return False
try:
attachment_urls = upload_evidence(
files, evidence_dir, source_repo, session_token
)
except Exception as exc:
body = replace_evidence_marker(
str(comment.get("body", "")), render_upload_failure(exc)
)
_api_request(
f"{API_BASE}/repos/{source_repo}/issues/comments/{comment['id']}",
token,
method="PATCH",
payload={"body": body},
)
raise
evidence = render_evidence(files, attachment_urls)
body = replace_evidence_marker(str(comment.get("body", "")), evidence)
_api_request(
f"{API_BASE}/repos/{source_repo}/issues/comments/{comment['id']}",
token,
method="PATCH",
payload={"body": body},
)
print(f"Published {len(files)} E2E evidence image attachment(s).")
return True
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--evidence-dir", type=Path, required=True)
parser.add_argument("--source-repo", required=True)
parser.add_argument("--pr-number", required=True)
args = parser.parse_args()
token = os.environ.get("GITHUB_TOKEN", "")
if not token:
parser.error("GITHUB_TOKEN is required")
session_token = os.environ.get("GH_SESSION_TOKEN", "")
if not session_token:
parser.error("GH_SESSION_TOKEN is required")
publish(token, args.source_repo, args.evidence_dir, args.pr_number, session_token)
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/ci/test_install_ps1_cli_launchers.ps1
+141
View File
@@ -0,0 +1,141 @@
# Behavioral test for install.ps1's hermes launcher staging (PR #92092,
# reworked for the managed-binary-dir layout).
#
# Run: powershell.exe -NoProfile -File scripts/ci/test_install_ps1_cli_launchers.ps1
#
# The test lifts the real Install-HermesCommandLaunchers function from the
# PowerShell AST and executes it against a temporary install tree. It never
# reads or changes the user's PATH. The staging destination is passed in by
# the caller (Set-PathVariable passes $HermesHome\bin -- the managed binary
# dir OUTSIDE the git checkout); here it is a sibling temp dir, which also
# proves the function stages wherever it is pointed rather than assuming
# the legacy in-checkout location.
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$installPs1 = Join-Path (Join-Path $PSScriptRoot '..') 'install.ps1' | Resolve-Path
$ast = [System.Management.Automation.Language.Parser]::ParseFile(
$installPs1, [ref]$null, [ref]$null)
$fn = $ast.Find({
param($n)
$n -is [System.Management.Automation.Language.FunctionDefinitionAst] -and
$n.Name -eq 'Install-HermesCommandLaunchers'
}, $true)
if (-not $fn) {
throw "Install-HermesCommandLaunchers not found in $installPs1"
}
Invoke-Expression $fn.Extent.Text
$tempBase = [System.IO.Path]::GetFullPath([System.IO.Path]::GetTempPath())
$caseRoot = [System.IO.Path]::GetFullPath((Join-Path $tempBase (
'hermes-cli-launcher-test-' + [guid]::NewGuid().ToString('N')
)))
if (-not $caseRoot.StartsWith($tempBase, [System.StringComparison]::OrdinalIgnoreCase)) {
throw "Refusing to create test directory outside the system temp directory: $caseRoot"
}
$script:Failures = 0
function Assert-True {
param([bool]$Condition, [string]$Name)
if ($Condition) {
Write-Host " PASS $Name"
} else {
Write-Host " FAIL $Name"
$script:Failures++
}
}
function Assert-BytesEqual {
param([byte[]]$Expected, [byte[]]$Actual, [string]$Name)
$same = $Expected.Length -eq $Actual.Length
if ($same) {
for ($i = 0; $i -lt $Expected.Length; $i++) {
if ($Expected[$i] -ne $Actual[$i]) {
$same = $false
break
}
}
}
Assert-True $same $Name
}
try {
$installRoot = Join-Path $caseRoot 'hermes-agent'
$binDir = Join-Path $caseRoot 'bin'
New-Item -ItemType Directory -Force -Path $installRoot | Out-Null
# Fail-before-PATH-mutation: a missing required source must throw and
# must not leave an empty destination for the caller to put on PATH.
$missingThrew = $false
try {
Install-HermesCommandLaunchers -Root $installRoot -Destination $binDir | Out-Null
} catch {
$missingThrew = $_.Exception.Message -like '*required launcher not found*'
}
Assert-True $missingThrew 'missing hermes.exe fails the launcher stage'
Assert-True (-not (Test-Path -LiteralPath $binDir)) `
'failure does not create an empty PATH directory'
$scriptsDir = Join-Path $installRoot 'venv\Scripts'
New-Item -ItemType Directory -Force -Path $scriptsDir | Out-Null
$hermesV1 = [byte[]](77, 90, 1)
$hermesV2 = [byte[]](77, 90, 2)
$acp = [byte[]](77, 90, 3)
[System.IO.File]::WriteAllBytes((Join-Path $scriptsDir 'hermes.exe'), $hermesV1)
Set-Content -Path (Join-Path $installRoot 'venv\pyvenv.cfg') `
-Value "home = X" -Encoding Ascii
$staged = Install-HermesCommandLaunchers -Root $installRoot -Destination $binDir
Assert-True ($staged -eq $binDir) 'returns the destination it staged into'
Assert-BytesEqual $hermesV1 `
([System.IO.File]::ReadAllBytes((Join-Path $binDir 'hermes.exe'))) `
'normal venv: exe copy lands in the destination'
Assert-True (-not (Test-Path -LiteralPath (Join-Path $binDir 'hermes-acp.exe'))) `
'optional ACP launcher may be absent'
[System.IO.File]::WriteAllBytes((Join-Path $scriptsDir 'hermes.exe'), $hermesV2)
[System.IO.File]::WriteAllBytes((Join-Path $scriptsDir 'hermes-acp.exe'), $acp)
Install-HermesCommandLaunchers -Root $installRoot -Destination $binDir | Out-Null
Assert-BytesEqual $hermesV2 `
([System.IO.File]::ReadAllBytes((Join-Path $binDir 'hermes.exe'))) `
'installer refreshes an existing Hermes launcher'
Assert-BytesEqual $acp `
([System.IO.File]::ReadAllBytes((Join-Path $binDir 'hermes-acp.exe'))) `
'installer copies the optional ACP launcher when present'
# Relocatable venv: exe trampolines die when copied out of venv\Scripts
# ('uv trampoline failed to canonicalize script path'), so the stage
# must emit .cmd delegators and clear the stale exe copies.
Set-Content -Path (Join-Path $installRoot 'venv\pyvenv.cfg') `
-Value "home = X`r`nrelocatable = true" -Encoding Ascii
Install-HermesCommandLaunchers -Root $installRoot -Destination $binDir | Out-Null
Assert-True (Test-Path -LiteralPath (Join-Path $binDir 'hermes.cmd')) `
'relocatable venv: .cmd delegator staged'
Assert-True (-not (Test-Path -LiteralPath (Join-Path $binDir 'hermes.exe'))) `
'relocatable venv: stale exe copy removed'
$cmdBody = [System.IO.File]::ReadAllText((Join-Path $binDir 'hermes.cmd'))
Assert-True ($cmdBody.Contains((Join-Path $scriptsDir 'hermes.exe')) -and $cmdBody.Contains('%*')) `
'delegator invokes the in-venv exe and forwards args'
} finally {
if (Test-Path -LiteralPath $caseRoot) {
$resolvedCase = [System.IO.Path]::GetFullPath($caseRoot)
if (-not $resolvedCase.StartsWith($tempBase, [System.StringComparison]::OrdinalIgnoreCase)) {
throw "Refusing to remove test directory outside the system temp directory: $resolvedCase"
}
Remove-Item -LiteralPath $resolvedCase -Recurse -Force
}
}
if ($script:Failures -gt 0) {
Write-Host ""
Write-Host "$script:Failures assertion(s) failed"
exit 1
}
Write-Host ""
Write-Host "all assertions passed"
scripts/ci/test_install_ps1_path_migration.ps1
+125
View File
@@ -0,0 +1,125 @@
# Behavioral test for install.ps1's persisted-User-PATH migration.
#
# Run: pwsh -NoProfile -File scripts/ci/test_install_ps1_path_migration.ps1
#
# Not wired into the default CI lane — the Linux runners have no PowerShell
# host. It runs on any machine with pwsh (including via nixpkgs#powershell),
# and on a Windows runner if one is ever added.
#
# This is NOT a source-regex test. It parses install.ps1, lifts the real
# Set-ManagedNodeFirstOnUserPath body out of the AST, and rewrites *only* the
# two registry calls into an in-memory store so the actual shipped logic —
# split, dedupe, prepend, change-detection — executes for real. Rewriting from
# the AST rather than hand-copying the body means the test cannot silently
# drift away from the function it claims to cover.
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$installPs1 = Join-Path $PSScriptRoot '..' 'install.ps1' | Resolve-Path
$ast = [System.Management.Automation.Language.Parser]::ParseFile(
$installPs1, [ref]$null, [ref]$null)
$fn = $ast.Find({
param($n)
$n -is [System.Management.Automation.Language.FunctionDefinitionAst] -and
$n.Name -eq 'Set-ManagedNodeFirstOnUserPath'
}, $true)
if (-not $fn) {
throw "Set-ManagedNodeFirstOnUserPath not found in $installPs1"
}
# Swap the two registry calls for the in-memory store. Both must match, or the
# function has changed shape and this harness is no longer exercising it.
# Rewrite the whole definition extent (which already carries `function <name>
# { param(...) ... }`) so the shipped param block and body run verbatim.
$definition = $fn.Extent.Text
$reads = ([regex]'\[Environment\]::GetEnvironmentVariable\("Path", "User"\)').Matches($definition).Count
$writes = ([regex]'\[Environment\]::SetEnvironmentVariable\("Path", ([^,]+), "User"\)').Matches($definition).Count
if ($reads -ne 1 -or $writes -ne 1) {
throw "expected exactly one User PATH read and one write in the function body; found $reads read(s), $writes write(s). Update this harness."
}
$definition = $definition -replace `
'\[Environment\]::GetEnvironmentVariable\("Path", "User"\)', '$script:FakeUserPath'
$definition = $definition -replace `
'\[Environment\]::SetEnvironmentVariable\("Path", ([^,]+), "User"\)', '$script:FakeUserPath = $1; $script:FakeWrites++'
Invoke-Expression $definition
$NODE = 'C:\Users\me\AppData\Local\hermes\node'
$script:Failures = 0
function Invoke-Migration {
param([string]$Start, [string]$NodeDir = $NODE)
$script:FakeUserPath = $Start
$script:FakeWrites = 0
Set-ManagedNodeFirstOnUserPath $NodeDir
}
function Assert-Equal {
param($Expected, $Actual, [string]$Name)
if ($Expected -ceq $Actual) {
Write-Host " PASS $Name"
} else {
Write-Host " FAIL $Name"
Write-Host " expected: [$Expected]"
Write-Host " actual: [$Actual]"
$script:Failures++
}
}
Write-Host "install.ps1 Set-ManagedNodeFirstOnUserPath"
# The regression this function exists for: an install made by an older
# install.ps1, which *appended*. A system Node leads and the managed dir is
# stranded at the tail, so every new shell resolves the wrong node.exe. An
# add-if-missing check would see the entry present and leave it there forever.
Invoke-Migration "C:\Program Files\nodejs;C:\Users\me\bin;$NODE"
Assert-Equal "$NODE;C:\Program Files\nodejs;C:\Users\me\bin" $script:FakeUserPath `
'upgrade from appending installer: managed dir becomes first entry'
Assert-Equal 1 (@($script:FakeUserPath -split ';' | Where-Object { $_ -eq $NODE }).Count) `
'upgrade: managed dir is not duplicated'
Assert-Equal "C:\Program Files\nodejs;C:\Users\me\bin" `
(($script:FakeUserPath -split ';' | Where-Object { $_ -ne $NODE }) -join ';') `
'upgrade: unrelated entries keep their relative order'
Assert-Equal 1 $script:FakeWrites 'upgrade: persists exactly once'
Invoke-Migration "$NODE;C:\Program Files\nodejs"
Assert-Equal "$NODE;C:\Program Files\nodejs" $script:FakeUserPath 'already correct: unchanged'
Assert-Equal 0 $script:FakeWrites 'already correct: no registry write'
Invoke-Migration "C:\Program Files\nodejs"
Assert-Equal "$NODE;C:\Program Files\nodejs" $script:FakeUserPath 'fresh install: prepended'
# Empty segments are legal in a real User PATH (a trailing ';' is common) and
# the installer's other PATH code preserves them. Migration must not quietly
# rewrite parts of PATH it was not asked to touch.
Invoke-Migration "C:\Program Files\nodejs;;C:\Users\me\bin;"
Assert-Equal "$NODE;C:\Program Files\nodejs;;C:\Users\me\bin;" $script:FakeUserPath `
'empty segments are preserved'
# Windows paths are case-insensitive, and -ne on strings is too.
Invoke-Migration "C:\Program Files\nodejs;c:\users\me\appdata\local\HERMES\Node"
Assert-Equal "$NODE;C:\Program Files\nodejs" $script:FakeUserPath `
'existing entry in different case is replaced, not duplicated'
Invoke-Migration "$NODE;C:\Program Files\nodejs;$NODE"
Assert-Equal "$NODE;C:\Program Files\nodejs" $script:FakeUserPath 'duplicates collapse'
Invoke-Migration ""
Assert-Equal $NODE $script:FakeUserPath 'empty User PATH'
Invoke-Migration "C:\Program Files\nodejs" ""
Assert-Equal "C:\Program Files\nodejs" $script:FakeUserPath 'empty NodeDir is a no-op'
Assert-Equal 0 $script:FakeWrites 'empty NodeDir does not write'
if ($script:Failures -gt 0) {
Write-Host ""
Write-Host "$script:Failures assertion(s) failed"
exit 1
}
Write-Host ""
Write-Host "all assertions passed"
scripts/ci/timings_report.py
+1085
View File
File diff suppressed because it is too large Load Diff
scripts/contributor_audit.py
+500
View File
@@ -0,0 +1,500 @@
#!/usr/bin/env python3
"""Contributor Audit Script
Cross-references git authors, Co-authored-by trailers, and salvaged PR
descriptions to find any contributors missing from the release notes.
Usage:
# Basic audit since a tag
python scripts/contributor_audit.py --since-tag v2026.4.8
# Audit with a custom endpoint
python scripts/contributor_audit.py --since-tag v2026.4.8 --until v2026.4.13
# Compare against a release notes file
python scripts/contributor_audit.py --since-tag v2026.4.8 --release-file RELEASE_v0.9.0.md
"""
import argparse
import json
import re
import subprocess
import sys
from collections import defaultdict
from pathlib import Path
# ---------------------------------------------------------------------------
# Import AUTHOR_MAP and resolve_author from the sibling release.py module
# ---------------------------------------------------------------------------
SCRIPT_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))
from release import resolve_author # noqa: E402
REPO_ROOT = SCRIPT_DIR.parent
# ---------------------------------------------------------------------------
# AI assistants, bots, and machine accounts to exclude from contributor lists
# ---------------------------------------------------------------------------
IGNORED_PATTERNS = [
re.compile(r"^Claude", re.IGNORECASE),
re.compile(r"^Copilot$", re.IGNORECASE),
re.compile(r"^Cursor(\s+Agent)?$", re.IGNORECASE),
re.compile(r"^Codex$", re.IGNORECASE),
re.compile(r"^OpenAI Codex$", re.IGNORECASE),
re.compile(r"^CommandCode", re.IGNORECASE),
re.compile(r"^github-advanced-security(\[bot\])?$", re.IGNORECASE),
re.compile(r"^GitHub\s*Actions?$", re.IGNORECASE),
re.compile(r"^github-actions(\[bot\])?$", re.IGNORECASE),
re.compile(r"^dependabot", re.IGNORECASE),
re.compile(r"^renovate", re.IGNORECASE),
re.compile(r"^Hermes\s+(Agent|Audit)$", re.IGNORECASE),
re.compile(r"^nousbot(-eng)?$", re.IGNORECASE),
re.compile(r"^Ubuntu$", re.IGNORECASE),
# v0.20.0 audit additions:
re.compile(r"^Blut-?Agent$", re.IGNORECASE), # self-described AI agent account
re.compile(r".*\[bot\]$", re.IGNORECASE), # any GitHub [bot] suffix (hermes-seaeye[bot] etc.)
re.compile(r"^TRON$", re.IGNORECASE), # AgentMail agent
re.compile(r"^Happy$", re.IGNORECASE), # happy.engineering AI agent
re.compile(r"^Orca$", re.IGNORECASE), # Stably AI agent
# v0.21.0 audit additions:
re.compile(r"^Junie$", re.IGNORECASE), # JetBrains Junie AI agent
re.compile(r"^GPT-[\d.]+\s*Codex$", re.IGNORECASE), # OpenAI Codex model trailer
re.compile(r"^cursoragent$", re.IGNORECASE), # Cursor AI GitHub account
]
IGNORED_EMAILS = {
"noreply@anthropic.com",
"noreply@github.com",
"noreply@nousresearch.com",
"cursoragent@cursor.com",
"hermes@nousresearch.com",
"hermes-audit@example.com",
"nousbot@nousresearch.com",
"hermes@habibilabs.dev",
"omx@oh-my-codex.dev",
"codex@openai.com",
"noreply@commandcode.ai",
# v0.20.0 audit additions — AI-agent co-author trailers:
"tron-agent@agentmail.to", # TRON (AgentMail agent)
"yesreply@happy.engineering", # Happy (AI coding agent)
"help@stably.ai", # Orca (Stably AI agent)
# v0.21.0 audit additions — AI-agent co-author trailers:
"junie@jetbrains.com", # JetBrains Junie
"noreply@openai.com", # GPT-x Codex model trailer
}
def is_ignored(handle: str, email: str = "") -> bool:
"""Return True if this contributor is a bot/AI/machine account."""
if email in IGNORED_EMAILS:
return True
for pattern in IGNORED_PATTERNS:
if pattern.search(handle):
return True
return False
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def git(*args, cwd=None):
"""Run a git command and return stdout."""
result = subprocess.run(
["git"] + list(args),
capture_output=True,
text=True, encoding='utf-8', errors='replace',
cwd=cwd or str(REPO_ROOT),
)
if result.returncode != 0:
print(f" [warn] git {' '.join(args)} failed: {result.stderr.strip()}", file=sys.stderr)
return ""
return result.stdout.strip()
def gh_pr_list():
"""Fetch merged PRs from GitHub using the gh CLI.
Returns a list of dicts with keys: number, title, body, author.
Returns an empty list if gh is not available or the call fails.
"""
try:
result = subprocess.run(
[
"gh", "pr", "list",
"--repo", "NousResearch/hermes-agent",
"--state", "merged",
"--json", "number,title,body,author,mergedAt",
"--limit", "300",
],
capture_output=True,
text=True, encoding='utf-8', errors='replace',
timeout=60,
)
if result.returncode != 0:
print(f" [warn] gh pr list failed: {result.stderr.strip()}", file=sys.stderr)
return []
return json.loads(result.stdout)
except FileNotFoundError:
print(" [warn] 'gh' CLI not found — skipping salvaged PR scan.", file=sys.stderr)
return []
except subprocess.TimeoutExpired:
print(" [warn] gh pr list timed out — skipping salvaged PR scan.", file=sys.stderr)
return []
except json.JSONDecodeError:
print(" [warn] gh pr list returned invalid JSON — skipping salvaged PR scan.", file=sys.stderr)
return []
# ---------------------------------------------------------------------------
# Contributor collection
# ---------------------------------------------------------------------------
# Patterns that indicate salvaged/cherry-picked/co-authored work in PR bodies
SALVAGE_PATTERNS = [
# "Salvaged from @username" or "Salvaged from #123"
re.compile(r"[Ss]alvaged\s+from\s+@(\w[\w-]*)"),
re.compile(r"[Ss]alvaged\s+from\s+#(\d+)"),
# "Cherry-picked from @username"
re.compile(r"[Cc]herry[- ]?picked\s+from\s+@(\w[\w-]*)"),
# "Based on work by @username"
re.compile(r"[Bb]ased\s+on\s+work\s+by\s+@(\w[\w-]*)"),
# "Original PR by @username"
re.compile(r"[Oo]riginal\s+PR\s+by\s+@(\w[\w-]*)"),
# "Co-authored with @username"
re.compile(r"[Cc]o[- ]?authored\s+with\s+@(\w[\w-]*)"),
]
# Pattern for Co-authored-by trailers in commit messages
CO_AUTHORED_RE = re.compile(
r"Co-authored-by:\s*(.+?)\s*<([^>]+)>",
re.IGNORECASE,
)
def collect_commit_authors(since_tag, until="HEAD"):
"""Collect contributors from git commit authors.
Returns:
contributors: dict mapping github_handle -> set of source labels
unknown_emails: dict mapping email -> git name (for emails not in AUTHOR_MAP)
"""
range_spec = f"{since_tag}..{until}"
log = git(
"log", range_spec,
"--format=%H|%an|%ae|%s",
"--no-merges",
)
contributors = defaultdict(set)
unknown_emails = {}
if not log:
return contributors, unknown_emails
for line in log.split("\n"):
if not line.strip():
continue
parts = line.split("|", 3)
if len(parts) != 4:
continue
_sha, name, email, _subject = parts
handle = resolve_author(name, email)
# resolve_author returns "@handle" or plain name
if handle.startswith("@"):
contributors[handle.lstrip("@")].add("commit")
else:
# Could not resolve — record as unknown
contributors[handle].add("commit")
unknown_emails[email] = name
return contributors, unknown_emails
def collect_co_authors(since_tag, until="HEAD"):
"""Collect contributors from Co-authored-by trailers in commit messages.
Returns:
contributors: dict mapping github_handle -> set of source labels
unknown_emails: dict mapping email -> git name
"""
range_spec = f"{since_tag}..{until}"
# Get full commit messages to scan for trailers
log = git(
"log", range_spec,
"--format=__COMMIT__%H%n%b",
"--no-merges",
)
contributors = defaultdict(set)
unknown_emails = {}
if not log:
return contributors, unknown_emails
for line in log.split("\n"):
match = CO_AUTHORED_RE.search(line)
if match:
name = match.group(1).strip()
email = match.group(2).strip()
handle = resolve_author(name, email)
if handle.startswith("@"):
contributors[handle.lstrip("@")].add("co-author")
else:
contributors[handle].add("co-author")
unknown_emails[email] = name
return contributors, unknown_emails
def collect_salvaged_contributors(since_tag, until="HEAD"):
"""Scan merged PR bodies for salvage/cherry-pick/co-author attribution.
Uses the gh CLI to fetch PRs, then filters to the date range defined
by since_tag..until and scans bodies for salvage patterns.
Returns:
contributors: dict mapping github_handle -> set of source labels
pr_refs: dict mapping github_handle -> list of PR numbers where found
"""
contributors = defaultdict(set)
pr_refs = defaultdict(list)
# Determine the date range from git tags/refs
since_date = git("log", "-1", "--format=%aI", since_tag)
if until == "HEAD":
until_date = git("log", "-1", "--format=%aI", "HEAD")
else:
until_date = git("log", "-1", "--format=%aI", until)
if not since_date:
print(f" [warn] Could not resolve date for {since_tag}", file=sys.stderr)
return contributors, pr_refs
prs = gh_pr_list()
if not prs:
return contributors, pr_refs
for pr in prs:
# Filter by merge date if available
merged_at = pr.get("mergedAt", "")
if merged_at and since_date:
if merged_at < since_date:
continue
if until_date and merged_at > until_date:
continue
body = pr.get("body") or ""
pr_number = pr.get("number", "?")
# Also credit the PR author
pr_author = pr.get("author", {})
pr_author_login = pr_author.get("login", "") if isinstance(pr_author, dict) else ""
for pattern in SALVAGE_PATTERNS:
for match in pattern.finditer(body):
value = match.group(1)
# If it's a number, it's a PR reference — skip for now
# (would need another API call to resolve PR author)
if value.isdigit():
continue
contributors[value].add("salvage")
pr_refs[value].append(pr_number)
return contributors, pr_refs
# ---------------------------------------------------------------------------
# Release file comparison
# ---------------------------------------------------------------------------
def check_release_file(release_file, all_contributors):
"""Check which contributors are mentioned in the release file.
Returns:
mentioned: set of handles found in the file
missing: set of handles NOT found in the file
"""
try:
content = Path(release_file).read_text(encoding="utf-8")
except FileNotFoundError:
print(f" [error] Release file not found: {release_file}", file=sys.stderr)
return set(), set(all_contributors)
mentioned = set()
missing = set()
content_lower = content.lower()
for handle in all_contributors:
# Check for @handle or just handle (case-insensitive)
if f"@{handle.lower()}" in content_lower or handle.lower() in content_lower:
mentioned.add(handle)
else:
missing.add(handle)
return mentioned, missing
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
parser = argparse.ArgumentParser(
description="Audit contributors across git history, co-author trailers, and salvaged PRs.",
)
parser.add_argument(
"--since-tag",
required=True,
help="Git tag to start from (e.g., v2026.4.8)",
)
parser.add_argument(
"--until",
default="HEAD",
help="Git ref to end at (default: HEAD)",
)
parser.add_argument(
"--release-file",
default=None,
help="Path to a release notes file to check for missing contributors",
)
parser.add_argument(
"--strict",
action="store_true",
help="Exit with code 1 if new unmapped emails are found (for CI)",
)
parser.add_argument(
"--diff-base",
default=None,
help="Git ref to diff against (only flag emails from commits after this ref)",
)
args = parser.parse_args()
print(f"=== Contributor Audit: {args.since_tag}..{args.until} ===")
print()
# ---- 1. Git commit authors ----
print("[1/3] Scanning git commit authors...")
commit_contribs, commit_unknowns = collect_commit_authors(args.since_tag, args.until)
print(f" Found {len(commit_contribs)} contributor(s) from commits.")
# ---- 2. Co-authored-by trailers ----
print("[2/3] Scanning Co-authored-by trailers...")
coauthor_contribs, coauthor_unknowns = collect_co_authors(args.since_tag, args.until)
print(f" Found {len(coauthor_contribs)} contributor(s) from co-author trailers.")
# ---- 3. Salvaged PRs ----
print("[3/3] Scanning salvaged/cherry-picked PR descriptions...")
salvage_contribs, salvage_pr_refs = collect_salvaged_contributors(args.since_tag, args.until)
print(f" Found {len(salvage_contribs)} contributor(s) from salvaged PRs.")
# ---- Merge all contributors ----
all_contributors = defaultdict(set)
for handle, sources in commit_contribs.items():
all_contributors[handle].update(sources)
for handle, sources in coauthor_contribs.items():
all_contributors[handle].update(sources)
for handle, sources in salvage_contribs.items():
all_contributors[handle].update(sources)
# Merge unknown emails
all_unknowns = {}
all_unknowns.update(commit_unknowns)
all_unknowns.update(coauthor_unknowns)
# Filter out AI assistants, bots, and machine accounts
ignored = {h for h in all_contributors if is_ignored(h)}
for h in ignored:
del all_contributors[h]
# Also filter unknowns by email
all_unknowns = {e: n for e, n in all_unknowns.items() if not is_ignored(n, e)}
# ---- Output ----
print()
print(f"=== All Contributors ({len(all_contributors)}) ===")
print()
# Sort by handle, case-insensitive
for handle in sorted(all_contributors.keys(), key=str.lower):
sources = sorted(all_contributors[handle])
source_str = ", ".join(sources)
extra = ""
if handle in salvage_pr_refs:
pr_nums = salvage_pr_refs[handle]
extra = f" (PRs: {', '.join(f'#{n}' for n in pr_nums)})"
print(f" @{handle} [{source_str}]{extra}")
# ---- Unknown emails ----
if all_unknowns:
print()
print(f"=== Unknown Emails ({len(all_unknowns)}) ===")
print("These emails have no mapping and should be added via:")
print()
for email, name in sorted(all_unknowns.items()):
print(f" python3 scripts/add_contributor.py {email} <github-username> # {name}")
# ---- Strict mode: fail CI if new unmapped emails are introduced ----
if args.strict and all_unknowns:
# In strict mode, check if ANY unknown emails come from commits in this
# PR's diff range (new unmapped emails that weren't there before).
# This is the CI gate: existing unknowns are grandfathered, but new
# commits must have their author email in AUTHOR_MAP.
new_unknowns = {}
if args.diff_base:
# Only flag emails from commits after diff_base
new_commits_output = git(
"log", f"{args.diff_base}..HEAD",
"--format=%ae", "--no-merges",
)
new_emails = set(new_commits_output.splitlines()) if new_commits_output else set()
for email, name in all_unknowns.items():
if email in new_emails:
new_unknowns[email] = name
else:
new_unknowns = all_unknowns
if new_unknowns:
print()
print(f"=== STRICT MODE FAILURE: {len(new_unknowns)} new unmapped email(s) ===")
print("Add mapping files before merging (do NOT edit AUTHOR_MAP):")
print()
for email, name in sorted(new_unknowns.items()):
print(f" python3 scripts/add_contributor.py {email} <github-username> # {name}")
print()
print("To find the GitHub username:")
print(" gh api 'search/users?q=EMAIL+in:email' --jq '.items[0].login'")
strict_failed = True
else:
strict_failed = False
else:
strict_failed = False
# ---- Release file comparison ----
if args.release_file:
print()
print(f"=== Release File Check: {args.release_file} ===")
print()
mentioned, missing = check_release_file(args.release_file, all_contributors.keys())
print(f" Mentioned in release notes: {len(mentioned)}")
print(f" Missing from release notes: {len(missing)}")
if missing:
print()
print(" Contributors NOT mentioned in the release file:")
for handle in sorted(missing, key=str.lower):
sources = sorted(all_contributors[handle])
print(f" @{handle} [{', '.join(sources)}]")
else:
print()
print(" All contributors are mentioned in the release file!")
print()
print("Done.")
if strict_failed:
sys.exit(1)
if __name__ == "__main__":
main()
scripts/desktop-update.ps1
+11
View File
@@ -0,0 +1,11 @@
# COMPAT FORWARDER — do not add logic here.
#
# The hand-off moved to scripts/desktop-update/windows.ps1. This forwarder
# exists for exactly one consumer: an already-installed Desktop whose asar
# is one update behind and still spawns scripts/desktop-update.ps1 (see
# resolveUpdateScriptHandoff in apps/desktop/electron/updater-process.ts).
# Without it, that Desktop would silently fall back to the frozen staged
# Tauri binary for one update cycle — the exact rot this script family
# exists to escape.
& (Join-Path $PSScriptRoot "desktop-update\windows.ps1") @args
exit $LASTEXITCODE
scripts/desktop-update/posix.sh
Executable
+808
View File
@@ -0,0 +1,808 @@
#!/bin/bash
# posix.sh -- repo-owned macOS/Linux Desktop update hand-off.
#
# The whole job: wait for the Desktop to exit, run `hermes update`, tell the
# shim how it went, reopen the app. The Desktop spawns this detached and
# quits; because it lives in the checkout, every update refreshes the code
# that drives the next one. Replaces the in-app updater
# (applyUpdatesPosixInApp) -- with the app gone before the update starts,
# the HERMES_DESKTOP_CHILD_PID reaper-exclusion dance dies with it.
#
# CONTRACT (keep in sync with apps/desktop/electron/main.ts):
# bash scripts/desktop-update/posix.sh
# --install-root <path> repo checkout (HERMES_HOME/hermes-agent)
# --branch <ref> branch to update against
# --desktop-pid <pid> the Electron main process to wait out
# [--relaunch-target <p>] mac: running .app to swap+reopen;
# linux: running binary (omit = no relaunch)
# [--relaunch-cwd <p>] linux: working directory to restore on relaunch
# [--sandbox-fallback] linux: the caller vouches for a sandbox opt-out
# (ELECTRON_DISABLE_SANDBOX / --no-sandbox launch)
# [--no-ui] [--no-marker-cleanup] [--self-test-ui] [--self-test-gate]
# [--self-test-marker]
# [-- <args...>] linux: filtered launch args to replay
#
# The shim (ui.html in a chromeless browser app window) is decoration: it
# polls /progress for the current stage or a terminal event and reacts. The
# stages come from the gates below, never from child output. It owns nothing --
# relaunch, result file, marker hygiene all happen here, identically, when
# no renderer exists. No chromium-family browser found = no UI, fine.
#
# ORDERING (the durable-truth rule): swap and relaunch are DECIDED AND
# EXECUTED before the result file is written, the marker is removed, or a
# terminal event reaches the shim. Nothing user-visible may claim an outcome
# the filesystem hasn't already delivered.
set -u
ORIGINAL_ARGS=("$@")
INSTALL_ROOT="" BRANCH="main" DESKTOP_PID=0 RELAUNCH_TARGET=""
RELAUNCH_CWD="" SANDBOX_FALLBACK=0 RELAUNCH_ARGS=()
NO_UI=0 NO_MARKER_CLEANUP=0 SELF_TEST_UI=0 SELF_TEST_GATE=0 SELF_TEST_MARKER=0
SELF_TEST_TCC_HEAL=0
HANDOFF_DAEMONIZED=0
while [ $# -gt 0 ]; do
case "$1" in
--install-root) INSTALL_ROOT="$2"; shift 2 ;;
--branch) BRANCH="$2"; shift 2 ;;
--desktop-pid) DESKTOP_PID="$2"; shift 2 ;;
--relaunch-target) RELAUNCH_TARGET="$2"; shift 2 ;;
--relaunch-cwd) RELAUNCH_CWD="$2"; shift 2 ;;
--sandbox-fallback) SANDBOX_FALLBACK=1; shift ;;
--no-ui) NO_UI=1; shift ;;
--no-marker-cleanup) NO_MARKER_CLEANUP=1; shift ;;
--self-test-ui) SELF_TEST_UI=1; shift ;;
--self-test-gate) SELF_TEST_GATE=1; shift ;;
--self-test-tcc-heal) SELF_TEST_TCC_HEAL=1; shift ;;
--daemonized) HANDOFF_DAEMONIZED=1; shift ;;
--self-test-marker) SELF_TEST_MARKER=1; NO_UI=1; NO_MARKER_CLEANUP=1; shift ;;
--) shift; RELAUNCH_ARGS=("$@"); shift $# ;;
*) echo "unknown arg: $1" >&2; exit 64 ;;
esac
done
[ "$SELF_TEST_UI" -eq 1 ] || [ -n "$INSTALL_ROOT" ] || { echo "--install-root is required" >&2; exit 64; }
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
HERMES_HOME="${INSTALL_ROOT:+$(dirname "$INSTALL_ROOT")}"
HERMES_HOME="${HERMES_HOME:-${TMPDIR:-/tmp}}"
MARKER="$HERMES_HOME/.hermes-update-in-progress"
LOG_DIR="$HERMES_HOME/logs"; mkdir -p "$LOG_DIR" 2>/dev/null || true
LOG="$LOG_DIR/desktop-update-handoff.log"
RESULT="$HERMES_HOME/.hermes-update-result.json"
STATUS="${TMPDIR:-/tmp}/hermes-update-status.$$"
STARTED_AT="$(date +%s)" # the shim's elapsed clock; see serve-ui.py
UI_SERVER_PID="" UI_BROWSER_PID="" FINAL_CODE=1
FINAL_MSG="update did not complete"
DONE_NOTE="" # set when the update succeeded but the app will NOT reopen itself
log() { echo "$(date +%Y-%m-%dT%H:%M:%S%z) $1" | tee -a "$LOG" 2>/dev/null; }
# Keep a durable signal breadcrumb. A detached hand-off used to leave only the
# generic FINAL_MSG when it was terminated while the updater child was running,
# which erased the one fact needed to diagnose the failure.
TERM_TEARDOWN_IGNORED=0
on_signal() {
local sig="$1" pgid="unknown"
pgid="$(ps -o pgid= -p $$ 2>/dev/null | tr -d '[:space:]')"
# Electron sends one final TERM to the detached hand-off process group while
# quitting, even after the orchestrator has been re-parented to PID 1. That
# TERM is teardown noise, not a user cancellation. Ignore it once only when
# the originating desktop PID is already gone; a later TERM still stops us.
if [ "$sig" = "TERM" ] && [ "$HANDOFF_DAEMONIZED" -eq 1 ] \
&& [ "$TERM_TEARDOWN_IGNORED" -eq 0 ] && ! kill -0 "$DESKTOP_PID" 2>/dev/null; then
TERM_TEARDOWN_IGNORED=1
log "SIGNAL: TERM ignored after desktop teardown pid=$$ ppid=$PPID pgid=${pgid:-unknown} desktopPid=$DESKTOP_PID"
return 0
fi
log "SIGNAL: $sig pid=$$ ppid=$PPID pgid=${pgid:-unknown}"
FINAL_MSG="Update hand-off was interrupted by $sig (pid $$)."
case "$sig" in
HUP) FINAL_CODE=129 ;;
INT) FINAL_CODE=130 ;;
QUIT) FINAL_CODE=131 ;;
TERM) FINAL_CODE=143 ;;
esac
exit "$FINAL_CODE"
}
trap 'on_signal HUP' HUP
trap 'on_signal INT' INT
trap 'on_signal QUIT' QUIT
trap 'on_signal TERM' TERM
# ── shim ────────────────────────────────────────────────────────────────────
json_escape() { # minimal JSON string escape: \ " and control whitespace
local s=${1//\\/\\\\}
s=${s//\"/\\\"}
s=${s//$'\n'/\\n}
s=${s//$'\r'/\\r}
s=${s//$'\t'/\\t}
printf '%s' "$s"
}
notify_fallback() { # status message — renderer-free recovery surface.
# Fires only when there is no shim window. BEST-EFFORT immediate channel:
# each rung requires EXECUTION acceptance, not existence — notify-send's
# exit code is its acceptance (fire-and-forget), zenity/kdialog must
# survive their first second (a dialog that dies instantly had no display
# and must not eat the message). The GUARANTEED channel is the result
# file: a manual/error outcome is durably marked and the next Desktop
# boot surfaces it in a dialog (handoff-result.ts + main.ts).
case "$1" in manual|error) ;; *) return 0 ;; esac
if [ "$(uname)" = "Darwin" ]; then
/usr/bin/osascript -e "display notification \"$(printf '%s' "$2" | sed 's/"/\\"/g')\" with title \"Hermes update\"" 2>/dev/null && return 0
else
if command -v notify-send >/dev/null 2>&1; then
notify-send -u critical "Hermes update" "$2" 2>/dev/null && return 0
fi
local p
if command -v zenity >/dev/null 2>&1; then
zenity --warning --title="Hermes update" --text="$2" 2>/dev/null &
p=$!; sleep 1
kill -0 "$p" 2>/dev/null && return 0
wait "$p" 2>/dev/null
fi
if command -v kdialog >/dev/null 2>&1; then
kdialog --title "Hermes update" --sorry "$2" 2>/dev/null &
p=$!; sleep 1
kill -0 "$p" 2>/dev/null && return 0
wait "$p" 2>/dev/null
fi
fi
# No immediate surface landed. The durable channel takes over: the result
# is marked manual/failed and the next boot shows it in a real dialog.
log "NOTICE: no notification surface accepted; outcome reaches the user via the result dialog on next launch: $2"
}
write_status() { # status message -- atomic replace; the server reads per poll
printf '{"status":"%s","message":"%s"}' "$(json_escape "$1")" "$(json_escape "$2")" > "$STATUS.tmp" \
&& mv -f "$STATUS.tmp" "$STATUS" 2>/dev/null || true
}
publish_stage() { # a long wait the orchestrator is already gating on. No poll
# beat (that would add a second per stage to every update) and no
# notification fallback (there is nothing here for the user to act on).
write_status "running" "$1"
}
publish() { # terminal event -- the page must render it before teardown
write_status "$1" "$2"
[ -n "$UI_SERVER_PID" ] && sleep 1 # one poll beat to render the state
[ -z "$UI_SERVER_PID" ] && notify_fallback "$1" "$2"
}
find_browser() {
local c
# No Microsoft Edge and no Brave, on purpose. Edge's OS-level
# Microsoft-account integration signs a fresh throwaway profile into the
# user's MSA and renders its own "syncing your data" notification — MSA
# email included — inside this window that is titled "Hermes" (#88410).
# Brave paints its own P3A privacy-notice bar over the progress page in
# the same window — cramped to unreadability at the shim's small size
# (#88682). The throwaway --user-data-dir below cannot block either; the
# remaining Chromium-family browsers carry no first-run chrome of their
# own into a fresh profile.
if [ "$(uname)" = "Darwin" ]; then
for c in "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \
"/Applications/Chromium.app/Contents/MacOS/Chromium"; do
[ -x "$c" ] && { echo "$c"; return; }
done
else
for c in google-chrome google-chrome-stable chromium chromium-browser; do
command -v "$c" 2>/dev/null && return
done
fi
}
# The shim is decoration; launching a browser the user does NOT use is not.
# A Safari/Firefox/Helium user who merely has Chrome installed watched Chrome
# open on every update — a "why is Chrome opening?" surprise (community
# report, Aug 2026). Only render the shim when the system DEFAULT browser is
# itself Chromium-family; otherwise skip the window and let notify_fallback +
# the durable result file carry the outcome. Best-effort on purpose: any
# detection failure keeps today's behavior (0 = allowed).
default_browser_is_chromium() {
local py="$1" handler=""
if [ "$(uname)" = "Darwin" ]; then
local plist="$HOME/Library/Preferences/com.apple.LaunchServices/com.apple.launchservices.secure.plist"
# No explicit https handler registered = the OS default (Safari).
[ -f "$plist" ] || return 1
handler="$("$py" -c '
import plistlib, sys
with open(sys.argv[1], "rb") as f:
data = plistlib.load(f)
for entry in data.get("LSHandlers", []):
if entry.get("LSHandlerURLScheme") == "https":
print(entry.get("LSHandlerRoleAll", ""))
break
' "$plist" 2>/dev/null)" || return 0
# Parsed but empty = no https override = Safari default.
[ -n "$handler" ] || return 1
case "$handler" in
com.google.[Cc]hrome*|org.chromium.[Cc]hromium*) return 0 ;;
*) return 1 ;;
esac
fi
# Linux: xdg-settings is the authority; missing tool = permissive.
command -v xdg-settings >/dev/null 2>&1 || return 0
handler="$(xdg-settings get default-web-browser 2>/dev/null)" || return 0
[ -n "$handler" ] || return 0
case "$handler" in
*chrome*|*chromium*|*Chrome*|*Chromium*) return 0 ;;
*) return 1 ;;
esac
}
start_ui() {
[ "$NO_UI" -eq 1 ] && return
local html="$SCRIPT_DIR/ui.html" py browser port="" i
py="${INSTALL_ROOT:+$INSTALL_ROOT/venv/bin/python3}"
[ -x "${py:-/nonexistent}" ] || py="$(command -v python3 2>/dev/null)"
browser="$(find_browser)"
if [ -n "$browser" ] && [ -n "$py" ] && ! default_browser_is_chromium "$py"; then
log "shim: default browser is not Chromium-family; skipping UI window"
browser=""
fi
{ [ -f "$html" ] && [ -n "$py" ] && [ -n "$browser" ]; } || { log "shim: no renderer; skipping UI"; return; }
publish_stage ""
# The Desktop's final teardown targets the updater process group. Put both
# UI processes in their own sessions so neither the HTTP server nor a Chrome
# renderer becomes collateral damage (Chrome surfaces that renderer death as
# an "Aw, Snap!" page with error code 15 even while /progress still returns
# HTTP 200). The Python wrapper immediately execs the real process, so $!
# remains the PID that stop_ui can terminate.
# TERM/HUP stay IGNORED in the server (SIG_IGN survives execv): a stray
# teardown TERM killed the shim ~1s into `hermes update` (2026-08-14 16:44,
# window showed ERR_CONNECTION_REFUSED for the whole run; upstream #66753).
# stop_ui ends the server with SIGKILL instead — it is stateless HTTP.
"$py" -c 'import os, signal, sys; os.setsid(); signal.signal(signal.SIGTERM, signal.SIG_IGN); signal.signal(signal.SIGHUP, signal.SIG_IGN); os.execv(sys.argv[1], sys.argv[1:])' \
"$py" "$SCRIPT_DIR/serve-ui.py" "$html" "$STATUS" "$STARTED_AT" > "$LOG_DIR/desktop-update-ui-port" 2>>"$LOG" &
UI_SERVER_PID=$!
for i in $(seq 1 10); do
port="$(tr -cd '0-9' < "$LOG_DIR/desktop-update-ui-port" 2>/dev/null)"
[ -n "$port" ] && break
sleep 0.2
done
[ -n "$port" ] || { kill -9 "$UI_SERVER_PID" 2>/dev/null; UI_SERVER_PID=""; return; }
# Throwaway profile: new window/process we own; user's browser untouched.
"$py" -c 'import os, signal, sys; os.setsid(); signal.signal(signal.SIGTERM, signal.SIG_DFL); os.execv(sys.argv[1], sys.argv[1:])' \
"$browser" --app="http://127.0.0.1:$port/" --user-data-dir="${TMPDIR:-/tmp}/hermes-update-ui-$$" \
--no-first-run --no-default-browser-check --window-size=280,320 >/dev/null 2>&1 &
UI_BROWSER_PID=$!
log "shim: app window on 127.0.0.1:$port"
}
stop_ui() { # error/manual outcomes keep the window up briefly so a watching
# user can read the message, then close it. The outcome is also durably
# written to the result file and surfaced in a dialog on the next Desktop
# boot (handoff-result.ts), so the shim window never lingers indefinitely —
# before this, each aborted update left another orphan browser window on
# screen until the user closed it by hand.
if [ "${1:-}" = "leave-window" ]; then
sleep "${HERMES_UPDATE_SHIM_GRACE_SECONDS:-15}"
fi
if [ -n "$UI_SERVER_PID" ]; then
# The server ignores TERM/HUP (see start_ui) — KILL is its off switch.
{ kill -9 "$UI_SERVER_PID" && wait "$UI_SERVER_PID"; } 2>/dev/null
fi
if [ -n "$UI_BROWSER_PID" ]; then
{ kill "$UI_BROWSER_PID" && wait "$UI_BROWSER_PID"; } 2>/dev/null
fi
UI_SERVER_PID="" UI_BROWSER_PID=""
}
# ── relaunch ────────────────────────────────────────────────────────────────
# Linux relaunch gate -- an exact port of the deleted update-relaunch.ts
# decision (#45205/#37541), not a loosened rewrite:
# * the running binary must live under THIS checkout's rebuilt
# apps/desktop/release/linux-unpacked (anchored, path-segment-aware --
# proof the update we just ran replaced the selected executable);
# * chrome-sandbox ABSENT is fine (namespace-sandbox build; nothing to
# block on), PRESENT means root-owned AND setuid or Electron refuses to
# boot ("quit and never came back");
# * a user sandbox opt-out (ELECTRON_DISABLE_SANDBOX=1/true in our
# inherited env, --no-sandbox among the replayed launch args, or the
# Desktop vouching via --sandbox-fallback) makes the relaunch safe
# despite a failed preflight.
# Outcomes mirror decideRelaunchOutcome: relaunch | skew | manual.
GATE="" GATE_MSG=""
linux_gate() {
local unpacked="$INSTALL_ROOT/apps/desktop/release/linux-unpacked" sb arg
case "$RELAUNCH_TARGET" in
"$unpacked"/*) ;;
*) GATE=skew GATE_MSG="Backend updated, but the desktop app package (AppImage/deb/rpm) was not changed. Update or reinstall it to match."; return ;;
esac
sb="$unpacked/chrome-sandbox"
if [ ! -e "$sb" ]; then GATE=relaunch; return; fi
if [ -u "$sb" ] && [ "$(stat -c %u "$sb" 2>/dev/null)" = "0" ]; then GATE=relaunch; return; fi
# Namespace sandbox usable => Electron never consults the setuid helper,
# so a non-root chrome-sandbox does not block relaunch (mirrors the
# _desktop_linux_userns_sandbox_available() probe in hermes_cli/main.py).
if unshare --user --map-root-user true 2>/dev/null; then GATE=relaunch; return; fi
case "${ELECTRON_DISABLE_SANDBOX:-}" in 1|true|TRUE|True) GATE=relaunch; return ;; esac
[ "$SANDBOX_FALLBACK" -eq 1 ] && { GATE=relaunch; return; }
for arg in ${RELAUNCH_ARGS[@]+"${RELAUNCH_ARGS[@]}"}; do
[ "$arg" = "--no-sandbox" ] && { GATE=relaunch; return; }
done
GATE=manual GATE_MSG="Update complete, but the rebuilt app can't relaunch itself (its sandbox helper needs root ownership). Reopen Hermes to finish."
}
mac_swap() {
local rebuilt="" c
for c in "$INSTALL_ROOT/apps/desktop/release/mac-arm64/Hermes.app" \
"$INSTALL_ROOT/apps/desktop/release/mac/Hermes.app"; do
[ -d "$c" ] && { rebuilt="$c"; break; }
done
# Transactional swap: stage a full copy, move the old bundle aside, move
# the copy in. Every step checked; a failed final move ROLLS BACK so the
# user always has a launchable app, and the result file tells the truth.
if [ "$FINAL_CODE" -eq 0 ] && [ -n "$rebuilt" ] && [ -d "$RELAUNCH_TARGET" ] && [ "$rebuilt" != "$RELAUNCH_TARGET" ]; then
publish_stage "Installing the new app"
rm -rf "$RELAUNCH_TARGET.new" "$RELAUNCH_TARGET.old" 2>/dev/null || true
if ! /usr/bin/ditto "$rebuilt" "$RELAUNCH_TARGET.new"; then
rm -rf "$RELAUNCH_TARGET.new" 2>/dev/null || true
DONE_NOTE="Update complete, but the new app could not be staged; the previous version was kept. Run the update again."
log "WARNING: bundle copy failed; keeping existing app"
elif ! mv "$RELAUNCH_TARGET" "$RELAUNCH_TARGET.old"; then
rm -rf "$RELAUNCH_TARGET.new" 2>/dev/null || true
DONE_NOTE="Update complete, but the new app could not replace the old one; the previous version was kept. Run the update again."
log "WARNING: could not move old bundle aside; keeping existing app"
elif ! mv "$RELAUNCH_TARGET.new" "$RELAUNCH_TARGET"; then
if mv "$RELAUNCH_TARGET.old" "$RELAUNCH_TARGET"; then
rm -rf "$RELAUNCH_TARGET.new" 2>/dev/null || true
DONE_NOTE="Update complete, but the new app could not be installed; the previous version was restored. Run the update again."
log "WARNING: bundle install failed; rolled back to the previous app"
else
FINAL_CODE=7 FINAL_MSG="The update finished but installing the new app failed and the previous app could not be restored. Reinstall Hermes (the rebuilt app is at $rebuilt)."
log "ERROR: bundle install failed AND rollback failed"
fi
else
rm -rf "$RELAUNCH_TARGET.old" 2>/dev/null || true
log "swapped app bundle"
fi
fi
}
deliver_outcome() { # the truth-determining half: swap bundles / gate the relaunch
[ -n "$RELAUNCH_TARGET" ] || return 0
if [ "$(uname)" = "Darwin" ]; then
mac_swap
else
linux_gate
if [ "$GATE" != "relaunch" ] && [ "$FINAL_CODE" -eq 0 ]; then
DONE_NOTE="$GATE_MSG"
log "no relaunch ($GATE): $GATE_MSG"
fi
fi
}
launch_app() { # attempted BEFORE the terminal event (launch acceptance is
# part of the outcome — gille's review). Returns nonzero when a launch
# was due but did not verifiably happen; caller downgrades to manual.
[ -n "$RELAUNCH_TARGET" ] || return 0
if [ "$(uname)" = "Darwin" ]; then
# A supplied target that no longer exists is a REJECTED launch (the
# swap failed badly or the bundle vanished) — not "no launch due".
[ -d "$RELAUNCH_TARGET" ] || { log "WARNING: relaunch target missing: $RELAUNCH_TARGET"; return 1; }
/usr/bin/xattr -dr com.apple.quarantine "$RELAUNCH_TARGET" 2>/dev/null || true
# `open` talks to launchd and FAILS LOUDLY on a broken/unlaunchable
# bundle — its exit code IS launch acceptance here.
/usr/bin/open "$RELAUNCH_TARGET" || { log "WARNING: open rejected the app"; return 1; }
elif [ "$GATE" = "relaunch" ]; then
# setsid only proves the wrapper shell started, so verify acceptance:
# spawn, then confirm the child is still alive shortly after — an
# immediate exec failure (ENOENT, ELF mismatch, dead sandbox) dies
# within the window and downgrades to manual instead of lying.
(cd "${RELAUNCH_CWD:-/}" 2>/dev/null || cd /
setsid "$RELAUNCH_TARGET" ${RELAUNCH_ARGS[@]+"${RELAUNCH_ARGS[@]}"} >/dev/null 2>&1 &
echo $! > "$STATUS.launchpid") || { log "WARNING: relaunch spawn failed"; return 1; }
local lp
lp="$(cat "$STATUS.launchpid" 2>/dev/null)"; rm -f "$STATUS.launchpid" 2>/dev/null
[ -n "$lp" ] || { log "WARNING: relaunch pid unknown"; return 1; }
sleep 1.5
kill -0 "$lp" 2>/dev/null || { log "WARNING: relaunched app exited immediately"; return 1; }
fi
}
MANUAL=0 # 1 = update landed but the user must act (result protocol field)
write_result() {
printf '{"ok":%s,"exit_code":%s,"manual":%s,"message":"%s","branch":"%s","finished_at":%s}' \
"$([ "$FINAL_CODE" -eq 0 ] && echo true || echo false)" "$FINAL_CODE" \
"$([ "$MANUAL" -eq 1 ] && echo true || echo false)" \
"$(json_escape "$FINAL_MSG")" "$(json_escape "$BRANCH")" "$(date +%s)" \
> "$RESULT.tmp" 2>/dev/null && mv -f "$RESULT.tmp" "$RESULT" 2>/dev/null || true
}
finish() {
# Ordering (gille's reviews, both rounds):
# 1. deliver the outcome (swap/gate) so the truth exists;
# 2. durable result + marker removal (the relaunched app consumes the
# result on boot and must not park on our marker — this must be on
# disk BEFORE any launch attempt);
# 3. attempt the launch and require ACCEPTANCE;
# 4. only then the terminal shim event — done means "the app is coming
# back", manual means "it is not, here's what to do", error is error.
# A rejected launch rewrites the result (nothing consumed it — the app
# never started) so the next boot tells the truth too.
deliver_outcome
[ "$FINAL_CODE" -eq 0 ] && [ -n "$DONE_NOTE" ] && { FINAL_MSG="$DONE_NOTE"; MANUAL=1; }
write_result
if [ "$NO_MARKER_CLEANUP" -eq 0 ] && [ "$(head -1 "$MARKER" 2>/dev/null | tr -d '[:space:]')" = "$$" ]; then
rm -f "$MARKER" 2>/dev/null || true
fi
if [ "$FINAL_CODE" -ne 0 ]; then
publish "error" "$FINAL_MSG"; stop_ui leave-window
launch_app || true # error path still tries to bring the app back
rm -f "$STATUS" "$STATUS.tmp" "$LOG_DIR/desktop-update-ui-port" 2>/dev/null || true
return
fi
if [ -n "$DONE_NOTE" ]; then
if [ "$(uname)" = "Darwin" ]; then
# mac DONE_NOTE = swap failed but the PREVIOUS bundle was kept/rolled
# back — bring it back up; the note still tells the user to re-run.
# A gated linux outcome (skew/manual) skips the launch BY DESIGN.
if ! launch_app; then
# Even the kept bundle didn't come back: the durable message must
# carry BOTH facts (update ok, previous app not reopened).
FINAL_MSG="$DONE_NOTE Hermes also could not reopen itself - open it manually."
write_result
fi
fi
publish "manual" "$FINAL_MSG"; stop_ui leave-window
elif launch_app; then
publish "done" ""; stop_ui
else
# Launch was due and did not land. Downgrade: truthful result for the
# next boot, manual state held on screen now.
FINAL_MSG="Update complete. Reopen Hermes to finish (it could not restart itself)."
MANUAL=1
write_result
publish "manual" "$FINAL_MSG"; stop_ui leave-window
fi
rm -f "$STATUS" "$STATUS.tmp" "$LOG_DIR/desktop-update-ui-port" 2>/dev/null || true
}
trap finish EXIT
# ── legacy macOS TCC anchor self-heal (#95759) ──────────────────────────────
# The reverted TCC interpreter anchor (#95425/#95541) left some installs with
# a real-file `venv/bin/python` copy plus a `.tcc-anchor-source` marker, and
# `python3`/`python3.N` aliases that die at interpreter init ("No module
# named 'encodings'"). On those installs EVERY normal CLI entrypoint is dead
# (`venv/bin/hermes` has a `python3` shebang), so no Python-side heal —
# doctor OR in-update — can ever run. This shell is the last surface that
# still executes, so the heal lives here (recovery design after @aeonsong's
# #96231; heal-point observation by @ahrazzle / @tokenfires on #95759).
#
# Ping-pong coherence with the re-landed forward anchor
# (hermes_cli/macos_tcc_anchor.ensure_tcc_anchor, which re-anchors whenever
# `venv/bin/python` is a uv-managed symlink): this heal is gated on the
# interpreter FAILING its boot probe, so a healthy anchored install is never
# touched; and when it does restore symlinks, the very `hermes update` run it
# unblocks re-installs a boot-gated healthy anchor — a one-shot convergence,
# not a loop.
tcc_probe_python() { # interpreter path → 0 iff it boots a real stdlib.
# PYTHONHOME/PYTHONPATH are scrubbed: an inherited PYTHONHOME papers over
# exactly the prefix-resolution failure this probe exists to detect.
[ -x "$1" ] || return 1
env -u PYTHONHOME -u PYTHONPATH -u PYTHONSTARTUP -u __PYVENV_LAUNCHER__ \
"$1" -c 'import encodings' >/dev/null 2>&1
}
TCC_HEAL_STATE="not-run"
tcc_heal_rollback() { # restore every .tcc-heal-old.$$ backup in a bin dir
local b
for b in "$1"/*.tcc-heal-old.$$; do
[ -e "$b" ] || [ -L "$b" ] || continue
mv -f "$b" "${b%.tcc-heal-old.$$}" 2>/dev/null || true
done
}
tcc_heal_cleanup() { # heal landed: drop the backups
local b
for b in "$1"/*.tcc-heal-old.$$; do
[ -e "$b" ] || [ -L "$b" ] || continue
rm -f "$b" 2>/dev/null || true
done
}
tcc_alias_names() { # existing python3 / python3.N entries in a bin dir
local a
for a in "$1"/python3 "$1"/python3.*; do
[ -e "$a" ] || [ -L "$a" ] || continue
case "${a##*/}" in
python3|python3.[0-9]|python3.[0-9][0-9]) printf '%s\n' "$a" ;;
esac
done
}
tcc_anchor_heal() { # $1 = venv bin dir. 0 iff python3 boots on exit.
local bin="$1" marker src alias staged
local py="$bin/python" py3="$bin/python3"
if tcc_probe_python "$py3"; then TCC_HEAL_STATE="healthy"; return 0; fi
marker="$bin/.tcc-anchor-source"
[ -f "$marker" ] || { TCC_HEAL_STATE="no-marker"; return 1; }
src="$(head -1 "$marker" 2>/dev/null)"
case "$src" in
"$bin"/*) TCC_HEAL_STATE="unsafe-source"; return 1 ;;
/*) ;;
*) TCC_HEAL_STATE="invalid-marker"; return 1 ;;
esac
if tcc_probe_python "$py"; then
# #95541 alias-brick: the anchored copy itself boots; only the aliases
# are dead. Aliases over a real-file anchor must be REAL FILES (hard
# link, else copy) — an alias *symlink* onto the copy is the exact
# crash shape being healed here.
TCC_HEAL_STATE="healed-aliases"
while IFS= read -r alias; do
[ -n "$alias" ] || continue
mv "$alias" "$alias.tcc-heal-old.$$" 2>/dev/null \
|| { tcc_heal_rollback "$bin"; TCC_HEAL_STATE="failed"; return 1; }
if ! { ln "$py" "$alias" 2>/dev/null || cp -p "$py" "$alias" 2>/dev/null; }; then
tcc_heal_rollback "$bin"; TCC_HEAL_STATE="failed"; return 1
fi
done <<EOF_ALIASES
$(tcc_alias_names "$bin")
EOF_ALIASES
elif [ -x "$src" ] && tcc_probe_python "$src"; then
# Full restore to the pre-anchor layout: python → symlink to the
# marker-recorded store interpreter, aliases → symlinks to python.
TCC_HEAL_STATE="healed-symlinks"
mv "$py" "$py.tcc-heal-old.$$" 2>/dev/null \
|| { TCC_HEAL_STATE="failed"; return 1; }
if ! ln -s "$src" "$py" 2>/dev/null; then
tcc_heal_rollback "$bin"; TCC_HEAL_STATE="failed"; return 1
fi
while IFS= read -r alias; do
[ -n "$alias" ] || continue
staged="$alias.tcc-heal-new.$$"
mv "$alias" "$alias.tcc-heal-old.$$" 2>/dev/null \
|| { tcc_heal_rollback "$bin"; TCC_HEAL_STATE="failed"; return 1; }
if ! { ln -s python "$staged" 2>/dev/null && mv "$staged" "$alias" 2>/dev/null; }; then
rm -f "$staged" 2>/dev/null
tcc_heal_rollback "$bin"; TCC_HEAL_STATE="failed"; return 1
fi
done <<EOF_ALIASES
$(tcc_alias_names "$bin")
EOF_ALIASES
else
# Recorded source gone or itself unbootable (the vanished-uv-store
# class from #95759): fail closed, touch nothing.
TCC_HEAL_STATE="source-missing"; return 1
fi
if tcc_probe_python "$py3"; then
if [ "$TCC_HEAL_STATE" = "healed-symlinks" ]; then
# Marker asserts the anchored layout; the symlink layout is done
# with it. (The alias-heal branch KEEPS it: real-file python +
# real-file aliases is exactly the layout ensure_tcc_anchor marks.)
rm -f "$marker" 2>/dev/null || true
fi
tcc_heal_cleanup "$bin"
return 0
fi
tcc_heal_rollback "$bin"
TCC_HEAL_STATE="failed"
return 1
}
tcc_pick_update_invoke() { # sets UPDATE_INVOKE; safety net past a failed heal
# Last-resort class: aliases still dead but the anchored copy boots. The
# launchd gateway proves `venv/bin/python -m hermes_cli.main` works when
# every alias entrypoint is bricked — drive the update the same way.
local bin="$1"
UPDATE_INVOKE=("$bin/hermes")
if ! tcc_probe_python "$bin/python3" && tcc_probe_python "$bin/python"; then
UPDATE_INVOKE=("$bin/python" -m hermes_cli.main)
fi
}
# ── self-tests: no update, touch nothing ────────────────────────────────────
if [ "$SELF_TEST_TCC_HEAL" -eq 1 ]; then
# Runs the REAL heal + invoke selection against --install-root and reports;
# tests/test_desktop_update_tcc_heal.py drives the state matrix through it.
trap - EXIT
tcc_anchor_heal "$INSTALL_ROOT/venv/bin" || true
tcc_pick_update_invoke "$INSTALL_ROOT/venv/bin"
echo "state=$TCC_HEAL_STATE invoke=${UPDATE_INVOKE[*]}"
exit 0
fi
if [ "$SELF_TEST_GATE" -eq 1 ]; then
# Prints the gate decision for the given --install-root/--relaunch-target
# and exits; scripts/desktop-update/repro.sh gate asserts the matrix.
trap - EXIT
linux_gate
echo "$GATE${GATE_MSG:+:$GATE_MSG}"
exit 0
fi
if [ "$SELF_TEST_UI" -eq 1 ]; then
start_ui
log "SELF-TEST: shim simulation (no update will run)"
sleep "${HERMES_SELFTEST_HOLD_SECONDS:-6}"
RELAUNCH_TARGET=""
if [ -n "${HERMES_SELFTEST_FAIL:-}" ]; then FINAL_MSG="self-test error state"
else FINAL_CODE=0 FINAL_MSG="self-test complete"; fi
exit "$FINAL_CODE"
fi
# ── the actual job ──────────────────────────────────────────────────────────
# Electron's macOS quit teardown sends SIGTERM to its still-parented updater
# child on this machine. `detached + unref` gives the child a process group but
# does not re-parent it before `before-quit` runs, so the hand-off consistently
# died two seconds after starting `hermes update`. Re-exec through a one-shot
# setsid child and let this direct Electron child exit first. The real
# orchestrator is then owned by launchd (PPID 1) and is outside Electron's quit
# teardown, while retaining the same marker/result protocol.
if [ "$HANDOFF_DAEMONIZED" -ne 1 ]; then
# This launcher is disposable. In particular it must not run finish() on
# EXIT: that would publish a false failure and relaunch Hermes while the
# re-parented orchestrator is only just starting.
trap - EXIT HUP INT QUIT TERM
# --daemonized must precede ORIGINAL_ARGS, not follow it: ORIGINAL_ARGS may
# contain a `--` separator (Linux relaunch args), and anything appended
# after that point is swallowed into RELAUNCH_ARGS instead of being parsed
# as a flag. Appending here previously left HANDOFF_DAEMONIZED unset on
# every re-exec, causing this block to re-fire forever (self-exec loop,
# unbounded argv growth) whenever relaunch args were present.
/usr/bin/nohup /usr/bin/python3 -c '
import os, sys
env = os.environ.copy()
os.setsid()
os.execve("/bin/bash", ["/bin/bash", sys.argv[1], *sys.argv[2:]], env)
' "$SCRIPT_DIR/posix.sh" --daemonized "${ORIGINAL_ARGS[@]}" >/dev/null 2>&1 &
exit 0
fi
# Electron terminates the entire detached updater process group during quit,
# including the loopback status server. Arm TERM immunity before `start_ui`
# so the shim server and the later `hermes update` subprocess both inherit
# SIG_IGN. The orchestrator restores its normal TERM handler after the update
# command has returned; the already-running server keeps the inherited setting
# until normal cleanup closes it.
trap '' TERM
log "hand-off start: root=$INSTALL_ROOT branch=$BRANCH desktopPid=$DESKTOP_PID pid=$$"
rm -f "$RESULT" 2>/dev/null || true
# Marker claim: same cross-process lock contract as windows.ps1 /
# update_lock.py (the `hermes update` child adopts it via process ancestry).
# The Desktop supplies one acquisition time for the whole ownership chain.
NOW="$(date +%s)"
STARTED_AT="${HERMES_UPDATE_STARTED_AT:-$NOW}"
case "$STARTED_AT" in ''|*[!0-9]*) STARTED_AT="$NOW" ;; esac
MIN_STARTED_AT=$((NOW - 1200))
# Compare the validated decimal strings before doing arithmetic. Shell integer
# expansion can wrap on an attacker-controlled value wider than signed 64-bit.
if [ "${#STARTED_AT}" -ne "${#NOW}" ] \
|| [[ "$STARTED_AT" > "$NOW" || "$STARTED_AT" < "$MIN_STARTED_AT" ]]; then
STARTED_AT="$NOW"
fi
printf '%s\n%s\n' "$$" "$STARTED_AT" > "$MARKER" 2>/dev/null || log "WARNING: could not write update marker"
if [ "$SELF_TEST_MARKER" -eq 1 ]; then
trap - EXIT
exit 0
fi
# Wait out the Desktop (FAIL CLOSED: updating under live backends bricks).
if [ "$DESKTOP_PID" -gt 0 ] 2>/dev/null; then
for _ in $(seq 1 100); do kill -0 "$DESKTOP_PID" 2>/dev/null || break; sleep 0.3; done
if kill -0 "$DESKTOP_PID" 2>/dev/null; then
FINAL_CODE=4 FINAL_MSG="Update aborted: the Hermes window (pid $DESKTOP_PID) did not exit within 30s. Nothing was changed. Close Hermes fully and try again."
log "$FINAL_MSG"; exit "$FINAL_CODE"
fi
fi
# Do not create Chrome until Electron has fully left. During its 2.5s quit
# dwell/before-quit teardown macOS can terminate descendants of the hand-off;
# a Chrome renderer reports that SIGTERM as "Aw, Snap!" error code 15. The
# update marker above prevents a second click during this short UI-less gap.
sleep 1
start_ui
HERMES_BIN="$INSTALL_ROOT/venv/bin/hermes"
[ -x "$HERMES_BIN" ] || { FINAL_CODE=3 FINAL_MSG="Update aborted: $HERMES_BIN is missing. The install needs repair (run the Hermes installer or hermes doctor)."; log "$FINAL_MSG"; exit 3; }
# Heal a venv the reverted TCC anchor left bricked BEFORE invoking the CLI:
# venv/bin/hermes execs venv/bin/python3, so a dead alias kills every attempt
# and its retry identically (#95759). macOS-only artifact; probe is cheap.
if [ "$(uname)" = "Darwin" ]; then
if tcc_anchor_heal "$INSTALL_ROOT/venv/bin"; then
case "$TCC_HEAL_STATE" in
healed-*) log "TCC anchor self-heal repaired the venv interpreter ($TCC_HEAL_STATE)" ;;
esac
else
log "TCC anchor self-heal could not repair the venv ($TCC_HEAL_STATE)"
fi
fi
tcc_pick_update_invoke "$INSTALL_ROOT/venv/bin"
if [ "${UPDATE_INVOKE[0]}" != "$HERMES_BIN" ]; then
log "venv/bin/python3 still unbootable; invoking the update via ${UPDATE_INVOKE[*]}"
fi
# Run FROM the install root: `hermes update` resolves the tree it mutates
# from the working directory, and we inherit the Desktop's cwd (which can be
# an unrelated repo — updating THAT instead of the install is the failure
# the sandbox repro caught). FAIL CLOSED: set -u without set -e means a
# failed cd would otherwise continue in the wrong tree — the exact class
# this correction exists to eliminate.
cd "$INSTALL_ROOT" || {
FINAL_CODE=3 FINAL_MSG="Update aborted: cannot enter the install root ($INSTALL_ROOT). Nothing was changed."
log "$FINAL_MSG"; exit 3
}
export PYTHONUNBUFFERED=1
# --keep-stash: never re-apply local source edits after the update (they stay
# parked in git stash). Probe --help first: older installed backends don't
# know the flag and argparse would abort with exit 2, which collides with the
# "close all Hermes windows" sentinel.
KEEP_STASH=""
if "${UPDATE_INVOKE[@]}" update --help 2>/dev/null | grep -q -- '--keep-stash'; then
KEEP_STASH="--keep-stash"
else
log "installed hermes predates --keep-stash; running without it"
fi
log "running: ${UPDATE_INVOKE[*]} update --yes --gateway $KEEP_STASH --branch $BRANCH"
publish_stage "Updating code and dependencies"
OUT="$("${UPDATE_INVOKE[@]}" update --yes --gateway $KEEP_STASH --branch "$BRANCH" 2>&1)"; CODE=$?
printf '%s\n' "$OUT" >> "$LOG" 2>/dev/null
log "hermes update exit code: $CODE"
if [ "$CODE" -ne 0 ] && [ "$CODE" -ne 2 ]; then
# Retry once: update-boundary class (fresh code on disk, stale in memory).
# Exit 2 ("close all Hermes windows") is not retryable.
#
# A parked-branch SKIP (checkout on a feature branch with unmerged
# commits) is also deterministic — the retry would hit the exact same
# branch state and skip again, so it only wastes time. Detect the skip
# by its banner, skip the retry, and surface an honest message with a
# dedicated exit code (8) so callers can distinguish "skipped" from a
# real failure.
if printf '%s' "$OUT" | grep -q "CODE UPDATE SKIPPED"; then
log "hermes update skipped (checkout parked on a non-target branch); not retrying"
FINAL_CODE=8
FINAL_MSG="Update skipped: the git checkout is on a branch that isn't fully merged into $BRANCH. Switch to the target branch and update again (see the terminal output for the exact commands)."
exit 8
fi
log "retrying once (freshly pulled fix loads on the second run)"
publish_stage "Retrying update"
OUT="$("${UPDATE_INVOKE[@]}" update --yes --gateway $KEEP_STASH --branch "$BRANCH" 2>&1)"; CODE=$?
printf '%s\n' "$OUT" >> "$LOG" 2>/dev/null
log "retry exit code: $CODE"
fi
trap 'on_signal TERM' TERM
# Truthful completion: `hermes update` calls a GUI build failure non-fatal
# (exit 0). For a Desktop-driven update that would relaunch the OLD build
# and call it success -- retry the build once, propagate honestly.
if [ "$CODE" -eq 0 ] && printf '%s' "$OUT" | grep -q "Desktop build failed"; then
log "desktop build failed inside hermes update; retrying build"
publish_stage "Rebuilding Desktop"
"${UPDATE_INVOKE[@]}" desktop --force-build --build-only >> "$LOG" 2>&1 || {
FINAL_CODE=6 FINAL_MSG="Code and dependencies updated, but the Desktop app rebuild failed - you are running the previous build. Run hermes desktop --force-build from a terminal to retry."
exit 6
}
fi
if [ "$CODE" -eq 0 ]; then FINAL_CODE=0 FINAL_MSG="Update complete."
else
FINAL_CODE="$CODE" FINAL_MSG="Update failed (exit $CODE). Run hermes debug share in a terminal to send a report."
# The bricked-venv class is fixable and must not read as a generic exit 1:
# a dead interpreter with a failed/impossible heal means retrying can never
# succeed — tell the user what is actually wrong (#95759).
if ! tcc_probe_python "$INSTALL_ROOT/venv/bin/python3" \
&& ! tcc_probe_python "$INSTALL_ROOT/venv/bin/python"; then
FINAL_MSG="Update failed: the Python interpreter inside $INSTALL_ROOT/venv cannot start (heal state: $TCC_HEAL_STATE). Reinstall the runtime with the Hermes installer, or run hermes doctor --fix from a terminal if any hermes command still works."
fi
fi
exit "$FINAL_CODE"
scripts/desktop-update/repro.sh
Executable
+187
View File
@@ -0,0 +1,187 @@
#!/bin/bash
# repro.sh -- reproduce desktop-update paths against a sandboxed HERMES_HOME.
#
# Nothing here touches your real ~/.hermes or checkout. Each mode builds (or
# reuses) a disposable install under /tmp and drives the REAL code path --
# the actual installer, the actual orchestrator, the actual `hermes update`.
#
# repro.sh shim shim UI only: success event after 6s
# repro.sh shim-fail shim UI only: error event after 6s
# repro.sh fresh fresh install into a sandbox HERMES_HOME
# (scripts/install.sh, the literal user path)
# repro.sh behind [N] sandbox install rewound N commits (default 25),
# then the posix orchestrator drives it forward --
# the "user who hasn't updated in a while" path
# repro.sh error orchestrator against a broken install (missing
# venv) -- exercises abort + result-file + shim error
# repro.sh gate linux relaunch-gate decision matrix (anchoring,
# sandbox preflight, opt-out fallbacks) -- asserts
# every outcome without touching a real install
#
# The sandbox persists between runs (~/tmp is fine to nuke): fresh reuses
# nothing, behind/error reuse the last sandbox install when present because
# a from-scratch install is minutes.
#
# npm entry points (apps/desktop/package.json):
# npm run update:shim / update:shim:fail / update:repro:fresh /
# update:repro:behind [-- N] / update:repro:error
set -euo pipefail
MODE="${1:-help}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
SANDBOX="${HERMES_UPDATE_REPRO_HOME:-/tmp/hermes-update-repro}"
SANDBOX_ROOT="$SANDBOX/hermes-agent"
say() { printf '\n\033[1m== %s ==\033[0m\n' "$1"; }
ensure_sandbox_install() {
if [ -x "$SANDBOX_ROOT/venv/bin/hermes" ]; then
say "reusing sandbox install at $SANDBOX_ROOT"
return
fi
say "fresh sandbox install into $SANDBOX (this takes a while)"
rm -rf "$SANDBOX"
mkdir -p "$SANDBOX"
# The literal user path: install.sh against a clone of THIS checkout, so
# the repro reproduces what you're about to ship, not origin/main.
git clone --quiet "$REPO_ROOT" "$SANDBOX_ROOT"
HERMES_HOME="$SANDBOX" bash "$SANDBOX_ROOT/scripts/install.sh" --non-interactive --skip-setup --hermes-home "$SANDBOX"
}
case "$MODE" in
shim)
HERMES_SELFTEST_HOLD_SECONDS="${HERMES_SELFTEST_HOLD_SECONDS:-6}" \
bash "$SCRIPT_DIR/posix.sh" --self-test-ui
;;
shim-fail)
HERMES_SELFTEST_FAIL=1 HERMES_SELFTEST_HOLD_SECONDS="${HERMES_SELFTEST_HOLD_SECONDS:-6}" \
bash "$SCRIPT_DIR/posix.sh" --self-test-ui
;;
fresh)
rm -rf "$SANDBOX"
ensure_sandbox_install
say "fresh install OK: $("$SANDBOX_ROOT/venv/bin/hermes" --version 2>/dev/null || echo '?')"
;;
behind)
N="${2:-25}"
ensure_sandbox_install
say "rewinding sandbox checkout $N commits"
git -C "$SANDBOX_ROOT" fetch --quiet origin main || true
git -C "$SANDBOX_ROOT" checkout --quiet main
git -C "$SANDBOX_ROOT" reset --hard --quiet "HEAD~$N"
say "sandbox now at: $(git -C "$SANDBOX_ROOT" log --oneline -1)"
say "driving the orchestrator (watch the shim; log: $SANDBOX/logs/desktop-update-handoff.log)"
HERMES_HOME="$SANDBOX" bash "$SCRIPT_DIR/posix.sh" \
--install-root "$SANDBOX_ROOT" --branch main --desktop-pid 0 || true
say "result file:"
cat "$SANDBOX/.hermes-update-result.json" 2>/dev/null || echo "(none written)"
echo
say "sandbox after update: $(git -C "$SANDBOX_ROOT" log --oneline -1)"
;;
error)
ensure_sandbox_install
say "breaking the sandbox venv, then driving the orchestrator"
mv "$SANDBOX_ROOT/venv" "$SANDBOX_ROOT/venv.hidden"
HERMES_HOME="$SANDBOX" bash "$SCRIPT_DIR/posix.sh" \
--install-root "$SANDBOX_ROOT" --branch main --desktop-pid 0 || true
mv "$SANDBOX_ROOT/venv.hidden" "$SANDBOX_ROOT/venv"
say "result file (expect ok:false, exit 3):"
cat "$SANDBOX/.hermes-update-result.json" 2>/dev/null || echo "(none written)"
echo
;;
gate)
# Pure-decision matrix for the linux relaunch gate. Builds a fake
# checkout layout under /tmp; --self-test-gate prints the decision and
# exits without running an update.
G="/tmp/hermes-gate-test.$$"
UNPACKED="$G/hermes-agent/apps/desktop/release/linux-unpacked"
mkdir -p "$UNPACKED"
touch "$UNPACKED/hermes" && chmod +x "$UNPACKED/hermes"
fails=0
expect() { # name expected actual
if [ "$2" = "$3" ]; then printf 'ok %s -> %s\n' "$1" "$3"
else printf 'FAIL %s -> %s (want %s)\n' "$1" "$3" "$2"; fails=$((fails+1)); fi
}
decide() { bash "$SCRIPT_DIR/posix.sh" --self-test-gate --install-root "$G/hermes-agent" "$@" | cut -d: -f1; }
expect "appimage (not under unpacked)" skew "$(decide --relaunch-target /opt/Hermes/hermes)"
expect "sibling-prefix dir not fooled" skew "$(decide --relaunch-target "$UNPACKED-evil/hermes")"
expect "no chrome-sandbox (namespace)" relaunch "$(decide --relaunch-target "$UNPACKED/hermes")"
touch "$UNPACKED/chrome-sandbox"
expect "sandbox not root/setuid" manual "$(decide --relaunch-target "$UNPACKED/hermes")"
expect "opt-out: --sandbox-fallback" relaunch "$(decide --relaunch-target "$UNPACKED/hermes" --sandbox-fallback)"
expect "opt-out: --no-sandbox launch arg" relaunch "$(decide --relaunch-target "$UNPACKED/hermes" -- --no-sandbox)"
expect "opt-out: ELECTRON_DISABLE_SANDBOX" relaunch "$(ELECTRON_DISABLE_SANDBOX=1 decide --relaunch-target "$UNPACKED/hermes")"
# Result JSON must survive hostile strings (git allows `"` in branch
# names; messages carry arbitrary text) -- parse it back with python.
QHOME="$G/qhome"; mkdir -p "$QHOME/hermes-agent"
bash "$SCRIPT_DIR/posix.sh" --no-ui --no-marker-cleanup --desktop-pid 0 \
--install-root "$QHOME/hermes-agent" --branch 'evil"branch\n$(x)' >/dev/null 2>&1 || true
if python3 -c "import json,sys; d=json.load(open('$QHOME/.hermes-update-result.json')); sys.exit(0 if d['branch']=='evil\"branch\\\\n\$(x)' and d['ok']==False else 1)"; then
printf 'ok result JSON escapes hostile branch/message\n'
else
printf 'FAIL result JSON escaping\n'; fails=$((fails+1))
fi
rm -rf "$G"
[ "$fails" -eq 0 ] && say "gate matrix: all pass" || { say "gate matrix: $fails FAILED"; exit 1; }
;;
launch)
# Terminal-lifecycle matrix (gille round 2): launch acceptance is part
# of the outcome. Each case runs the REAL orchestrator (--no-ui) against
# a fake install whose `hermes` stub exits 0 instantly, so the flow
# reaches finish() with FINAL_CODE=0 and exercises the launch leg.
L="/tmp/hermes-launch-test.$$"
fails=0
expect_msg() { # name python-expr
if python3 -c "import json,sys; d=json.load(open('$L/.hermes-update-result.json')); sys.exit(0 if ($2) else 1)"; then
printf 'ok %s\n' "$1"
else
printf 'FAIL %s -> %s\n' "$1" "$(cat "$L/.hermes-update-result.json" 2>/dev/null)"; fails=$((fails+1))
fi
}
stub_install() { # creates a fake install whose hermes update succeeds
rm -rf "$L"; mkdir -p "$L/hermes-agent/venv/bin"
printf '#!/bin/sh\nexit 0\n' > "$L/hermes-agent/venv/bin/hermes"
chmod +x "$L/hermes-agent/venv/bin/hermes"
}
# 1. linux relaunch target dies instantly -> manual downgrade in result
stub_install
UNPACKED="$L/hermes-agent/apps/desktop/release/linux-unpacked"
mkdir -p "$UNPACKED"
printf '#!/bin/sh\nexit 1\n' > "$UNPACKED/hermes"; chmod +x "$UNPACKED/hermes"
if [ "$(uname)" != "Darwin" ]; then
bash "$SCRIPT_DIR/posix.sh" --no-ui --desktop-pid 0 --install-root "$L/hermes-agent" \
--relaunch-target "$UNPACKED/hermes" >/dev/null 2>&1 || true
expect_msg "instant-exit relaunch downgrades to manual" "d['ok']==True and d['manual']==True and 'Reopen Hermes' in d['message']"
else
# mac: a SUPPLIED target that is missing is a REJECTED launch and
# must downgrade to manual — never a clean "Update complete."
bash "$SCRIPT_DIR/posix.sh" --no-ui --desktop-pid 0 --install-root "$L/hermes-agent" \
--relaunch-target "$L/NoSuch.app" >/dev/null 2>&1 || true
expect_msg "missing bundle downgrades to manual" "d['ok']==True and d['manual']==True and 'Reopen Hermes' in d['message']"
fi
# 2. gated skew: success result carries the skew message (the manual
# event's payload), never a bare "Update complete."
stub_install
bash "$SCRIPT_DIR/posix.sh" --no-ui --desktop-pid 0 --install-root "$L/hermes-agent" \
--relaunch-target /opt/Hermes/hermes >/dev/null 2>&1 || true
if [ "$(uname)" != "Darwin" ]; then
expect_msg "skew outcome surfaces in result message" "d['ok']==True and d['manual']==True and 'was not changed' in d['message']"
fi
rm -rf "$L"
[ "$fails" -eq 0 ] && say "launch matrix: all pass" || { say "launch matrix: $fails FAILED"; exit 1; }
;;
*)
sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'
exit 64
;;
esac
scripts/desktop-update/retry-policy.ps1
+16
View File
@@ -0,0 +1,16 @@
function Test-HermesUpdateShouldRetry {
param(
[int]$ExitCode,
[string]$InstallRoot
)
if ($ExitCode -eq 0) { return $false }
if ($ExitCode -ne 2) { return $true }
# Exit 2 is shared by non-retryable safety refusals and the self-lock
# deferral. Only the latter writes this marker. The handoff treats it as a
# retry signal for one fresh-process attempt, whose early-recovery pass
# completes core dependencies before native modules load.
$deferredInstallMarker = Join-Path $InstallRoot ".update-incomplete"
return Test-Path -LiteralPath $deferredInstallMarker
}
scripts/desktop-update/serve-ui.py
+63
View File
@@ -0,0 +1,63 @@
"""Loopback shim server for the desktop update hand-off.
Two GET routes: / serves ui.html, /progress serves the status file the
orchestrator script writes ({"status": "running"|"done"|"error", ...}).
Exists because a file:// page cannot receive events from a detached
process. Prints the chosen ephemeral port on stdout, serves until killed.
`elapsed_seconds` is stamped per request, not read from the status file:
stages are minutes apart, so a value frozen at the last publish would sit
still through the longest waits -- exactly the stall the page exists to
disprove. Windows' in-process listener computes it the same way.
"""
import http.server
import json
import socketserver
import sys
import time
html_path, status_path = sys.argv[1], sys.argv[2]
started_at = float(sys.argv[3]) if len(sys.argv) > 3 else time.time()
with open(html_path, "rb") as f:
HTML = f.read()
def progress_body():
try:
with open(status_path, "rb") as f:
state = json.loads(f.read())
if not isinstance(state, dict):
raise ValueError(state)
except Exception:
state = {"status": "running", "message": ""}
state["elapsed_seconds"] = max(0, int(time.time() - started_at))
return json.dumps(state).encode("utf-8")
class Handler(http.server.BaseHTTPRequestHandler):
def log_message(self, format, *args): # noqa: A002 - base class signature
pass
def do_GET(self):
if self.path.startswith("/progress"):
body = progress_body()
ctype = "application/json; charset=utf-8"
elif self.path == "/":
body, ctype = HTML, "text/html; charset=utf-8"
else:
self.send_response(404)
self.end_headers()
return
self.send_response(200)
self.send_header("Content-Type", ctype)
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(body)
with socketserver.TCPServer(("127.0.0.1", 0), Handler) as srv:
print(srv.server_address[1], flush=True)
srv.serve_forever()
scripts/desktop-update/ui.html
+259
View File
@@ -0,0 +1,259 @@
<!doctype html>
<!--
Quiet shim page for the desktop update hand-off (Windows + POSIX).
Served over loopback by the orchestrator (windows.ps1 / posix.sh) into a
chromeless browser app window. Pure veneer: polls /progress for the current
hand-off stage or a terminal event and reacts; owns nothing (relaunch,
result file, marker hygiene all live in the orchestrator, which runs
identically with no UI at all).
The visual is PR #75895's update hand-off screen, ported verbatim:
- Loader: the desktop's "Fourier Flow" curve. Math + tuning lifted from
apps/bootstrap-installer/src/components/loader.tsx (itself lifted from
apps/desktop/src/components/ui/loader.tsx 'fourier-flow'). Keep the
constants in sync if the desktop's curve is retuned.
- Layout: loader (size-20) + one title + one muted stage/elapsed line. No
progress bar, stage list, log pane, or cancel (see #75895 for the
arguments). Elapsed comes from the orchestrator or is omitted -- a clock
started here would measure when this window painted, not the update.
- Appearance follows the OS. Dark seeds are the installer's neutral
charcoal (#232323 base, foreground #d6d6d6) — never brand blue.
-->
<html lang="en">
<head>
<meta charset="utf-8">
<title>Hermes</title>
<style>
:root {
--background: #ffffff;
--foreground: #1a1a1a;
--muted-foreground: #737373;
}
@media (prefers-color-scheme: dark) {
:root {
--background: #232323;
--foreground: #d6d6d6;
--muted-foreground: #8a8a8a;
}
}
html, body {
margin: 0;
height: 100%;
background: var(--background);
color: var(--foreground);
font-family: system-ui, 'Segoe UI', sans-serif;
-webkit-font-smoothing: antialiased;
user-select: none;
cursor: default;
overflow: hidden;
}
/* UpdateScreen: flex h-full flex-col items-center justify-center gap-4
px-6 text-center (routes/progress.tsx) */
.wrap {
height: 100%;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
gap: 16px;
padding: 0 24px;
box-sizing: border-box;
text-align: center;
}
@keyframes hermes-fade-in {
from { opacity: 0; transform: translateY(4px); }
to { opacity: 1; transform: translateY(0); }
}
.wrap { animation: hermes-fade-in 0.45s ease-out both; }
/* Loader size-20 (5rem); svg overflow-visible; curve path opacity .1 */
#loader { width: 80px; height: 80px; color: var(--foreground); }
#loader svg { width: 100%; height: 100%; overflow: visible; }
#glyph {
width: 80px;
height: 80px;
display: none;
align-items: center;
justify-content: center;
font-size: 44px;
line-height: 1;
}
/* text-lg font-semibold tracking-tight */
h2 {
margin: 0;
font-size: 18px;
font-weight: 600;
letter-spacing: -0.025em;
}
/* text-xs text-muted-foreground */
p {
margin: 0;
max-width: 80%;
font-size: 12px;
line-height: 1.5;
color: var(--muted-foreground);
white-space: pre-line;
}
p code {
font-family: ui-monospace, SFMono-Regular, Consolas, monospace;
font-size: 11px;
color: var(--foreground);
}
body.done #loader, body.error #loader { display: none; }
body.done #glyph, body.error #glyph { display: flex; }
</style>
</head>
<body>
<div class="wrap">
<div id="loader" role="status" aria-label="Updating"></div>
<div id="glyph"></div>
<h2 id="title">Updating Hermes</h2>
<p id="line">Hermes will open once done.</p>
</div>
<script>
/* ── Fourier Flow loader, ported verbatim from loader.tsx ─────────────── */
const TWO_PI = Math.PI * 2
const CURVE = {
durationMs: 2200,
particleCount: 92,
pulseDurationMs: 2000,
strokeWidth: 4.2,
trailSpan: 0.31,
point(progress, detailScale) {
const t = progress * TWO_PI
const mix = 1 + detailScale * 0.16
const x = 17 * Math.cos(t) + 7.5 * Math.cos(3 * t + 0.6 * mix) + 3.2 * Math.sin(5 * t - 0.4)
const y = 15 * Math.sin(t) + 8.2 * Math.sin(2 * t + 0.25) - 4.2 * Math.cos(4 * t - 0.5 * mix)
return { x: 50 + x, y: 50 + y }
}
}
const PATH_STEPS = 240
const norm = progress => ((progress % 1) + 1) % 1
function detailScaleFor(time, phaseOffset) {
const p = ((time + phaseOffset * CURVE.pulseDurationMs) % CURVE.pulseDurationMs) / CURVE.pulseDurationMs
return 0.52 + ((Math.sin(p * TWO_PI + 0.55) + 1) / 2) * 0.48
}
function buildPath(detailScale, steps) {
return Array.from({ length: steps + 1 }, (_, i) => {
const { x, y } = CURVE.point(i / steps, detailScale)
return `${i === 0 ? 'M' : 'L'} ${x.toFixed(2)} ${y.toFixed(2)}`
}).join(' ')
}
function particleFor(index, progress, detailScale) {
const tail = index / (CURVE.particleCount - 1)
const { x, y } = CURVE.point(norm(progress - tail * CURVE.trailSpan), detailScale)
const fade = (1 - tail) ** 0.56
return { x, y, opacity: 0.04 + fade * 0.96, radius: 0.9 + fade * 2.7 }
}
const SVG_NS = 'http://www.w3.org/2000/svg'
const svg = document.createElementNS(SVG_NS, 'svg')
svg.setAttribute('viewBox', '0 0 100 100')
svg.setAttribute('fill', 'none')
svg.setAttribute('aria-hidden', 'true')
const curvePath = document.createElementNS(SVG_NS, 'path')
curvePath.setAttribute('opacity', '0.1')
curvePath.setAttribute('stroke', 'currentColor')
curvePath.setAttribute('stroke-linecap', 'round')
curvePath.setAttribute('stroke-linejoin', 'round')
curvePath.setAttribute('stroke-width', String(CURVE.strokeWidth))
svg.appendChild(curvePath)
const particles = Array.from({ length: CURVE.particleCount }, () => {
const c = document.createElementNS(SVG_NS, 'circle')
c.setAttribute('fill', 'currentColor')
svg.appendChild(c)
return c
})
document.getElementById('loader').appendChild(svg)
let frame = 0
const startedAt = performance.now()
const phaseOffset = Math.random()
function render(now) {
const time = now - startedAt
const progress = ((time + phaseOffset * CURVE.durationMs) % CURVE.durationMs) / CURVE.durationMs
const detailScale = detailScaleFor(time, phaseOffset)
curvePath.setAttribute('d', buildPath(detailScale, PATH_STEPS))
particles.forEach((node, index) => {
const p = particleFor(index, progress, detailScale)
node.setAttribute('cx', p.x.toFixed(2))
node.setAttribute('cy', p.y.toFixed(2))
node.setAttribute('r', p.radius.toFixed(2))
node.setAttribute('opacity', p.opacity.toFixed(3))
})
frame = window.requestAnimationFrame(render)
}
render(performance.now())
/* ── Event listener: running stage or terminal outcome ───────────────── */
const titleEl = document.getElementById('title')
const lineEl = document.getElementById('line')
const glyphEl = document.getElementById('glyph')
const defaultLine = lineEl.textContent /* what a stage-less run says */
let settled = false
const elapsedText = s =>
s < 60 ? `${s}s elapsed` : `${Math.floor(s / 60)}m ${s % 60}s elapsed`
function settle(state) {
settled = true
window.cancelAnimationFrame(frame)
document.body.className = state
}
function apply(state) {
if (settled) return
if (state.status === 'running') {
const stage = state.message || defaultLine
const elapsed = Number(state.elapsed_seconds)
lineEl.textContent = Number.isFinite(elapsed) && elapsed >= 0
? `${stage}\n${elapsedText(Math.floor(elapsed))}`
: stage
} else if (state.status === 'done') {
settle('done')
glyphEl.textContent = '\u2713'
lineEl.textContent = 'Opening Hermes\u2026'
} else if (state.status === 'manual') {
// Update landed but Hermes will NOT reopen itself (package skew,
// sandbox helper, launch rejected). The orchestrator leaves this
// window up; the message says what to do.
settle('done')
glyphEl.textContent = '\u2713'
titleEl.textContent = 'Update complete'
lineEl.textContent = state.message || 'Reopen Hermes to finish.'
} else if (state.status === 'error') {
settle('error')
glyphEl.textContent = '\u2715'
titleEl.textContent = 'Failed to update'
lineEl.innerHTML = 'Run <code>hermes debug share</code> in a terminal to send a report.'
}
}
async function poll() {
try {
const res = await fetch('/progress', { cache: 'no-store' })
if (res.ok) apply(await res.json())
} catch {
// Server gone: hold the last known state. The orchestrator owns
// closing this window; the relaunched Desktop owns the result.
}
if (!settled) setTimeout(poll, 400)
}
poll()
</script>
</body>
</html>
scripts/desktop-update/windows.ps1
+1619
View File
File diff suppressed because it is too large Load Diff
scripts/dev-sandbox.sh
Executable
+591
View File
@@ -0,0 +1,591 @@
#!/usr/bin/env bash
# Run a command in a disposable, network-isolated fake Internet.
#
# The command runs in private user, mount, PID, and network namespaces. This
# script is stage 1: it builds the sandbox tree, mints the fake CA, and creates
# the user+network namespaces with `unshare` (see the namespace plan further
# down), then re-execs into scripts/sandbox/stage2-run.sh, which adds the
# mount/pid namespaces with bubblewrap and runs the payload. Its only writable
# filesystem is SANDBOX_ROOT. HTTP(S) goes to a local static MITM proxy;
# github.com SSH uses a sandbox-local git-upload-pack shim; neither transport
# can reach the host network.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Helper files the sandbox needs: the stage-2 script it re-execs into, plus the
# files it copies in (the fake-internet proxy, the ssh shim, the openssl config).
# They sit next to this script in the repo, but the Nix wrapper installs the
# script into the store on its own, so it exports DEV_SANDBOX_ASSETS to point
# here.
SANDBOX_ASSETS="${DEV_SANDBOX_ASSETS:-$SCRIPT_DIR/sandbox}"
for asset in proxy.py ssh-shim.sh openssl.cnf stage2-run.sh; do
[ -f "$SANDBOX_ASSETS/$asset" ] || {
echo "error: missing sandbox asset: $SANDBOX_ASSETS/$asset" >&2
exit 1
}
done
print_help() {
cat <<'EOF'
Usage: dev-sandbox.sh [options] [--] <command...>
dev-sandbox.sh install [options] [--] [installer arguments...]
Run COMMAND in a throwaway chroot-like bubblewrap sandbox. The sandbox has no
writable host mounts: only its own root, mounted at /work, is writable.
Options:
--persistent Keep the whole sandbox under .hermes-sandbox/.
--delete Delete the persistent sandbox (asks first).
--root Install as uid 0 with the root FHS layout: code in
/usr/local/lib/hermes-agent, command in
/usr/local/bin. Default is the user-level layout.
--from DIR One-time copy of DIR into the sandbox's $HOME.
Existing persistent sandboxes are never overwritten.
--http-root DIR Copy DIR into the fake web server root for this run.
Requests map to DIR/<host>/<path>; no URL is forwarded.
--installer PATH With `install`, serve PATH at the canonical install.sh
URL. Default: scripts/install.sh in this worktree.
--from-main With `install`, fetch the real upstream main installer
and repository, then advance fake main to this folder
after a successful install for update testing.
Shorthand for --install-ref refs/heads/main.
--install-ref REF Like --from-main, but installs REF instead of main:
a branch, a tag (v2026.7.7), or a SHA reachable from main.
Use it to test updating from an older release, not just
from the tip.
-h, --help Show this help.
Option order matters: every option above is consumed by THIS script, and
parsing stops at the first argument it does not recognize. Everything from
that point on is passed through to the command (or, with `install`, to the
installer). Put sandbox options first and separate installer arguments with
`--`, otherwise they arrive here and fail:
# WRONG — --from-main reaches install.sh, which rejects it
scripts/dev-sandbox.sh install --skip-setup --from-main
# RIGHT
scripts/dev-sandbox.sh install --from-main -- --skip-setup
Install layout: `install.sh` picks its layout from `id -u` alone, so uid is what
separates the two real-world Linux installs. By default the sandbox runs as an
unprivileged `hermes` user, giving the layout most people have —
$HERMES_HOME/hermes-agent plus a ~/.local/bin launcher. Pass --root for the FHS
one. Both are worth testing; they differ in more than paths (root also relocates
uv's Python to /usr/local/share for world-readability).
The fake web server signs certificates with a CA trusted only inside this
sandbox. HTTP_PROXY/HTTPS_PROXY send fixture URLs there first; other HTTP(S)
requests pass through the sandbox's rootless outbound network. SSH to github.com
runs a sandbox-local upload-pack shim, never your SSH config, agent,
known-hosts file, or authorized keys.
Fake github main always comes from this folder. If it has staged, unstaged, or
non-ignored untracked changes, the sandbox warns and creates a temporary local
commit containing them; it never stages or commits the real worktree.
Environment:
HERMES_DEV_SANDBOX_DIR Sandbox directory name, relative to the repo root
(default: .hermes-sandbox).
Examples:
# create a sandbox, install this branch as `main`, and then drop to a shell,
# skipping `hermes setup` & the browser tools for speed.
scripts/dev-sandbox.sh install --persistent -- --skip-setup --skip-browser
# Install the official upstream main. You're dropped into a shell where
# you can run `hermes update`.
scripts/dev-sandbox.sh install --persistent --from-main
EOF
}
PERSISTENT=false
DELETE=false
RUN_AS_USER=true
SEED_DIR=""
HTTP_ROOT=""
INSTALL_SHORTCUT=false
INSTALLER_PATH=""
# Which upstream commit the sandbox installs before the update routes run.
# Empty means "install this worktree's own installer" (no upstream fetch); set,
# it is anything git can resolve -- a branch, a tag (v2026.7.7), or a SHA
# reachable from main -- so "can a user two releases back still update?" is
# expressible. --from-main is shorthand for refs/heads/main.
INSTALL_REF=""
UPSTREAM_URL="${HERMES_DEV_SANDBOX_UPSTREAM:-https://github.com/NousResearch/hermes-agent.git}"
if [ "${1:-}" = install ]; then
INSTALL_SHORTCUT=true
shift
fi
while [ "$#" -gt 0 ]; do
case "$1" in
--persistent) PERSISTENT=true; shift ;;
--delete) DELETE=true; shift ;;
--root) RUN_AS_USER=false; shift ;;
--user) RUN_AS_USER=true; shift ;; # the default; accepted for symmetry
--from)
[ "$#" -ge 2 ] || { echo 'error: --from needs a directory' >&2; exit 1; }
SEED_DIR="$2"; shift 2 ;;
--http-root)
[ "$#" -ge 2 ] || { echo 'error: --http-root needs a directory' >&2; exit 1; }
HTTP_ROOT="$2"; shift 2 ;;
--installer)
[ "$#" -ge 2 ] || { echo 'error: --installer needs a file' >&2; exit 1; }
INSTALLER_PATH="$2"; shift 2 ;;
--from-main) INSTALL_REF="refs/heads/main"; shift ;;
--install-ref)
[ "$#" -ge 2 ] || { echo 'error: --install-ref needs a value' >&2; exit 1; }
INSTALL_REF="$2"
shift 2 ;;
--from=*|--http-root=*|--installer=*|--install-ref=*)
key="${1%%=*}"; value="${1#*=}"
[ -n "$value" ] || { echo "error: $key needs a value" >&2; exit 1; }
case "$key" in
--from) SEED_DIR="$value" ;;
--http-root) HTTP_ROOT="$value" ;;
--installer) INSTALLER_PATH="$value" ;;
--install-ref) INSTALL_REF="$value" ;;
esac
shift ;;
-h|--help) print_help; exit 0 ;;
--) shift; break ;;
*) break ;;
esac
done
if [ "$INSTALL_SHORTCUT" = false ] && [ "$#" -eq 0 ]; then
print_help >&2
exit 1
fi
if [ -n "$INSTALLER_PATH" ] && [ "$INSTALL_SHORTCUT" = false ]; then
echo 'error: --installer is only valid with the install shortcut' >&2
exit 1
fi
if [ -n "$INSTALL_REF" ] && [ "$INSTALL_SHORTCUT" = false ]; then
echo 'error: --from-main / --install-ref are only valid with the install shortcut' >&2
exit 1
fi
if [ -n "$INSTALL_REF" ] && [ -n "$INSTALLER_PATH" ]; then
echo 'error: --from-main / --install-ref cannot be combined with --installer' >&2
exit 1
fi
for dir in "$SEED_DIR" "$HTTP_ROOT"; do
[ -z "$dir" ] || [ -d "$dir" ] || { echo "error: directory '$dir' does not exist" >&2; exit 1; }
done
GIT_ROOT="${HERMES_SANDBOX_SOURCE_ROOT:-$(git rev-parse --show-toplevel)}"
GIT_ROOT="$(cd "$GIT_ROOT" && pwd)"
if [ "$INSTALL_SHORTCUT" = true ] && [ -z "$INSTALL_REF" ] && [ -z "$INSTALLER_PATH" ]; then
INSTALLER_PATH="$GIT_ROOT/scripts/install.sh"
fi
if [ -n "$INSTALLER_PATH" ] && [ ! -f "$INSTALLER_PATH" ]; then
echo "error: installer '$INSTALLER_PATH' does not exist" >&2
exit 1
fi
COMMIT="$(git -C "$GIT_ROOT" rev-parse --verify 'HEAD^{commit}')" || {
echo "error: current folder has no HEAD commit" >&2
exit 1
}
SANDBOX_DIR_NAME="${HERMES_DEV_SANDBOX_DIR:-.hermes-sandbox}"
PERSISTENT_ROOT="$GIT_ROOT/$SANDBOX_DIR_NAME"
if [ "$DELETE" = true ]; then
if [ ! -d "$PERSISTENT_ROOT" ]; then
echo "[sandbox] nothing to delete at $PERSISTENT_ROOT" >&2
exit 0
fi
read -r -p "[sandbox] delete $PERSISTENT_ROOT? [y/N] " reply
case "$reply" in
y|Y|yes|YES) rm -rf -- "$PERSISTENT_ROOT" ;;
*) echo '[sandbox] aborted' >&2; exit 1 ;;
esac
exit 0
fi
if [ "$PERSISTENT" = true ]; then
SANDBOX_ROOT="$PERSISTENT_ROOT"
else
SANDBOX_ROOT="$(mktemp -d -t hermes-sandbox.XXXXXX)"
cleanup() { chmod -R u+w "$SANDBOX_ROOT"; rm -rf -- "$SANDBOX_ROOT"; }
trap cleanup EXIT INT TERM
fi
mkdir -p "$SANDBOX_ROOT"/{root,home,etc}
UPSTREAM_REPO=""
UPSTREAM_COMMIT=""
if [ -n "$INSTALL_REF" ]; then
echo "[sandbox] fetching upstream $INSTALL_REF for installer/update test" >&2
UPSTREAM_REPO="$(mktemp -d -t hermes-sandbox-upstream.XXXXXX)"
git -C "$UPSTREAM_REPO" init -q
# Fetch the ref as given. A branch or tag name resolves on its own; a raw SHA
# needs the remote to allow fetching it directly, so fall back to fetching
# main and resolving the SHA locally (which works for any commit that is an
# ancestor of main -- the interesting case for "update from N versions ago").
#
# Peel to ^{commit} in both cases: an annotated tag fetches as a tag OBJECT,
# and using it directly fails later with "trying to write non-commit object
# ... to branch 'refs/heads/main'".
if git -C "$UPSTREAM_REPO" fetch -q "$UPSTREAM_URL" "$INSTALL_REF" 2>/dev/null; then
UPSTREAM_COMMIT="$(git -C "$UPSTREAM_REPO" rev-parse "FETCH_HEAD^{commit}")"
elif git -C "$UPSTREAM_REPO" fetch -q "$UPSTREAM_URL" refs/heads/main \
&& UPSTREAM_COMMIT="$(git -C "$UPSTREAM_REPO" rev-parse --verify -q "$INSTALL_REF^{commit}")"; then
:
else
rm -rf -- "$UPSTREAM_REPO"
echo "error: could not resolve upstream ref: $INSTALL_REF" >&2
echo ' Use a branch (main), a tag (v2026.7.7), or a SHA reachable from main.' >&2
exit 1
fi
fi
if [ ! -e "$SANDBOX_ROOT/root/repo/.sandbox-source" ]; then
mkdir -p "$SANDBOX_ROOT/root/repo"
# Persistent roots live under the worktree, so copying with cp would recurse
# into the sandbox itself. tar also lets us exclude a worktree's .git file,
# which can point at the host's shared worktree metadata.
tar -C "$GIT_ROOT" --exclude='./.git' --exclude="./$SANDBOX_DIR_NAME" -cf - . \
| tar -C "$SANDBOX_ROOT/root/repo" -xf -
: > "$SANDBOX_ROOT/root/repo/.sandbox-source"
fi
if [ -n "$SEED_DIR" ] && [ ! -e "$SANDBOX_ROOT/.seeded" ]; then
echo "[sandbox] seeding home from $SEED_DIR" >&2
cp -a "$SEED_DIR/." "$SANDBOX_ROOT/home/"
: > "$SANDBOX_ROOT/.seeded"
fi
rm -rf "$SANDBOX_ROOT/root/http"
mkdir -p "$SANDBOX_ROOT/root/http"
if [ -n "$HTTP_ROOT" ]; then
cp -a "$HTTP_ROOT/." "$SANDBOX_ROOT/root/http/"
fi
if [ "$INSTALL_SHORTCUT" = true ]; then
mkdir -p "$SANDBOX_ROOT/root/http/hermes-agent.nousresearch.com"
if [ -n "$INSTALL_REF" ]; then
git -C "$UPSTREAM_REPO" show "$UPSTREAM_COMMIT:scripts/install.sh" \
> "$SANDBOX_ROOT/root/http/hermes-agent.nousresearch.com/install.sh"
else
cp -a "$INSTALLER_PATH" "$SANDBOX_ROOT/root/http/hermes-agent.nousresearch.com/install.sh"
fi
set -- bash -c '
set +e
curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash -s -- "$@"
install_status=$?
if [ "$install_status" -eq 0 ] && [ -f /work/promote-main ]; then
next_main=$(cat /work/promote-main)
if git --git-dir=/work/repos/hermes-agent.git update-ref refs/heads/main "$next_main"; then
rm -f /work/promote-main
printf "[sandbox] fake main advanced to this folder for update testing\n" >&2
else
printf "[sandbox] failed to advance fake main after install\n" >&2
install_status=1
fi
fi
if [ "$DEV_SANDBOX_INTERACTIVE" = true ]; then
printf "\n[sandbox] installer exited %s; entering sandbox shell\n" "$install_status" >&2
exec </dev/tty >/dev/tty 2>&1
exec bash -i
fi
exit "$install_status"
' sandbox-installer "$@"
fi
mkdir -p "$SANDBOX_ROOT/root"/{bin,certs,lib64,logs,repos,ssh,usr/bin,usr/local}
REAL_CA_CERT="${DEV_SANDBOX_REAL_CA_CERT:-}"
if [ -z "$REAL_CA_CERT" ]; then
for candidate in /etc/ssl/certs/ca-certificates.crt /etc/ssl/cert.pem; do
if [ -f "$candidate" ]; then
REAL_CA_CERT="$candidate"
break
fi
done
fi
if [ ! -f "$REAL_CA_CERT" ]; then
echo 'error: no system CA bundle found for outbound sandbox HTTPS' >&2
exit 1
fi
if [ ! -f "$SANDBOX_ROOT/root/certs/real-ca.pem" ]; then
cp "$REAL_CA_CERT" "$SANDBOX_ROOT/root/certs/real-ca.pem"
fi
printf 'nameserver 10.0.2.3\n' > "$SANDBOX_ROOT/etc/resolv.conf"
SANDBOX_SHELL="$(command -v bash)"
DYNAMIC_LINKER="${DEV_SANDBOX_DYNAMIC_LINKER:-}"
if [ -z "$DYNAMIC_LINKER" ]; then
# Nix store first: NixOS also ships a /lib64/ld-linux-x86-64.so.2 compat stub,
# so probing FHS paths first would quietly switch which loader a bare script
# invocation uses on this host. Globs that match nothing expand to themselves,
# so every candidate is -f tested. The FHS paths cover Debian/Ubuntu (where
# the loader is under /lib64 or a multiarch /lib dir), which is what CI runs.
for candidate in \
/nix/store/*-glibc-*/lib/ld-linux-*.so.* \
/lib64/ld-linux-x86-64.so.2 \
/lib/ld-linux-aarch64.so.1 \
/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 \
/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1
do
if [ -f "$candidate" ]; then
DYNAMIC_LINKER="$candidate"
break
fi
done
fi
if [ ! -f "$DYNAMIC_LINKER" ]; then
echo 'error: no glibc dynamic linker found for sandboxed release binaries' >&2
echo ' Set DEV_SANDBOX_DYNAMIC_LINKER to its path.' >&2
exit 1
fi
ln -sf "$SANDBOX_SHELL" "$SANDBOX_ROOT/root/bin/sh"
ln -sf "$(command -v ls)" "$SANDBOX_ROOT/root/bin/ls"
ln -sf "$(command -v env)" "$SANDBOX_ROOT/root/usr/bin/env"
ln -sf "$DYNAMIC_LINKER" "$SANDBOX_ROOT/root/lib64/$(basename "$DYNAMIC_LINKER")"
# Identity inside the sandbox. install.sh chooses its layout from `id -u`
# alone (see resolve_install_layout), so the uid here is what decides between
# the root FHS install and a user-level one.
if [ "$RUN_AS_USER" = true ]; then
SANDBOX_UID=1000
SANDBOX_GID=1000
SANDBOX_USER=hermes
SANDBOX_HOME=/home/hermes
else
SANDBOX_UID=0
SANDBOX_GID=0
SANDBOX_USER=root
SANDBOX_HOME=/root
fi
{
printf 'root:x:0:0:Sandbox Root:/root:%s\n' "$SANDBOX_SHELL"
if [ "$RUN_AS_USER" = true ]; then
printf '%s:x:%s:%s:Sandbox User:%s:%s\n' \
"$SANDBOX_USER" "$SANDBOX_UID" "$SANDBOX_GID" "$SANDBOX_HOME" "$SANDBOX_SHELL"
fi
} > "$SANDBOX_ROOT/etc/passwd"
{
printf 'root:x:0:\n'
if [ "$RUN_AS_USER" = true ]; then
printf '%s:x:%s:\n' "$SANDBOX_USER" "$SANDBOX_GID"
fi
} > "$SANDBOX_ROOT/etc/group"
# A user-level install writes the `hermes` launcher to ~/.local/bin and the
# checkout to $HERMES_HOME; both live under the sandbox HOME, which is bound
# from $SANDBOX_ROOT/home. bwrap maps our real uid to $SANDBOX_UID, so the
# host-side ownership of that directory is what the sandbox sees as its own.
printf 'hosts: files dns\n' > "$SANDBOX_ROOT/etc/nsswitch.conf"
printf '127.0.0.1 localhost\n' > "$SANDBOX_ROOT/etc/hosts"
SOURCE_REPO="$GIT_ROOT"
SOURCE_REF="$COMMIT"
SNAPSHOT_REPO=""
FAKE_REPO="$SANDBOX_ROOT/root/repos/hermes-agent.git"
git -C "$SANDBOX_ROOT/root/repos" init --bare -q hermes-agent.git
if [ -n "$INSTALL_REF" ]; then
git --git-dir="$FAKE_REPO" fetch -q --force "$UPSTREAM_REPO" \
"$UPSTREAM_COMMIT:refs/heads/main"
fi
if [ -n "$(git -C "$GIT_ROOT" status --porcelain)" ]; then
echo '[sandbox] warning: current folder is dirty; creating a temporary fake commit for main' >&2
SNAPSHOT_REPO="$(mktemp -d -t hermes-sandbox-snapshot.XXXXXX)"
git -C "$SNAPSHOT_REPO" init -q
git -C "$SNAPSHOT_REPO" fetch -q "$GIT_ROOT" "$COMMIT"
git -C "$SNAPSHOT_REPO" config user.name 'Hermes sandbox'
git -C "$SNAPSHOT_REPO" config user.email 'sandbox@invalid'
GIT_DIR="$SNAPSHOT_REPO/.git" GIT_WORK_TREE="$GIT_ROOT" git read-tree "$COMMIT"
GIT_DIR="$SNAPSHOT_REPO/.git" GIT_WORK_TREE="$GIT_ROOT" \
git add -A -- .
SNAPSHOT_TREE="$(GIT_DIR="$SNAPSHOT_REPO/.git" git write-tree)"
SNAPSHOT_PARENT="$COMMIT"
if EXISTING_MAIN="$(git --git-dir="$FAKE_REPO" rev-parse --verify refs/heads/main 2>/dev/null)"; then
git -C "$SNAPSHOT_REPO" fetch -q "$FAKE_REPO" "$EXISTING_MAIN"
SNAPSHOT_PARENT="$EXISTING_MAIN"
fi
SOURCE_REF="$(GIT_DIR="$SNAPSHOT_REPO/.git" git commit-tree "$SNAPSHOT_TREE" -p "$SNAPSHOT_PARENT" \
-m 'sandbox snapshot of dirty worktree')"
SOURCE_REPO="$SNAPSHOT_REPO"
fi
if [ -n "$INSTALL_REF" ]; then
git --git-dir="$FAKE_REPO" fetch -q --force "$SOURCE_REPO" \
"$SOURCE_REF:refs/hermes-sandbox/next"
printf '%s\n' "$SOURCE_REF" > "$SANDBOX_ROOT/root/promote-main"
else
git --git-dir="$FAKE_REPO" fetch -q --force "$SOURCE_REPO" \
"$SOURCE_REF:refs/heads/main"
fi
git --git-dir="$FAKE_REPO" symbolic-ref HEAD refs/heads/main
if [ -n "$SNAPSHOT_REPO" ]; then
# Best-effort: it is a mktemp directory the OS will reap, and failing the whole
# run over a leftover object file would be worse than leaking it. Concurrent
# git activity in the worktree can still be writing here as we delete.
rm -rf -- "$SNAPSHOT_REPO" 2>/dev/null || true
fi
if [ -n "$UPSTREAM_REPO" ]; then
rm -rf -- "$UPSTREAM_REPO"
fi
# openssl reads a config even for `req -addext`, and its compiled-in path is a
# symlink into /etc/ssl on Debian/Ubuntu -- which the sandbox replaces. Ship our
# own and point OPENSSL_CONF at it, both here and inside the sandbox.
cp "$SANDBOX_ASSETS/openssl.cnf" "$SANDBOX_ROOT/root/certs/openssl.cnf"
if [ ! -f "$SANDBOX_ROOT/root/certs/ca.pem" ]; then
if ! ca_error="$(OPENSSL_CONF="$SANDBOX_ROOT/root/certs/openssl.cnf" \
openssl req -x509 -newkey rsa:2048 -nodes -days 2 \
-subj '/CN=Hermes dev sandbox CA' \
-extensions sandbox_ca_ext \
-keyout "$SANDBOX_ROOT/root/certs/ca.key" \
-out "$SANDBOX_ROOT/root/certs/ca.pem" 2>&1 >/dev/null)"; then
echo 'error: could not create the sandbox CA:' >&2
printf '%s\n' "$ca_error" >&2
exit 1
fi
fi
GIT_UPLOAD_PACK="$(command -v git-upload-pack)"
sed "s|@GIT_UPLOAD_PACK@|$GIT_UPLOAD_PACK|" "$SANDBOX_ASSETS/ssh-shim.sh" \
> "$SANDBOX_ROOT/root/usr/bin/ssh"
chmod 700 "$SANDBOX_ROOT/root/usr/bin/ssh"
# The fake-internet proxy and the ssh shim are real files under
# scripts/sandbox/ rather than heredocs, so they can be linted, syntax-checked
# and diffed like any other source. Copy them into the sandbox tree.
cp "$SANDBOX_ASSETS/proxy.py" "$SANDBOX_ROOT/root/proxy.py"
if [ -n "$INSTALL_REF" ]; then
echo "[sandbox] fake main: upstream $INSTALL_REF ($UPSTREAM_COMMIT)" >&2
echo "[sandbox] prepared update: current folder ($SOURCE_REF)" >&2
else
echo "[sandbox] fake main: current folder ($SOURCE_REF)" >&2
fi
echo "[sandbox] root: $SANDBOX_ROOT" >&2
echo "[sandbox] http root: $SANDBOX_ROOT/root/http" >&2
if [ "$RUN_AS_USER" = true ]; then
echo "[sandbox] identity: $SANDBOX_USER (uid $SANDBOX_UID) — installs are user-level under $SANDBOX_HOME" >&2
else
echo '[sandbox] identity: root (uid 0) — installs use the /usr/local FHS layout' >&2
fi
[ "$PERSISTENT" = true ] && echo '[sandbox] persistent' >&2 || echo '[sandbox] ephemeral' >&2
for command in awk bash bwrap curl git openssl python3 slirp4netns tar unshare; do
command -v "$command" >/dev/null || {
echo "error: missing required command: $command" >&2
exit 1
}
done
INTERACTIVE=false
if [ -t 0 ] && [ -t 1 ]; then
INTERACTIVE=true
fi
NODE_DIR="${DEV_SANDBOX_NODE_DIR:-}"
if [ -z "$NODE_DIR" ] && command -v node >/dev/null; then
NODE_DIR="$(dirname "$(dirname "$(command -v node)")")"
fi
WAYLAND_SOCKET=""
if [ -n "${XDG_RUNTIME_DIR:-}" ] && [ -n "${WAYLAND_DISPLAY:-}" ] \
&& [ -S "$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" ]; then
WAYLAND_SOCKET="$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY"
fi
# Namespace plan (stage 1 -> stage 2).
#
# slirp4netns joins the target's userns and setuids to root before configuring
# the netns, so the userns MUST map a uid 0. bwrap's own --unshare-user maps
# exactly one uid, so it cannot both run the payload as uid 1000 and offer slirp
# a root to become: that combination fails with
# setns(CLONE_NEWNET): Operation not permitted.
#
# So stage 1 builds the namespaces here with two ranges:
# inner 0 <- a subuid, unused by the payload, present only so slirp can
# become root inside the namespace
# inner $SANDBOX_UID <- our real host uid, so everything the sandbox writes
# stays owned by us and `rm -rf` on a persistent sandbox needs
# no privileges or chown dance
# The payload then runs in stage 2, where bwrap adds the mount/pid namespaces
# without creating a userns at all.
#
# The root layout needs no subuid at all: inner 0 IS the host uid there.
netns_args=(--user --net)
if [ "$RUN_AS_USER" = true ]; then
host_user="$(id -un)"
subuid_base="$(awk -F: -v u="$host_user" '$1 == u {print $2; exit}' /etc/subuid)"
subgid_base="$(awk -F: -v u="$host_user" '$1 == u {print $2; exit}' /etc/subgid)"
if [ -z "$subuid_base" ] || [ -z "$subgid_base" ]; then
echo "error: no /etc/subuid or /etc/subgid range for $host_user" >&2
echo ' A user-level sandbox needs one spare subordinate id to host' >&2
echo " its internal root. Add e.g. '$host_user:100000:65536' to both," >&2
echo ' or use --root.' >&2
exit 1
fi
netns_args+=(
--map-users="0:$subuid_base:1" --map-users="$SANDBOX_UID:$(id -u):1"
--map-groups="0:$subgid_base:1" --map-groups="$SANDBOX_GID:$(id -g):1"
)
else
netns_args+=(--map-root-user)
fi
sandbox_pid_file="$SANDBOX_ROOT/root/logs/sandbox.pid"
slirp_ready="$SANDBOX_ROOT/root/logs/slirp.ready"
slirp_log="$SANDBOX_ROOT/root/logs/slirp.log"
: > "$sandbox_pid_file"
: > "$slirp_ready"
env \
DEV_SANDBOX_ROOT="$SANDBOX_ROOT" \
DEV_SANDBOX_BASH="$(command -v bash)" \
DEV_SANDBOX_REAL_CA_CERT="$REAL_CA_CERT" \
DEV_SANDBOX_INTERACTIVE="$INTERACTIVE" \
DEV_SANDBOX_USER="$SANDBOX_USER" \
DEV_SANDBOX_HOME="$SANDBOX_HOME" \
DEV_SANDBOX_NODE_DIR="$NODE_DIR" \
DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH="${DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH:-}" \
DEV_SANDBOX_XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-}" \
DEV_SANDBOX_WAYLAND_DISPLAY="${WAYLAND_DISPLAY:-}" \
DEV_SANDBOX_WAYLAND_SOCKET="$WAYLAND_SOCKET" \
unshare "${netns_args[@]}" \
"$SANDBOX_ASSETS/stage2-run.sh" "$@" &
sandbox_launcher=$!
for _ in $(seq 1 200); do
[ -s "$sandbox_pid_file" ] && break
if ! kill -0 "$sandbox_launcher" 2>/dev/null; then
wait "$sandbox_launcher"
exit $?
fi
sleep 0.05
done
sandbox_pid="$(tr -dc '0-9' < "$sandbox_pid_file")"
if [ -z "$sandbox_pid" ]; then
echo 'error: sandbox did not report its PID' >&2
exit 1
fi
slirp4netns --configure --disable-host-loopback --ready-fd=3 \
--userns-path="/proc/$sandbox_pid/ns/user" "$sandbox_pid" tap0 \
3>"$slirp_ready" >"$slirp_log" 2>&1 &
slirp_pid=$!
cleanup_slirp() {
kill "$slirp_pid" 2>/dev/null || true
wait "$slirp_pid" 2>/dev/null || true
}
trap cleanup_slirp EXIT INT TERM
for _ in $(seq 1 200); do
[ -s "$slirp_ready" ] && break
if ! kill -0 "$slirp_pid" 2>/dev/null; then
cat "$slirp_log" >&2 || true
exit 1
fi
sleep 0.05
done
if [ ! -s "$slirp_ready" ]; then
echo 'error: timed out waiting for sandbox network setup' >&2
exit 1
fi
wait "$sandbox_launcher"
exit $?
scripts/discord-voice-doctor.py
Executable
+396
View File
@@ -0,0 +1,396 @@
#!/usr/bin/env python3
"""Discord Voice Doctor — diagnostic tool for voice channel support.
Checks all dependencies, configuration, and bot permissions needed
for Discord voice mode to work correctly.
Usage:
python scripts/discord-voice-doctor.py
.venv/bin/python scripts/discord-voice-doctor.py
"""
import os
import sys
import shutil
from pathlib import Path
# Resolve project root
SCRIPT_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = SCRIPT_DIR.parent
sys.path.insert(0, str(PROJECT_ROOT))
HERMES_HOME = Path(os.getenv("HERMES_HOME", Path.home() / ".hermes"))
ENV_FILE = HERMES_HOME / ".env"
OK = "\033[92m\u2713\033[0m"
FAIL = "\033[91m\u2717\033[0m"
WARN = "\033[93m!\033[0m"
# Track whether discord.py is available for later sections
_discord_available = False
def mask(value):
"""Mask sensitive value: show only first 4 chars."""
if not value or len(value) < 8:
return "****"
return f"{value[:4]}{'*' * (len(value) - 4)}"
def check(label, ok, detail=""):
symbol = OK if ok else FAIL
msg = f" {symbol} {label}"
if detail:
msg += f" ({detail})"
print(msg)
return ok
def warn(label, detail=""):
msg = f" {WARN} {label}"
if detail:
msg += f" ({detail})"
print(msg)
def section(title):
print(f"\n\033[1m{title}\033[0m")
def check_packages():
"""Check Python package dependencies. Returns True if all critical deps OK."""
global _discord_available
section("Python Packages")
ok = True
# discord.py
try:
import discord
_discord_available = True
check("discord.py", True, f"v{discord.__version__}")
except ImportError:
check("discord.py", False, "pip install discord.py[voice]")
ok = False
# PyNaCl
try:
import nacl
ver = getattr(nacl, "__version__", "unknown")
try:
import nacl.secret
nacl.secret.Aead(bytes(32))
check("PyNaCl", True, f"v{ver}")
except (AttributeError, Exception):
check("PyNaCl (Aead)", False, f"v{ver} — need >=1.5.0")
ok = False
except ImportError:
check("PyNaCl", False, "pip install PyNaCl>=1.5.0")
ok = False
# davey (DAVE E2EE)
try:
import davey
check("davey (DAVE E2EE)", True, f"v{getattr(davey, '__version__', '?')}")
except ImportError:
check("davey (DAVE E2EE)", False, "pip install davey")
ok = False
# Optional: local STT
try:
import faster_whisper
check("faster-whisper (local STT)", True)
except ImportError:
warn("faster-whisper (local STT)", "not installed — local STT unavailable")
# Optional: TTS providers
try:
import edge_tts
check("edge-tts", True)
except ImportError:
warn("edge-tts", "not installed — edge TTS unavailable")
try:
import elevenlabs
check("elevenlabs SDK", True)
except ImportError:
warn("elevenlabs SDK", "not installed — premium TTS unavailable")
return ok
def check_system_tools():
"""Check system-level tools (opus, ffmpeg). Returns True if all OK."""
section("System Tools")
ok = True
# Opus codec
if _discord_available:
try:
import discord
opus_loaded = discord.opus.is_loaded()
if not opus_loaded:
import ctypes.util
opus_path = ctypes.util.find_library("opus")
if not opus_path:
# Platform-specific fallback paths
candidates = [
"/opt/homebrew/lib/libopus.dylib", # macOS Apple Silicon
"/usr/local/lib/libopus.dylib", # macOS Intel
"/usr/lib/x86_64-linux-gnu/libopus.so.0", # Debian/Ubuntu x86
"/usr/lib/aarch64-linux-gnu/libopus.so.0", # Debian/Ubuntu ARM
"/usr/lib/libopus.so", # Arch Linux
"/usr/lib64/libopus.so", # RHEL/Fedora
]
for p in candidates:
if os.path.isfile(p):
opus_path = p
break
if opus_path:
discord.opus.load_opus(opus_path)
opus_loaded = discord.opus.is_loaded()
if opus_loaded:
check("Opus codec", True)
else:
check("Opus codec", False, "brew install opus / apt install libopus0")
ok = False
except Exception as e:
check("Opus codec", False, str(e))
ok = False
else:
warn("Opus codec", "skipped — discord.py not installed")
# ffmpeg
ffmpeg_path = shutil.which("ffmpeg")
if ffmpeg_path:
check("ffmpeg", True, ffmpeg_path)
else:
check("ffmpeg", False, "brew install ffmpeg / apt install ffmpeg")
ok = False
return ok
def check_env_vars():
"""Check environment variables. Returns (ok, token, groq_key, eleven_key)."""
section("Environment Variables")
# Load .env
try:
from hermes_cli.env_loader import load_hermes_dotenv
load_hermes_dotenv(
hermes_home=ENV_FILE.parent,
project_env=PROJECT_ROOT / ".env",
)
except ImportError:
pass
ok = True
token = os.getenv("DISCORD_BOT_TOKEN", "")
if token:
check("DISCORD_BOT_TOKEN", True, mask(token))
else:
check("DISCORD_BOT_TOKEN", False, "not set")
ok = False
# Allowed users — resolve usernames if possible
allowed = os.getenv("DISCORD_ALLOWED_USERS", "")
if allowed:
users = [u.strip() for u in allowed.split(",") if u.strip()]
user_labels = []
for uid in users:
label = mask(uid)
if token and uid.isdigit():
try:
import requests
r = requests.get(
f"https://discord.com/api/v10/users/{uid}",
headers={"Authorization": f"Bot {token}"},
timeout=3,
)
if r.status_code == 200:
label = f"{r.json().get('username', '?')} ({mask(uid)})"
except Exception:
pass
user_labels.append(label)
check("DISCORD_ALLOWED_USERS", True, f"{len(users)} user(s): {', '.join(user_labels)}")
else:
warn("DISCORD_ALLOWED_USERS", "not set — all users can use voice")
groq_key = os.getenv("GROQ_API_KEY", "")
eleven_key = os.getenv("ELEVENLABS_API_KEY", "")
if groq_key:
check("GROQ_API_KEY (STT)", True, mask(groq_key))
else:
warn("GROQ_API_KEY", "not set — Groq STT unavailable")
if eleven_key:
check("ELEVENLABS_API_KEY (TTS)", True, mask(eleven_key))
else:
warn("ELEVENLABS_API_KEY", "not set — ElevenLabs TTS unavailable")
return ok, token, groq_key, eleven_key
def check_config(groq_key, eleven_key):
"""Check hermes config.yaml."""
section("Configuration")
config_path = HERMES_HOME / "config.yaml"
if config_path.exists():
try:
import yaml
with open(config_path, encoding="utf-8") as f:
cfg = yaml.safe_load(f) or {}
stt_provider = cfg.get("stt", {}).get("provider", "local")
tts_provider = cfg.get("tts", {}).get("provider", "edge")
check("STT provider", True, stt_provider)
check("TTS provider", True, tts_provider)
if stt_provider == "groq" and not groq_key:
warn("STT config says groq but GROQ_API_KEY is missing")
if stt_provider == "mistral" and not os.getenv("MISTRAL_API_KEY"):
warn("STT config says mistral but MISTRAL_API_KEY is missing")
if tts_provider == "elevenlabs" and not eleven_key:
warn("TTS config says elevenlabs but ELEVENLABS_API_KEY is missing")
if tts_provider == "mistral" and not os.getenv("MISTRAL_API_KEY"):
warn("TTS config says mistral but MISTRAL_API_KEY is missing")
except Exception as e:
warn("config.yaml", f"parse error: {e}")
else:
warn("config.yaml", "not found — using defaults")
# Voice mode state
voice_mode_path = HERMES_HOME / "gateway_voice_mode.json"
if voice_mode_path.exists():
try:
import json
modes = json.loads(voice_mode_path.read_text(encoding="utf-8"))
off_count = sum(1 for v in modes.values() if v == "off")
all_count = sum(1 for v in modes.values() if v == "all")
check("Voice mode state", True, f"{all_count} on, {off_count} off, {len(modes)} total")
except Exception:
warn("Voice mode state", "parse error")
else:
check("Voice mode state", True, "no saved state (fresh)")
def check_bot_permissions(token):
"""Check bot permissions via Discord API. Returns True if all OK."""
section("Bot Permissions")
if not token:
warn("Bot permissions", "no token — skipping")
return True
try:
import requests
except ImportError:
warn("Bot permissions", "requests not installed — skipping")
return True
VOICE_PERMS = {
"Priority Speaker": 8,
"Stream": 9,
"View Channel": 10,
"Send Messages": 11,
"Embed Links": 14,
"Attach Files": 15,
"Read Message History": 16,
"Connect": 20,
"Speak": 21,
"Mute Members": 22,
"Deafen Members": 23,
"Move Members": 24,
"Use VAD": 25,
"Send Voice Messages": 46,
}
REQUIRED_PERMS = {"Connect", "Speak", "View Channel", "Send Messages"}
ok = True
try:
headers = {"Authorization": f"Bot {token}"}
r = requests.get("https://discord.com/api/v10/users/@me", headers=headers, timeout=5)
if r.status_code == 401:
check("Bot login", False, "invalid token (401)")
return False
if r.status_code != 200:
check("Bot login", False, f"HTTP {r.status_code}")
return False
bot = r.json()
bot_name = bot.get("username", "?")
check("Bot login", True, f"{bot_name[:3]}{'*' * (len(bot_name) - 3)}")
# Check guilds
r2 = requests.get("https://discord.com/api/v10/users/@me/guilds", headers=headers, timeout=5)
if r2.status_code != 200:
warn("Guilds", f"HTTP {r2.status_code}")
return ok
guilds = r2.json()
check("Guilds", True, f"{len(guilds)} guild(s)")
for g in guilds[:5]:
perms = int(g.get("permissions", 0))
is_admin = bool(perms & (1 << 3))
if is_admin:
print(f" {OK} {g['name']}: Administrator (all permissions)")
continue
has = []
missing = []
for name, bit in sorted(VOICE_PERMS.items(), key=lambda x: x[1]):
if perms & (1 << bit):
has.append(name)
elif name in REQUIRED_PERMS:
missing.append(name)
if missing:
print(f" {FAIL} {g['name']}: missing {', '.join(missing)}")
ok = False
else:
print(f" {OK} {g['name']}: {', '.join(has)}")
except requests.exceptions.Timeout:
warn("Bot permissions", "Discord API timeout")
except requests.exceptions.ConnectionError:
warn("Bot permissions", "cannot reach Discord API")
except Exception as e:
warn("Bot permissions", f"check failed: {e}")
return ok
def main():
print()
print("\033[1m" + "=" * 50 + "\033[0m")
print("\033[1m Discord Voice Doctor\033[0m")
print("\033[1m" + "=" * 50 + "\033[0m")
all_ok = True
all_ok &= check_packages()
all_ok &= check_system_tools()
env_ok, token, groq_key, eleven_key = check_env_vars()
all_ok &= env_ok
check_config(groq_key, eleven_key)
all_ok &= check_bot_permissions(token)
# Summary
print()
print("\033[1m" + "-" * 50 + "\033[0m")
if all_ok:
print(f" {OK} \033[92mAll checks passed — voice mode ready!\033[0m")
else:
print(f" {FAIL} \033[91mSome checks failed — fix issues above.\033[0m")
print()
if __name__ == "__main__":
main()
scripts/docker_config_migrate.py
+110
View File
@@ -0,0 +1,110 @@
#!/usr/bin/env python3
"""Run Docker boot-time config migrations safely."""
from __future__ import annotations
import shutil
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Iterable
from hermes_cli.config import (
check_config_version,
get_config_path,
get_env_path,
migrate_config,
)
from hermes_cli.config_migrations import (
SUPPORT_FLOOR_VERSION,
support_floor_message,
)
from utils import env_var_enabled
def _backup_path(path: Path, stamp: str) -> Path:
base = path.with_name(f"{path.name}.bak-{stamp}")
if not base.exists():
return base
for index in range(1, 1000):
candidate = path.with_name(f"{path.name}.bak-{stamp}.{index}")
if not candidate.exists():
return candidate
raise RuntimeError(f"could not choose a backup path for {path}")
def _backup_existing(paths: Iterable[Path]) -> dict[Path, Path]:
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
backups: dict[Path, Path] = {}
for path in paths:
if not path.is_file():
continue
dest = _backup_path(path, stamp)
shutil.copy2(path, dest)
backups[path] = dest
return backups
def _restore_backups(backups: dict[Path, Path]) -> list[Path]:
restored: list[Path] = []
for original, backup in backups.items():
if not backup.is_file():
continue
shutil.copy2(backup, original)
restored.append(original)
return restored
def main() -> int:
if env_var_enabled("HERMES_SKIP_CONFIG_MIGRATION"):
print("[config-migrate] HERMES_SKIP_CONFIG_MIGRATION is set; skipping config migration")
return 0
current_ver, latest_ver = check_config_version()
if current_ver >= latest_ver:
return 0
# Below the auto-migration support floor: migrate_config() refuses (and
# leaves the file untouched), so don't run the backup/verify dance that
# would raise "did not advance config version" and block the boot.
# Warn-and-continue matches the CLI's fail-safe posture.
if current_ver < SUPPORT_FLOOR_VERSION:
print(
f"[config-migrate] WARNING: {support_floor_message()}",
file=sys.stderr,
)
return 0
backups = _backup_existing((get_config_path(), get_env_path()))
backup_text = ", ".join(str(path) for path in backups.values()) if backups else "none"
print(
f"[config-migrate] Migrating config schema {current_ver} -> {latest_ver}; "
f"backups: {backup_text}"
)
try:
migrate_config(interactive=False, quiet=False)
except Exception:
restored = _restore_backups(backups)
if restored:
print(
"[config-migrate] Migration failed; restored "
+ ", ".join(str(path) for path in restored)
)
raise
post_ver, _ = check_config_version()
if post_ver < latest_ver:
restored = _restore_backups(backups)
restored_text = ", ".join(str(path) for path in restored) if restored else "none"
raise RuntimeError(
f"migration did not advance config version to {latest_ver} "
f"(still {post_ver}); restored: {restored_text}"
)
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except Exception as exc:
print(f"[config-migrate] ERROR: {exc}", file=sys.stderr)
raise SystemExit(1)
scripts/docker_rebootstrap_nous_session.py
+227
View File
@@ -0,0 +1,227 @@
#!/usr/bin/env python3
"""Boot-time re-seed of a terminally-dead Nous bootstrap session.
Background
----------
A Nous bootstrap session (client_id ``hermes-cli-vps``) can take a terminal
``invalid_grant`` and be quarantined locally — the refresh path clears the dead
tokens from ``auth.json`` and stamps
``providers.nous.last_auth_error.relogin_required = true``. From then on every
inference turn hard-fails with a provider-auth error until the credential is
replaced, even though the gateway and dashboard otherwise look healthy.
``stage2-hook.sh`` seeds ``auth.json`` from ``HERMES_AUTH_JSON_BOOTSTRAP`` only
on a *blank* volume (``[ ! -f auth.json ]``) — that guard is load-bearing: it
stops a container restart from clobbering a healthy, rotated refresh token. So a
plain restart with a fresh seed env can NOT recover a container whose volume
already has an auth.json.
This script is the narrow, safe exception. An orchestrator that manages the
container can supply a freshly-issued bootstrap session via
``HERMES_AUTH_JSON_REBOOTSTRAP`` (plus a restart). On boot we re-seed the Nous
provider entry from that env when the on-disk entry is provably terminal, or
when the orchestrator seed's ``obtained_at`` is newer than the local session.
The latter matters because an orchestrator may revoke the previous session
before restart while its still-present local tokens look healthy. Older or
incomparable seeds remain no-ops, so a retained env cannot roll auth backward.
Design constraints
------------------
- Pure stdlib, no hermes_cli imports: runs early in the boot hook, before the
app venv/modules are guaranteed importable, as its own subprocess.
- Surgical: replaces ONLY ``providers.nous`` in the existing auth.json, leaving
every other provider, the version, and any other top-level state untouched.
- Fail-safe: any parse/IO error leaves auth.json exactly as-is and exits 0 (a
failed re-seed must never take the container further down than it already is).
"""
from __future__ import annotations
import json
import os
import sys
from datetime import datetime, timezone
from typing import Any, Optional
# Env var the orchestrator sets to the re-seed payload. Deliberately DISTINCT
# from HERMES_AUTH_JSON_BOOTSTRAP (create-only, blank-volume seed) so the two
# paths can never be confused: BOOTSTRAP seeds a fresh volume; REBOOTSTRAP
# overwrites a terminally-dead Nous entry on an existing volume.
REBOOTSTRAP_ENV = "HERMES_AUTH_JSON_REBOOTSTRAP"
BOOTSTRAP_CLIENT_ID = "hermes-cli-vps"
def _nous_entry_is_terminal(nous_state: Any) -> bool:
"""True iff the on-disk Nous provider entry is in the terminal/quarantined
state AND holds no usable credential.
Mirrors the ``terminal`` predicate in ``hermes_cli.auth.get_nous_session_validity``:
a persisted ``last_auth_error.relogin_required`` with the token material
already cleared. Keeping this in lockstep is what guarantees we only re-seed
a session that is genuinely dead.
"""
if not isinstance(nous_state, dict):
return False
last_err = nous_state.get("last_auth_error")
if not (isinstance(last_err, dict) and last_err.get("relogin_required")):
return False
# Only terminal while there is no usable credential left. If a live token is
# somehow present, treat it as healthy and do NOT clobber it.
if nous_state.get("access_token") or nous_state.get("refresh_token"):
return False
return True
def _extract_nous_from_seed(seed_raw: str) -> Optional[dict]:
"""Pull the ``providers.nous`` block out of a HERMES_AUTH_JSON_REBOOTSTRAP
payload. The payload is a full auth.json document (same shape as
HERMES_AUTH_JSON_BOOTSTRAP). Returns None unless it carries the expected VPS
bootstrap client plus non-empty access and refresh tokens — caller treats
None as "nothing to do"."""
try:
seed = json.loads(seed_raw)
except (ValueError, TypeError):
return None
if not isinstance(seed, dict):
return None
providers = seed.get("providers")
if not isinstance(providers, dict):
return None
nous = providers.get("nous")
if not isinstance(nous, dict) or not nous:
return None
if nous.get("client_id") != BOOTSTRAP_CLIENT_ID:
return None
if not (
isinstance(nous.get("access_token"), str)
and nous["access_token"].strip()
and isinstance(nous.get("refresh_token"), str)
and nous["refresh_token"].strip()
):
return None
return nous
def _parse_timestamp(value: Any) -> Optional[datetime]:
"""Parse an OAuth timestamp without guessing when either side is malformed."""
if not isinstance(value, str) or not value.strip():
return None
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except (ValueError, OverflowError):
return None
if parsed.tzinfo is None:
return None
try:
return parsed.astimezone(timezone.utc)
except (OverflowError, ValueError):
return None
def _seed_is_newer(local_nous: Any, seed_nous: dict) -> bool:
"""Whether NAS supplied a bootstrap session newer than the local one.
NAS mints the replacement before restarting the machine and revokes the
previous session. A healthy-looking local entry can therefore already be
stale. ``obtained_at`` is the server-issued ordering signal that lets boot
apply a genuinely newer replacement without allowing an old retained env
value to roll credentials back on later restarts.
"""
if not isinstance(local_nous, dict):
return False
local_obtained = _parse_timestamp(local_nous.get("obtained_at"))
seed_obtained = _parse_timestamp(seed_nous.get("obtained_at"))
return bool(
local_obtained is not None
and seed_obtained is not None
and seed_obtained > local_obtained
)
def reseed_if_terminal(auth_path: str, seed_raw: str) -> str:
"""Core logic. Returns a short status string for logging/testing:
- "no_seed" — seed env empty/absent
- "bad_seed" — seed present but unparseable / no nous entry
- "no_auth_file" — auth.json absent (blank volume → let the normal
HERMES_AUTH_JSON_BOOTSTRAP path handle it)
- "auth_unreadable" — auth.json present but unparseable (leave as-is)
- "not_terminal" — local entry is healthy and at least as new → no-op
- "reseeded" — terminal entry replaced from seed
- "reseeded_newer" — healthy-but-stale entry replaced by a newer seed
"""
if not seed_raw:
return "no_seed"
seed_nous = _extract_nous_from_seed(seed_raw)
if seed_nous is None:
return "bad_seed"
if not os.path.exists(auth_path):
# Blank volume — this is the normal first-boot case, not a re-seed.
return "no_auth_file"
try:
with open(auth_path, "r", encoding="utf-8") as fh:
store = json.load(fh)
except (OSError, ValueError):
# Corrupt/unreadable auth.json: do NOT overwrite blindly. A separate
# concern; leave it for the operator / other recovery paths.
return "auth_unreadable"
if not isinstance(store, dict):
return "auth_unreadable"
providers = store.get("providers")
if not isinstance(providers, dict):
providers = {}
store["providers"] = providers
local_nous = providers.get("nous")
terminal = _nous_entry_is_terminal(local_nous)
newer_seed = _seed_is_newer(local_nous, seed_nous)
if not terminal and not newer_seed:
# Healthy and at least as new as the seed, or incomparable. Never roll a
# session back merely because an old rebootstrap env remains configured.
return "not_terminal"
# Surgical replacement: swap ONLY providers.nous, preserve everything else.
providers["nous"] = seed_nous
tmp_path = f"{auth_path}.rebootstrap.tmp"
with open(tmp_path, "w", encoding="utf-8") as fh:
json.dump(store, fh)
os.replace(tmp_path, auth_path)
try:
os.chmod(auth_path, 0o600)
except OSError:
pass
return "reseeded" if terminal else "reseeded_newer"
def main() -> int:
auth_path = sys.argv[1] if len(sys.argv) > 1 else ""
if not auth_path:
home = os.environ.get("HERMES_HOME", "")
auth_path = os.path.join(home, "auth.json") if home else "auth.json"
seed_raw = os.environ.get(REBOOTSTRAP_ENV, "")
try:
result = reseed_if_terminal(auth_path, seed_raw)
except Exception as exc: # never let a re-seed error fail the boot
print(f"[rebootstrap] error (ignored): {exc!r}", file=sys.stderr)
return 0
if result == "reseeded":
print("[rebootstrap] Nous bootstrap session was terminal; re-seeded auth.json from "
f"{REBOOTSTRAP_ENV}")
elif result == "reseeded_newer":
print("[rebootstrap] Applied newer orchestrator-issued Nous bootstrap session from "
f"{REBOOTSTRAP_ENV}")
else:
# Quiet by default for the common no-op cases; still emit a breadcrumb.
print(f"[rebootstrap] no-op ({result})")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/e2e_shared_metrics_staging.py
+198
View File
@@ -0,0 +1,198 @@
"""Live staging E2E for the shared-metrics exporter.
Sends REAL packages through the REAL sender to the REAL staging ingest
service, then reports what the service acknowledged. Uses a throwaway
HERMES_HOME so the operator's own telemetry state is untouched.
Usage:
.venv/bin/python scripts/e2e_shared_metrics_staging.py
"""
from __future__ import annotations
import json
import os
import sys
import tempfile
import uuid
from datetime import datetime, timezone
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO))
STAGING = "https://telemetry.staging-nousresearch.com/v1/telemetry"
def main() -> int:
scratch = Path(tempfile.mkdtemp(prefix="hermes-telemetry-e2e-"))
os.environ["HERMES_HOME"] = str(scratch)
# Staging is selected by writing config into the THROWAWAY profile, not by
# an environment override: a runtime env var that can retarget consented
# telemetry would be a consent hazard in production.
(scratch / "config.yaml").write_text(
"telemetry:\n"
" shared_metrics:\n"
" enabled: true\n"
" send: true\n"
f" endpoint: {STAGING}\n",
encoding="utf-8",
)
from hermes_cli.observability.shared_metrics import SharedMetricsStore
from hermes_cli.observability.shared_metrics_send_config import (
resolve_send_config,
)
from hermes_cli.observability.shared_metrics_sender import SharedMetricsSender
# Resolve through the real config path so this exercises what a user gets.
import yaml
resolved = resolve_send_config(
yaml.safe_load((scratch / "config.yaml").read_text(encoding="utf-8"))
)
if not resolved.send or resolved.endpoint != STAGING:
print(f"FAIL: config did not resolve to staging: {resolved}")
return 1
store = SharedMetricsStore(
database_path=scratch / "metrics.sqlite3",
outbox_directory=scratch / "outbox",
)
today = datetime.now(timezone.utc).date().isoformat()
# The generator only exports COMPLETED periods, so the realistic E2E
# package is yesterday's. It also has to be: the consent gate only
# releases a package once its whole period is confirmed consented, and
# today's period cannot be confirmed before it ends.
from datetime import timedelta
period_day = (
datetime.now(timezone.utc).date() - timedelta(days=1)
).isoformat()
# Open the consent window before the period, confirm it after — exactly
# what the runtime reconciler does across two days of hook fires.
from hermes_cli.observability.shared_metrics_sender import (
reconcile_send_consent,
)
from hermes_cli.sqlite_util import write_txn
with store._connection() as connection:
with write_txn(connection):
reconcile_send_consent(
connection,
True,
now=datetime.now(timezone.utc) - timedelta(days=2),
)
reconcile_send_consent(connection, True)
real_install_id = str(uuid.uuid4())
packages = []
# Two packages for today's period: the "head" and a later "tail", which is
# the real shape the outbox produces and the case the period gate exists
# for. One is large enough to exercise gzip.
for index, metric_count in ((0, 3), (1, 140)):
package_id = str(uuid.uuid4())
payload = {
"schema_version": "hermes.shared_metrics.v2",
"package_id": package_id,
"install_id": real_install_id,
"generated_at": datetime.now(timezone.utc).isoformat().replace(
"+00:00", "Z"
),
"period_start": f"{period_day}T00:00:00Z",
"period_end": f"{period_day}T23:59:59Z",
"resource": {
"hermes_version": "e2e-test",
"os_family": "macos",
"architecture": "arm64",
"install_method": "git",
},
"metrics": [
{
"name": f"hermes.e2e.metric.{i}",
"type": "counter",
"dimensions": {"outcome": "ok", "surface": "e2e"},
"value": i + 1,
}
for i in range(metric_count)
],
}
with store._connection() as connection:
connection.execute(
"""
INSERT INTO package_outbox(
package_id, period_start, period_end, payload_json,
created_at, exported_at
) VALUES (?, ?, ?, ?, ?, ?)
""",
(
package_id,
f"{period_day}T00:00:00Z",
f"{period_day}T23:59:59Z",
json.dumps(payload),
f"{period_day}T0{index}:00:00Z",
f"{period_day}T0{index}:00:01Z",
),
)
packages.append((package_id, metric_count))
print(f"scratch HERMES_HOME : {scratch}")
print(f"endpoint : {STAGING}")
print(f"local install_id : {real_install_id}")
print(f"packages queued : {len(packages)}")
for package_id, count in packages:
print(f" - {package_id} ({count} metrics)")
print()
outcome = SharedMetricsSender(store, resolved.endpoint).send_pending()
print(f"outcome: sent={outcome.sent} rejected={outcome.rejected} "
f"deferred={outcome.deferred}")
print()
failures = []
with store._connection() as connection:
rows = connection.execute(
"""
SELECT package_id, send_state, sent_at, send_attempts,
sent_install_id, last_error
FROM package_outbox ORDER BY created_at
"""
).fetchall()
for row in rows:
print(f"package : {row[0]}")
print(f" send_state : {row[1]}")
print(f" sent_at : {row[2]}")
print(f" attempts : {row[3]}")
print(f" transmitted : {row[4]}")
print(f" last_error : {row[5]}")
if row[1] != "sent":
failures.append(f"{row[0]} is {row[1]}: {row[5]}")
# Product decision 2026-08-27: the stable install_id is transmitted
# as-is; the transmitted value must be exactly the local id.
if row[4] != real_install_id:
failures.append(
f"{row[0]} transmitted {row[4]!r}, expected the install_id"
)
print()
if failures:
print("FAILURES:")
for failure in failures:
print(f" ✗ {failure}")
return 1
print("PASS: every package acknowledged 202 with the stable install_id.")
print()
print("Verify the objects in S3 with the package ids above:")
print(" aws s3 ls --recursive "
"s3://hermes-agent-telemetry-staging-767397871023-us-west-2-an/raw/ "
"| tail -20")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/generate_conformance_vectors.py
+266
View File
@@ -0,0 +1,266 @@
#!/usr/bin/env python3
"""Conformance-vector generator — the native adapters as executable spec.
Renders a shared corpus (markdown grid + scar tissue + adversarial agent
output) through the NATIVE platform renderers and dumps input→output JSON
vectors, stamped with the oracle commit. The gateway-gateway connector
commits these under conformance/vectors/ and its vitest runner asserts the
CONNECTOR's constructed REST payloads against them — so cross-repo renderer
drift breaks a test instead of a user's formatting.
Oracles (all imported, never reimplemented):
telegram plugins.platforms.telegram.adapter.TelegramAdapter.format_message
(standard markdown → Telegram MarkdownV2)
slack plugins.platforms.slack.adapter.SlackAdapter.format_message
(standard markdown → Slack mrkdwn)
whatsapp gateway.platforms.whatsapp_common.WhatsAppBehaviorMixin
.format_message (standard markdown → WhatsApp formatting)
discord plugins.platforms.discord.adapter.DiscordAdapter.format_message
(GFM tables → bullet groups; otherwise identity)
Expect semantics (consumed by the gg runner):
parity connector render must BYTE-EQUAL native_output
(Slack / WhatsApp — same-dialect ports; most Discord).
semantic connector renders a DIFFERENT representation on purpose
(Telegram: connector sends HTML, native sends MarkdownV2);
the runner asserts plain-text content equivalence instead.
divergent documented no-parity (e.g. Discord tables: native converts to
bullets, connector passes raw markdown through). The runner
asserts the DOCUMENTED connector behavior named in `note`.
Run: python scripts/generate_conformance_vectors.py [--out DIR]
Determinism is covered by tests/conformance/test_vector_generator.py.
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from pathlib import Path
from typing import Any, Callable, Dict, List
REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT))
GENERATOR_VERSION = 1
# ── corpus ───────────────────────────────────────────────────────────────
# Every entry: (id, category, input). Categories: grid | scar | adversarial.
# Ids are STABLE API — the runner and divergence notes key on them.
GRID: List[tuple] = [
("plain-text", "Just a plain sentence."),
("bold", "This is **bold** text."),
("italic", "This is *italic* text."),
("bold-italic", "Mix of **bold** and *italic* in one line."),
("strikethrough", "This is ~~struck~~ text."),
("inline-code", "Run `pip install hermes` to start."),
("fenced-code", "```\nprint('hello')\n```"),
("fenced-code-lang", "```python\ndef f(x):\n return x * 2\n```"),
("link", "See [the docs](https://example.com/docs) for more."),
("link-parens-url", "See [spec](https://example.com/a_(b)) here."),
("header-h1", "# Big Title\nBody follows."),
("header-h2", "## Section\nBody follows."),
("header-h3", "### Sub-section\nBody follows."),
("ul-list", "- first\n- second\n- third"),
("ol-list", "1. first\n2. second\n3. third"),
("nested-list", "- outer\n - inner one\n - inner two\n- outer two"),
("blockquote", "> quoted wisdom\nregular line"),
("hrule", "above\n\n---\n\nbelow"),
(
"table-simple",
"| name | value |\n|------|-------|\n| a | 1 |\n| b | 2 |",
),
("emoji", "Done ✅ with 🎉 emoji 👀 test."),
("cjk", "中文测试：**粗体** 和 `代码` 混排。"),
("bare-url", "Visit https://example.com/path?q=1&r=2 today."),
(
"mixed-document",
"## Report\n\nStatus: **green**. Details in `runbook.md`.\n\n"
"- item *one*\n- item **two**\n\n```sh\nmake deploy\n```\n\n"
"See [dashboard](https://grafana.example.com/d/x).",
),
]
SCAR: List[tuple] = [
# MarkdownV2 reserved characters in prose — the classic Telegram 400.
("mdv2-reserved-chars", "Price is 3.50 (was 4.00) — save ~12%! #deal +tax = win."),
("mdv2-underscores", "snake_case_name and file_name.py in prose."),
("mdv2-brackets", "Array[0] and dict{key} and (parens) live here."),
# Slack: **bold** must become *bold*; [t](u) must become <u|t>.
("slack-bold-conversion", "**important** word"),
("slack-link-conversion", "[click here](https://example.com)"),
# Slack broadcast-mention escape (model output must not ping @everyone).
("slack-broadcast-mention", "Hey <!everyone> and <!channel> and <!here>!"),
# Fence language tag handling (Slack renders the tag literally).
("fence-lang-tag-slack", "```text\nliteral first line issue\n```"),
# Backslashes inside code must survive doubling rules.
("backslash-in-code", "`C:\\Users\\ben\\file.txt` and ```\npath = \"a\\\\b\"\n```"),
("backtick-in-fence", "```\nuse `inline` inside fence\n```"),
# Headers containing bold markers (native strips redundant **).
("header-with-bold", "## The **Real** Deal"),
# Table with CJK cells (display-width alignment scar in Slack).
(
"table-cjk",
"| 名前 | 値 |\n|------|----|\n| 中文 | 42 |\n| b | 2 |",
),
# Link display text that itself needs escaping.
("link-display-escapes", "[v2.0 (beta)](https://example.com/v2)"),
]
ADVERSARIAL: List[tuple] = [
("media-tag", "Here you go\nMEDIA:/tmp/output.png\ndone"),
("unclosed-fence", "```python\nprint('never closed')"),
("pathological-nesting", "**bold *italic ~~struck `code` struck~~ italic* bold**"),
("placeholder-injection", "sneaky \x00PH0\x00 token and \x00SL1\x00 too"),
("triple-markers", "***what is this*** and ____that____"),
("empty-string", ""),
("whitespace-only", " \n\t\n "),
("long-line", "word " * 500),
("many-fences", "```\na\n```\nmid\n```\nb\n```\nend ```inline``` tail"),
]
def corpus() -> List[Dict[str, str]]:
rows: List[Dict[str, str]] = []
for cid, text in GRID:
rows.append({"id": cid, "category": "grid", "input": text})
for cid, text in SCAR:
rows.append({"id": cid, "category": "scar", "input": text})
for cid, text in ADVERSARIAL:
rows.append({"id": cid, "category": "adversarial", "input": text})
return rows
# ── oracles ──────────────────────────────────────────────────────────────
def _oracles() -> Dict[str, Callable[[str], str]]:
from plugins.platforms.telegram.adapter import TelegramAdapter
from plugins.platforms.slack.adapter import SlackAdapter
from plugins.platforms.discord.adapter import DiscordAdapter
from gateway.platforms.whatsapp_common import WhatsAppBehaviorMixin
wa = object.__new__(WhatsAppBehaviorMixin) # format_message needs no __init__
return {
# These format_message implementations are self-free (asserted by
# tests/conformance/test_vector_generator.py) — invoked unbound.
"telegram": lambda s: TelegramAdapter.format_message(None, s), # type: ignore[arg-type]
"slack": lambda s: SlackAdapter.format_message(None, s), # type: ignore[arg-type]
"discord": lambda s: DiscordAdapter.format_message(None, s), # type: ignore[arg-type]
"whatsapp": wa.format_message,
}
# Per-platform expect overrides (default: parity, except telegram=semantic).
# Keyed by vector id; value = (expect, note).
_EXPECT_OVERRIDES: Dict[str, Dict[str, tuple]] = {
"telegram": {
# Native wraps pipe tables into row groups; the connector's HTML lane
# renders tables as <pre>. Same content, structurally different enough
# that plain-text comparison is noise — documented divergence.
"table-simple": ("divergent", "native wraps tables into row groups; connector renders <pre> — content preserved, layout differs"),
"table-cjk": ("divergent", "same as table-simple (CJK width alignment is native-only)"),
"unclosed-fence": ("divergent", "unterminated fence: native escapes as prose, connector HTML may close the block — degraded either way, never a 400"),
"placeholder-injection": ("divergent", "NUL placeholder tokens are renderer-internal; each side neutralizes its own pattern"),
"whitespace-only": ("divergent", "native collapses to empty-ish prose, connector HTML preserves — cosmetic"),
},
"slack": {
"placeholder-injection": ("divergent", "\\x00SL tokens are the native renderer's own placeholder alphabet; connector uses a different scheme"),
"unclosed-fence": ("divergent", "unterminated fence handling differs; both degrade without dropping content"),
},
"whatsapp": {
"placeholder-injection": ("divergent", "placeholder alphabets are renderer-internal"),
"unclosed-fence": ("divergent", "unterminated fence handling differs; both degrade without dropping content"),
},
"discord": {
"table-simple": ("divergent", "native converts GFM tables to bullet groups; connector passes raw markdown through (port deferred — parity report Phase 4/oracle section)"),
"table-cjk": ("divergent", "same as table-simple"),
},
}
def _expect_for(platform: str, vector_id: str) -> tuple:
default = ("semantic", "") if platform == "telegram" else ("parity", "")
return _EXPECT_OVERRIDES.get(platform, {}).get(vector_id, default)
def _oracle_commit() -> str:
try:
return (
subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=REPO_ROOT,
capture_output=True,
text=True,
check=True,
).stdout.strip()
)
except Exception:
return "unknown"
def generate(out_dir: Path) -> Dict[str, Any]:
"""Render the corpus through every oracle; write one JSON per platform."""
oracles = _oracles()
commit = _oracle_commit()
rows = corpus()
summary: Dict[str, Any] = {}
out_dir.mkdir(parents=True, exist_ok=True)
for platform, render in sorted(oracles.items()):
vectors = []
for row in rows:
expect, note = _expect_for(platform, row["id"])
entry: Dict[str, Any] = {
"id": row["id"],
"category": row["category"],
"expect": expect,
"input": row["input"],
"native_output": render(row["input"]),
}
if note:
entry["note"] = note
vectors.append(entry)
doc = {
"$comment": (
"GENERATED — do not hand-edit. Regenerate with "
"hermes-agent scripts/generate_conformance_vectors.py; the "
"native renderers are the oracle (executable spec)."
),
"oracle": {
"repo": "NousResearch/hermes-agent",
"commit": commit,
"generator": "scripts/generate_conformance_vectors.py",
"generator_version": GENERATOR_VERSION,
},
"platform": platform,
"vectors": vectors,
}
path = out_dir / f"{platform}.json"
path.write_text(
json.dumps(doc, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
summary[platform] = len(vectors)
return summary
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--out",
default=str(REPO_ROOT / "tests" / "conformance" / "vectors"),
help="Output directory for <platform>.json vector files",
)
args = parser.parse_args()
summary = generate(Path(args.out))
for platform, count in sorted(summary.items()):
print(f"{platform}: {count} vectors")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/hermes-gateway
Executable
+423
View File
@@ -0,0 +1,423 @@
#!/usr/bin/env python3
"""
Hermes Gateway - Standalone messaging platform integration.
This is the proper entry point for running the gateway as a service.
NOT tied to the CLI - runs independently.
Usage:
# Run in foreground (for testing)
./scripts/hermes-gateway
# Install as systemd service
./scripts/hermes-gateway install
# Manage the service
./scripts/hermes-gateway start
./scripts/hermes-gateway stop
./scripts/hermes-gateway restart
./scripts/hermes-gateway status
# Uninstall
./scripts/hermes-gateway uninstall
"""
import argparse
import asyncio
import os
import subprocess
import sys
from pathlib import Path
# Add parent directory to path
SCRIPT_DIR = Path(__file__).parent.resolve()
PROJECT_DIR = SCRIPT_DIR.parent
sys.path.insert(0, str(PROJECT_DIR))
# Load .env file
from dotenv import load_dotenv
env_path = PROJECT_DIR / '.env'
if env_path.exists():
load_dotenv(dotenv_path=env_path)
# =============================================================================
# Service Configuration
# =============================================================================
SERVICE_NAME = "hermes-gateway"
SERVICE_DESCRIPTION = "Hermes Agent Gateway - Messaging Platform Integration"
def get_systemd_unit_path() -> Path:
"""Get the path for the systemd user service file."""
return Path.home() / ".config" / "systemd" / "user" / f"{SERVICE_NAME}.service"
def get_launchd_plist_path() -> Path:
"""Get the path for the launchd plist file (macOS)."""
return Path.home() / "Library" / "LaunchAgents" / f"ai.hermes.gateway.plist"
def get_python_path() -> str:
"""Get the path to the Python interpreter."""
# Prefer the venv if it exists
venv_python = PROJECT_DIR / "venv" / "bin" / "python"
if venv_python.exists():
return str(venv_python)
return sys.executable
def get_gateway_script_path() -> str:
"""Get the path to this script."""
return str(Path(__file__).resolve())
# =============================================================================
# Systemd Service (Linux)
# =============================================================================
def generate_systemd_unit() -> str:
"""Generate the systemd unit file content."""
python_path = get_python_path()
script_path = get_gateway_script_path()
working_dir = str(PROJECT_DIR)
return f"""[Unit]
Description={SERVICE_DESCRIPTION}
After=network.target
StartLimitIntervalSec=600
StartLimitBurst=5
[Service]
Type=simple
ExecStart={python_path} {script_path} run
WorkingDirectory={working_dir}
Restart=on-failure
RestartSec=30
StandardOutput=journal
StandardError=journal
# Environment (optional - can also use .env file)
# Environment="TELEGRAM_BOT_TOKEN=your_token"
# Environment="DISCORD_BOT_TOKEN=your_token"
[Install]
WantedBy=default.target
"""
def install_systemd():
"""Install the systemd user service."""
unit_path = get_systemd_unit_path()
unit_path.parent.mkdir(parents=True, exist_ok=True)
print(f"Installing systemd service to: {unit_path}")
unit_path.write_text(generate_systemd_unit())
# Reload systemd
subprocess.run(["systemctl", "--user", "daemon-reload"], check=True)
# Enable the service (start on boot)
subprocess.run(["systemctl", "--user", "enable", SERVICE_NAME], check=True)
print(f"✓ Service installed and enabled")
print(f"")
print(f"To start the service:")
print(f" systemctl --user start {SERVICE_NAME}")
print(f"")
print(f"To view logs:")
print(f" journalctl --user -u {SERVICE_NAME} -f")
print(f"")
print(f"To enable lingering (keeps service running after logout):")
print(f" sudo loginctl enable-linger $USER")
def uninstall_systemd():
"""Uninstall the systemd user service."""
unit_path = get_systemd_unit_path()
# Stop and disable first
subprocess.run(["systemctl", "--user", "stop", SERVICE_NAME], check=False)
subprocess.run(["systemctl", "--user", "disable", SERVICE_NAME], check=False)
# Remove the unit file
if unit_path.exists():
unit_path.unlink()
print(f"✓ Removed {unit_path}")
# Reload systemd
subprocess.run(["systemctl", "--user", "daemon-reload"], check=True)
print(f"✓ Service uninstalled")
def systemd_status():
"""Show systemd service status."""
subprocess.run(["systemctl", "--user", "status", SERVICE_NAME])
def systemd_start():
"""Start the systemd service."""
subprocess.run(["systemctl", "--user", "start", SERVICE_NAME], check=True)
print(f"✓ Service started")
def systemd_stop():
"""Stop the systemd service."""
subprocess.run(["systemctl", "--user", "stop", SERVICE_NAME], check=True)
print(f"✓ Service stopped")
def systemd_restart():
"""Restart the systemd service."""
subprocess.run(["systemctl", "--user", "restart", SERVICE_NAME], check=True)
print(f"✓ Service restarted")
# =============================================================================
# Launchd Service (macOS)
# =============================================================================
def generate_launchd_plist() -> str:
"""Generate the launchd plist file content."""
python_path = get_python_path()
script_path = get_gateway_script_path()
working_dir = str(PROJECT_DIR)
log_dir = Path.home() / ".hermes" / "logs"
return f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>ai.hermes.gateway</string>
<key>ProgramArguments</key>
<array>
<string>{python_path}</string>
<string>{script_path}</string>
<string>run</string>
</array>
<key>WorkingDirectory</key>
<string>{working_dir}</string>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<dict>
<key>SuccessfulExit</key>
<false/>
</dict>
<key>StandardOutPath</key>
<string>{log_dir}/gateway.log</string>
<key>StandardErrorPath</key>
<string>{log_dir}/gateway.error.log</string>
<key>EnvironmentVariables</key>
<dict>
<key>PATH</key>
<string>/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin</string>
</dict>
</dict>
</plist>
"""
def install_launchd():
"""Install the launchd service (macOS)."""
plist_path = get_launchd_plist_path()
plist_path.parent.mkdir(parents=True, exist_ok=True)
# Ensure log directory exists
log_dir = Path.home() / ".hermes" / "logs"
log_dir.mkdir(parents=True, exist_ok=True)
print(f"Installing launchd service to: {plist_path}")
plist_path.write_text(generate_launchd_plist())
# Load the service
subprocess.run(["launchctl", "load", str(plist_path)], check=True)
print(f"✓ Service installed and loaded")
print(f"")
print(f"To view logs:")
print(f" tail -f ~/.hermes/logs/gateway.log")
print(f"")
print(f"To manage the service:")
print(f" launchctl start ai.hermes.gateway")
print(f" launchctl stop ai.hermes.gateway")
def uninstall_launchd():
"""Uninstall the launchd service (macOS)."""
plist_path = get_launchd_plist_path()
# Unload first
subprocess.run(["launchctl", "unload", str(plist_path)], check=False)
# Remove the plist file
if plist_path.exists():
plist_path.unlink()
print(f"✓ Removed {plist_path}")
print(f"✓ Service uninstalled")
def launchd_status():
"""Show launchd service status."""
subprocess.run(["launchctl", "list", "ai.hermes.gateway"])
def launchd_start():
"""Start the launchd service."""
subprocess.run(["launchctl", "start", "ai.hermes.gateway"], check=True)
print(f"✓ Service started")
def launchd_stop():
"""Stop the launchd service."""
subprocess.run(["launchctl", "stop", "ai.hermes.gateway"], check=True)
print(f"✓ Service stopped")
def launchd_restart():
"""Restart the launchd service."""
launchd_stop()
launchd_start()
# =============================================================================
# Platform Detection
# =============================================================================
def is_linux() -> bool:
return sys.platform.startswith('linux')
def is_macos() -> bool:
return sys.platform == 'darwin'
def is_windows() -> bool:
return sys.platform == 'win32'
# =============================================================================
# Gateway Runner
# =============================================================================
def run_gateway():
"""Run the gateway in foreground."""
# Startup-liveness watchdog (OOF-298): arm before importing the gateway
# graph so an import-time or pre-loop deadlock still gets respawned.
try:
from hermes_startup_watchdog import arm_startup_watchdog
arm_startup_watchdog()
except Exception:
pass
from gateway.run import start_gateway
print("Starting Hermes Gateway...")
print("Press Ctrl+C to stop.")
print()
asyncio.run(start_gateway())
# =============================================================================
# Main CLI
# =============================================================================
def main():
parser = argparse.ArgumentParser(
description="Hermes Gateway - Messaging Platform Integration",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
# Run in foreground (for testing)
./scripts/hermes-gateway run
# Install as system service
./scripts/hermes-gateway install
# Manage the service
./scripts/hermes-gateway start
./scripts/hermes-gateway stop
./scripts/hermes-gateway restart
./scripts/hermes-gateway status
# Uninstall
./scripts/hermes-gateway uninstall
Configuration:
Set environment variables in .env file or system environment:
- TELEGRAM_BOT_TOKEN
- DISCORD_BOT_TOKEN
- WHATSAPP_ENABLED
Or create ~/.hermes/gateway.json for advanced configuration.
"""
)
parser.add_argument(
"command",
choices=["run", "install", "uninstall", "start", "stop", "restart", "status"],
nargs="?",
default="run",
help="Command to execute (default: run)"
)
parser.add_argument(
"--verbose", "-v",
action="store_true",
help="Verbose output"
)
args = parser.parse_args()
# Detect platform and dispatch command
if args.command == "run":
run_gateway()
elif args.command == "install":
if is_linux():
install_systemd()
elif is_macos():
install_launchd()
else:
print("Service installation not supported on this platform.")
print("Please run manually: ./scripts/hermes-gateway run")
sys.exit(1)
elif args.command == "uninstall":
if is_linux():
uninstall_systemd()
elif is_macos():
uninstall_launchd()
else:
print("Service uninstallation not supported on this platform.")
sys.exit(1)
elif args.command == "start":
if is_linux():
systemd_start()
elif is_macos():
launchd_start()
else:
print("Not supported on this platform.")
sys.exit(1)
elif args.command == "stop":
if is_linux():
systemd_stop()
elif is_macos():
launchd_stop()
else:
print("Not supported on this platform.")
sys.exit(1)
elif args.command == "restart":
if is_linux():
systemd_restart()
elif is_macos():
launchd_restart()
else:
print("Not supported on this platform.")
sys.exit(1)
elif args.command == "status":
if is_linux():
systemd_status()
elif is_macos():
launchd_status()
else:
print("Not supported on this platform.")
sys.exit(1)
if __name__ == "__main__":
main()
scripts/install.cmd
+28
View File
@@ -0,0 +1,28 @@
@echo off
REM ============================================================================
REM Hermes Agent Installer for Windows (CMD wrapper)
REM ============================================================================
REM This batch file launches the PowerShell installer for users running CMD.
REM
REM Usage:
REM curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.cmd -o install.cmd && install.cmd && del install.cmd
REM
REM Or if you're already in PowerShell, use the direct command instead:
REM iex (irm https://hermes-agent.nousresearch.com/install.ps1)
REM ============================================================================
echo.
echo Hermes Agent Installer
echo Launching PowerShell installer...
echo.
powershell -ExecutionPolicy ByPass -NoProfile -Command "iex (irm https://hermes-agent.nousresearch.com/install.ps1)"
if %ERRORLEVEL% NEQ 0 (
echo.
echo Installation failed. Please try running PowerShell directly:
echo powershell -ExecutionPolicy ByPass -c "iex (irm https://hermes-agent.nousresearch.com/install.ps1)"
echo.
pause
exit /b 1
)
scripts/install.ps1
+5070
View File
File diff suppressed because it is too large Load Diff
scripts/install.sh
Executable
+3890
View File
File diff suppressed because it is too large Load Diff
scripts/install_psutil_android.py
Executable
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""Install psutil on Termux/Android by patching upstream platform detection.
psutil's setup currently gates Linux sources behind
``sys.platform.startswith('linux')``. On Termux, Python reports
``sys.platform == 'android'``, so ``pip install psutil`` aborts with
"platform android is not supported" — even though psutil compiles fine
when the Linux source path is reused.
This script downloads the official psutil sdist, applies a one-line
patch (``LINUX = sys.platform.startswith(("linux", "android"))``), and
installs the patched tree with ``pip install --no-build-isolation``.
Usage:
python scripts/install_psutil_android.py [--pip "/path/to/pip"] [--uv]
When neither flag is given, the script auto-detects ``uv`` on PATH and
falls back to ``<sys.executable> -m pip``.
This is a stopgap. Remove once psutil upstream merges
https://github.com/giampaolo/psutil/pull/2762 and ships a release.
"""
from __future__ import annotations
import argparse
import shutil
import subprocess
import sys
import tempfile
import urllib.request
from pathlib import Path
# Keep sibling imports working when invoked as
# ``python scripts/install_psutil_android.py`` from the repo checkout.
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from hermes_cli.psutil_android import (
PSUTIL_URL,
PsutilAndroidInstallError,
prepare_patched_psutil_sdist,
)
def _resolve_install_cmd(pip_arg: str | None, prefer_uv: bool) -> list[str]:
if pip_arg:
return pip_arg.split()
if prefer_uv:
uv = shutil.which("uv")
if not uv:
sys.exit("--uv requested but no uv on PATH")
return [uv, "pip"]
auto_uv = shutil.which("uv")
if auto_uv:
return [auto_uv, "pip"]
return [sys.executable, "-m", "pip"]
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--pip",
help="Explicit installer command (e.g. '/usr/bin/uv pip' or 'python -m pip')",
)
parser.add_argument(
"--uv",
action="store_true",
help="Force using uv (errors out if uv is not on PATH)",
)
args = parser.parse_args()
install_cmd_prefix = _resolve_install_cmd(args.pip, args.uv)
print(
"→ Termux/Android: prebuilding psutil with Linux source path "
"compatibility shim (see psutil#2762)..."
)
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
archive = tmp_path / "psutil.tar.gz"
urllib.request.urlretrieve(PSUTIL_URL, archive)
try:
src_root = prepare_patched_psutil_sdist(archive, tmp_path)
except PsutilAndroidInstallError as exc:
sys.exit(str(exc))
cmd = install_cmd_prefix + ["install", "--no-build-isolation", str(src_root)]
print(f" $ {' '.join(cmd)}")
result = subprocess.run(cmd)
if result.returncode != 0:
return result.returncode
print("✓ psutil installed via Android compatibility shim")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/iso-certify.py
Executable
+625
View File
@@ -0,0 +1,625 @@
#!/usr/bin/env python3
"""iso-certify — AC-4 dashboard turn-isolation certify harness.
Certifies Mechanism-B process isolation
(``docs/desktop/2026-07-04-dashboard-process-isolation-PRD.md``, AC-4): under
6 concurrent heavy agent turns, the dashboard's HTTP/ws SERVING plane must stay
responsive (p99 < 1s) with zero event-loop stalls.
What it does
------------
1. Spawns a SCRATCH dashboard (``hermes dashboard``) bound to loopback on a
free port, with an ISOLATED ``HERMES_HOME`` (temp dir, minimal seeded state).
It NEVER touches the live :9119 dashboard / ai.hermes.dashboard / live
state.db. Loopback bind ⇒ no auth gate (web_server.should_require_auth).
2. Arms the synthetic GIL-heavy turn seam (``HERMES_ISO_CERTIFY_SYNTH_TURN=1``,
see ``tui_gateway/synthetic_turn.py``) so 6 concurrent turns reproduce the
``take_gil`` interpreter-contention regime WITHOUT real model calls. A
network/sleep stub would release the GIL and NOT reproduce the incident, so
it would be a fake green — the synthetic turn is pure-Python CPU on purpose.
3. Drives 6 concurrent heavy turns over ws (session.create → prompt.submit),
and CONCURRENTLY probes the serving path — a ws ``session.list`` round-trip
AND a REST ``GET /api/status`` — every 500ms, timing each.
4. Reports p50/p95/p99 latency for both probes + the count of probes over the
1s threshold ("serving stalls") + the count of ``event loop stalled`` /
``ws write slow`` lines the dashboard logged during the run.
Verdict
-------
The AC-4 verdict comes ONLY from the heavy run: PASS iff serving p99 < 1s AND
zero serving stalls AND zero ``event loop stalled`` log lines over the sustained
window. ``--dry-run`` runs ONE short light turn as a plumbing smoke test and is
explicitly NOT a verdict (a dry-run green is a fake green — the spec says so).
Turn isolation is controlled by the ``dashboard.turn_isolation`` config knob in
the scratch HERMES_HOME; ``--isolation on|off`` sets it. Run BOTH:
iso-certify --isolation off # baseline: expect stalls
iso-certify --isolation on # the measurement that decides AC-4
"""
from __future__ import annotations
import argparse
import json
import os
import re
import socket
import subprocess
import sys
import tempfile
import threading
import time
import shutil
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any
try:
from websockets.sync.client import connect as ws_connect
except Exception as exc: # pragma: no cover - dependency guard
print(f"iso-certify requires the 'websockets' package: {exc}", file=sys.stderr)
raise
REPO_ROOT = Path(__file__).resolve().parents[1]
_READY_RE = re.compile(r"HERMES_(?:DASHBOARD|BACKEND)_READY port=(\d+)")
_STALL_LOG_RE = re.compile(r"event loop stalled|ws write slow \(loop stalled")
# ── stats ──────────────────────────────────────────────────────────────
def percentile(values: list[float], pct: float) -> float:
if not values:
return 0.0
ordered = sorted(values)
if len(ordered) == 1:
return ordered[0]
rank = (len(ordered) - 1) * (pct / 100.0)
lo = int(rank)
hi = min(lo + 1, len(ordered) - 1)
frac = rank - lo
return ordered[lo] * (1.0 - frac) + ordered[hi] * frac
def summarize(values: list[float]) -> dict[str, float]:
return {
"count": len(values),
"p50_ms": percentile(values, 50),
"p95_ms": percentile(values, 95),
"p99_ms": percentile(values, 99),
"max_ms": max(values) if values else 0.0,
}
def free_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("127.0.0.1", 0))
return int(s.getsockname()[1])
# ── scratch HERMES_HOME ─────────────────────────────────────────────────
def seed_scratch_home(home: Path, *, isolation: str, heartbeat_secs: int, respawn_max: int) -> None:
"""Write a minimal config.yaml with the isolation knob set."""
home.mkdir(parents=True, exist_ok=True)
(home / "state").mkdir(parents=True, exist_ok=True)
(home / "logs").mkdir(parents=True, exist_ok=True)
cfg = {
# Pin the model/provider to what SyntheticHeavyAgent reports so the
# per-turn _sync_agent_model_with_config sees a match and no-ops — a
# mismatch would try (and fail) a real model switch, erroring the turn
# before its heavy loop runs (which would be a FALSE green: serving
# stays responsive because no heavy compute happened). The synthetic
# seam means no real API call is ever made regardless.
"provider": "synthetic",
"model": "synthetic-heavy",
"dashboard": {
"turn_isolation": (isolation == "on"),
"compute_host_heartbeat_secs": heartbeat_secs,
"compute_host_respawn_max": respawn_max,
},
# Keep memory/mem0/skills side-machinery from reaching out.
"memory": {"enabled": False},
}
# config.yaml is the canonical config; write it directly.
import yaml # provided by the runtime venv
(home / "config.yaml").write_text(yaml.safe_dump(cfg, sort_keys=True), encoding="utf-8")
# A stub .env so credential resolution doesn't spelunk the real home.
(home / ".env").write_text("OPENAI_API_KEY=sk-synthetic-not-used\n", encoding="utf-8")
# ── dashboard process ───────────────────────────────────────────────────
class ScratchDashboard:
def __init__(
self,
*,
home: Path,
port: int,
isolation: str,
env_extra: dict[str, str],
) -> None:
self.home = home
self.port = port
self.isolation = isolation
self.env_extra = env_extra
self.proc: subprocess.Popen[str] | None = None
self.actual_port = port
self.log_lines: list[str] = []
self._log_lock = threading.Lock()
self._ready = threading.Event()
@property
def stall_log_count(self) -> int:
with self._log_lock:
return sum(1 for ln in self.log_lines if _STALL_LOG_RE.search(ln))
def _drain(self, stream: Any) -> None:
for raw in stream:
line = raw.rstrip("\n")
with self._log_lock:
self.log_lines.append(line)
m = _READY_RE.search(line)
if m:
self.actual_port = int(m.group(1))
self._ready.set()
def __enter__(self) -> "ScratchDashboard":
venv_py = REPO_ROOT / "venv" / "bin" / "python"
python = str(venv_py) if venv_py.exists() else sys.executable
env = dict(os.environ)
env.update(self.env_extra)
env["HERMES_HOME"] = str(self.home)
env["HOME"] = str(self.home.parent) if str(self.home.parent) else env.get("HOME", "")
env["HERMES_HOME"] = str(self.home)
env["PYTHONPATH"] = str(REPO_ROOT) + os.pathsep + env.get("PYTHONPATH", "")
env["HERMES_ISO_CERTIFY_SYNTH_TURN"] = "1"
cmd = [
python, "-m", "hermes_cli.main", "dashboard",
"--no-open", "--host", "127.0.0.1", "--port", str(self.port),
]
self.proc = subprocess.Popen(
cmd, cwd=str(REPO_ROOT), env=env,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", bufsize=1,
)
threading.Thread(target=self._drain, args=(self.proc.stdout,), name="dash-log", daemon=True).start()
if not self._ready.wait(timeout=90.0):
self._dump_tail()
raise RuntimeError("scratch dashboard did not become ready within 90s")
# Give uvicorn a beat to actually bind the ws route.
time.sleep(1.0)
return self
def __exit__(self, *_exc: object) -> None:
if self.proc is None:
return
try:
self.proc.terminate()
self.proc.wait(timeout=10)
except Exception:
try:
self.proc.kill()
except Exception:
pass
def _dump_tail(self, n: int = 40) -> None:
with self._log_lock:
tail = self.log_lines[-n:]
sys.stderr.write("---- scratch dashboard log tail ----\n")
for ln in tail:
sys.stderr.write(ln + "\n")
sys.stderr.write("------------------------------------\n")
# ── ws client (one connection = one lane) ───────────────────────────────
class WSClient:
def __init__(self, port: int, token: str) -> None:
self.url = f"ws://127.0.0.1:{port}/api/ws?token={token}"
self.ws = ws_connect(
self.url,
open_timeout=15,
max_size=None,
additional_headers={"Origin": f"http://127.0.0.1:{port}"},
)
self._id = 0
self._lock = threading.Lock()
# Drain the gateway.ready event.
self._recv_until(lambda o: o.get("method") == "event", timeout=10)
def _next_id(self) -> str:
with self._lock:
self._id += 1
return f"r{self._id}"
def _recv_until(self, pred, timeout: float) -> dict:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
raw = self.ws.recv(timeout=max(0.05, deadline - time.monotonic()))
except TimeoutError:
continue
try:
obj = json.loads(raw)
except Exception:
continue
if pred(obj):
return obj
raise TimeoutError("ws recv predicate timed out")
def rpc(self, method: str, params: dict, timeout: float = 30.0) -> dict:
rid = self._next_id()
self.ws.send(json.dumps({"jsonrpc": "2.0", "id": rid, "method": method, "params": params}))
return self._recv_until(lambda o: o.get("id") == rid, timeout=timeout)
def send_only(self, method: str, params: dict) -> str:
rid = self._next_id()
self.ws.send(json.dumps({"jsonrpc": "2.0", "id": rid, "method": method, "params": params}))
return rid
def close(self) -> None:
try:
self.ws.close()
except Exception:
pass
# ── heavy-turn lane ─────────────────────────────────────────────────────
def drive_heavy_turn(port: int, token: str, turn_spec: dict, stop_at: float, results: list[dict]) -> None:
"""One lane: create a session, submit heavy turns until the deadline."""
lane: dict[str, Any] = {"turns": 0, "errors": [], "turn_durations_s": [], "min_deltas": None}
try:
cli = WSClient(port, token)
except Exception as exc:
lane["errors"].append(f"connect: {exc}")
results.append(lane)
return
try:
resp = cli.rpc("session.create", {"cols": 80, "source": "iso-certify"}, timeout=30)
sid = ((resp.get("result") or {}).get("session_id")) or ((resp.get("result") or {}).get("id"))
if not sid:
lane["errors"].append(f"session.create bad resp: {resp}")
results.append(lane)
return
lane["sid"] = sid
spec_text = json.dumps(turn_spec)
while time.monotonic() < stop_at:
# prompt.submit returns immediately ("streaming"); the turn runs async.
r = cli.rpc("prompt.submit", {"session_id": sid, "text": spec_text}, timeout=30)
if r.get("error"):
lane["errors"].append(f"prompt.submit: {r['error']}")
break
# Wait for the REAL turn boundary. The isolated path emits
# session.info MID-turn (metadata mirror), so waiting on it would
# false-complete a turn in <1s and inflate the count while NO heavy
# compute ran — the acceptance-gate "proxy not effect" trap. The
# turn is done only on message.complete. Count deltas + duration so
# a fast-erroring turn (e.g. a failed model switch) is caught, not
# masked as sustained load.
per_turn_budget = turn_spec.get("duration_s", 8.0) + 30.0
deadline = time.monotonic() + per_turn_budget
turn_start = time.monotonic()
deltas = 0
started = False
done = False
errored = False
while time.monotonic() < deadline:
try:
o = cli._recv_until(
lambda o: o.get("method") == "event"
and (o.get("params") or {}).get("type")
in {"message.start", "message.delta", "message.complete", "error"},
timeout=max(0.5, deadline - time.monotonic()),
)
except TimeoutError:
break
ptype = (o.get("params") or {}).get("type")
if ptype == "message.start":
started = True
turn_start = time.monotonic()
deltas = 0
continue
# prompt.submit's RPC response may race with a duplicate terminal
# event from the previous turn. Only events after this turn's
# message.start can satisfy or score its load boundary.
if not started:
continue
if ptype == "message.delta":
deltas += 1
continue
if ptype == "error":
msg = str(((o.get("params") or {}).get("payload") or {}).get("message") or "")
lane["errors"].append(f"turn error: {msg[:160]}")
errored = True
break
if ptype == "message.complete":
done = True
break
turn_dur = time.monotonic() - turn_start
if done:
lane["turns"] += 1
lane["turn_durations_s"].append(round(turn_dur, 2))
lane["min_deltas"] = deltas if lane["min_deltas"] is None else min(lane["min_deltas"], deltas)
if errored:
break
except Exception as exc:
lane["errors"].append(f"lane: {exc}")
finally:
cli.close()
results.append(lane)
# ── serving-path probes ─────────────────────────────────────────────────
def warmup_serving(port: int, token: str, rounds: int = 6) -> None:
"""Prime serving-path caches before the measured window.
The first ``/api/status`` on a freshly-booted process pays one-time
cold-start cost (config-version check, gateway-health probe, DB connect) that
a real dashboard — warm for hours before the incident regime — never pays
during a stall. AC-4 measures sustained-load responsiveness, not cold boot,
so we hit both serving endpoints a few times UNMEASURED first. This is not a
green-washing shortcut: the measured window still runs the full 6-lane heavy
load; warmup only removes a one-time boot artifact from the p99.
"""
rest_url = f"http://127.0.0.1:{port}/api/status"
warm_ws: WSClient | None = None
try:
warm_ws = WSClient(port, token)
except Exception:
warm_ws = None
for _ in range(rounds):
try:
with urllib.request.urlopen(rest_url, timeout=30) as fh:
fh.read()
except Exception:
pass
if warm_ws is not None:
try:
warm_ws.rpc("session.list", {"limit": 20}, timeout=30)
except Exception:
pass
time.sleep(0.2)
if warm_ws is not None:
warm_ws.close()
def probe_loop(port: int, token: str, stop_at: float, cadence_s: float, ws_samples: list[float], rest_samples: list[float]) -> None:
"""Probe ws session.list + REST /api/status every ``cadence_s`` seconds."""
probe_ws: WSClient | None = None
try:
probe_ws = WSClient(port, token)
except Exception:
probe_ws = None
rest_url = f"http://127.0.0.1:{port}/api/status"
next_tick = time.monotonic()
while time.monotonic() < stop_at:
# ws round-trip (session.list — the serving-plane DB read AC-4 protects).
if probe_ws is not None:
t0 = time.perf_counter()
try:
probe_ws.rpc("session.list", {"limit": 20}, timeout=30)
ws_samples.append((time.perf_counter() - t0) * 1000.0)
except Exception:
# A failed/timed-out probe is itself a serving stall; record the
# elapsed as the sample so it counts against p99.
ws_samples.append((time.perf_counter() - t0) * 1000.0)
try:
probe_ws.close()
probe_ws = WSClient(port, token)
except Exception:
probe_ws = None
# REST round-trip.
t1 = time.perf_counter()
try:
with urllib.request.urlopen(rest_url, timeout=30) as fh:
fh.read()
rest_samples.append((time.perf_counter() - t1) * 1000.0)
except Exception:
rest_samples.append((time.perf_counter() - t1) * 1000.0)
next_tick += cadence_s
sleep_for = next_tick - time.monotonic()
if sleep_for > 0:
time.sleep(sleep_for)
else:
next_tick = time.monotonic()
if probe_ws is not None:
probe_ws.close()
# ── run ─────────────────────────────────────────────────────────────────
def run_certify(args: argparse.Namespace) -> dict[str, Any]:
port = free_port()
import secrets
token = secrets.token_urlsafe(24)
parent_tmp = Path(tempfile.mkdtemp(prefix="iso-certify-"))
home = parent_tmp / "hermes-home"
seed_scratch_home(
home,
isolation=args.isolation,
heartbeat_secs=args.heartbeat_secs,
respawn_max=args.respawn_max,
)
concurrency = 1 if args.dry_run else args.concurrency
duration_s = 3.0 if args.dry_run else args.duration_s
turn_duration = 0.5 if args.dry_run else args.turn_duration_s
threshold_ms = args.threshold_ms
turn_spec = {
"duration_s": turn_duration,
"delta_interval_s": args.delta_interval_s,
"tokens_per_delta": args.tokens_per_delta,
"chunk": args.chunk,
}
result: dict[str, Any] = {
"mode": "dry-run" if args.dry_run else "heavy",
"isolation": args.isolation,
"concurrency": concurrency,
"run_duration_s": duration_s,
"turn_spec": turn_spec,
"threshold_ms": threshold_ms,
"scratch_home": str(home),
"port": port,
}
try:
with ScratchDashboard(
home=home, port=port, isolation=args.isolation,
env_extra={"HERMES_DASHBOARD_SESSION_TOKEN": token},
) as dash:
actual_port = dash.actual_port
result["port"] = actual_port
ws_samples: list[float] = []
rest_samples: list[float] = []
lane_results: list[dict] = []
stall_before = dash.stall_log_count
# Prime serving-path caches so a one-time cold-start artifact does
# not count as a serving stall (only for the real heavy run; a
# dry-run stays a raw plumbing smoke).
if not args.dry_run:
warmup_serving(actual_port, token)
stall_before = dash.stall_log_count
stop_at = time.monotonic() + duration_s
probe_thread = threading.Thread(
target=probe_loop,
args=(actual_port, token, stop_at, args.cadence_s, ws_samples, rest_samples),
name="probe", daemon=True,
)
probe_thread.start()
with ThreadPoolExecutor(max_workers=concurrency) as pool:
for _ in range(concurrency):
pool.submit(drive_heavy_turn, actual_port, token, turn_spec, stop_at, lane_results)
# Pool context waits for all lanes.
probe_thread.join(timeout=30)
stall_after = dash.stall_log_count
total_turns = sum(l.get("turns", 0) for l in lane_results)
lane_errors = [e for l in lane_results for e in l.get("errors", [])]
all_durations = [d for l in lane_results for d in l.get("turn_durations_s", [])]
min_deltas = [l["min_deltas"] for l in lane_results if l.get("min_deltas") is not None]
lanes_with_turn = sum(1 for l in lane_results if l.get("turns", 0) > 0)
ws_stat = summarize(ws_samples)
rest_stat = summarize(rest_samples)
ws_over = sum(1 for v in ws_samples if v > threshold_ms)
rest_over = sum(1 for v in rest_samples if v > threshold_ms)
serving_stalls = ws_over + rest_over
log_stalls = stall_after - stall_before
# Load validity — the run only certifies anything if the offered load
# was REAL: every lane completed ≥1 turn, and the completed turns
# actually held ~the requested heavy duration and streamed deltas.
# A fast-erroring/short turn is NOT sustained GIL load, so a green off
# it would be a proxy (serving stays fast because nothing burned).
median_turn_dur = percentile(all_durations, 50) if all_durations else 0.0
min_turn_dur = min(all_durations) if all_durations else 0.0
worst_min_deltas = min(min_deltas) if min_deltas else 0
expected_turn_s = float(turn_spec.get("duration_s", 8.0))
load_valid = (
lanes_with_turn >= concurrency
and total_turns >= concurrency
and min_turn_dur >= 0.7 * expected_turn_s
and worst_min_deltas >= 5
)
result.update({
"ws_probe": ws_stat,
"rest_probe": rest_stat,
"ws_probes_over_threshold": ws_over,
"rest_probes_over_threshold": rest_over,
"serving_stalls": serving_stalls,
"event_loop_stall_log_lines": log_stalls,
"heavy_turns_completed": total_turns,
"lanes_with_turn": lanes_with_turn,
"median_turn_duration_s": round(median_turn_dur, 2),
"min_turn_duration_s": round(min_turn_dur, 2),
"worst_lane_min_deltas": worst_min_deltas,
"load_valid": load_valid,
"lane_errors": lane_errors[:20],
})
# AC-4 verdict — heavy run only. A dry-run reports but never PASSes.
serving_p99 = max(ws_stat["p99_ms"], rest_stat["p99_ms"])
result["serving_p99_ms"] = serving_p99
if args.dry_run:
result["verdict"] = "SMOKE-OK" if (total_turns > 0 and not lane_errors) else "SMOKE-FAIL"
result["is_verdict"] = False
else:
serving_ok = (
serving_p99 < threshold_ms
and serving_stalls == 0
and log_stalls == 0
and probe_thread_samples_ok(ws_samples, rest_samples)
)
if not load_valid:
# Cannot certify: the offered load was not the incident
# regime. Never a PASS; report INCONCLUSIVE, not FAIL, so it
# is not read as "isolation broke serving".
result["verdict"] = "INCONCLUSIVE"
result.setdefault("notes", []).append(
"load invalid: lanes/turn-duration/deltas below the sustained-heavy-load floor — "
"not the AC-4 incident regime, verdict cannot certify"
)
else:
result["verdict"] = "PASS" if serving_ok else "FAIL"
result["is_verdict"] = True
finally:
if not args.keep_home:
shutil.rmtree(parent_tmp, ignore_errors=True)
else:
result["scratch_home_kept"] = str(parent_tmp)
return result
def probe_thread_samples_ok(ws_samples: list[float], rest_samples: list[float]) -> bool:
"""Guard against a blind gate: require the probes actually ran.
A run that produced no probe samples saw NOTHING — it must not PASS. This is
the tri-state INCONCLUSIVE floor (an empty timeline is never a green).
"""
return len(ws_samples) >= 3 and len(rest_samples) >= 3
def main(argv: list[str] | None = None) -> int:
p = argparse.ArgumentParser(description="AC-4 dashboard turn-isolation certify harness")
p.add_argument("--isolation", choices=["on", "off"], default="on",
help="set dashboard.turn_isolation in the scratch HERMES_HOME")
p.add_argument("--dry-run", action="store_true",
help="1 short light turn plumbing smoke — NOT an AC-4 verdict")
p.add_argument("--concurrency", type=int, default=6, help="concurrent heavy-turn lanes (AC-4: 6)")
p.add_argument("--duration-s", type=float, default=600.0, dest="duration_s",
help="sustained run window seconds (AC-4: ~600 = 10 min)")
p.add_argument("--turn-duration-s", type=float, default=12.0, dest="turn_duration_s",
help="wall seconds of GIL-holding compute per heavy turn")
p.add_argument("--delta-interval-s", type=float, default=0.05, dest="delta_interval_s",
help="streamed-delta cadence per heavy turn")
p.add_argument("--tokens-per-delta", type=int, default=512, dest="tokens_per_delta")
p.add_argument("--chunk", type=int, default=20000, help="pure-Python ops per interrupt-check chunk")
p.add_argument("--cadence-s", type=float, default=0.5, dest="cadence_s",
help="serving-path probe cadence (AC-4: 500ms)")
p.add_argument("--threshold-ms", type=float, default=1000.0, dest="threshold_ms",
help="serving p99 threshold (AC-4: <1s)")
p.add_argument("--heartbeat-secs", type=int, default=15, dest="heartbeat_secs")
p.add_argument("--respawn-max", type=int, default=3, dest="respawn_max")
p.add_argument("--keep-home", action="store_true", help="do not delete the scratch HERMES_HOME on exit")
p.add_argument("--json-out", type=Path, help="write JSON metrics to this path")
args = p.parse_args(argv)
result = run_certify(args)
text = json.dumps(result, indent=2, sort_keys=True)
print(text)
if args.json_out:
args.json_out.parent.mkdir(parents=True, exist_ok=True)
args.json_out.write_text(text + "\n", encoding="utf-8")
# Exit code: 0 only on a real PASS (or a clean dry-run smoke); non-zero
# otherwise. A dry-run is never treated as a verdict for automation.
if result.get("is_verdict"):
return 0 if result.get("verdict") == "PASS" else 1
return 0 if result.get("verdict") == "SMOKE-OK" else 1
if __name__ == "__main__":
raise SystemExit(main())
scripts/keystroke_diagnostic.py
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env python3
"""Diagnose how prompt_toolkit identifies keystrokes in the current terminal.
Useful when adding a keybinding to Hermes (or any prompt_toolkit app) and you
need to know what the terminal actually delivers — particularly on Windows,
where terminals can collapse, intercept, or silently remap key combinations.
Usage:
# POSIX
python scripts/keystroke_diagnostic.py
# Windows (PowerShell / git-bash / cmd)
python scripts\\keystroke_diagnostic.py
Press the key combinations you care about. Each keystroke prints the
prompt_toolkit `Keys.*` identifier and the raw escape bytes the terminal
sent. The last 20 keystrokes stay on screen. Ctrl+Q or Ctrl+C to quit.
Common questions this answers:
- Does my terminal distinguish Ctrl+Enter from plain Enter?
(On Windows Terminal: yes, Ctrl+Enter → c-j, Enter → c-m.)
- Does Alt+Enter reach the app, or does the terminal eat it?
(Windows Terminal eats it for fullscreen; mintty may too.)
- Does Shift+Enter register as a separate key?
(Almost never — most terminals collapse it to Enter.)
- What byte sequence does Home/End/PageUp/etc. produce?
Example output for Ctrl+Enter on Windows Terminal + PowerShell:
key=<Keys.ControlJ: 'c-j'> data='\\n'
Then in Hermes, bind the newline behaviour to that key:
@kb.add('c-j')
def handle_ctrl_enter(event):
event.current_buffer.insert_text('\\n')
"""
from prompt_toolkit import Application
from prompt_toolkit.key_binding import KeyBindings
from prompt_toolkit.layout import Layout
from prompt_toolkit.layout.containers import Window
from prompt_toolkit.layout.controls import FormattedTextControl
_HISTORY: list[str] = []
def _header() -> list[str]:
return [
"Keystroke diagnostic — press keys to see how prompt_toolkit sees them.",
"Try: Enter, Ctrl+Enter, Shift+Enter, Alt+Enter, Ctrl+J, Ctrl+M, arrows, Home/End.",
"Ctrl+Q or Ctrl+C to quit. Last 20 keystrokes shown.",
"",
]
def _render_text() -> str:
return "\n".join(_header() + _HISTORY[-20:])
def main() -> None:
kb = KeyBindings()
@kb.add("<any>")
def _on_any(event): # noqa: ANN001 — prompt_toolkit event type
parts = []
for kp in event.key_sequence:
parts.append(f"key={kp.key!r} data={kp.data!r}")
_HISTORY.append(" | ".join(parts))
event.app.invalidate()
@kb.add("c-q")
@kb.add("c-c")
def _quit(event): # noqa: ANN001
event.app.exit()
control = FormattedTextControl(text=_render_text)
layout = Layout(Window(content=control))
Application(layout=layout, key_bindings=kb, full_screen=False).run()
if __name__ == "__main__":
main()
scripts/kill_modal.sh
Executable
+34
View File
@@ -0,0 +1,34 @@
#!/bin/bash
# Kill all running Modal apps (sandboxes, deployments, etc.)
#
# Usage:
# bash scripts/kill_modal.sh # Stop hermes-agent sandboxes
# bash scripts/kill_modal.sh --all # Stop ALL Modal apps
set -uo pipefail
echo "Fetching Modal app list..."
APP_LIST=$(modal app list 2>/dev/null)
if [[ "${1:-}" == "--all" ]]; then
echo "Stopping ALL Modal apps..."
echo "$APP_LIST" | grep -oE 'ap-[A-Za-z0-9]+' | sort -u | while read app_id; do
echo " Stopping $app_id"
modal app stop "$app_id" 2>/dev/null || true
done
else
echo "Stopping hermes-agent sandboxes..."
APPS=$(echo "$APP_LIST" | grep 'hermes-agent' | grep -oE 'ap-[A-Za-z0-9]+' || true)
if [[ -z "$APPS" ]]; then
echo " No hermes-agent apps found."
else
echo "$APPS" | while read app_id; do
echo " Stopping $app_id"
modal app stop "$app_id" 2>/dev/null || true
done
fi
fi
echo ""
echo "Current hermes-agent status:"
modal app list 2>/dev/null | grep -E 'State|hermes-agent' || echo " (none)"
scripts/lib/node-bootstrap.sh
+461
View File
@@ -0,0 +1,461 @@
#!/usr/bin/env bash
# ============================================================================
# scripts/lib/node-bootstrap.sh
# ----------------------------------------------------------------------------
# Sourceable helper: ensure Node.js >= MIN_VERSION is available for the TUI
# (React + Ink), browser tools, and the WhatsApp bridge.
#
# Strategy (first hit wins — respects the user's existing tooling):
# 1. modern `node` already on PATH
# 2. ~/.hermes/node/ from a prior Hermes-managed install
# 3. fnm, proto, nvm (in that order) if the user already uses a version manager
# 4. Termux `pkg`, macOS Homebrew
# 5. pinned nodejs.org tarball into ~/.hermes/node/ (always works, zero shell rc edits)
#
# Usage:
# source scripts/lib/node-bootstrap.sh
# ensure_node # returns 0 on success, non-zero on failure
# if [ "$HERMES_NODE_AVAILABLE" = true ]; then ...; fi
#
# Env inputs (set before sourcing to override defaults):
# HERMES_NODE_MIN_VERSION (default: 20) — accepted on PATH
# HERMES_NODE_TARGET_MAJOR (default: 22) — installed when we install
# HERMES_HOME (default: $HOME/.hermes)
# ============================================================================
HERMES_NODE_MIN_VERSION="${HERMES_NODE_MIN_VERSION:-20}"
HERMES_NODE_TARGET_MAJOR="${HERMES_NODE_TARGET_MAJOR:-22}"
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
HERMES_NODE_AVAILABLE=false
# ---------------------------------------------------------------------------
# Logging — prefer the host script's log_* helpers when present
# ---------------------------------------------------------------------------
_nb_log() { declare -F log_info >/dev/null 2>&1 && log_info "$*" || printf '→ %s\n' "$*" >&2; }
_nb_ok() { declare -F log_success >/dev/null 2>&1 && log_success "$*" || printf '✓ %s\n' "$*" >&2; }
_nb_warn() { declare -F log_warn >/dev/null 2>&1 && log_warn "$*" || printf '⚠ %s\n' "$*" >&2; }
# ---------------------------------------------------------------------------
# Platform + version helpers
# ---------------------------------------------------------------------------
_nb_is_termux() {
[ -n "${TERMUX_VERSION:-}" ] || [[ "${PREFIX:-}" == *"com.termux/files/usr"* ]]
}
# Where to symlink node/npm/npx so they land on PATH.
# Mirrors get_command_link_dir() from install.sh: root FHS → /usr/local/bin,
# Termux → $PREFIX/bin, otherwise ~/.local/bin.
_nb_get_link_dir() {
if _nb_is_termux && [ -n "${PREFIX:-}" ]; then
echo "$PREFIX/bin"
elif [ "$(id -u)" = 0 ] && [ "$(uname -s)" = "Linux" ]; then
echo "/usr/local/bin"
else
echo "$HOME/.local/bin"
fi
}
# Redirect a Hermes-managed Node's `npm install -g` to the command link dir
# (already on PATH) instead of the default $HERMES_HOME/node/bin, which is off
# PATH and wiped on every Node upgrade. Scoped to the managed Node via its
# prefix-local global npmrc; the user's other Node installs / ~/.npmrc are
# untouched. Idempotent no-op when there's no managed npm.
_nb_configure_npm_prefix() {
[ -x "$HERMES_HOME/node/bin/npm" ] || return 0
local _link_dir
_link_dir="$(_nb_get_link_dir)"
mkdir -p "$HERMES_HOME/node/etc"
printf 'prefix=%s\n' "$(dirname "$_link_dir")" > "$HERMES_HOME/node/etc/npmrc"
}
_nb_node_major() {
local v
v=$(node --version 2>/dev/null | sed 's/^v//' | cut -d. -f1)
[[ "$v" =~ ^[0-9]+$ ]] && echo "$v" || echo 0
}
# The npm range the checkout's root package.json demands. Read from the
# manifest rather than duplicated here so the two can never drift; falls back
# to the current floor when the manifest is unreadable (vendored copy of this
# script, stripped install tree).
_nb_npm_range() {
if [ -n "${HERMES_NPM_TARGET_RANGE:-}" ]; then
printf '%s\n' "$HERMES_NPM_TARGET_RANGE"
return 0
fi
local repo_root manifest range
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)"
manifest="$repo_root/package.json"
if [ -r "$manifest" ]; then
# sed, not node: this runs before a usable node is guaranteed.
range=$(sed -n '/"engines"/,/}/p' "$manifest" \
| sed -n 's/.*"npm"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' \
| head -1)
if [ -n "$range" ]; then
printf '%s\n' "$range"
return 0
fi
fi
printf '>=12.0.0\n'
}
# Upgrade the managed tree's bundled npm into the checkout's engines.npm range.
#
# The nodejs.org tarball ships whatever npm that Node major bundles — Node
# 26.5.1 bundles npm 11.17.0, one minor below our own `engines.npm` floor of
# >=12. With `engine-strict=true` in the repo .npmrc that is fatal, not a
# warning, so a brand-new install died at the first `npm ci` with EBADENGINE.
# The Python side recovers through hermes_cli/npm_engine.py; the installer path
# had no such rung, so provision the right npm here instead of reacting later.
#
# Three details are load-bearing, all mirroring upgrade_managed_npm():
# - a temp cwd, so the checkout's own .npmrc (engine-strict, min-release-age)
# does not gate the very upgrade meant to satisfy it;
# - npm_config_min_release_age=0, which also neutralises a user ~/.npmrc;
# - an explicit --prefix at the managed tree, because
# _nb_configure_npm_prefix wrote prefix=~/.local into its etc/npmrc, and
# without the override this installs a second npm elsewhere while the
# managed tree stays stale.
#
# Best-effort: a failure here leaves a working Node with an old npm, which is
# strictly better than no Node at all, and npm_engine.py still covers the
# EBADENGINE that follows.
_nb_ensure_bundled_npm_range() {
local npm_bin="$HERMES_HOME/node/bin/npm"
[ -x "$npm_bin" ] || return 0
local range have want
range="$(_nb_npm_range)"
[ -n "$range" ] || return 0
# Skip the network round-trip when the bundled npm already satisfies the
# range. Only the ">=N" shape we actually author is checked; anything more
# exotic falls through to letting npm itself decide.
if [[ "$range" =~ ^\>=([0-9]+) ]]; then
want="${BASH_REMATCH[1]}"
have=$("$npm_bin" --version 2>/dev/null | cut -d. -f1)
if [[ "$have" =~ ^[0-9]+$ ]] && [ "$have" -ge "$want" ]; then
return 0
fi
fi
_nb_log "Upgrading bundled npm to satisfy $range..."
local tmp_cwd
tmp_cwd=$(mktemp -d)
if (
cd "$tmp_cwd" || exit 1
CI=1 npm_config_min_release_age=0 \
"$npm_bin" install --global \
--prefix "$HERMES_HOME/node" \
"npm@$range" \
--no-fund --no-audit --progress=false >/dev/null 2>&1
); then
rm -rf "$tmp_cwd"
_nb_ok "npm $("$npm_bin" --version 2>/dev/null) installed"
return 0
fi
rm -rf "$tmp_cwd"
_nb_warn "Could not upgrade bundled npm to $range — \`npm ci\` may fail with EBADENGINE."
_nb_warn "Fix manually: npm install -g --prefix \"$HERMES_HOME/node\" npm@\"$range\""
return 1
}
# A pre-release Node (…-alpha/-beta/-rc/-pre/-nightly) never counts as modern,
# however high its major. nodejs.org publishes a headers tarball only for final
# releases, so node-gyp cannot build node-pty — which has no Linux prebuild —
# against one. Mirrors node_satisfies_build() in install.sh.
_nb_node_is_prerelease() {
case "$(node --version 2>/dev/null)" in
*-*) return 0 ;;
*) return 1 ;;
esac
}
_nb_have_modern_node() {
command -v node >/dev/null 2>&1 || return 1
_nb_node_is_prerelease && return 1
[ "$(_nb_node_major)" -ge "$HERMES_NODE_MIN_VERSION" ]
}
# ---------------------------------------------------------------------------
# Version-manager paths — respect what the user already uses
# ---------------------------------------------------------------------------
_nb_try_fnm() {
command -v fnm >/dev/null 2>&1 || return 1
_nb_log "fnm detected — installing Node $HERMES_NODE_TARGET_MAJOR..."
eval "$(fnm env 2>/dev/null)" || true
fnm install "$HERMES_NODE_TARGET_MAJOR" >/dev/null 2>&1 || return 1
fnm use "$HERMES_NODE_TARGET_MAJOR" >/dev/null 2>&1 || return 1
_nb_have_modern_node || return 1
_nb_ok "Node $(node --version) activated via fnm"
return 0
}
_nb_try_proto() {
command -v proto >/dev/null 2>&1 || return 1
_nb_log "proto detected — installing Node $HERMES_NODE_TARGET_MAJOR..."
proto install node "$HERMES_NODE_TARGET_MAJOR" >/dev/null 2>&1 || return 1
_nb_have_modern_node || return 1
_nb_ok "Node $(node --version) activated via proto"
return 0
}
_nb_try_nvm() {
local nvm_sh="${NVM_DIR:-$HOME/.nvm}/nvm.sh"
[ -s "$nvm_sh" ] || return 1
# shellcheck source=/dev/null
\. "$nvm_sh" >/dev/null 2>&1 || return 1
_nb_log "nvm detected — installing Node $HERMES_NODE_TARGET_MAJOR..."
nvm install "$HERMES_NODE_TARGET_MAJOR" >/dev/null 2>&1 || return 1
nvm use "$HERMES_NODE_TARGET_MAJOR" >/dev/null 2>&1 || return 1
_nb_have_modern_node || return 1
_nb_ok "Node $(node --version) activated via nvm"
return 0
}
# ---------------------------------------------------------------------------
# Platform package managers
# ---------------------------------------------------------------------------
_nb_try_termux_pkg() {
_nb_is_termux || return 1
_nb_log "Installing Node.js via pkg..."
pkg install -y nodejs >/dev/null 2>&1 || return 1
_nb_have_modern_node || return 1
_nb_ok "Node $(node --version) installed via pkg"
return 0
}
_nb_try_brew() {
[ "$(uname -s)" = "Darwin" ] || return 1
command -v brew >/dev/null 2>&1 || return 1
_nb_log "Installing Node via Homebrew..."
brew install "node@${HERMES_NODE_TARGET_MAJOR}" >/dev/null 2>&1 \
|| brew install node >/dev/null 2>&1 \
|| return 1
brew link --overwrite --force "node@${HERMES_NODE_TARGET_MAJOR}" >/dev/null 2>&1 || true
_nb_have_modern_node || return 1
_nb_ok "Node $(node --version) installed via Homebrew"
return 0
}
# ---------------------------------------------------------------------------
# Bundled binary fallback — always works, no shell rc edits
# ---------------------------------------------------------------------------
_nb_install_bundled_node() {
local arch node_arch os_name node_os
arch=$(uname -m)
case "$arch" in
x86_64) node_arch="x64" ;;
aarch64|arm64) node_arch="arm64" ;;
armv7l) node_arch="armv7l" ;;
*)
_nb_warn "Unsupported arch ($arch) — install Node.js manually: https://nodejs.org/"
return 1
;;
esac
os_name=$(uname -s)
case "$os_name" in
Linux*) node_os="linux" ;;
Darwin*) node_os="darwin" ;;
*)
_nb_warn "Unsupported OS ($os_name) — install Node.js manually: https://nodejs.org/"
return 1
;;
esac
local index_url="https://nodejs.org/dist/latest-v${HERMES_NODE_TARGET_MAJOR}.x/"
local tarball
tarball=$(curl -fsSL "$index_url" \
| grep -oE "node-v${HERMES_NODE_TARGET_MAJOR}\.[0-9]+\.[0-9]+-${node_os}-${node_arch}\.tar\.xz" \
| head -1)
if [ -z "$tarball" ]; then
tarball=$(curl -fsSL "$index_url" \
| grep -oE "node-v${HERMES_NODE_TARGET_MAJOR}\.[0-9]+\.[0-9]+-${node_os}-${node_arch}\.tar\.gz" \
| head -1)
fi
if [ -z "$tarball" ]; then
_nb_warn "Could not resolve Node $HERMES_NODE_TARGET_MAJOR binary for $node_os-$node_arch"
return 1
fi
local tmp
tmp=$(mktemp -d)
_nb_log "Downloading $tarball..."
curl -fsSL "${index_url}${tarball}" -o "$tmp/$tarball" || {
_nb_warn "Download failed"; rm -rf "$tmp"; return 1
}
_nb_log "Extracting to $HERMES_HOME/node/..."
if [[ "$tarball" == *.tar.xz ]]; then
tar xf "$tmp/$tarball" -C "$tmp" || { rm -rf "$tmp"; return 1; }
else
tar xzf "$tmp/$tarball" -C "$tmp" || { rm -rf "$tmp"; return 1; }
fi
local extracted
extracted=$(find "$tmp" -maxdepth 1 -type d -name 'node-v*' 2>/dev/null | head -1)
if [ ! -d "$extracted" ]; then
_nb_warn "Extraction produced no node-v* directory"
rm -rf "$tmp"
return 1
fi
# Trust the binary, not the filename: a tarball named for a final release
# can still carry a pre-release build (latest-v26.x serves
# node-v26.8.0-<os>-<arch>.tar.xz stamped v26.8.0-alpha.0.0.0). Probe it
# before it replaces a working managed tree.
case "$("$extracted/bin/node" --version 2>/dev/null)" in
*-*)
_nb_warn "Node $("$extracted/bin/node" --version 2>/dev/null) is a pre-release build — native modules cannot be built against it"
rm -rf "$tmp"
return 1
;;
esac
mkdir -p "$HERMES_HOME"
rm -rf "$HERMES_HOME/node"
mv "$extracted" "$HERMES_HOME/node"
rm -rf "$tmp"
local _link_dir
_link_dir="$(_nb_get_link_dir)"
# HERMES_NODE_SKIP_LINKS=1: the caller only wants the private managed tree
# (e.g. the EBADENGINE recovery provisioning a runtime alongside a working
# system Node). Skipping the links keeps the user's own node/npm first on
# PATH instead of shadowing them with ours.
if [ "${HERMES_NODE_SKIP_LINKS:-0}" != "1" ]; then
mkdir -p "$_link_dir"
ln -sf "$HERMES_HOME/node/bin/node" "$_link_dir/node"
ln -sf "$HERMES_HOME/node/bin/npm" "$_link_dir/npm"
ln -sf "$HERMES_HOME/node/bin/npx" "$_link_dir/npx"
fi
_nb_configure_npm_prefix
export PATH="$HERMES_HOME/node/bin:$PATH"
_nb_have_modern_node || return 1
_nb_ok "Node $(node --version) installed to $HERMES_HOME/node/"
# The tarball's bundled npm is usually below the repo's engines.npm floor.
# Best-effort: an old npm still beats no Node.
_nb_ensure_bundled_npm_range || true
return 0
}
# ---------------------------------------------------------------------------
# Heal a broken Hermes-managed Node tree (partial upgrade / missing lib/)
# ---------------------------------------------------------------------------
_nb_managed_tool_broken() {
local tool="$1"
local probe
for probe in \
"$HERMES_HOME/node/bin/$tool" \
"$HERMES_HOME/node/${tool}.exe" \
"$HERMES_HOME/node/$tool"; do
if [ -x "$probe" ] || [ -f "$probe" ]; then
if ! "$probe" --version >/dev/null 2>&1; then
return 0
fi
fi
done
return 1
}
# The managed node runs but is below HERMES_NODE_TARGET_MAJOR — an old tree
# from a previous install (e.g. 22). Outdated heals the same way broken does,
# so existing users get upgraded on the next heal probe, not just on a full
# installer re-run. Mirrors _managed_node_tree_outdated() in
# hermes_constants.py.
_nb_managed_node_outdated() {
local probe ver major
for probe in "$HERMES_HOME/node/bin/node" "$HERMES_HOME/node/node"; do
[ -x "$probe" ] || continue
ver="$("$probe" --version 2>/dev/null)" || return 1
major="${ver#v}"; major="${major%%.*}"
case "$major" in ''|*[!0-9]*) return 1 ;; esac
[ "$major" -lt "$HERMES_NODE_TARGET_MAJOR" ] && return 0
return 1
done
return 1
}
_nb_managed_node_needs_heal() {
local tool
for tool in node npm npx; do
if _nb_managed_tool_broken "$tool"; then
return 0
fi
done
_nb_managed_node_outdated
}
# Redownload the pinned nodejs.org tarball when a managed tree exists but
# node/npm/npx fail a --version probe. No-op when the tree is healthy or
# absent. Used by hermes_constants.find_hermes_node_executable() and safe
# to call from install reruns.
heal_managed_node() {
[ -d "$HERMES_HOME/node" ] || return 1
if ! _nb_managed_node_needs_heal; then
return 0
fi
_nb_log "Hermes-managed Node is broken — redownloading to $HERMES_HOME/node/..."
_nb_install_bundled_node
}
# ---------------------------------------------------------------------------
# Public entry point
# ---------------------------------------------------------------------------
ensure_node() {
HERMES_NODE_AVAILABLE=false
# Repair pre-existing managed installs where `npm install -g` lands off
# PATH. No-op when there's no managed Node, so it's safe to run first.
_nb_configure_npm_prefix
if _nb_have_modern_node; then
_nb_ok "Node $(node --version) found"
HERMES_NODE_AVAILABLE=true
return 0
fi
if [ -x "$HERMES_HOME/node/bin/node" ]; then
export PATH="$HERMES_HOME/node/bin:$PATH"
if _nb_have_modern_node; then
_nb_ok "Node $(node --version) found (Hermes-managed)"
HERMES_NODE_AVAILABLE=true
# A tree from an older install still carries that Node major's
# bundled npm, and the upgrade in _nb_install_bundled_node is
# best-effort — one offline install leaves an at-target tree
# stranded below engines.npm forever, since heal only fires for a
# *broken* tree. Mirrors Update-ManagedNpm's reuse-path call in
# install.ps1. No-ops on a probe when the npm is already in range.
_nb_ensure_bundled_npm_range || true
return 0
fi
fi
# Version managers first — respect the user's existing setup.
_nb_try_fnm && { HERMES_NODE_AVAILABLE=true; return 0; }
_nb_try_proto && { HERMES_NODE_AVAILABLE=true; return 0; }
_nb_try_nvm && { HERMES_NODE_AVAILABLE=true; return 0; }
# Platform package managers.
_nb_try_termux_pkg && { HERMES_NODE_AVAILABLE=true; return 0; }
_nb_try_brew && { HERMES_NODE_AVAILABLE=true; return 0; }
# Last resort: pinned nodejs.org tarball.
_nb_install_bundled_node && { HERMES_NODE_AVAILABLE=true; return 0; }
_nb_warn "Node.js install failed — TUI and browser tools will be unavailable."
_nb_warn "Install manually: https://nodejs.org/en/download/ (or: \`brew install node\`, \`fnm install $HERMES_NODE_TARGET_MAJOR\`, etc.)"
return 1
}
scripts/lint_diff.py
Executable
+207
View File
@@ -0,0 +1,207 @@
#!/usr/bin/env python3
"""Diff ruff + ty diagnostic reports between two git refs.
Produces a Markdown summary suitable for `$GITHUB_STEP_SUMMARY` and for PR
comments. Compares issues by a stable key (file, rule, line) so line-only
shifts from unrelated edits are treated as the same issue.
Usage:
lint_diff.py \\
--base-ruff base/ruff.json --head-ruff head/ruff.json \\
--base-ty base/ty.json --head-ty head/ty.json \\
[--base-ref origin/main] [--head-ref HEAD]
Any of the four --{base,head}-{ruff,ty} files may be missing or empty; in that
case the tool treats it as "0 diagnostics" (e.g. if base/main doesn't have the
config yet, or a tool crashed).
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from collections import Counter
from pathlib import Path
def _load_json(path: Path | None) -> list[dict]:
if path is None or not path.exists() or path.stat().st_size == 0:
return []
try:
data = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
print(f"warning: could not parse {path}: {exc}", file=sys.stderr)
return []
if not isinstance(data, list):
return []
return data
def _normalize_ruff(entries: list[dict]) -> list[dict]:
"""Ruff JSON: {code, filename, location.row, message}."""
out: list[dict] = []
for e in entries:
code = e.get("code") or "unknown"
# ruff emits absolute paths; relativize to repo root if possible
filename = e.get("filename", "")
try:
filename = os.path.relpath(filename)
except ValueError:
pass
line = (e.get("location") or {}).get("row", 0)
out.append(
{
"tool": "ruff",
"rule": code,
"path": filename,
"line": line,
"message": e.get("message", ""),
}
)
return out
def _normalize_ty(entries: list[dict]) -> list[dict]:
"""ty gitlab JSON: {check_name, location.path, location.positions.begin.line, description}."""
out: list[dict] = []
for e in entries:
loc = e.get("location") or {}
begin = (loc.get("positions") or {}).get("begin") or {}
out.append(
{
"tool": "ty",
"rule": e.get("check_name", "unknown"),
"path": loc.get("path", ""),
"line": begin.get("line", 0),
"message": e.get("description", ""),
}
)
return out
def _key(d: dict) -> tuple[str, str, str]:
"""Stable diagnostic identity across commits: (path, rule, message)."""
# Intentionally omit line so unrelated edits above an issue don't flag it
# as "new". Same file + same rule + same message = same issue.
return (d["path"], d["rule"], d["message"])
def _diff(base: list[dict], head: list[dict]) -> tuple[list[dict], list[dict], list[dict]]:
base_map = {_key(d): d for d in base}
head_map = {_key(d): d for d in head}
base_keys = set(base_map)
head_keys = set(head_map)
new_keys = head_keys - base_keys
fixed_keys = base_keys - head_keys
unchanged_keys = base_keys & head_keys
# Return head entries for new (current line numbers), base entries for fixed
return (
[head_map[k] for k in new_keys],
[base_map[k] for k in fixed_keys],
[head_map[k] for k in unchanged_keys],
)
def _rule_counts(entries: list[dict]) -> list[tuple[str, int]]:
return Counter(e["rule"] for e in entries).most_common()
def _section(title: str, entries: list[dict], limit: int = 25) -> str:
if not entries:
return f"**{title}:** none\n"
lines = [f"**{title} ({len(entries)}):**\n"]
# Group by rule for readability
counts = _rule_counts(entries)
lines.append("| Rule | Count |")
lines.append("| --- | ---: |")
for rule, count in counts[:15]:
lines.append(f"| `{rule}` | {count} |")
if len(counts) > 15:
lines.append(f"| _+{len(counts) - 15} more rules_ | |")
lines.append("")
lines.append("<details><summary>First entries</summary>\n")
lines.append("```")
for e in entries[:limit]:
lines.append(f"{e['path']}:{e['line']}: [{e['rule']}] {e['message']}")
if len(entries) > limit:
lines.append(f"... and {len(entries) - limit} more")
lines.append("```")
lines.append("</details>\n")
return "\n".join(lines)
def _tool_report(
tool_name: str,
base: list[dict],
head: list[dict],
base_available: bool,
) -> str:
new, fixed, unchanged = _diff(base, head)
delta = len(head) - len(base)
delta_str = f"+{delta}" if delta > 0 else str(delta)
emoji = "🆕" if delta > 0 else ("✅" if delta < 0 else "➖")
lines = [f"## {tool_name}\n"]
if not base_available:
lines.append(
"_Base report unavailable (likely main has no config for this tool yet); "
"treating all head diagnostics as new._\n"
)
lines.append(
f"**Total:** {len(head)} on HEAD, {len(base)} on base "
f"({emoji} {delta_str})\n"
)
lines.append(_section("🆕 New issues", new))
lines.append(_section("✅ Fixed issues", fixed))
lines.append(
f"**Unchanged:** {len(unchanged)} pre-existing issues carried over.\n"
)
return "\n".join(lines)
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--base-ruff", type=Path, required=True)
ap.add_argument("--head-ruff", type=Path, required=True)
ap.add_argument("--base-ty", type=Path, required=True)
ap.add_argument("--head-ty", type=Path, required=True)
ap.add_argument("--base-ref", default="base")
ap.add_argument("--head-ref", default="HEAD")
ap.add_argument(
"--output", type=Path, help="Write summary to this file instead of stdout"
)
args = ap.parse_args()
base_ruff_raw = _load_json(args.base_ruff)
head_ruff_raw = _load_json(args.head_ruff)
base_ty_raw = _load_json(args.base_ty)
head_ty_raw = _load_json(args.head_ty)
base_ruff = _normalize_ruff(base_ruff_raw)
head_ruff = _normalize_ruff(head_ruff_raw)
base_ty = _normalize_ty(base_ty_raw)
head_ty = _normalize_ty(head_ty_raw)
base_ruff_avail = args.base_ruff.exists() and args.base_ruff.stat().st_size > 0
base_ty_avail = args.base_ty.exists() and args.base_ty.stat().st_size > 0
buf: list[str] = []
buf.append(f"# 🔎 Lint report: `{args.head_ref}` vs `{args.base_ref}`\n")
buf.append(_tool_report("ruff", base_ruff, head_ruff, base_ruff_avail))
buf.append(_tool_report("ty (type checker)", base_ty, head_ty, base_ty_avail))
buf.append(
"_Diagnostics are surfaced as warnings — this check never fails the build._\n"
)
summary = "\n".join(buf)
if args.output:
args.output.write_text(summary, encoding="utf-8")
else:
print(summary)
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/micro_compaction_report.py
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Summarize micro-compaction telemetry from Hermes logs.
Reads the content-free JSON lines emitted by
``ContextCompressor._emit_micro_compaction_telemetry`` and reports what the
feature actually bought you.
Usage:
python scripts/micro_compaction_report.py [LOGFILE ...]
python scripts/micro_compaction_report.py --per-session
With no LOGFILE, reads ``$HERMES_HOME/logs/agent.log`` (default ~/.hermes).
What to look at
---------------
The point of micro-compaction is not saving tokens or time. It is:
(a) amortizing the one long batch-compaction pause across many turns, and
(b) keeping the context window low enough that a session runs much further
before it needs a hard compaction at all.
So the headline numbers here are OCCUPANCY (how full the window is kept, as a
percentage of the compaction threshold) and BATCH COMPACTIONS (how often the
long pause actually fired). Net tokens saved is reported too, but it is the
least interesting figure -- a session can save nothing on paper and still be a
clear win because the stalls disappeared and the window never filled.
Caveat: running the test suite writes telemetry into the same log. Test lines
cluster inside a sub-second window and carry an empty session_id (they group
as "(unknown)"). Use --per-session to spot them.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from collections import defaultdict
from pathlib import Path
MICRO_MARKER = "micro compaction telemetry: "
BATCH_MARKER = "context compression attempt telemetry: "
def default_log() -> Path:
home = os.environ.get("HERMES_HOME") or str(Path.home() / ".hermes")
return Path(home) / "logs" / "agent.log"
def load(paths: list[Path]) -> tuple[list[dict], list[dict]]:
micro: list[dict] = []
batch: list[dict] = []
for path in paths:
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError as exc:
print(f"warning: cannot read {path}: {exc}", file=sys.stderr)
continue
for line in text.splitlines():
for marker, sink in ((MICRO_MARKER, micro), (BATCH_MARKER, batch)):
idx = line.find(marker)
if idx == -1:
continue
try:
sink.append(json.loads(line[idx + len(marker):]))
except ValueError:
pass
break
return micro, batch
def pct(values: list[float]) -> tuple[float, float, float] | None:
if not values:
return None
ordered = sorted(values)
return ordered[0], ordered[len(ordered) // 2], ordered[-1]
def fmt(n) -> str:
return "-" if n is None else f"{n:,}"
def report(micro: list[dict], batch: list[dict], per_session: bool) -> int:
if not micro:
print("No micro-compaction telemetry found.")
print("It may be disabled (compression.micro_compact), or no session")
print("has run long enough to trigger a pass yet.")
return 1
by_session: dict[str, list[dict]] = defaultdict(list)
for e in micro:
by_session[e.get("session_id") or "(unknown)"].append(e)
outcomes: dict[str, int] = defaultdict(int)
for e in micro:
outcomes[e.get("outcome", "?")] += 1
occupancies = [e["occupancy_pct"] for e in micro if e.get("occupancy_pct") is not None]
saved = sum(-(e.get("tokens_delta") or 0) for e in micro)
absorbed = [e for e in micro if e.get("outcome") == "absorbed"]
durations = [e.get("duration_ms") or 0 for e in micro]
if per_session:
print(f"{'session':<26} {'passes':>6} {'occupancy%':>18} {'batch':>6} {'saved':>10}")
print("-" * 72)
batch_by_session: dict[str, int] = defaultdict(int)
for b in batch:
batch_by_session[b.get("session_id") or "(unknown)"] += 1
for sid, evs in sorted(by_session.items(), key=lambda kv: -len(kv[1])):
occ = [e["occupancy_pct"] for e in evs if e.get("occupancy_pct") is not None]
spread = pct(occ)
occ_s = f"{spread[0]:.0f}-{spread[2]:.0f} (med {spread[1]:.0f})" if spread else "-"
s = sum(-(e.get("tokens_delta") or 0) for e in evs)
print(f"{sid[:26]:<26} {len(evs):>6} {occ_s:>18} "
f"{batch_by_session.get(sid, 0):>6} {s:>+10,}")
print()
print("-- headroom ----------------------------------")
spread = pct(occupancies)
if spread:
print(f"context occupancy min {spread[0]:.0f}% median {spread[1]:.0f}% max {spread[2]:.0f}%")
print(" (% of the batch-compaction threshold)")
else:
print("context occupancy unavailable (window not resolved when logged)")
print(f"batch compactions {len(batch):,}")
if batch:
print(f" micro passes each {len(micro) / len(batch):.1f}")
else:
print(" none fired -- the long pause never happened in this log")
print()
print("-- activity ----------------------------------")
print(f"sessions {len(by_session):,}")
print(f"passes {len(micro):,}")
for name, count in sorted(outcomes.items(), key=lambda kv: -kv[1]):
print(f" {name:<20} {count:,}")
if durations:
ordered = sorted(durations)
print(f"pass duration median {ordered[len(ordered) // 2]:,} ms "
f"max {ordered[-1]:,} ms")
print()
print("-- tokens (least interesting) ----------------")
print(f"net tokens saved {saved:+,}")
if absorbed:
sizes = [e.get("exchange_tokens") or 0 for e in absorbed]
print(f"exchanges absorbed {len(absorbed):,} "
f"(mean {sum(sizes) // len(absorbed):,} tokens each)")
print("note: the first pass in a session costs ~400 tokens of marker")
print("scaffolding; it pays back from the second pass on.")
return 0
def main() -> int:
ap = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("logs", nargs="*", type=Path, help="log files (default: agent.log)")
ap.add_argument("--per-session", action="store_true", help="break down by session")
args = ap.parse_args()
paths = args.logs or [default_log()]
for p in paths:
if not p.exists():
print(f"warning: {p} does not exist", file=sys.stderr)
micro, batch = load([p for p in paths if p.exists()])
return report(micro, batch, args.per_session)
if __name__ == "__main__":
sys.exit(main())
scripts/observability/gateway_health_export_probe.py
+92
View File
@@ -0,0 +1,92 @@
#!/usr/bin/env python3
"""Exercise Gateway Health & Diagnostics Export against a local OTLP capture collector."""
from __future__ import annotations
import argparse
import json
import logging
import os
import sys
import tempfile
import time
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--endpoint", default="http://127.0.0.1:4318/v1/traces")
parser.add_argument("--log", required=True, help="JSONL file written by otel_capture_collector.py")
parser.add_argument("--wait", type=float, default=7.0)
args = parser.parse_args()
hermes_home = Path(tempfile.mkdtemp(prefix="hermes-otel-smoke-"))
os.environ["HERMES_HOME"] = str(hermes_home)
from gateway.status import write_runtime_status
from agent.monitoring.gateway_health_export import start_gateway_health_export
from agent.monitoring import emitter
config = {
"monitoring": {
"local": True,
"gateway_health_export": {
"enabled": True,
"metrics_enabled": True,
"diagnostic_events_enabled": True,
"warning_error_events_enabled": True,
"export_interval_seconds": 5,
"logs_export_interval_seconds": 5,
"resource_attributes": {
"service.name": "hermes-gateway-smoke",
"deployment.environment.name": "local-smoke",
},
},
"export": {
"otlp": {
"enabled": True,
"endpoint": args.endpoint,
"headers_env": {},
}
},
}
}
runtime = start_gateway_health_export(config)
if not runtime.enabled:
raise SystemExit(f"gateway health exporter did not enable: {runtime.reason}")
write_runtime_status(gateway_state="starting", active_agents=0)
write_runtime_status(gateway_state="running", active_agents=2)
write_runtime_status(platform="slack", platform_state="running")
write_runtime_status(
platform="slack",
platform_state="fatal",
error_code="auth_failed",
error_message="Bearer *** rejected for smoke@example.com",
)
logging.getLogger("gateway.platforms.slack").warning("Slack token *** rejected for smoke@example.com")
emitter.get_emitter().flush(timeout=2.0)
time.sleep(args.wait)
write_runtime_status(gateway_state="stopped", active_agents=0)
runtime.shutdown()
emitter.get_emitter().flush(timeout=2.0)
log_path = Path(args.log)
rows = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()]
paths = {row["path"] for row in rows}
print(json.dumps({"hermes_home": str(hermes_home), "requests": len(rows), "paths": sorted(paths)}, indent=2))
if "/v1/traces" not in paths:
raise SystemExit("missing /v1/traces request")
if "/v1/logs" not in paths:
raise SystemExit("missing /v1/logs request")
if "/v1/metrics" not in paths:
raise SystemExit("missing /v1/metrics request")
if __name__ == "__main__":
main()
scripts/observability/otel_capture_collector.py
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
"""Tiny local OTLP/HTTP capture collector for Hermes gateway health smoke tests.
This is not a production collector. It accepts OTLP protobuf POSTs on /v1/traces,
/v1/metrics, and /v1/logs, records request metadata as JSONL, and returns 200 so
local exporters can be exercised without Docker or a vendor backend.
"""
from __future__ import annotations
import argparse
import json
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
class CaptureHandler(BaseHTTPRequestHandler):
log_path: Path
def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API
length = int(self.headers.get("content-length") or 0)
body = self.rfile.read(length) if length else b""
record = {
"ts": time.time(),
"path": self.path,
"content_type": self.headers.get("content-type"),
"content_length": length,
"body_prefix_hex": body[:24].hex(),
}
with self.log_path.open("a", encoding="utf-8") as fh:
fh.write(json.dumps(record, sort_keys=True) + "\n")
self.send_response(200)
self.end_headers()
self.wfile.write(b"{}")
def log_message(self, format: str, *args) -> None:
# Keep tmux panes clean; JSONL file is the assertion surface.
return
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=4318)
parser.add_argument("--log", required=True)
args = parser.parse_args()
log_path = Path(args.log).expanduser().resolve()
log_path.parent.mkdir(parents=True, exist_ok=True)
log_path.write_text("", encoding="utf-8")
CaptureHandler.log_path = log_path
server = ThreadingHTTPServer((args.host, args.port), CaptureHandler)
print(f"OTLP capture collector listening on http://{args.host}:{args.port}; log={log_path}", flush=True)
server.serve_forever()
if __name__ == "__main__":
main()
scripts/probe_active_session_exclusivity.py
+201
View File
@@ -0,0 +1,201 @@
"""Cross-process falsification of the per-session fence.
The unit tests share one interpreter, so they cannot see the failure this whole
change exists to prevent: two SEPARATE gateway processes, each with its own
snapshot of a conversation, both writing to it. That is how the defect was found
and it is the only way to prove it is closed.
Run against the fork's own HERMES_HOME so nothing here touches a real profile:
python scripts/probe_active_session_exclusivity.py
It drives two real ``python -m tui_gateway.entry`` processes over stdio JSON-RPC
and asserts the sequence the reviewer specified:
A resume S, submit -> claims the session
B resume S, submit -> typed SESSION_NOT_OWNED, no row, no turn
A exits -> its lease is pruned as a dead owner
B submit again -> succeeds
No provider is required. The fence is checked BEFORE the agent is built, so a
submit that later fails for want of a model still proves who owns the session --
which is the property under test, and keeps the probe free of credentials and of
inference cost.
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
import time
from pathlib import Path
REPO = Path(__file__).resolve().parent.parent
PYTHON = REPO / "venv" / "Scripts" / "python.exe"
if not PYTHON.exists(): # posix layout
PYTHON = REPO / "venv" / "bin" / "python"
class Gateway:
"""One gateway process, spoken to the way the TUI speaks to it."""
def __init__(self, name: str, home: Path):
env = dict(os.environ)
env["HERMES_HOME"] = str(home)
env["PYTHONUNBUFFERED"] = "1"
self.name = name
self.proc = subprocess.Popen(
[str(PYTHON), "-u", "-m", "tui_gateway.entry"],
cwd=str(REPO),
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
encoding="utf-8",
errors="replace",
)
self._next = 1
self.ready()
def _read(self):
line = self.proc.stdout.readline()
if not line:
raise RuntimeError(f"[{self.name}] gateway closed its pipe")
line = line.strip()
if not line:
return None
try:
return json.loads(line)
except json.JSONDecodeError:
return None
def ready(self, timeout: float = 180.0) -> None:
deadline = time.time() + timeout
while time.time() < deadline:
msg = self._read()
if msg and msg.get("method") == "event":
if msg.get("params", {}).get("type") == "gateway.ready":
return
raise RuntimeError(f"[{self.name}] never announced gateway.ready")
def call(self, method: str, params: dict, timeout: float = 180.0) -> dict:
rid = str(self._next)
self._next += 1
self.proc.stdin.write(json.dumps({"jsonrpc": "2.0", "id": rid, "method": method, "params": params}) + "\n")
self.proc.stdin.flush()
deadline = time.time() + timeout
while time.time() < deadline:
msg = self._read()
if msg and msg.get("id") == rid:
return msg
raise RuntimeError(f"[{self.name}] timed out calling {method}")
def close(self):
try:
self.proc.stdin.close()
except Exception:
pass
try:
self.proc.terminate()
self.proc.wait(timeout=15)
except Exception:
try:
self.proc.kill()
except Exception:
pass
def reason_of(response: dict):
return (response.get("error") or {}).get("data", {}).get("reason")
def registry(home: Path):
path = home / "runtime" / "active_sessions.json"
try:
return json.loads(path.read_text(encoding="utf-8")).get("entries", [])
except Exception:
return []
def main() -> int:
home = REPO / ".probe-home"
# A fresh profile each run: a lease left by a previous run would make the
# first check pass or fail for the wrong reason.
import shutil
shutil.rmtree(home, ignore_errors=True)
failures = []
def check(label: str, ok: bool, detail: str = ""):
print(f" {'PASS' if ok else 'FAIL'} {label}{(' -- ' + detail) if detail else ''}")
if not ok:
failures.append(label)
a = Gateway("A", home)
b = None
try:
created = a.call("session.create", {"cols": 80})
sid_a = created["result"]["session_id"]
# Opening a chat must not claim anything -- an idle composer is invisible
# and a slot held by one would fence a real turn for no reason.
check("session.create claims nothing", registry(home) == [], f"{len(registry(home))} entries")
a.call("prompt.submit", {"session_id": sid_a, "text": "probe: A takes the session"})
held = registry(home)
check("A's first turn claims a session", len(held) == 1, json.dumps(held)[:200])
if not held:
raise RuntimeError("A never claimed anything; nothing further can be tested")
# The STORED key, which only materialises when a turn is first submitted --
# and which is what the lease must be keyed on. A lease keyed on the live
# runtime id would fence nothing: two processes resuming one conversation
# have different runtime ids by construction.
key = held[0].get("session_id")
print(f"A live session {sid_a}, stored key {key}")
check("the lease is keyed on the STORED session, not the runtime handle",
bool(key) and key != sid_a, f"key={key} runtime={sid_a}")
b = Gateway("B", home)
resumed = b.call("session.resume", {"session_id": key})
check("B may still RESUME (reading is never fenced)", "result" in resumed,
json.dumps(resumed.get("error", ""))[:160])
sid_b = resumed.get("result", {}).get("session_id")
before = len(registry(home))
refused = b.call("prompt.submit", {"session_id": sid_b, "text": "probe: B must not write"})
check("B's submit is refused", refused.get("error") is not None,
json.dumps(refused.get("result", ""))[:120])
check("refusal is typed SESSION_NOT_OWNED", reason_of(refused) == "SESSION_NOT_OWNED",
str(reason_of(refused)))
check("refusal left the registry untouched", len(registry(home)) == before)
# A dies without releasing -- the crash case, not a clean handoff.
a.proc.kill()
a.proc.wait(timeout=30)
time.sleep(1.0)
retried = b.call("prompt.submit", {"session_id": sid_b, "text": "probe: B may write now"})
check("after A dies, B's retry is accepted", retried.get("error") is None,
json.dumps(retried.get("error", ""))[:200])
held = registry(home)
check("and B now owns the session", len(held) == 1 and held[0].get("session_id") == key,
json.dumps(held)[:160])
finally:
if b is not None:
b.close()
a.close()
print()
if failures:
print(f"FAILED: {len(failures)} check(s): {', '.join(failures)}")
return 1
print("All cross-process checks passed.")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/profile-tui.py
Executable
+625
View File
@@ -0,0 +1,625 @@
#!/usr/bin/env python3
"""Drive the Hermes TUI under HERMES_DEV_PERF and summarize the pipeline.
Usage:
scripts/profile-tui.py [--session SID] [--hold KEY] [--seconds N] [--rate HZ]
Defaults: picks the session with the most messages, holds PageUp for 8s at
~30 Hz (matching xterm key-repeat), summarizes ~/.hermes/perf.log on exit.
The --tui build must exist (run `npm run build` in ui-tui first). This script
launches `node dist/entry.js` directly with HERMES_TUI_RESUME set so it
bypasses the hermes_cli wrapper — we want repeatable timing, not the CLI's
session-picker flow.
Environment overrides:
HERMES_PERF_LOG (default ~/.hermes/perf.log)
HERMES_PERF_NODE (default node from $PATH)
HERMES_TUI_DIR (default: <repo>/ui-tui relative to this script)
Exit code is 0 if the harness ran and parsed results, 2 if the TUI crashed
or produced no perf data (suggests HERMES_DEV_PERF wiring is broken).
"""
from __future__ import annotations
import argparse
import json
import os
import pty
import select
import signal
import sqlite3
import sys
import time
from pathlib import Path
from typing import Any
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(_PROJECT_ROOT))
try:
from hermes_constants import get_hermes_home
except ImportError:
def get_hermes_home() -> Path: # type: ignore[misc]
val = (os.environ.get("HERMES_HOME") or "").strip()
return Path(val) if val else Path.home() / ".hermes"
DEFAULT_TUI_DIR = Path(
os.environ.get("HERMES_TUI_DIR")
or str(Path(__file__).resolve().parent.parent / "ui-tui")
)
DEFAULT_LOG = Path(os.environ.get("HERMES_PERF_LOG", str(get_hermes_home() / "perf.log")))
DEFAULT_STATE_DB = get_hermes_home() / "state.db"
# Keystroke escape sequences. Matches what xterm/VT220 send when the
# terminal has bracketed-paste disabled and the key-repeat handler fires.
KEYS = {
"page_up": b"\x1b[5~",
"page_down": b"\x1b[6~",
"wheel_up": b"\x1b[M`!!", # mouse wheel up (SGR-less) — best-effort
"shift_up": b"\x1b[1;2A",
"shift_down": b"\x1b[1;2B",
}
def pick_longest_session(db: Path) -> str:
conn = sqlite3.connect(db)
row = conn.execute(
"SELECT id FROM sessions s ORDER BY "
"(SELECT COUNT(*) FROM messages m WHERE m.session_id = s.id) DESC LIMIT 1"
).fetchone()
if not row:
sys.exit(f"no sessions in {db}")
return row[0]
def drain(fd: int, timeout: float) -> bytes:
"""Read whatever's available from fd within `timeout`, then return."""
chunks = []
end = time.monotonic() + timeout
while time.monotonic() < end:
r, _, _ = select.select([fd], [], [], max(0.0, end - time.monotonic()))
if not r:
break
try:
data = os.read(fd, 4096)
except OSError:
break
if not data:
break
chunks.append(data)
return b"".join(chunks)
def hold_key(fd: int, seq: bytes, seconds: float, rate_hz: int) -> int:
"""Write `seq` to fd at ~rate_hz for `seconds`. Returns keystrokes sent."""
interval = 1.0 / max(1, rate_hz)
end = time.monotonic() + seconds
sent = 0
while time.monotonic() < end:
try:
os.write(fd, seq)
sent += 1
except OSError:
break
# Drain stdout to keep the PTY buffer flowing; ignore content.
drain(fd, 0)
time.sleep(interval)
return sent
def summarize(log: Path, since_ts_ms: int) -> dict[str, Any]:
"""Parse perf.log, keep only events newer than since_ts_ms, return stats."""
react_events: list[dict[str, Any]] = []
frame_events: list[dict[str, Any]] = []
if not log.exists():
return {"error": f"no log at {log}", "react": [], "frame": []}
for line in log.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except json.JSONDecodeError:
continue
if int(row.get("ts", 0)) < since_ts_ms:
continue
src = row.get("src")
if src == "react":
react_events.append(row)
elif src == "frame":
frame_events.append(row)
return {
"react": react_events,
"frame": frame_events,
}
def pct(values: list[float], p: float) -> float:
if not values:
return 0.0
s = sorted(values)
idx = min(len(s) - 1, int(len(s) * p))
return s[idx]
def format_report(data: dict[str, Any]) -> str:
react = data.get("react") or []
frames = data.get("frame") or []
out = []
out.append("═══ React Profiler ═══")
if not react:
out.append(" (no react events — HERMES_DEV_PERF wired? threshold too high?)")
else:
by_id: dict[str, list[float]] = {}
for r in react:
by_id.setdefault(r["id"], []).append(r["actualMs"])
out.append(f" {'pane':<14} {'count':>6} {'p50':>8} {'p95':>8} {'p99':>8} {'max':>8}")
for pid, ms in sorted(by_id.items(), key=lambda kv: -pct(kv[1], 0.99)):
out.append(
f" {pid:<14} {len(ms):>6} {pct(ms,0.50):>8.2f} {pct(ms,0.95):>8.2f} "
f"{pct(ms,0.99):>8.2f} {max(ms):>8.2f}"
)
out.append("")
out.append("═══ Ink pipeline ═══")
if not frames:
out.append(" (no frame events — onFrame wiring broken?)")
else:
dur = [f["durationMs"] for f in frames]
phases_present = any(f.get("phases") for f in frames)
out.append(f" frames captured: {len(frames)}")
out.append(
f" durationMs p50={pct(dur,0.50):.2f} p95={pct(dur,0.95):.2f} "
f"p99={pct(dur,0.99):.2f} max={max(dur):.2f}"
)
# Effective FPS during the run: frames / elapsed seconds.
ts = sorted(f["ts"] for f in frames)
if len(ts) >= 2:
elapsed_s = (ts[-1] - ts[0]) / 1000.0
fps = len(frames) / elapsed_s if elapsed_s > 0 else float("inf")
out.append(f" throughput: {len(frames)} frames / {elapsed_s:.2f}s = {fps:.1f} fps")
if phases_present:
fields = ["yoga", "renderer", "diff", "optimize", "write", "commit"]
out.append("")
out.append(f" {'phase':<10} {'p50':>8} {'p95':>8} {'p99':>8} {'max':>8} (ms)")
for field in fields:
vals = [f["phases"][field] for f in frames if f.get("phases")]
if vals:
out.append(
f" {field:<10} {pct(vals,0.50):>8.2f} {pct(vals,0.95):>8.2f} "
f"{pct(vals,0.99):>8.2f} {max(vals):>8.2f}"
)
# Derived: sum of phases vs durationMs (reveals hidden time).
sum_ps = [
sum(f["phases"][k] for k in fields)
for f in frames if f.get("phases")
]
if sum_ps:
dur_match = [f["durationMs"] for f in frames if f.get("phases")]
deltas = [d - s for d, s in zip(dur_match, sum_ps)]
out.append(
f" {'dur-Σphases':<10} {pct(deltas,0.50):>8.2f} {pct(deltas,0.95):>8.2f} "
f"{pct(deltas,0.99):>8.2f} {max(deltas):>8.2f} (unaccounted-for time)"
)
# Yoga counters
visited = [f["phases"]["yogaVisited"] for f in frames if f.get("phases")]
measured = [f["phases"]["yogaMeasured"] for f in frames if f.get("phases")]
cache_hits = [f["phases"]["yogaCacheHits"] for f in frames if f.get("phases")]
live = [f["phases"]["yogaLive"] for f in frames if f.get("phases")]
out.append("")
out.append(" Yoga counters (per frame):")
for name, vals in (
("visited", visited),
("measured", measured),
("cacheHits", cache_hits),
("live", live),
):
if vals:
out.append(f" {name:<11} p50={pct(vals,0.5):.0f} p99={pct(vals,0.99):.0f} max={max(vals)}")
# Patch counts — proxy for "how much changed each frame"
patches = [f["phases"]["patches"] for f in frames if f.get("phases")]
if patches:
out.append(
f" patches p50={pct(patches,0.5):.0f} p99={pct(patches,0.99):.0f} "
f"max={max(patches)} total={sum(patches)}"
)
optimized = [
f["phases"].get("optimizedPatches", 0)
for f in frames if f.get("phases")
]
if any(optimized):
out.append(
f" optimized p50={pct(optimized,0.5):.0f} p99={pct(optimized,0.99):.0f} "
f"max={max(optimized)} total={sum(optimized)}"
f" (ratio: {sum(optimized)/max(1,sum(patches)):.2f})"
)
# Write bytes + drain telemetry — the outer-terminal bottleneck gauge.
bytes_written = [
f["phases"].get("writeBytes", 0)
for f in frames if f.get("phases")
]
if any(bytes_written):
total_b = sum(bytes_written)
kb = total_b / 1024
out.append(
f" writeBytes p50={pct(bytes_written,0.5):.0f}B p99={pct(bytes_written,0.99):.0f}B "
f"max={max(bytes_written)}B total={kb:.1f}KB"
)
drains = [
f["phases"].get("prevFrameDrainMs", 0)
for f in frames if f.get("phases")
]
if any(d > 0 for d in drains):
nonzero = [d for d in drains if d > 0]
out.append(
f" drainMs p50={pct(nonzero,0.5):.2f} p95={pct(nonzero,0.95):.2f} "
f"p99={pct(nonzero,0.99):.2f} max={max(nonzero):.2f} (terminal flush latency)"
)
backpressure = sum(1 for f in frames if f.get("phases", {}).get("backpressure"))
if backpressure:
out.append(
f" backpressure: {backpressure}/{len(frames)} frames "
f"({100*backpressure/len(frames):.0f}%) (Node stdout buffer full — terminal slow)"
)
# Flickers
flicker_frames = [f for f in frames if f.get("flickers")]
if flicker_frames:
out.append("")
out.append(f" ⚠ flickers detected in {len(flicker_frames)} frames")
reasons: dict[str, int] = {}
for f in flicker_frames:
for fl in f["flickers"]:
reasons[fl["reason"]] = reasons.get(fl["reason"], 0) + 1
for reason, n in sorted(reasons.items(), key=lambda kv: -kv[1]):
out.append(f" {reason}: {n}")
return "\n".join(out)
def key_metrics(data: dict[str, Any]) -> dict[str, float]:
"""Flatten the report into a dict of scalar metrics for A/B diffing."""
metrics: dict[str, float] = {}
frames = data.get("frame") or []
react = data.get("react") or []
if frames:
durs = [f["durationMs"] for f in frames]
metrics["frames"] = len(frames)
metrics["dur_p50"] = pct(durs, 0.50)
metrics["dur_p95"] = pct(durs, 0.95)
metrics["dur_p99"] = pct(durs, 0.99)
metrics["dur_max"] = max(durs)
ts = sorted(f["ts"] for f in frames)
if len(ts) >= 2:
elapsed = (ts[-1] - ts[0]) / 1000.0
metrics["fps_throughput"] = len(frames) / elapsed if elapsed > 0 else 0.0
# Interframe gaps distribution — complementary view to throughput:
gaps = [ts[i] - ts[i - 1] for i in range(1, len(ts))]
if gaps:
metrics["gap_p50_ms"] = pct(gaps, 0.50)
metrics["gap_p99_ms"] = pct(gaps, 0.99)
metrics["gaps_under_16ms"] = sum(1 for g in gaps if g < 16)
metrics["gaps_over_200ms"] = sum(1 for g in gaps if g >= 200)
for phase in ("renderer", "yoga", "diff", "write"):
vals = [f["phases"][phase] for f in frames if f.get("phases")]
if vals:
metrics[f"{phase}_p99"] = pct(vals, 0.99)
metrics[f"{phase}_max"] = max(vals)
patches = [f["phases"]["patches"] for f in frames if f.get("phases")]
if patches:
metrics["patches_total"] = sum(patches)
metrics["patches_p99"] = pct(patches, 0.99)
optimized = [
f["phases"].get("optimizedPatches", 0) for f in frames if f.get("phases")
]
if any(optimized):
metrics["optimized_total"] = sum(optimized)
bytes_list = [
f["phases"].get("writeBytes", 0) for f in frames if f.get("phases")
]
if any(bytes_list):
metrics["writeBytes_total"] = sum(bytes_list)
drains = [
f["phases"].get("prevFrameDrainMs", 0)
for f in frames if f.get("phases")
]
drain_nonzero = [d for d in drains if d > 0]
if drain_nonzero:
metrics["drain_p99"] = pct(drain_nonzero, 0.99)
metrics["drain_max"] = max(drain_nonzero)
bp = sum(1 for f in frames if f.get("phases", {}).get("backpressure"))
metrics["backpressure_frames"] = bp
if react:
for pid in {e["id"] for e in react}:
ms = [e["actualMs"] for e in react if e["id"] == pid]
metrics[f"react_{pid}_p99"] = pct(ms, 0.99)
metrics[f"react_{pid}_max"] = max(ms)
return metrics
def format_diff(before: dict[str, float], after: dict[str, float]) -> str:
"""Render a side-by-side A/B comparison table."""
keys = sorted(set(before) | set(after))
lines = [f"{'metric':<28} {'before':>12} {'after':>12} {'delta':>12} {'%':>6}"]
lines.append("─" * 76)
for k in keys:
b = before.get(k, 0.0)
a = after.get(k, 0.0)
d = a - b
pct_change = ((a / b) - 1) * 100 if b not in {0, 0.0} else float("inf") if a else 0
# Flag improvements vs regressions. For _p99 / _max / _total / gaps_over /
# patches / writeBytes / backpressure, LOWER is better. For fps / gaps_under,
# HIGHER is better.
lower_is_better = any(
token in k
for token in (
"p50",
"p95",
"p99",
"_max",
"_total",
"gaps_over",
"backpressure",
"drain",
)
)
higher_is_better = "fps_" in k or "gaps_under" in k
mark = ""
if d and not (lower_is_better or higher_is_better):
mark = ""
elif d < 0 and lower_is_better:
mark = "↓"
elif d > 0 and higher_is_better:
mark = "↑"
elif d > 0 and lower_is_better:
mark = "↑" # regression
elif d < 0 and higher_is_better:
mark = "↓" # regression
pct_str = "—" if pct_change == float("inf") else f"{pct_change:+6.1f}%"
lines.append(
f"{k:<28} {b:>12.2f} {a:>12.2f} {d:>+12.2f} {pct_str} {mark}"
)
return "\n".join(lines)
def run_once(args: argparse.Namespace) -> dict[str, Any]:
tui_dir = Path(args.tui_dir).resolve()
entry = tui_dir / "dist" / "entry.js"
if not entry.exists():
sys.exit(f"{entry} missing — run `npm run build` in {tui_dir} first")
sid = args.session or pick_longest_session(DEFAULT_STATE_DB)
print(f"• session: {sid}")
print(f"• hold: {args.hold} x {args.rate}Hz for {args.seconds}s after {args.warmup}s warmup")
print(f"• terminal: {args.cols}x{args.rows}")
log = Path(args.log)
if not args.keep_log and log.exists():
log.unlink()
since_ms = int(time.time() * 1000)
env = os.environ.copy()
env["HERMES_DEV_PERF"] = "1"
env["HERMES_DEV_PERF_MS"] = str(args.threshold_ms)
env["HERMES_DEV_PERF_LOG"] = str(log)
env["HERMES_TUI_RESUME"] = sid
env["COLUMNS"] = str(args.cols)
env["LINES"] = str(args.rows)
env["TERM"] = env.get("TERM", "xterm-256color")
# Pass through extra flags the TUI wrapper recognizes (e.g. --no-fullscreen).
# Stored on args as `extra_flags` list.
node = os.environ.get("HERMES_PERF_NODE", "node")
node_args = [node, str(entry), *getattr(args, "extra_flags", [])]
pid, fd = pty.fork()
if pid == 0:
os.execvpe(node, node_args, env)
try:
import fcntl, struct, termios
winsize = struct.pack("HHHH", args.rows, args.cols, 0, 0)
fcntl.ioctl(fd, termios.TIOCSWINSZ, winsize)
print(f"• pid: {pid} fd: {fd}")
print(f"• warmup {args.warmup}s (drain startup output)…")
drain(fd, args.warmup)
print(f"• holding {args.hold}…")
sent = hold_key(fd, KEYS[args.hold], args.seconds, args.rate)
print(f" sent {sent} keystrokes")
drain(fd, 0.5)
finally:
try:
os.kill(pid, signal.SIGTERM)
for _ in range(10):
pid_done, _ = os.waitpid(pid, os.WNOHANG)
if pid_done == pid:
break
time.sleep(0.1)
else:
os.kill(pid, signal.SIGKILL) # windows-footgun: ok — POSIX-only script (imports pty at top)
os.waitpid(pid, 0)
except (ProcessLookupError, ChildProcessError):
pass
try:
os.close(fd)
except OSError:
pass
time.sleep(0.2)
return summarize(log, since_ms)
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--session", help="session id to resume (default: longest in db)")
p.add_argument("--hold", default="page_up", choices=sorted(KEYS.keys()), help="key to hold")
p.add_argument("--seconds", type=float, default=8.0, help="how long to hold the key")
p.add_argument("--rate", type=int, default=30, help="keystrokes per second")
p.add_argument("--warmup", type=float, default=3.0, help="seconds to wait after launch before input")
p.add_argument("--threshold-ms", type=float, default=0.0, help="HERMES_DEV_PERF_MS (0 = capture all)")
p.add_argument("--cols", type=int, default=120)
p.add_argument("--rows", type=int, default=40)
p.add_argument("--keep-log", action="store_true", help="don't wipe perf.log before run")
p.add_argument("--tui-dir", default=str(DEFAULT_TUI_DIR))
p.add_argument("--log", default=str(DEFAULT_LOG))
p.add_argument("--save", metavar="LABEL",
help="save the final metrics as /tmp/perf-<LABEL>.json for later --compare")
p.add_argument("--compare", metavar="LABEL",
help="diff against /tmp/perf-<LABEL>.json after running")
p.add_argument("--loop", action="store_true",
help="watch for source changes, rebuild, rerun, and diff vs previous run")
p.add_argument("--extra-flag", dest="extra_flags", action="append", default=[],
help="pass through to node dist/entry.js (repeatable)")
args = p.parse_args()
if args.loop:
return loop_mode(args)
# Single-shot path.
data = run_once(args)
print()
print(format_report(data))
metrics = key_metrics(data)
if args.save:
path = Path(f"/tmp/perf-{args.save}.json")
path.write_text(json.dumps(metrics, indent=2), encoding="utf-8")
print(f"\n• saved: {path}")
if args.compare:
path = Path(f"/tmp/perf-{args.compare}.json")
if not path.exists():
print(f"\n⚠ no baseline at {path} — run with --save {args.compare} first")
else:
before = json.loads(path.read_text(encoding="utf-8"))
print(f"\n═══ A/B diff vs /tmp/perf-{args.compare}.json ═══")
print(format_diff(before, metrics))
if not data["react"] and not data["frame"]:
return 2
return 0
def loop_mode(args: argparse.Namespace) -> int:
"""Watch source files, rebuild, rerun, print A/B diff against previous run.
Keeps a rolling 'previous run' baseline in memory so each iteration
reports delta vs the last one — visibility into whether the last
edit moved the needle. Press Ctrl+C to stop.
"""
import subprocess
tui_dir = Path(args.tui_dir).resolve()
src_root = tui_dir / "src"
pkg_root = tui_dir / "packages" / "hermes-ink" / "src"
def collect_mtimes() -> dict[str, float]:
mtimes: dict[str, float] = {}
for root in (src_root, pkg_root):
if not root.exists():
continue
for path in root.rglob("*"):
if path.suffix in {".ts", ".tsx"} and "__tests__" not in str(path):
try:
mtimes[str(path)] = path.stat().st_mtime
except OSError:
pass
return mtimes
previous_metrics: dict[str, float] | None = None
previous_mtimes = collect_mtimes()
iteration = 0
print(f"• loop mode — watching {src_root} + {pkg_root} for *.ts(x) changes")
print("• edit any TS file, the harness rebuilds + reruns automatically")
print("• Ctrl+C to stop\n")
try:
while True:
iteration += 1
print(f"\n{'═' * 76}")
print(f"Iteration {iteration} @ {time.strftime('%H:%M:%S')}")
print("═" * 76)
if iteration > 1:
print("• rebuilding…")
result = subprocess.run(
["npm", "run", "build"],
cwd=tui_dir,
capture_output=True,
text=True, encoding='utf-8', errors='replace',
)
if result.returncode != 0:
print("✗ build failed:")
print(result.stdout[-2000:])
print(result.stderr[-2000:])
print("\n• waiting for source changes to retry…")
previous_mtimes = wait_for_change(previous_mtimes, collect_mtimes)
continue
print("✓ build ok")
data = run_once(args)
metrics = key_metrics(data)
print()
print(format_report(data))
if previous_metrics is not None:
print(f"\n═══ A/B diff vs iteration {iteration - 1} ═══")
print(format_diff(previous_metrics, metrics))
previous_metrics = metrics
print("\n• waiting for source changes…")
previous_mtimes = wait_for_change(previous_mtimes, collect_mtimes)
except KeyboardInterrupt:
print("\n• loop stopped")
return 0
def wait_for_change(prev: dict[str, float], collect) -> dict[str, float]:
"""Poll every 1s until a watched file's mtime changes. Debounced 500ms."""
while True:
time.sleep(1)
current = collect()
changed = [
path for path, mtime in current.items() if prev.get(path) != mtime
]
if changed:
print(f" ↻ {len(changed)} file(s) changed:")
for path in changed[:5]:
print(f" {path}")
# Debounce — editor save bursts can take ~500ms to settle
time.sleep(0.5)
return collect()
if __name__ == "__main__":
sys.exit(main())
scripts/release.py
Executable
+2643
View File
File diff suppressed because it is too large Load Diff
scripts/run_tests.sh
Executable
+183
View File
@@ -0,0 +1,183 @@
#!/usr/bin/env bash
# Canonical test runner for hermes-agent. Run this instead of calling
# `pytest` directly to guarantee your local run matches CI behavior.
#
# What this script enforces:
# * Per-file isolation via scripts/run_tests_parallel.py — each test
# file runs in its own freshly-spawned `python -m pytest <file>`
# subprocess. No xdist, no shared workers, no module-level leakage
# between files.
# * TZ=UTC, LANG=C.UTF-8, PYTHONHASHSEED=0 (deterministic)
# * Env vars blanked (conftest.py also does this, but this
# is belt-and-suspenders for anyone running pytest outside our
# conftest path — e.g. on a single file)
# * Proper venv activation (probes .venv, venv, then ~/.hermes/...)
#
# Usage:
# scripts/run_tests.sh # full suite
# scripts/run_tests.sh -j 4 # cap parallelism
# scripts/run_tests.sh tests/agent/ # discover only here
# scripts/run_tests.sh tests/agent/ tests/acp/ # multiple roots
# scripts/run_tests.sh tests/foo.py # single file
# scripts/run_tests.sh tests/foo.py -q # path + bare pytest flag
# scripts/run_tests.sh tests/foo.py -v --tb=long # bare flags "just work"
# scripts/run_tests.sh -k 'pattern' # value flags pass through too
# scripts/run_tests.sh tests/foo.py -- --tb=long # explicit '--' still works
#
# Bare pytest flags (anything starting with '-' that isn't one of this
# runner's own options: -j/--jobs, --paths, --slice, --file-timeout, etc.)
# are forwarded to each per-file pytest invocation automatically — no '--'
# separator required. The explicit '--' form still works and stacks with
# bare flags. Positional path arguments override the default discovery
# root (tests/).
set -euo pipefail
# ── Locate repo root ────────────────────────────────────────────────────────
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
# ── Locate python ───────────────────────────────────────────────────────────
# Probe local venvs first; fall back to the Nix devShell's editable venv
# (HERMES_PYTHON is exported by the devShell hook and ships [dev] extras:
# pytest, pytest-asyncio, pytest-timeout, ruff, ty).
#
# A candidate must have pytest INSTALLED, not merely exist. The release venv
# at ~/.hermes/hermes-agent/venv has bin/activate but no pytest, so an
# existence-only probe selected it in checkouts/worktrees without a local
# .venv — every file then died with "No module named pytest" and the run
# reported "0 tests passed" (which reads green at a glance even though the
# exit code is 1). Skip such a venv and keep probing instead.
VENV=""
VENV_PYTHON=""
SKIPPED_VENVS=""
for candidate in "$REPO_ROOT/.venv" "$REPO_ROOT/venv" "$HOME/.hermes/hermes-agent/venv"; do
if [ -f "$candidate/bin/activate" ]; then
if "$candidate/bin/python" -c 'import pytest' 2>/dev/null; then
VENV="$candidate"
VENV_PYTHON="$candidate/bin/python"
break
fi
SKIPPED_VENVS="$SKIPPED_VENVS $candidate"
fi
# Native Windows venv layout: python.exe and activate live under
# Scripts/, and there is no bin/. Anyone running this script from
# Git Bash / MSYS with a `python -m venv`- or uv-created venv hits
# this branch — without it the canonical runner refuses to start.
if [ -f "$candidate/Scripts/activate" ]; then
if "$candidate/Scripts/python.exe" -c 'import pytest' 2>/dev/null; then
VENV="$candidate"
VENV_PYTHON="$candidate/Scripts/python.exe"
break
fi
SKIPPED_VENVS="$SKIPPED_VENVS $candidate"
fi
done
if [ -n "$SKIPPED_VENVS" ]; then
for skipped in $SKIPPED_VENVS; do
echo "▶ skipping venv without pytest: $skipped" >&2
done
fi
if [ -n "$VENV" ]; then
PYTHON="$VENV_PYTHON"
elif [ -n "${HERMES_PYTHON:-}" ] && [ -x "$HERMES_PYTHON" ] \
&& "$HERMES_PYTHON" -c 'import pytest' 2>/dev/null; then
# Guard with an import check: HERMES_PYTHON may point at the RELEASE
# venv (no pytest) when inherited from a wrapped `hermes` binary rather
# than the devShell hook.
PYTHON="$HERMES_PYTHON"
echo "▶ no local venv — using Nix dev venv via HERMES_PYTHON: $PYTHON"
else
echo "error: no virtualenv with pytest found in $REPO_ROOT/.venv or $REPO_ROOT/venv," >&2
echo " and HERMES_PYTHON is not a python with pytest (enter the Nix devShell or create a venv)" >&2
if [ -n "$SKIPPED_VENVS" ]; then
echo " (skipped for missing pytest:$SKIPPED_VENVS — install dev extras there, or create $REPO_ROOT/.venv)" >&2
fi
exit 1
fi
# ── Live-gateway plugin (computed before we drop env) ───────────────────────
EXTRA_PYTHONPATH=""
EXTRA_PYTEST_PLUGINS=""
if [ -f "$HOME/.hermes/pytest_live_guard.py" ]; then
EXTRA_PYTHONPATH="$HOME/.hermes"
EXTRA_PYTEST_PLUGINS="pytest_live_guard"
fi
# ── Windows location variables (computed before we drop env) ───────────────
# `env -i` forwards HOME, which is enough on POSIX. Native Windows CPython
# resolves Path.home() from USERPROFILE (or HOMEDRIVE+HOMEPATH), stdlib
# platform paths come from LOCALAPPDATA/APPDATA, ssl/sockets need SYSTEMROOT,
# and tempfile needs TEMP/TMP. Dropping them breaks collection on native
# Windows (issues #67385, #70813). These are location variables, not
# credentials, so forwarding them keeps the isolation intent intact. Each is
# only forwarded when actually set, so POSIX runs are byte-for-byte unchanged.
WIN_ENV=()
for _win_var in USERPROFILE HOMEDRIVE HOMEPATH LOCALAPPDATA APPDATA SYSTEMROOT TEMP TMP; do
if [ -n "${!_win_var:-}" ]; then
WIN_ENV+=("$_win_var=${!_win_var}")
fi
done
# ── Test-runner knobs (computed before we drop env) ────────────────────────
# The runner's own documented environment knobs must survive the hermetic
# `env -i` below, or they are silent no-ops for anyone invoking this script:
#
# * HERMES_TEST_WORKERS / PATHS / FILE_TIMEOUT / FILE_RETRIES / SLICE are
# read by run_tests_parallel.py at argparse-default time — inside the
# stripped environment.
# * HERMES_TEST_IMAGE is read by tests/docker/conftest.py to skip its
# session-scoped `docker build`. CI's docker.yml sets it to the image
# the build step just loaded; stripping it made every per-file pytest
# subprocess rebuild the 5GB image from a cold builder cache instead
# (~4 min per worker per run, and the rebuilt image lacked the
# HERMES_GIT_SHA build-arg the workflow bakes in).
#
# These are test-infrastructure knobs, not credentials — same class as the
# HERMES_RUN_SLOW_PET_TESTS / HERMES_E2E_BROWSER opt-ins already forwarded.
# Keep this an explicit allowlist (no HERMES_TEST_* glob) so the "no
# credential can leak" property stays auditable at a glance.
TEST_ENV=()
for _test_var in HERMES_TEST_IMAGE HERMES_TEST_WORKERS HERMES_TEST_PATHS \
HERMES_TEST_FILE_TIMEOUT HERMES_TEST_FILE_RETRIES HERMES_TEST_SLICE; do
if [ -n "${!_test_var:-}" ]; then
TEST_ENV+=("$_test_var=${!_test_var}")
fi
done
# ── Run in hermetic env ──────────────────────────────────────────────────────
# env -i: start with empty environment, opt-in only what we need.
# No credential var can leak — you'd have to explicitly add it here.
echo "▶ running per-file parallel test suite via run_tests_parallel.py"
echo " (TZ=UTC LANG=C.UTF-8 PYTHONHASHSEED=0; clean env)"
cd "$REPO_ROOT"
# ── Pre-compile .pyc bytecode cache ─────────────────────────────────────────
# Each test file runs in its own subprocess via run_tests_parallel.py.
# Pre-building the bytecode cache once here (instead of each subprocess
# compiling on first import) avoids redundant work across ~2000 processes.
# Uses git to list tracked .py files (skips venv, node_modules, etc).
echo "▶ pre-compiling bytecode cache"
"$PYTHON" -m compileall -q -j 0 -- $(git ls-files '*.py') >/dev/null 2>&1 || true
echo "▶ launching test runner"
exec env -i \
PATH="$PATH" \
HOME="$HOME" \
${WIN_ENV[@]+"${WIN_ENV[@]}"} \
${TEST_ENV[@]+"${TEST_ENV[@]}"} \
TZ=UTC \
LANG=C.UTF-8 \
LC_ALL=C.UTF-8 \
PYTHONHASHSEED=0 \
PYTHONUTF8=1 \
${HERMES_RUN_SLOW_PET_TESTS:+HERMES_RUN_SLOW_PET_TESTS="$HERMES_RUN_SLOW_PET_TESTS"} \
${HERMES_E2E_BROWSER:+HERMES_E2E_BROWSER="$HERMES_E2E_BROWSER"} \
${EXTRA_PYTHONPATH:+PYTHONPATH="$EXTRA_PYTHONPATH"} \
${EXTRA_PYTEST_PLUGINS:+PYTEST_PLUGINS="$EXTRA_PYTEST_PLUGINS"} \
"$PYTHON" "$SCRIPT_DIR/run_tests_parallel.py" "$@"
scripts/run_tests_parallel.py
Executable
+1262
View File
File diff suppressed because it is too large Load Diff
scripts/sample_and_compress.py
+409
View File
@@ -0,0 +1,409 @@
#!/usr/bin/env python3
"""
Sample and Compress HuggingFace Datasets
Downloads trajectories from multiple HuggingFace datasets, randomly samples them,
and runs trajectory compression to fit within a target token budget.
Usage:
python scripts/sample_and_compress.py
# Custom sample size
python scripts/sample_and_compress.py --total_samples=5000
# Custom output name
python scripts/sample_and_compress.py --output_name=compressed_16k
"""
import json
import random
from pathlib import Path
from typing import List, Dict, Any, Tuple
import fire
# Load environment variables
from dotenv import load_dotenv
load_dotenv()
# Default datasets to sample from
DEFAULT_DATASETS = [
"NousResearch/swe-terminus-agent-glm-kimi-minimax",
"NousResearch/hermes-agent-megascience-sft1",
"NousResearch/Hermes-Agent-Thinking-GLM-4.7-SFT2",
"NousResearch/Hermes-Agent-Thinking-GLM-4.7-SFT1",
"NousResearch/terminal-tasks-glm-hermes-agent"
]
def load_dataset_from_hf(dataset_name: str) -> List[Dict[str, Any]]:
"""
Load a dataset from HuggingFace.
Args:
dataset_name: HuggingFace dataset name (e.g., "NousResearch/dataset-name")
Returns:
List of trajectory entries
"""
from datasets import load_dataset
print(f" Loading {dataset_name}...")
try:
# Try loading with default config
ds = load_dataset(dataset_name, split="train")
except Exception as e:
print(f" ⚠️ Error loading {dataset_name}: {e}")
return []
# Convert to list of dicts
entries = []
for item in ds:
# Handle different possible formats
if "conversations" in item:
entries.append({"conversations": item["conversations"]})
elif "messages" in item:
# Convert messages format to conversations format if needed
entries.append({"conversations": item["messages"]})
else:
# Assume the whole item is the entry
entries.append(dict(item))
print(f" ✅ Loaded {len(entries):,} entries from {dataset_name}")
return entries
# Global tokenizer for multiprocessing (set in worker init)
_TOKENIZER = None
def _init_tokenizer_worker(tokenizer_name: str):
"""Initialize tokenizer in worker process."""
global _TOKENIZER
from transformers import AutoTokenizer
_TOKENIZER = AutoTokenizer.from_pretrained(tokenizer_name, trust_remote_code=True)
def _count_tokens_for_entry(entry: Dict) -> Tuple[Dict, int]:
"""
Count tokens for a single entry (used in parallel processing).
Args:
entry: Trajectory entry with 'conversations' field
Returns:
Tuple of (entry, token_count)
"""
global _TOKENIZER
conversations = entry.get("conversations", [])
if not conversations:
return entry, 0
total = 0
for turn in conversations:
value = turn.get("value", "")
if value:
try:
total += len(_TOKENIZER.encode(value))
except Exception:
# Fallback to character estimate
total += len(value) // 4
return entry, total
def sample_from_datasets(
datasets: List[str],
total_samples: int,
min_tokens: int = 16000,
tokenizer_name: str = "moonshotai/Kimi-K2-Thinking",
seed: int = 42,
num_proc: int = 8
) -> List[Dict[str, Any]]:
"""
Load all datasets, filter by token count, then randomly sample from combined pool.
Args:
datasets: List of HuggingFace dataset names
total_samples: Total number of samples to collect
min_tokens: Minimum token count to include (only sample trajectories >= this)
tokenizer_name: HuggingFace tokenizer for counting tokens
seed: Random seed for reproducibility
num_proc: Number of parallel processes for tokenization
Returns:
List of sampled trajectory entries
"""
from multiprocessing import Pool
random.seed(seed)
print(f"\n📥 Loading {len(datasets)} datasets...")
print(f" Minimum tokens: {min_tokens:,} (filtering smaller trajectories)")
print(f" Parallel workers: {num_proc}")
print()
# Load ALL entries from all datasets into one pool
all_entries = []
for dataset_name in datasets:
entries = load_dataset_from_hf(dataset_name)
if not entries:
print(f" ⚠️ Skipping {dataset_name} (no entries loaded)")
continue
# Add source metadata to each entry
for entry in entries:
entry["_source_dataset"] = dataset_name
all_entries.extend(entries)
print(f"\n📊 Total entries loaded: {len(all_entries):,}")
# Filter by token count using parallel processing
print(f"\n🔍 Filtering trajectories with >= {min_tokens:,} tokens (using {num_proc} workers)...")
filtered_entries = []
token_counts = []
# Use multiprocessing for token counting
with Pool(
processes=num_proc,
initializer=_init_tokenizer_worker,
initargs=(tokenizer_name,)
) as pool:
# Process in chunks and show progress
chunk_size = 1000
processed = 0
for result in pool.imap_unordered(_count_tokens_for_entry, all_entries, chunksize=100):
entry, token_count = result
processed += 1
if processed % chunk_size == 0:
print(f" Processed {processed:,}/{len(all_entries):,}...", end="\r")
if token_count >= min_tokens:
entry["_original_tokens"] = token_count
filtered_entries.append(entry)
token_counts.append(token_count)
print(f"\n ✅ Found {len(filtered_entries):,} trajectories >= {min_tokens:,} tokens")
if token_counts:
avg_tokens = sum(token_counts) / len(token_counts)
print(f" 📈 Token stats: min={min(token_counts):,}, max={max(token_counts):,}, avg={avg_tokens:,.0f}")
# Random sample from the filtered pool
if len(filtered_entries) <= total_samples:
print(f"\n⚠️ Only {len(filtered_entries):,} trajectories available, using all of them")
sampled = filtered_entries
else:
sampled = random.sample(filtered_entries, total_samples)
print(f"\n✅ Randomly sampled {len(sampled):,} trajectories from pool of {len(filtered_entries):,}")
# Show source distribution
source_counts = {}
for entry in sampled:
source = entry.get("_source_dataset", "unknown").split("/")[-1]
source_counts[source] = source_counts.get(source, 0) + 1
print("\n📌 Sample distribution by source:")
for source, count in sorted(source_counts.items()):
print(f" {source}: {count:,}")
# Shuffle
random.shuffle(sampled)
return sampled
def save_samples_for_compression(
samples: List[Dict[str, Any]],
output_dir: Path,
batch_size: int = 100
):
"""
Save samples to JSONL files for trajectory compression.
Args:
samples: List of trajectory entries
output_dir: Directory to save JSONL files
batch_size: Number of entries per file
"""
output_dir.mkdir(parents=True, exist_ok=True)
# Split into batches
num_batches = (len(samples) + batch_size - 1) // batch_size
print(f"\n💾 Saving {len(samples)} samples to {output_dir}")
print(f" Batch size: {batch_size}, Total batches: {num_batches}")
for i in range(num_batches):
start_idx = i * batch_size
end_idx = min((i + 1) * batch_size, len(samples))
batch = samples[start_idx:end_idx]
output_file = output_dir / f"batch_{i}.jsonl"
with open(output_file, 'w', encoding='utf-8') as f:
for entry in batch:
f.write(json.dumps(entry, ensure_ascii=False) + '\n')
print(f" ✅ Saved {num_batches} batch files")
def run_compression(input_dir: Path, output_dir: Path, config_path: str):
"""
Run trajectory compression on the sampled data.
Args:
input_dir: Directory containing JSONL files to compress
output_dir: Directory for compressed output
config_path: Path to compression config YAML
"""
# Import the compressor
import sys
sys.path.insert(0, str(Path(__file__).parent.parent))
from trajectory_compressor import TrajectoryCompressor, CompressionConfig
print("\n🗜️ Running trajectory compression...")
print(f" Input: {input_dir}")
print(f" Output: {output_dir}")
print(f" Config: {config_path}")
# Load config
config = CompressionConfig.from_yaml(config_path)
# Initialize compressor
compressor = TrajectoryCompressor(config)
# Run compression
compressor.process_directory(input_dir, output_dir)
def merge_output_to_single_jsonl(input_dir: Path, output_file: Path):
"""
Merge all JSONL files in a directory into a single JSONL file.
Args:
input_dir: Directory containing JSONL files
output_file: Output JSONL file path
"""
print(f"\n📦 Merging output files into {output_file.name}...")
all_entries = []
for jsonl_file in sorted(input_dir.glob("*.jsonl")):
if jsonl_file.name == output_file.name:
continue
with open(jsonl_file, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if line:
all_entries.append(json.loads(line))
# Write merged file
with open(output_file, 'w', encoding='utf-8') as f:
for entry in all_entries:
f.write(json.dumps(entry, ensure_ascii=False) + '\n')
print(f" ✅ Merged {len(all_entries):,} entries into {output_file.name}")
return output_file
def main(
total_samples: int = 2500,
output_name: str = "compressed_agentic",
datasets: str = None,
config: str = "configs/trajectory_compression.yaml",
seed: int = 42,
batch_size: int = 100,
min_tokens: int = 16000,
num_proc: int = 8,
skip_download: bool = False,
):
"""
Sample trajectories from HuggingFace datasets and run compression.
Args:
total_samples: Total number of samples to collect (default: 2500)
output_name: Name for output directory/file (default: "compressed_agentic")
datasets: Comma-separated list of dataset names (uses defaults if not provided)
config: Path to compression config YAML
seed: Random seed for reproducibility
batch_size: Number of entries per JSONL file during processing
min_tokens: Minimum token count to filter trajectories (default: 16000)
num_proc: Number of parallel workers for tokenization (default: 8)
skip_download: Skip download and use existing sampled data
"""
print("=" * 70)
print("📊 TRAJECTORY SAMPLING AND COMPRESSION")
print("=" * 70)
# Parse datasets
if datasets:
dataset_list = [d.strip() for d in datasets.split(",")]
else:
dataset_list = DEFAULT_DATASETS
print("\n📋 Configuration:")
print(f" Total samples: {total_samples:,}")
print(f" Min tokens filter: {min_tokens:,}")
print(f" Parallel workers: {num_proc}")
print(f" Datasets: {len(dataset_list)}")
for ds in dataset_list:
print(f" - {ds}")
print(f" Output name: {output_name}")
print(f" Config: {config}")
print(f" Seed: {seed}")
# Setup paths
base_dir = Path(__file__).parent.parent
sampled_dir = base_dir / "data" / f"{output_name}_raw"
compressed_dir = base_dir / "data" / f"{output_name}_batches"
final_output = base_dir / "data" / f"{output_name}.jsonl"
if not skip_download:
# Step 1: Download, filter by token count, and sample from combined pool
samples = sample_from_datasets(
dataset_list,
total_samples,
min_tokens=min_tokens,
seed=seed,
num_proc=num_proc
)
if not samples:
print("❌ No samples collected. Exiting.")
return
# Step 2: Save to JSONL files
save_samples_for_compression(samples, sampled_dir, batch_size)
else:
print(f"\n⏭️ Skipping download, using existing data in {sampled_dir}")
# Step 3: Run compression
config_path = base_dir / config
if not config_path.exists():
print(f"❌ Config not found: {config_path}")
return
run_compression(sampled_dir, compressed_dir, str(config_path))
# Step 4: Merge into single JSONL file
merge_output_to_single_jsonl(compressed_dir, final_output)
print("\n" + "=" * 70)
print("✅ COMPLETE!")
print("=" * 70)
print(f"\n📁 Raw samples: {sampled_dir}")
print(f"📁 Compressed batches: {compressed_dir}")
print(f"📁 Final output: {final_output}")
print("\nTo upload to HuggingFace:")
print(f" huggingface-cli upload NousResearch/{output_name} {final_output}")
if __name__ == "__main__":
fire.Fire(main)
scripts/sandbox/openssl.cnf
+43
View File
@@ -0,0 +1,43 @@
# Minimal openssl config for the dev sandbox.
#
# The sandbox replaces /etc wholesale, and on Debian/Ubuntu
# /usr/lib/ssl/openssl.cnf (openssl's compiled-in OPENSSLDIR) is a symlink into
# /etc/ssl -- so the config openssl insists on reading disappears and every
# `openssl req` fails with:
#
# Can't open "/usr/lib/ssl/openssl.cnf" for reading
#
# which surfaces to the payload as a bare `curl: (35) Recv failure`. Rather than
# reconstruct each distro's /etc/ssl, point OPENSSL_CONF at this file: the proxy
# only needs enough config for `req -addext` and `x509 -copy_extensions`.
[ req ]
distinguished_name = req_distinguished_name
[ req_distinguished_name ]
# Used by `req -x509` for the sandbox's own CA. Without an explicit
# basicConstraints the generated certificate is not a CA, and every leaf it
# signs is rejected by the client with "invalid CA certificate (79)".
[ sandbox_ca_ext ]
basicConstraints = critical,CA:true
keyUsage = critical,keyCertSign,cRLSign
subjectKeyIdentifier = hash
[ ca ]
default_ca = sandbox_ca
[ sandbox_ca ]
default_md = sha256
policy = policy_anything
email_in_dn = no
preserve = no
[ policy_anything ]
commonName = optional
countryName = optional
stateOrProvinceName = optional
localityName = optional
organizationName = optional
organizationalUnitName = optional
emailAddress = optional
scripts/sandbox/pick-release-tags.sh
Executable
+110
View File
@@ -0,0 +1,110 @@
#!/usr/bin/env bash
# Pick the release tags the install/update E2E should update FROM.
#
# Emits a JSON array of tag names on stdout, suitable for a GitHub Actions
# matrix (`fromJSON`). Choosing at runtime rather than hardcoding keeps the
# matrix honest as releases land: a pinned list silently stops covering the
# newest release the day after it ships, and pins the "oldest" forever even
# after it stops being a version anyone still runs.
#
# Selection: the newest tag, the oldest tag, and evenly spaced tags in between.
# Newest catches "did the last release break updating?", oldest is the longest
# upgrade jump anyone can still make, and the spread samples the migrations in
# between (config-schema bumps, venv layout changes, dependency floors).
#
# Usage:
# scripts/sandbox/pick-release-tags.sh [--count N] [--repo DIR]
#
# --count how many tags to emit (default 5, minimum 1). Fewer tags than
# requested emits all of them.
# --repo repository to read tags from (default: this checkout).
#
# Reads tags from the local checkout, so it needs one fetched with tags
# (actions/checkout with fetch-depth: 0, or `fetch-tags: true`). A shallow
# checkout has no tags and this exits non-zero rather than silently emitting an
# empty matrix.
#
# Only vYYYY.M.D[.N] release tags are considered; the repo also carries
# backup/* and one-off tags that are not releases.
set -euo pipefail
COUNT=5
# Default to the repository containing this script, resolved through its real
# path so a symlinked or copied script still reads the checkout it lives in
# rather than whatever repo the caller happens to be standing in.
REPO=""
while [ "$#" -gt 0 ]; do
case "$1" in
--count)
[ "$#" -ge 2 ] || { echo 'error: --count needs a value' >&2; exit 1; }
COUNT="$2"; shift 2 ;;
--repo)
[ "$#" -ge 2 ] || { echo 'error: --repo needs a value' >&2; exit 1; }
REPO="$2"; shift 2 ;;
-h|--help) sed -n '2,30p' "$0"; exit 0 ;;
*) echo "error: unknown argument: $1" >&2; exit 1 ;;
esac
done
case "$COUNT" in
''|*[!0-9]*) echo "error: --count must be a positive integer: $COUNT" >&2; exit 1 ;;
esac
[ "$COUNT" -ge 1 ] || { echo 'error: --count must be at least 1' >&2; exit 1; }
# Resolve the script's own location through symlinks, then ask git which
# worktree that path belongs to. Deriving the repo from the script rather than
# from $PWD means a copied script cannot silently report a different checkout's
# tags, and --show-toplevel keeps it correct when invoked from a subdirectory.
if [ -z "$REPO" ]; then
script_path="${BASH_SOURCE[0]}"
if command -v readlink >/dev/null 2>&1; then
script_path="$(readlink -f "$script_path" 2>/dev/null || printf '%s' "$script_path")"
fi
script_dir="$(cd "$(dirname "$script_path")" && pwd)"
REPO="$(git -C "$script_dir" rev-parse --show-toplevel 2>/dev/null || printf '%s' "$script_dir")"
fi
# sort -V orders v2026.4.8 before v2026.4.13 (numeric), which a plain
# lexicographic sort gets wrong.
mapfile -t tags < <(
git -C "$REPO" tag --list 'v*' \
| grep -E '^v[0-9]{4}\.[0-9]+\.[0-9]+(\.[0-9]+)?$' \
| sort -V
)
total="${#tags[@]}"
if [ "$total" -eq 0 ]; then
echo "error: no release tags found in $REPO" >&2
echo ' A shallow clone has no tags: fetch with tags (actions/checkout' >&2
echo ' with fetch-depth: 0, or fetch-tags: true).' >&2
exit 1
fi
if [ "$total" -le "$COUNT" ]; then
picked=("${tags[@]}")
elif [ "$COUNT" -eq 1 ]; then
# One slot means the newest release; there is no span to spread across.
picked=("${tags[$((total - 1))]}")
else
# Evenly spaced indices across [0, total-1], endpoints included, so the
# oldest and newest are always present and the rest are spread between them.
picked=()
for slot in $(seq 0 $((COUNT - 1))); do
# Round to nearest rather than truncate, so the spacing does not bunch
# toward the oldest end.
index=$(( (slot * (total - 1) * 2 + (COUNT - 1)) / ((COUNT - 1) * 2) ))
candidate="${tags[$index]}"
# Guard against a duplicate if rounding lands twice on the same tag.
case " ${picked[*]-} " in
*" $candidate "*) continue ;;
esac
picked+=("$candidate")
done
fi
printf '['
for i in "${!picked[@]}"; do
[ "$i" -eq 0 ] || printf ','
printf '"%s"' "${picked[$i]}"
done
printf ']\n'
scripts/sandbox/proxy.py
+237
View File
@@ -0,0 +1,237 @@
"""MITM proxy backing the dev sandbox's fake Internet.
Listens on 127.0.0.1:8080 and is pointed at by http_proxy/https_proxy inside
the sandbox. For each request it either serves a fixture from the filesystem or
forwards to the real host:
* ``<root>/<host>/<path>`` exists -> serve it. This is how the sandbox answers
the canonical install URL with the installer under test, so the payload can
run the true ``curl -fsSL https://…/install.sh | bash`` one-liner.
* otherwise -> forward upstream, verifying against the real CA bundle. The
sandbox is isolated from the *host*, not from the internet: a real install
still has to reach PyPI and npm.
HTTPS is intercepted by minting a per-host certificate from the sandbox's own
throwaway CA, which the payload trusts via CURL_CA_BUNDLE / SSL_CERT_FILE.
Usage: proxy.py <fixture-root> <certs-dir> <real-ca-bundle>
"""
import os
import pathlib
import socket
import ssl
import subprocess
import sys
import threading
from urllib.parse import unquote, urlsplit
ROOT, CERTS, REAL_CA = map(pathlib.Path, sys.argv[1:])
LISTEN_ADDRESS = ('127.0.0.1', 8080)
MAX_REQUEST_BYTES = 65536
UPSTREAM_TIMEOUT_SECONDS = 30
CERT_VALIDITY_DAYS = 2
def read_request(conn):
data = b""
while b"\r\n\r\n" not in data and len(data) < MAX_REQUEST_BYTES:
part = conn.recv(4096)
if not part:
return b""
data += part
return data
def run_openssl(args):
"""Run openssl, raising with its stderr when it fails.
Discarding stderr here costs real debugging time: the caller sees only a
dropped connection (``curl: (35) Recv failure``) and the log holds nothing
but the argv, so an unwritable directory, a missing CA key, and an option
the host's openssl rejects all look identical.
"""
done = subprocess.run(
['openssl', *args], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
)
if done.returncode != 0:
detail = done.stderr.decode('utf-8', 'replace').strip()
raise RuntimeError(
f'openssl {args[0]} failed (exit {done.returncode}): {detail}'
)
_CERT_LOCK = threading.Lock()
def cert_for(host):
"""Return a (cert, key) pair for host, minting it from the sandbox CA.
Minting is serialized and published atomically. The proxy is threaded, so
two concurrent requests for the same host would otherwise both run openssl
into the same paths, and a reader could pick up a finished certificate
beside a key from the other writer -- which TLS rejects as
``[X509: KEY_VALUES_MISMATCH] key values mismatch``.
"""
safe = ''.join(char if char.isalnum() or char in '.-' else '_' for char in host)
cert, key = CERTS / f'{safe}.pem', CERTS / f'{safe}.key'
if cert.exists() and key.exists():
return cert, key
with _CERT_LOCK:
# Re-check: another thread may have finished while we waited.
if cert.exists() and key.exists():
return cert, key
# Build under unique temp names, then rename into place. os.replace is
# atomic, so a reader sees either the old pair or the new one, never a
# half-written mix. The key lands first: the certificate's existence is
# what everything else keys off.
stamp = f'{os.getpid()}.{threading.get_ident()}'
tmp_key = CERTS / f'{safe}.key.{stamp}'
tmp_cert = CERTS / f'{safe}.pem.{stamp}'
csr = CERTS / f'{safe}.csr.{stamp}'
run_openssl([
'req', '-newkey', 'rsa:2048', '-nodes',
'-subj', f'/CN={host}',
'-addext', f'subjectAltName=DNS:{host}',
'-keyout', str(tmp_key), '-out', str(csr),
])
run_openssl([
'x509', '-req', '-days', str(CERT_VALIDITY_DAYS), '-in', str(csr),
'-CA', str(CERTS / 'ca.pem'), '-CAkey', str(CERTS / 'ca.key'),
'-CAcreateserial', '-copy_extensions', 'copy', '-out', str(tmp_cert),
])
csr.unlink(missing_ok=True)
os.replace(tmp_key, key)
os.replace(tmp_cert, cert)
return cert, key
def file_for(host, target):
"""Resolve a request to a fixture file, or None to forward upstream."""
path = urlsplit(target).path or '/'
parts = pathlib.PurePosixPath(unquote(path)).parts
if '..' in parts:
return None
candidate = ROOT / host / pathlib.PurePosixPath(*[p for p in parts if p != '/'])
if candidate.is_dir():
candidate /= 'index.html'
return candidate if candidate.is_file() else None
def respond_fixture(conn, found):
body = found.read_bytes()
headers = (
f'Content-Length: {len(body)}\r\nConnection: close\r\n\r\n'.encode()
)
conn.sendall(b'HTTP/1.1 200 OK\r\n' + headers + body)
def close_request(request, target=None):
"""Rewrite a proxied request for a direct upstream connection."""
headers, separator, body = request.partition(b'\r\n\r\n')
lines = headers.split(b'\r\n')
if target is not None:
method, _, version = lines[0].split(b' ', 2)
lines[0] = b' '.join((method, target.encode(), version))
lines = [
line for line in lines
if not line.lower().startswith(b'proxy-connection:')
]
lines.append(b'Connection: close')
return b'\r\n'.join(lines) + separator + body
def relay(source, destination):
while True:
chunk = source.recv(MAX_REQUEST_BYTES)
if not chunk:
return
destination.sendall(chunk)
def forward_https(conn, host, port, request):
context = ssl.create_default_context(cafile=str(REAL_CA))
with socket.create_connection((host, port), timeout=UPSTREAM_TIMEOUT_SECONDS) as raw:
with context.wrap_socket(raw, server_hostname=host) as upstream:
upstream.sendall(close_request(request))
relay(upstream, conn)
def forward_http(conn, host, port, request, target):
parsed = urlsplit(target)
path = parsed.path or '/'
if parsed.query:
path += f'?{parsed.query}'
with socket.create_connection((host, port), timeout=UPSTREAM_TIMEOUT_SECONDS) as upstream:
upstream.sendall(close_request(request, path))
relay(upstream, conn)
def handle_connect(conn, target):
"""Intercept a CONNECT tunnel, terminating TLS with a minted cert."""
host, _, port_text = target.rpartition(':')
port = int(port_text or '443')
conn.sendall(b'HTTP/1.1 200 Connection Established\r\n\r\n')
cert, key = cert_for(host)
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain(cert, key)
with context.wrap_socket(conn, server_side=True) as tls:
nested = read_request(tls)
if not nested:
return
line = nested.split(b'\r\n', 1)[0].decode('iso-8859-1')
nested_target = line.split(' ', 2)[1]
found = file_for(host, nested_target)
if found is not None:
respond_fixture(tls, found)
else:
forward_https(tls, host, port, nested)
def host_from_headers(request):
for header in request.split(b'\r\n')[1:]:
if header.lower().startswith(b'host:'):
value = header.split(b':', 1)[1].strip().decode()
return value.split(':', 1)[0]
return None
def handle_request(conn):
with conn:
request = read_request(conn)
if not request:
return
line = request.split(b'\r\n', 1)[0].decode('iso-8859-1')
method, target, _ = line.split(' ', 2)
if method.upper() == 'CONNECT':
handle_connect(conn, target)
return
parsed = urlsplit(target)
host = parsed.hostname or host_from_headers(request) or 'unknown'
found = file_for(host, target)
if found is not None:
respond_fixture(conn, found)
else:
forward_http(conn, host, parsed.port or 80, request, target)
def handle(conn):
try:
handle_request(conn)
except Exception as error:
print(f'proxy request failed: {error!r}', file=sys.stderr, flush=True)
def main():
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server:
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(LISTEN_ADDRESS)
server.listen()
while True:
conn, _ = server.accept()
threading.Thread(target=handle, args=(conn,), daemon=True).start()
if __name__ == '__main__':
main()
scripts/sandbox/ssh-shim.sh
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env bash
# Stand-in for ssh inside the dev sandbox.
#
# install.sh and `hermes update` clone over ssh first (git@github.com:...), so
# the sandbox needs an `ssh` that answers. Rather than run a real sshd, this
# ignores the host, user, and command git asked for and speaks the
# upload-pack protocol directly against the sandbox's bare repo -- which is
# what makes the ssh-first code path exercisable with no keys, no known_hosts,
# and no network.
#
# GIT_UPLOAD_PACK is substituted by dev-sandbox.sh when it installs this shim,
# because the host's git-upload-pack is not necessarily on the sandbox PATH.
exec @GIT_UPLOAD_PACK@ /work/repos/hermes-agent.git
scripts/sandbox/stage2-run.sh
Executable
+251
View File
@@ -0,0 +1,251 @@
#!/usr/bin/env bash
# Stage 2 of the dev sandbox: build the mounts and run the payload.
#
# Not called directly. scripts/dev-sandbox.sh (stage 1) creates the user and
# network namespaces with `unshare` and re-execs into this script inside them,
# so by the time this runs we are already at the target uid with a private
# netns. bwrap therefore does NOT create a userns here -- it only adds the
# mount and pid namespaces. (`unshare --user` grants its creator full
# capabilities in the new userns regardless of which uid it maps, which is what
# lets bwrap mount as a non-root uid.)
#
# The whole interface with stage 1 is the DEV_SANDBOX_* environment, asserted
# below: there are no shared functions or variables between the two stages.
# Stage 1 locates this script alongside the other sandbox assets (see
# DEV_SANDBOX_ASSETS in dev-sandbox.sh), so the Nix wrapper's store copy and a
# plain repo checkout both work.
set -euo pipefail
: "${DEV_SANDBOX_ROOT:?missing DEV_SANDBOX_ROOT}"
: "${DEV_SANDBOX_BASH:?missing DEV_SANDBOX_BASH}"
: "${DEV_SANDBOX_INTERACTIVE:?missing DEV_SANDBOX_INTERACTIVE}"
: "${DEV_SANDBOX_USER:?missing DEV_SANDBOX_USER}"
: "${DEV_SANDBOX_HOME:?missing DEV_SANDBOX_HOME}"
# Announce our pid so stage 1 can point slirp4netns at these namespaces,
# then hold until it reports the network is up.
slirp_ready="$DEV_SANDBOX_ROOT/root/logs/slirp.ready"
printf '%s\n' "$$" > "$DEV_SANDBOX_ROOT/root/logs/sandbox.pid"
for _ in $(seq 1 200); do
[ -s "$slirp_ready" ] && break
sleep 0.05
done
if [ ! -s "$slirp_ready" ]; then
echo 'error: timed out waiting for sandbox network setup' >&2
cat "$DEV_SANDBOX_ROOT/root/logs/slirp.log" >&2 || true
exit 1
fi
# The sandbox HOME is /root for a root install and /home/<user> for a
# user-level one. Only the latter needs its parent created first; --dir /
# is not a thing bwrap accepts.
home_mounts=()
home_parent="$(dirname "$DEV_SANDBOX_HOME")"
if [ "$home_parent" != / ]; then
home_mounts+=(--dir "$home_parent")
fi
home_mounts+=(--bind "$DEV_SANDBOX_ROOT/home" "$DEV_SANDBOX_HOME")
node_env=()
if [ -n "${DEV_SANDBOX_NODE_DIR:-}" ]; then
node_env+=(--setenv npm_config_nodedir "$DEV_SANDBOX_NODE_DIR")
fi
electron_env=()
if [ -n "${DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH:-}" ]; then
electron_env+=(
--setenv LD_LIBRARY_PATH "$DEV_SANDBOX_ELECTRON_LD_LIBRARY_PATH"
--setenv HERMES_DESKTOP_DISABLE_GPU 1
)
fi
gui_mounts=()
if [ -n "${DEV_SANDBOX_WAYLAND_SOCKET:-}" ]; then
runtime_dir="${DEV_SANDBOX_XDG_RUNTIME_DIR:?missing DEV_SANDBOX_XDG_RUNTIME_DIR}"
runtime_parent="$(dirname "$runtime_dir")"
runtime_grandparent="$(dirname "$runtime_parent")"
gui_mounts+=(
--dir "$runtime_grandparent"
--dir "$runtime_parent"
--dir "$runtime_dir"
--bind "$DEV_SANDBOX_WAYLAND_SOCKET" "$DEV_SANDBOX_WAYLAND_SOCKET"
--setenv XDG_RUNTIME_DIR "$runtime_dir"
--setenv WAYLAND_DISPLAY "${DEV_SANDBOX_WAYLAND_DISPLAY:?missing DEV_SANDBOX_WAYLAND_DISPLAY}"
)
fi
# How the sandbox gets a usable runtime, and where its own shims go.
#
# On Nix, every binary lives under /nix/store, so the sandbox can own /bin,
# /lib64 and /usr/bin outright and fill them with symlinks into the store.
#
# Elsewhere the runtime IS /usr, /bin, /lib, /lib64 -- so binding the
# sandbox's near-empty versions over them hides the real thing, and bwrap
# dies with `execvp /usr/bin/bash: No such file or directory`. Keep the host
# directories read-only and override only the individual files we shim.
#
# The same answer decides how /etc is handled further down.
if [ -d /nix ] && [[ "$(readlink -f "$DEV_SANDBOX_BASH")" == /nix/* ]]; then
USE_HOST_RUNTIME=false
else
USE_HOST_RUNTIME=true
fi
runtime_mounts=()
shim_mounts=()
if [ "$USE_HOST_RUNTIME" = false ]; then
runtime_mounts+=(--ro-bind /nix /nix)
shim_mounts+=(
--dir /usr
--dir /bin
--dir /lib64
--bind "$DEV_SANDBOX_ROOT/root/bin" /bin
--bind "$DEV_SANDBOX_ROOT/root/lib64" /lib64
--bind "$DEV_SANDBOX_ROOT/root/usr/bin" /usr/bin
)
else
for path in /usr /bin /sbin /lib /lib64; do
[ -e "$path" ] && runtime_mounts+=(--ro-bind "$path" "$path")
done
# The git-upload-pack shim standing in for github.com is the only file that
# must beat the host's copy; sh/ls/env are already there for real.
shim_mounts+=(--bind "$DEV_SANDBOX_ROOT/root/usr/bin/ssh" /usr/bin/ssh)
fi
# /etc: start from a copy of the host's and overwrite only the files we fake.
#
# Replacing the whole directory with a five-file one is the tempting shortcut
# and it is wrong: a distro puts things under /etc that binaries outside /etc
# depend on, so hiding all of it breaks tools that look fine on PATH. Two real
# examples, both Debian/Ubuntu: openssl's compiled-in openssl.cnf is a symlink
# into /etc/ssl, and /usr/bin/awk is a symlink to /etc/alternatives/awk -- with
# /etc replaced, openssl cannot mint a certificate and awk reports "not found".
# Those are two symptoms of one cause, and nothing says there are only two.
#
# Copying rather than mount-overlaying the individual files, because several of
# these are symlinks in the wild (resolv.conf -> ../run/systemd/... on Ubuntu,
# hosts and nsswitch.conf -> /etc/static/... on NixOS) and bwrap cannot bind a
# file onto a symlink whose target does not exist inside the sandbox.
#
# Symlinks are copied as symlinks, never dereferenced: on NixOS /etc/static
# points into the store and following it would copy gigabytes per sandbox. The
# store is already mounted at /nix on that path, and the host runtime dirs are
# mounted at their own paths, so absolute symlinks still resolve.
#
# The five we override, and why each must differ from the host's:
# passwd, group the sandbox identity, which does not exist on the host
# resolv.conf slirp4netns's DNS, not the host resolver
# nsswitch.conf files+dns only, so nothing consults host NSS modules
# hosts minimal, so no host entry leaks in
#
# os-release is removed rather than replaced. Installers branch on it to reach
# for a package manager -- `install.sh` reads ID from it and, on debian/ubuntu,
# offers to apt-get build tools, prompting on /dev/tty when sudo exists but is
# not passwordless. That prompt cannot be satisfied here (no terminal) and it is
# fatal under `set -e`. Inheriting the host's file would make the sandbox claim
# to be a distro whose package manager it cannot actually use; absent means
# DISTRO="unknown" and the apt path is skipped, which is the truth.
etc_mounts=()
if [ "$USE_HOST_RUNTIME" = true ] && [ -d /etc ]; then
sandbox_etc="$DEV_SANDBOX_ROOT/etc-merged"
rm -rf -- "$sandbox_etc"
mkdir -p "$sandbox_etc"
# -a keeps symlinks as symlinks; unreadable entries (shadow, sudoers) are
# skipped rather than failing the run.
cp -a /etc/. "$sandbox_etc/" 2>/dev/null || true
for etc_file in passwd group resolv.conf nsswitch.conf hosts; do
[ -f "$DEV_SANDBOX_ROOT/etc/$etc_file" ] || continue
rm -f "$sandbox_etc/$etc_file"
cp "$DEV_SANDBOX_ROOT/etc/$etc_file" "$sandbox_etc/$etc_file"
done
rm -f "$sandbox_etc/os-release" "$sandbox_etc/lsb-release"
etc_mounts+=(--ro-bind "$sandbox_etc" /etc)
else
etc_mounts+=(--bind "$DEV_SANDBOX_ROOT/etc" /etc)
fi
# /dev without a tty, so a script guarding on `[ -e /dev/tty ]` takes its
# no-terminal path.
#
# bwrap's --dev creates a /dev/tty NODE, but nothing in here has a controlling
# terminal, so opening it fails with "No such device or address". That is the
# worst of both: the guard passes and the read then fails. Under `set -e` --
# which install.sh uses -- a failed read inside a function aborts the whole
# installer, which is exactly how older releases died here while prompting for
# sudo to install ripgrep/ffmpeg.
#
# Making the tty real is not the fix: with an openable terminal that prompt
# blocks forever waiting for input nobody will type. Absent is what a headless
# machine looks like, and what every prompt in here should assume.
#
# --dev cannot be used with the node removed afterwards (bwrap refuses to mount
# a directory over a device node), so /dev is assembled explicitly.
dev_mounts=(
--tmpfs /dev
--dev-bind /dev/null /dev/null
--dev-bind /dev/zero /dev/zero
--dev-bind /dev/full /dev/full
--dev-bind /dev/random /dev/random
--dev-bind /dev/urandom /dev/urandom
--symlink /proc/self/fd /dev/fd
--symlink /proc/self/fd/0 /dev/stdin
--symlink /proc/self/fd/1 /dev/stdout
--symlink /proc/self/fd/2 /dev/stderr
)
if [ "$DEV_SANDBOX_INTERACTIVE" = true ]; then
# An interactive shell is deliberately given a terminal; keep bwrap's /dev.
dev_mounts=(--dev /dev)
fi
exec bwrap \
--unshare-pid \
--die-with-parent --proc /proc --tmpfs /tmp \
"${dev_mounts[@]}" \
"${gui_mounts[@]}" \
"${runtime_mounts[@]}" \
--bind "$DEV_SANDBOX_ROOT/root" /work \
"${shim_mounts[@]}" \
--bind "$DEV_SANDBOX_ROOT/root/usr/local" /usr/local \
"${home_mounts[@]}" \
"${etc_mounts[@]}" \
--chdir /work/repo \
--clearenv \
--setenv PATH "$DEV_SANDBOX_HOME/.local/bin:/usr/local/bin:/usr/bin:$PATH" \
--setenv HOME "$DEV_SANDBOX_HOME" \
--setenv USER "$DEV_SANDBOX_USER" \
--setenv LOGNAME "$DEV_SANDBOX_USER" \
--setenv CURL_CA_BUNDLE /work/certs/ca.pem \
--setenv SSL_CERT_FILE /work/certs/ca.pem \
--setenv GIT_SSL_CAINFO /work/certs/ca.pem \
--setenv NODE_EXTRA_CA_CERTS /work/certs/real-ca.pem \
--setenv OPENSSL_CONF /work/certs/openssl.cnf \
--setenv HTTP_PROXY http://127.0.0.1:8080 \
--setenv HTTPS_PROXY http://127.0.0.1:8080 \
--setenv ALL_PROXY http://127.0.0.1:8080 \
--setenv NO_PROXY '' \
--setenv DEV_SANDBOX_INTERACTIVE "$DEV_SANDBOX_INTERACTIVE" \
--setenv ELECTRON_DISABLE_SANDBOX 1 \
"${node_env[@]}" \
"${electron_env[@]}" \
-- "$DEV_SANDBOX_BASH" -ceu '
python3 /work/proxy.py /work/http /work/certs /work/certs/real-ca.pem >/work/logs/proxy.log 2>&1 &
proxy_pid=$!
cleanup() {
kill "$proxy_pid" 2>/dev/null || true
wait "$proxy_pid" 2>/dev/null || true
}
trap cleanup EXIT INT TERM
# Bash opens /dev/tcp itself, so the readiness probe needs no netcat --
# one less binary the sandbox has to find on the host (GitHub runners
# ship no `nc`).
proxy_up() { (exec 3<>/dev/tcp/127.0.0.1/8080) 2>/dev/null; }
for _ in $(seq 1 100); do
proxy_up && break
sleep 0.05
done
if ! proxy_up; then
echo "error: the sandbox fake-internet proxy never came up" >&2
cat /work/logs/proxy.log >&2 || true
exit 1
fi
"$@"
' sandbox-command "$@"
scripts/smoke_nemo_relay_shared_metrics.py
+739
View File
@@ -0,0 +1,739 @@
"""Run a real Hermes CLI turn and validate the Relay shared-metrics output."""
from __future__ import annotations
import argparse
import json
import os
import shutil
import sqlite3
import subprocess
import sys
import tempfile
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any
PROMPT_CANARY = "relay-smoke-sensitive-prompt"
MODEL_CANARY = "gpt-relay-smoke-sensitive-model"
RESPONSE_CANARY = "relay-smoke-sensitive-response"
TOOL_CALL_CANARY = "relay-smoke-sensitive-tool-call"
TOOL_RESULT_CANARY = "relay-smoke-sensitive-tool-result"
TOOL_FILE = "relay-smoke-input.txt"
SKILL_CANARY = "relay-smoke-private-agent-skill"
INSTALLED_SKILL_CANARY = "relay-smoke-private-installed-skill"
def _resolve_hermes_executable(hermes_repo: Path) -> Path:
for relative_path in (
Path(".venv") / "bin" / "hermes",
Path(".venv") / "Scripts" / "hermes.exe",
):
candidate = hermes_repo / relative_path
if candidate.is_file():
return candidate
discovered = shutil.which("hermes")
if discovered:
return Path(discovered)
raise SystemExit(
"Hermes executable not found in the repository virtual environment "
"or on PATH"
)
class _ModelHandler(BaseHTTPRequestHandler):
"""Minimal OpenAI-compatible model server for one deterministic turn."""
protocol_version = "HTTP/1.1"
requests: list[dict[str, Any]] = []
def do_GET(self) -> None: # noqa: N802
if self.path.rstrip("/") != "/v1/models":
self.send_error(404)
return
self._write_json({
"object": "list",
"data": [
{
"id": MODEL_CANARY,
"object": "model",
"created": 0,
"owned_by": "smoke-test",
}
],
})
def do_POST(self) -> None: # noqa: N802
if self.path.rstrip("/") != "/v1/chat/completions":
self.send_error(404)
return
length = int(self.headers.get("Content-Length", "0"))
request = json.loads(self.rfile.read(length) or b"{}")
type(self).requests.append(request)
request_tool = not any(
message.get("role") == "tool"
for message in request.get("messages") or []
if isinstance(message, dict)
)
if request.get("stream"):
self._write_stream(request_tool=request_tool)
else:
self._write_json(self._completion(request_tool=request_tool))
def _completion(self, *, request_tool: bool) -> dict[str, Any]:
message: dict[str, Any] = {
"role": "assistant",
"content": "" if request_tool else RESPONSE_CANARY,
}
finish_reason = "tool_calls" if request_tool else "stop"
if request_tool:
message["tool_calls"] = [
{
"id": TOOL_CALL_CANARY,
"type": "function",
"function": {
"name": "read_file",
"arguments": json.dumps({"path": TOOL_FILE}),
},
}
]
return {
"id": "chatcmpl-relay-smoke",
"object": "chat.completion",
"created": int(time.time()),
"model": MODEL_CANARY,
"choices": [
{
"index": 0,
"message": message,
"finish_reason": finish_reason,
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 1,
"total_tokens": 11,
},
}
def log_message(self, format: str, *args: Any) -> None:
return
def _write_json(self, payload: dict[str, Any]) -> None:
body = json.dumps(payload).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("Connection", "close")
self.end_headers()
self.wfile.write(body)
self.close_connection = True
def _write_stream(self, *, request_tool: bool) -> None:
now = int(time.time())
chunks: list[dict[str, Any]] = [
{
"id": "chatcmpl-relay-smoke",
"object": "chat.completion.chunk",
"created": now,
"model": MODEL_CANARY,
"choices": [
{
"index": 0,
"delta": {
"role": "assistant",
"content": "",
},
"finish_reason": None,
}
],
}
]
if request_tool:
chunks.append({
"id": "chatcmpl-relay-smoke",
"object": "chat.completion.chunk",
"created": now,
"model": MODEL_CANARY,
"choices": [
{
"index": 0,
"delta": {
"tool_calls": [
{
"index": 0,
"id": TOOL_CALL_CANARY,
"type": "function",
"function": {
"name": "read_file",
"arguments": json.dumps({"path": TOOL_FILE}),
},
}
]
},
"finish_reason": None,
}
],
})
else:
chunks.append({
"id": "chatcmpl-relay-smoke",
"object": "chat.completion.chunk",
"created": now,
"model": MODEL_CANARY,
"choices": [
{
"index": 0,
"delta": {"content": RESPONSE_CANARY},
"finish_reason": None,
}
],
})
chunks.extend([
{
"id": "chatcmpl-relay-smoke",
"object": "chat.completion.chunk",
"created": now,
"model": MODEL_CANARY,
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": "tool_calls" if request_tool else "stop",
}
],
},
{
"id": "chatcmpl-relay-smoke",
"object": "chat.completion.chunk",
"created": now,
"model": MODEL_CANARY,
"choices": [],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 1,
"total_tokens": 11,
},
},
])
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "close")
self.end_headers()
for chunk in chunks:
self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode("utf-8"))
self.wfile.flush()
self.wfile.write(b"data: [DONE]\n\n")
self.wfile.flush()
self.close_connection = True
def _arguments() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--hermes-repo",
type=Path,
default=Path.cwd(),
help="Hermes source checkout containing .venv/bin/hermes",
)
parser.add_argument(
"--relay-python",
type=Path,
default=None,
help="Optional NeMo Relay checkout's python directory",
)
parser.add_argument(
"--output-dir",
type=Path,
default=None,
help="Directory for the isolated HERMES_HOME and captured output",
)
return parser.parse_args()
def _write_config(home: Path, port: int) -> None:
home.mkdir(parents=True, exist_ok=True)
(home / "config.yaml").write_text(
f"""model:
default: {MODEL_CANARY}
provider: custom
base_url: http://127.0.0.1:{port}/v1
api_mode: chat_completions
api_key: no-key-required
security:
tirith_enabled: false
telemetry:
shared_metrics:
enabled: true
""",
encoding="utf-8",
)
def _validate_store(database_path: Path) -> list[dict[str, Any]]:
if not database_path.is_file():
raise AssertionError(f"Metrics database was not created: {database_path}")
with sqlite3.connect(database_path) as connection:
rows = connection.execute(
"""
SELECT metric_name, dimensions_json, value, packaged_value
FROM counter_aggregates
ORDER BY metric_name, dimensions_json
"""
).fetchall()
counters = [
{
"name": name,
"dimensions": json.loads(dimensions),
"value": value,
"packaged_value": packaged_value,
}
for name, dimensions, value, packaged_value in rows
]
by_name: dict[str, list[dict[str, Any]]] = {}
for counter in counters:
by_name.setdefault(counter["name"], []).append(counter)
if set(by_name) != {
"hermes.client.active",
"hermes.model_route.count",
"hermes.skill.lifecycle.count",
"hermes.skill.load.count",
"hermes.task_run.finished",
"hermes.task_run.started",
"hermes.tool_call.count",
}:
raise AssertionError(
f"Unexpected SQLite counters:\n{json.dumps(counters, indent=2)}"
)
if by_name["hermes.client.active"] != [
{
"name": "hermes.client.active",
"dimensions": {},
"value": 1,
"packaged_value": 1,
}
]:
raise AssertionError(
f"Unexpected client-active counter: {by_name['hermes.client.active']}"
)
[model] = by_name["hermes.model_route.count"]
expected_model = {
"name": "hermes.model_route.count",
"dimensions": {
"model": MODEL_CANARY,
"provider": "custom",
},
"value": 2,
"packaged_value": 2,
}
if model != expected_model:
raise AssertionError(
f"Unexpected model counter: {by_name['hermes.model_route.count']}"
)
expected_start = {
"name": "hermes.task_run.started",
"dimensions": {
"entrypoint": "interactive",
"execution_surface": "cli",
},
"value": 1,
"packaged_value": 1,
}
if by_name["hermes.task_run.started"] != [expected_start]:
raise AssertionError(
f"Unexpected task start: {by_name['hermes.task_run.started']}"
)
[terminal] = by_name["hermes.task_run.finished"]
expected_terminal_dimensions = {
"duration_bucket": terminal["dimensions"].get("duration_bucket"),
"end_reason": "completed",
"entrypoint": "interactive",
"execution_surface": "cli",
"model_call_count_bucket": "2",
"outcome": "success",
"retry_count_bucket": "0",
"termination": "none",
"tool_call_count_bucket": "1",
}
if (
terminal["dimensions"] != expected_terminal_dimensions
or terminal["value"] != 1
or terminal["packaged_value"] != 1
):
raise AssertionError(f"Unexpected task terminal counter: {terminal}")
[tool] = by_name["hermes.tool_call.count"]
expected_tool_dimensions = {
"approval_outcome": "not_required",
"latency_bucket": tool["dimensions"].get("latency_bucket"),
"outcome": "success",
"retry_count_bucket": "unknown",
"tool_category": "file",
}
if (
tool["dimensions"] != expected_tool_dimensions
or tool["dimensions"]["latency_bucket"] == "unknown"
or tool["value"] != 1
or tool["packaged_value"] != 1
):
raise AssertionError(f"Unexpected tool counter: {tool}")
lifecycle = by_name["hermes.skill.lifecycle.count"]
expected_actions = {
"archived",
"created",
"edited",
"installed",
"patched",
"restored",
"stale",
}
if (
{counter["dimensions"]["action"] for counter in lifecycle} != expected_actions
or any(counter["value"] != 1 for counter in lifecycle)
or any(counter["packaged_value"] != 1 for counter in lifecycle)
):
raise AssertionError(f"Unexpected skill lifecycle counters: {lifecycle}")
loads = by_name["hermes.skill.load.count"]
expected_load_states = {
("first_use", "not_applicable", "1"),
("reused", "no_new_patch", "2"),
("reused", "reused_after_patch", "3_to_5"),
}
observed_load_states = {
(
counter["dimensions"]["reuse_state"],
counter["dimensions"]["post_patch_state"],
counter["dimensions"]["use_count_bucket"],
)
for counter in loads
}
if (
observed_load_states != expected_load_states
or any(counter["value"] != 1 for counter in loads)
or any(counter["packaged_value"] != 1 for counter in loads)
):
raise AssertionError(f"Unexpected skill load counters: {loads}")
return counters
def _validate_packages(
outbox: Path,
schema_path: Path,
) -> tuple[list[Path], list[dict[str, Any]]]:
package_paths = sorted(outbox.glob("*.json"))
if len(package_paths) != 2:
raise AssertionError(
f"Expected two delta packages in {outbox}, found {len(package_paths)}"
)
try:
import jsonschema
except ImportError as exc:
raise RuntimeError(
"The Hermes development environment requires jsonschema"
) from exc
schema = json.loads(schema_path.read_text(encoding="utf-8"))
packages = [
json.loads(package_path.read_text(encoding="utf-8"))
for package_path in package_paths
]
for package in packages:
jsonschema.validate(package, schema)
if set(package["resource"]) != {
"architecture",
"hermes_version",
"install_method",
"os_family",
}:
raise AssertionError(f"Unexpected client resource: {package['resource']}")
serialized = json.dumps(packages)
for prohibited in (
PROMPT_CANARY,
RESPONSE_CANARY,
TOOL_CALL_CANARY,
TOOL_RESULT_CANARY,
SKILL_CANARY,
INSTALLED_SKILL_CANARY,
):
if prohibited in serialized:
raise AssertionError(
f"Exported package leaked prohibited value: {prohibited!r}"
)
metrics: dict[str, list[dict[str, Any]]] = {}
for package in packages:
for metric in package.get("metrics", []):
metrics.setdefault(metric["name"], []).append(metric)
if set(metrics) != {
"hermes.client.active",
"hermes.model_route.count",
"hermes.skill.lifecycle.count",
"hermes.skill.load.count",
"hermes.task_run.finished",
"hermes.task_run.started",
"hermes.tool_call.count",
}:
raise AssertionError(
f"Unexpected package metrics:\n{json.dumps(metrics, indent=2)}"
)
if metrics["hermes.client.active"] != [
{
"name": "hermes.client.active",
"type": "counter",
"dimensions": {},
"value": 1,
}
]:
raise AssertionError(
f"Unexpected client-active metric: {metrics['hermes.client.active']}"
)
[model] = metrics["hermes.model_route.count"]
if model["dimensions"] != {
"model": MODEL_CANARY,
"provider": "custom",
} or model["value"] != 2:
raise AssertionError(
f"Unexpected model metric: {metrics['hermes.model_route.count']}"
)
[terminal] = metrics["hermes.task_run.finished"]
if terminal["dimensions"] != {
"duration_bucket": terminal["dimensions"].get("duration_bucket"),
"end_reason": "completed",
"entrypoint": "interactive",
"execution_surface": "cli",
"model_call_count_bucket": "2",
"outcome": "success",
"retry_count_bucket": "0",
"termination": "none",
"tool_call_count_bucket": "1",
}:
raise AssertionError(f"Unexpected task terminal metric: {terminal}")
[tool] = metrics["hermes.tool_call.count"]
if (
tool["dimensions"]
!= {
"approval_outcome": "not_required",
"latency_bucket": tool["dimensions"].get("latency_bucket"),
"outcome": "success",
"retry_count_bucket": "unknown",
"tool_category": "file",
}
or tool["dimensions"]["latency_bucket"] == "unknown"
):
raise AssertionError(f"Unexpected tool metric: {tool}")
lifecycle = metrics["hermes.skill.lifecycle.count"]
if {metric["dimensions"]["action"] for metric in lifecycle} != {
"archived",
"created",
"edited",
"installed",
"patched",
"restored",
"stale",
}:
raise AssertionError(f"Unexpected skill lifecycle metrics: {lifecycle}")
loads = metrics["hermes.skill.load.count"]
if {
(
metric["dimensions"]["reuse_state"],
metric["dimensions"]["post_patch_state"],
metric["dimensions"]["use_count_bucket"],
)
for metric in loads
} != {
("first_use", "not_applicable", "1"),
("reused", "no_new_patch", "2"),
("reused", "reused_after_patch", "3_to_5"),
}:
raise AssertionError(f"Unexpected skill load metrics: {loads}")
return package_paths, packages
def main() -> int:
args = _arguments()
hermes_repo = args.hermes_repo.resolve()
relay_python = args.relay_python.resolve() if args.relay_python else None
hermes = _resolve_hermes_executable(hermes_repo)
if relay_python is not None and not any(
(relay_python / "nemo_relay").glob("_native.*")
):
raise SystemExit(
"Built NeMo Relay Python binding not found under "
f"{relay_python}; run the Relay Python build first"
)
if args.output_dir:
root = args.output_dir.resolve()
if root.exists():
raise SystemExit(f"Refusing to replace existing output directory: {root}")
root.mkdir(parents=True)
else:
root = Path(tempfile.mkdtemp(prefix="hermes-relay-shared-metrics-"))
home = root / "hermes-home"
workdir = root / "workspace"
workdir.mkdir()
(workdir / TOOL_FILE).write_text(TOOL_RESULT_CANARY, encoding="utf-8")
home.mkdir()
(home / ".no-bundled-skills").touch()
agent_skill = home / "skills" / SKILL_CANARY
agent_skill.mkdir(parents=True)
(agent_skill / "SKILL.md").write_text(
f"---\nname: {SKILL_CANARY}\ndescription: private smoke skill\n---\n",
encoding="utf-8",
)
installed_skill = home / "skills" / INSTALLED_SKILL_CANARY
installed_skill.mkdir(parents=True)
(installed_skill / "SKILL.md").write_text(
f"---\nname: {INSTALLED_SKILL_CANARY}\ndescription: installed smoke skill\n---\n",
encoding="utf-8",
)
hub_state = home / "skills" / ".hub"
hub_state.mkdir()
(hub_state / "lock.json").write_text(
json.dumps({
"version": 1,
"installed": {
INSTALLED_SKILL_CANARY: {"source": "smoke/local"},
},
}),
encoding="utf-8",
)
_ModelHandler.requests = []
server = ThreadingHTTPServer(("127.0.0.1", 0), _ModelHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
_write_config(home, server.server_port)
env = os.environ.copy()
env["HERMES_HOME"] = str(home)
python_paths = [str(hermes_repo)]
if relay_python is not None:
python_paths.append(str(relay_python))
python_paths.append(env.get("PYTHONPATH", ""))
env["PYTHONPATH"] = os.pathsep.join(python_paths).rstrip(os.pathsep)
result = subprocess.run(
[
str(hermes),
"chat",
"--query",
PROMPT_CANARY,
"--provider",
"custom",
"--model",
MODEL_CANARY,
"--quiet",
"--ignore-rules",
"--toolsets",
"file",
"--max-turns",
"2",
],
cwd=workdir,
env=env,
text=True,
capture_output=True,
timeout=120,
)
finally:
server.shutdown()
server.server_close()
thread.join(timeout=5)
(root / "hermes.stdout.txt").write_text(result.stdout, encoding="utf-8")
(root / "hermes.stderr.txt").write_text(result.stderr, encoding="utf-8")
if result.returncode != 0:
raise AssertionError(
f"Hermes exited with {result.returncode}\n"
f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
)
if len(_ModelHandler.requests) != 2:
raise AssertionError(
f"Expected two model requests, got {len(_ModelHandler.requests)}"
)
request = _ModelHandler.requests[0]
if request.get("model") != MODEL_CANARY:
raise AssertionError(f"Unexpected model request: {request.get('model')!r}")
if PROMPT_CANARY not in json.dumps(request.get("messages", [])):
raise AssertionError("Hermes model request did not contain the prompt canary")
follow_up = json.dumps(_ModelHandler.requests[1].get("messages", []))
if TOOL_CALL_CANARY not in follow_up or TOOL_RESULT_CANARY not in follow_up:
raise AssertionError("Hermes did not return the tool result to the model")
if RESPONSE_CANARY not in result.stdout:
raise AssertionError("Hermes did not print the mock model response")
skill_result = subprocess.run(
[
sys.executable,
"-c",
"\n".join([
"from hermes_cli.observability import relay_shared_metrics",
"from tools.skill_usage import (",
" STATE_ACTIVE, STATE_ARCHIVED, STATE_STALE, bump_patch,",
" bump_use, record_created, record_installed, set_state,",
")",
f"skill = {SKILL_CANARY!r}",
f"installed = {INSTALLED_SKILL_CANARY!r}",
"record_created(skill, agent_created=True)",
"bump_use(skill)",
"bump_use(skill)",
"bump_patch(skill)",
"bump_use(skill)",
"bump_patch(skill, action='edit')",
"set_state(skill, STATE_STALE)",
"set_state(skill, STATE_ACTIVE)",
"set_state(skill, STATE_ARCHIVED)",
"set_state(skill, STATE_ACTIVE)",
"record_installed(installed)",
"runtime = relay_shared_metrics._get_runtime()",
"assert runtime is not None",
"runtime.shutdown()",
# Production leaves same-day deltas pending. Force a package so
# this smoke can validate them without waiting for the next day.
"runtime.subscriber.store.create_and_export_package()",
]),
],
cwd=workdir,
env=env,
text=True,
capture_output=True,
timeout=60,
)
(root / "skills.stdout.txt").write_text(
skill_result.stdout,
encoding="utf-8",
)
(root / "skills.stderr.txt").write_text(
skill_result.stderr,
encoding="utf-8",
)
if skill_result.returncode != 0:
raise AssertionError(
f"Skill lifecycle probe exited with {skill_result.returncode}\n"
f"stdout:\n{skill_result.stdout}\nstderr:\n{skill_result.stderr}"
)
telemetry = home / "telemetry" / "shared_metrics"
counters = _validate_store(telemetry / "metrics.sqlite3")
package_paths, packages = _validate_packages(
telemetry / "outbox",
hermes_repo
/ "hermes_cli"
/ "observability"
/ "schemas"
/ "hermes.shared_metrics.v2.schema.json",
)
print("Hermes -> NeMo Relay shared-metrics smoke test passed")
print(f"Artifact directory: {root}")
print(f"Model requests: {len(_ModelHandler.requests)}")
print(f"SQLite counters: {json.dumps(counters, indent=2)}")
print(f"Export packages: {', '.join(str(path) for path in package_paths)}")
print(json.dumps(packages, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/tests/test-install-ps1-gitbash-compatibility.ps1
+120
View File
@@ -0,0 +1,120 @@
# Unit tests for install.ps1's Git Bash compatibility and Mandatory-ASLR
# guidance helpers. The installer itself is never executed: functions are
# extracted through the PowerShell AST to avoid downloads, PATH changes, or
# user-environment writes.
$ErrorActionPreference = "Stop"
$repoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path))
$installScript = Join-Path $repoRoot "scripts\install.ps1"
$failures = 0
function Assert-Equal {
param($Expected, $Actual, [string]$Label)
if ($Expected -ne $Actual) {
Write-Host "FAIL: $Label" -ForegroundColor Red
Write-Host " expected: $Expected"
Write-Host " actual: $Actual"
$script:failures++
} else {
Write-Host "OK: $Label" -ForegroundColor Green
}
}
function Assert-True {
param($Condition, [string]$Label)
if (-not $Condition) {
Write-Host "FAIL: $Label" -ForegroundColor Red
$script:failures++
} else {
Write-Host "OK: $Label" -ForegroundColor Green
}
}
$tokens = $null
$parseErrors = $null
$ast = [System.Management.Automation.Language.Parser]::ParseFile(
$installScript, [ref]$tokens, [ref]$parseErrors
)
if ($parseErrors.Count -gt 0) {
throw "install.ps1 has parse errors: $($parseErrors -join '; ')"
}
foreach ($name in @(
"Test-GitBashCompatibility",
"Test-MandatoryAslrEnabled",
"Get-GitRootFromBashPath",
"New-GitBashAslrFailureReason",
"Stage-Git"
)) {
$fnAst = $ast.FindAll(
{
param($node)
$node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and
$node.Name -eq $name
}, $true
) | Select-Object -First 1
if (-not $fnAst) { throw "$name not found in install.ps1" }
. ([scriptblock]::Create($fnAst.Extent.Text))
}
Write-Host ""
Write-Host "-- Git root resolution --"
Assert-Equal "C:\Program Files\Git" `
(Get-GitRootFromBashPath "C:\Program Files\Git\bin\bash.exe") `
"PortableGit/full Git bin layout"
Assert-Equal "C:\Program Files\Git" `
(Get-GitRootFromBashPath "C:\Program Files\Git\usr\bin\bash.exe") `
"usr/bin layout"
Write-Host ""
Write-Host "-- Mandatory ASLR detection --"
function Get-ProcessMitigation {
param([switch]$System)
[pscustomobject]@{ Aslr = [pscustomobject]@{ ForceRelocateImages = "ON" } }
}
Assert-Equal $true (Test-MandatoryAslrEnabled) "ForceRelocateImages ON is detected"
function Get-ProcessMitigation {
param([switch]$System)
[pscustomobject]@{ Aslr = [pscustomobject]@{ ForceRelocateImages = "NOTSET" } }
}
Assert-Equal $false (Test-MandatoryAslrEnabled) "ForceRelocateImages NOTSET is not diagnosed"
Write-Host ""
Write-Host "-- Actionable remediation --"
$reason = New-GitBashAslrFailureReason "C:\Program Files\Git\bin\bash.exe"
Assert-True ($reason -match "Mandatory ASLR") "reason identifies Mandatory ASLR"
Assert-True ($reason -match "Reinstalling Git will not change") "reason rejects ineffective reinstall"
Assert-True ($reason -match [regex]::Escape("C:\Program Files\Git")) "reason uses selected Git root"
Assert-True ($reason -match "Set-ProcessMitigation") "reason includes targeted mitigation command"
Write-Host ""
Write-Host "-- External-program probe --"
$gitCommand = Get-Command git -ErrorAction SilentlyContinue
$gitBash = $null
if ($gitCommand -and $gitCommand.Source) {
$gitRoot = Split-Path (Split-Path $gitCommand.Source -Parent) -Parent
foreach ($candidate in @("$gitRoot\bin\bash.exe", "$gitRoot\usr\bin\bash.exe")) {
if (Test-Path -LiteralPath $candidate) { $gitBash = $candidate; break }
}
}
if ($gitBash) {
Assert-Equal $true (Test-GitBashCompatibility $gitBash) `
"installed Git Bash launches external MSYS programs"
} else {
Write-Host "SKIP: no Git Bash found next to git.exe" -ForegroundColor Yellow
}
Write-Host ""
Write-Host "-- Stage failure propagation --"
function Install-Git { return $false }
$script:GitInstallFailureReason = "specific Git Bash failure"
$stageError = $null
try { Stage-Git } catch { $stageError = $_.Exception.Message }
Assert-Equal "specific Git Bash failure" $stageError "Git stage preserves actionable reason"
Write-Host ""
if ($failures -gt 0) {
Write-Host "FAILED: $failures assertion(s) failed" -ForegroundColor Red
exit 1
}
Write-Host "All Git Bash compatibility tests passed." -ForegroundColor Green
exit 0
scripts/tests/test-install-ps1-longpath.ps1
+323
View File
@@ -0,0 +1,323 @@
# Tests for install.ps1's 8.3 short-path normalization.
#
# Run from a PowerShell prompt:
#
# pwsh -NoProfile -ExecutionPolicy Bypass -File scripts/tests/test-install-ps1-longpath.ps1
#
# Background: when the Windows profile folder's name contains a space
# ("First Last"), a dot ("Stone.ZEN8"), or an accented character, Windows can
# expose %TEMP%, %LOCALAPPDATA% and friends as an 8.3 alias
# (C:\Users\FIRST~1.LAS\...). PowerShell's FileSystem provider chokes on the
# aliased component once it reaches a provider cmdlet (Tee-Object -FilePath),
# aborting the Node/Electron stages and the desktop post-build probe.
# install.ps1 expands those paths up front; this asserts that contract.
#
# HOW THIS RUNS THE CODE: by executing install.ps1 as a real subprocess with a
# crafted environment and reading what it reports back. `-ProtocolVersion` is a
# side-effect-free early exit that sits BELOW the normalization block, so the
# whole block -- including the script-level Add-Type the kernel32 resolver
# needs -- executes exactly as it does during an install. Nothing here parses
# install.ps1's source (AGENTS.md bans source-reading tests: they pass on
# broken code and fail on correct refactors).
#
# HERMETIC ENVIRONMENT: every case sets all five profile variables explicitly.
# GitHub's own Windows runners hand down a genuinely 8.3-aliased TEMP/TMP
# (C:\Users\RUNNER~1\AppData\Local\Temp), so an inherited variable is a live
# instance of the very bug under test and would contaminate any case that
# didn't override it.
#
# Portability: resolver 3 (profile-root substitution) is pure path arithmetic,
# so the substitution assertions run everywhere, including non-Windows CI. The
# kernel32 and COM resolvers only have anything to expand on a real Windows
# volume; on other hosts they no-op and fall through, which is itself the
# graceful-degradation contract asserted below.
$ErrorActionPreference = "Stop"
$repoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path))
$installScript = Join-Path $repoRoot "scripts/install.ps1"
if (-not (Test-Path $installScript)) {
throw "Could not locate install.ps1 at $installScript"
}
$failures = 0
$script:lastRaw = ''
function Assert-Equal {
param($Expected, $Actual, [Parameter(Mandatory = $true)][string]$Label)
if ($Expected -ne $Actual) {
Write-Host "FAIL: $Label" -ForegroundColor Red
Write-Host " expected: $Expected"
Write-Host " actual: $Actual"
if ($script:lastRaw) {
# The installer's own account of what it did, plus the environment
# it was handed. Without both, a failure on a host you cannot reach
# is pure guesswork.
Write-Host " installer reported: $script:lastRaw"
Write-Host " environment sent: $script:lastEnv"
}
$script:failures++
} else {
Write-Host "OK: $Label" -ForegroundColor Green
}
}
# --- Harness ---------------------------------------------------------------
# The real profile root the installer will substitute in, derived the same way
# install.ps1 derives it so these assertions hold on any host and any account.
$profileDir = [Environment]::GetFolderPath('UserProfile')
$usersDir = Split-Path -Parent $profileDir
$sep = [System.IO.Path]::DirectorySeparatorChar
# Starting guess for the baseline environment; replaced by the probe below with
# whatever root the installer itself resolves.
$script:baseRoot = $profileDir
# A profile alias that cannot resolve: no such folder exists, so kernel32 and
# COM both fail and only the profile-root substitution can handle it.
$shortProfile = Join-Path $usersDir 'FIRST~1.LAS'
function Join-Parts {
# Join path segments with the platform separator. Literal forward slashes
# inside a path would make Split-Path's behavior host-dependent, which is
# noise this suite doesn't need.
param([string[]]$Parts)
return ($Parts -join $sep)
}
# Ask install.ps1 what paths it resolves under a given environment.
#
# -ShowResolvedPaths prints a JSON object on STDOUT and exits without touching
# anything, so the whole normalization block -- including the script-level
# Add-Type the kernel32 resolver needs -- has already run by the time it is
# printed. Stdout, deliberately: three separate stderr capture mechanisms
# (ProcessStartInfo.RedirectStandardError, `2>$file`, and a merged `2>&1`
# pipeline) were each verified to come back EMPTY from the installer on a
# windows-latest runner while stdout arrived intact. The installer's human
# diagnostics still go to stderr; the machine-readable contract is on stdout,
# which is the only stream that survives everywhere.
#
# Environment overrides are applied to this process and restored afterwards,
# since that is what the child inherits.
function Invoke-Normalization {
param(
[hashtable]$Environment = @{},
[string[]]$ExtraArgs = @()
)
# Start from a long, self-consistent profile so nothing is inherited;
# callers override only the variables their case is about. $script:baseRoot
# is the test's best guess until the probe below replaces it with the root
# the installer actually resolves.
$root = $script:baseRoot
$env0 = @{
TEMP = (Join-Parts @($root, 'AppData', 'Local', 'Temp'))
TMP = (Join-Parts @($root, 'AppData', 'Local', 'Temp'))
LOCALAPPDATA = (Join-Parts @($root, 'AppData', 'Local'))
APPDATA = (Join-Parts @($root, 'AppData', 'Roaming'))
USERPROFILE = $root
HERMES_HOME = ''
}
foreach ($key in $Environment.Keys) { $env0[$key] = $Environment[$key] }
$psExe = (Get-Process -Id $PID).Path
$outFile = [System.IO.Path]::GetTempFileName()
$errFile = [System.IO.Path]::GetTempFileName()
$saved = @{}
foreach ($key in $env0.Keys) { $saved[$key] = [Environment]::GetEnvironmentVariable($key) }
try {
foreach ($key in $env0.Keys) { Set-Item -Path "Env:$key" -Value $env0[$key] }
$callArgs = @('-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', $installScript) + $ExtraArgs + @('-ShowResolvedPaths')
# The call operator, not Start-Process: on Windows Start-Process does
# not hand the parent's modified environment block to the child, so the
# installer saw the runner's real TEMP instead of the aliased one this
# case sets, and every rewrite assertion came back "not rewritten".
# `&` inherits the environment on every host.
#
# stderr is merged into the same file rather than redirected separately:
# Windows PowerShell 5.1 wraps ANY stderr from a native command in a
# NativeCommandError record, and a bare `2>$file` still emits that
# record into this script's error stream, which fails the 5.1 lane even
# under 'Continue'. Merging with 2>&1 keeps the bytes and produces no
# error record. The installer's stdout here is a single JSON object and
# its diagnostics are all `[hermes] `-prefixed, so the two separate
# cleanly on the way back out.
$prevEAP = $ErrorActionPreference
$ErrorActionPreference = 'Continue'
$global:LASTEXITCODE = 0
try {
& $psExe @callArgs *> $outFile
} finally {
$ErrorActionPreference = $prevEAP
}
$exitCode = $LASTEXITCODE
$raw = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue)
$stderr = ($raw | Where-Object { $_ -like '`[hermes`]*' }) -join "`n"
$stdout = ($raw | Where-Object { $_ -notlike '`[hermes`]*' }) -join "`n"
} finally {
foreach ($key in $saved.Keys) {
if ($null -eq $saved[$key]) {
Remove-Item -LiteralPath "Env:$key" -ErrorAction SilentlyContinue
} else {
Set-Item -Path "Env:$key" -Value $saved[$key]
}
}
Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue
}
if ($null -eq $stdout) { $stdout = '' }
$stdout = $stdout.Trim()
$script:lastRaw = if ($stdout) { $stdout } else { '(child produced no stdout)' }
$script:lastEnv = ($env0.Keys | Sort-Object | ForEach-Object { "$_=$($env0[$_])" }) -join '; '
$paths = $null
if ($stdout) {
try { $paths = $stdout | ConvertFrom-Json } catch { $paths = $null }
}
# normalized is an object keyed by variable name; flatten to a hashtable so
# callers can ask "was TEMP rewritten, and to what".
$rewrites = @{}
if ($paths -and $paths.normalized) {
foreach ($prop in $paths.normalized.PSObject.Properties) {
$rewrites[$prop.Name] = "$($prop.Value)"
}
}
return @{
ExitCode = $exitCode
Stdout = $stdout
Rewrites = $rewrites
InstallDir = $(if ($paths) { $paths.install_dir } else { $null })
HermesHome = $(if ($paths) { $paths.hermes_home } else { $null })
LongRoot = $(if ($paths) { $paths.long_profile_root } else { $null })
}
}
function Get-Rewrite {
# '' rather than $null for an untouched variable, so a failure prints
# something legible instead of a blank.
param($Result, [string]$Name)
if ($Result.Rewrites.ContainsKey($Name)) { return $Result.Rewrites[$Name] }
return '<not rewritten>'
}
# Ask the installer once, up front, which long root it resolves on this host,
# and assert every expectation against that. Deriving it independently in the
# test would only prove the two derivations agree, not that the fix works --
# and on GitHub's Windows runners they don't agree, because the runner hands
# down a genuinely 8.3-aliased profile.
$probe = Invoke-Normalization @{ USERPROFILE = $shortProfile }
$longRoot = $probe.LongRoot
Write-Host ""
Write-Host "-- the installer resolves a long profile root --"
if ([string]::IsNullOrEmpty($longRoot)) {
# Nothing below can mean anything without this, so show the child's whole
# output rather than leaving a bare assertion failure on an unreachable host.
Write-Host "FAIL: a long profile root is found" -ForegroundColor Red
Write-Host " probe exit code: $($probe.ExitCode)"
Write-Host " probe stdout: $($probe.Stdout)"
Write-Host " probe env: $script:lastEnv"
Write-Host " probe stdout (raw):"
foreach ($line in ($script:lastRaw -split "`r?`n")) {
if ($line.Trim()) { Write-Host " $line" }
}
Write-Host "FAILED: cannot continue without a long profile root" -ForegroundColor Red
exit 1
}
Write-Host "OK: a long profile root is found ($longRoot)" -ForegroundColor Green
Assert-Equal -Expected $false -Actual ($longRoot -match '~\d') -Label "the resolved root carries no 8.3 alias"
# Every subsequent case's baseline is now the installer's own root, so a
# "nothing to expand" case really has nothing to expand even on a runner whose
# inherited profile is itself aliased.
$script:baseRoot = $longRoot
Write-Host ""
Write-Host "-- normalization is a no-op for ordinary paths --"
# A profile name with a space is NOT itself a short path; nothing to expand.
$result = Invoke-Normalization
Assert-Equal -Expected 0 -Actual $result.ExitCode -Label "long paths: install.ps1 still reaches its early exit"
Assert-Equal -Expected 0 -Actual $result.Rewrites.Count -Label "long paths: nothing rewritten"
Assert-Equal -Expected $false -Actual ($result.InstallDir -match '~\d') -Label "long paths: InstallDir passes through clean"
Write-Host ""
Write-Host "-- an unresolvable profile alias is rebuilt on the long profile root --"
# The reported failure: TEMP under an 8.3 profile alias that no resolver can
# expand (8dot3 disabled, or a stale alias). GH #52842, GH #57526.
$shortTemp = Join-Parts @($shortProfile, 'AppData', 'Local', 'Temp')
$expectedTemp = Join-Parts @($profileDir, 'AppData', 'Local', 'Temp')
$result = Invoke-Normalization @{ TEMP = $shortTemp; TMP = $shortTemp }
Assert-Equal -Expected 0 -Actual $result.ExitCode -Label "short TEMP: install.ps1 still reaches its early exit"
$expectedTemp = "$longRoot${sep}AppData${sep}Local${sep}Temp"
Assert-Equal -Expected $expectedTemp -Actual (Get-Rewrite $result 'TEMP') -Label "short TEMP is rebuilt on the long profile root"
Assert-Equal -Expected $expectedTemp -Actual (Get-Rewrite $result 'TMP') -Label "short TMP is rebuilt on the long profile root"
Assert-Equal -Expected 2 -Actual $result.Rewrites.Count -Label "short TEMP: only the aliased variables are touched"
# The profile root itself, with no tail to reattach. USERPROFILE is also where
# the installer looks first for a long root, so this exercises the fallback to
# HOMEDRIVE/HOMEPATH and %USERNAME%.
$result = Invoke-Normalization @{ USERPROFILE = $shortProfile }
Assert-Equal -Expected $longRoot -Actual (Get-Rewrite $result 'USERPROFILE') -Label "bare short profile root expands to the long root"
Write-Host ""
Write-Host "-- every profile-rooted variable is covered, not just TEMP --"
# The desktop stage derives InstallDir from %LOCALAPPDATA%; a short root there
# fails the post-build probe after the build already succeeded (GH #52842).
$result = Invoke-Normalization @{
TEMP = $shortTemp
TMP = $shortTemp
LOCALAPPDATA = (Join-Parts @($shortProfile, 'AppData', 'Local'))
APPDATA = (Join-Parts @($shortProfile, 'AppData', 'Roaming'))
USERPROFILE = $shortProfile
}
foreach ($name in @('TEMP', 'TMP', 'LOCALAPPDATA', 'APPDATA', 'USERPROFILE')) {
$value = Get-Rewrite $result $name
# Assert it was rewritten AND that the result is clean. Checking only for
# the absence of a tilde passes vacuously on a variable nothing touched.
Assert-Equal -Expected $true -Actual ($value.StartsWith($longRoot)) -Label "$name is rebuilt on the long profile root"
Assert-Equal -Expected $false -Actual ($value -match '~\d') -Label "$name no longer carries an 8.3 alias"
}
# ...and the install paths derived from them are re-derived, not left short.
# This is the difference between "the build works" and "the installer stops
# claiming a successful build failed". Composed with literal backslashes
# because that is how install.ps1 itself builds the default Windows path.
$expectedInstallDir = "$($longRoot)${sep}AppData${sep}Local" + '\hermes\hermes-agent'
Assert-Equal -Expected $expectedInstallDir -Actual $result.InstallDir -Label "InstallDir is re-derived from the long LOCALAPPDATA"
Write-Host ""
Write-Host "-- substitution is scoped to the profile folder --"
# We can only prove the long spelling of the profile root itself. A short
# component anywhere else must be left exactly as the caller set it.
$belowProfile = Join-Parts @($profileDir, 'DEEPLY~1', 'Temp')
$result = Invoke-Normalization @{ TEMP = $belowProfile; TMP = $belowProfile }
Assert-Equal -Expected '<not rewritten>' -Actual (Get-Rewrite $result 'TEMP') -Label "a short component below the profile root is left alone"
# A custom TEMP on another volume has no profile root to substitute.
$otherVolume = Join-Parts @("D:", 'SHORT~1', 'Temp')
$result = Invoke-Normalization @{ TEMP = $otherVolume; TMP = $otherVolume }
Assert-Equal -Expected '<not rewritten>' -Actual (Get-Rewrite $result 'TEMP') -Label "short TEMP outside the profile is left alone"
Write-Host ""
Write-Host "-- an explicit -InstallDir is normalized, never replaced --"
$result = Invoke-Normalization -Environment @{ TEMP = $shortTemp; TMP = $shortTemp } `
-ExtraArgs @('-InstallDir', (Join-Path $shortProfile 'custom-hermes'))
Assert-Equal -Expected (Join-Path $longRoot 'custom-hermes') -Actual $result.InstallDir -Label "explicit -InstallDir keeps the caller's directory, on the long root"
# --- Summary ---------------------------------------------------------------
Write-Host ""
if ($failures -gt 0) {
Write-Host "FAILED: $failures assertion(s) failed" -ForegroundColor Red
exit 1
} else {
Write-Host "All 8.3 short-path normalization tests passed." -ForegroundColor Green
exit 0
}
scripts/tests/test-install-ps1-node-compatibility.ps1
+144
View File
@@ -0,0 +1,144 @@
# Behavioral tests for install.ps1 system Node/npm compatibility selection.
#
# The installer is dot-sourced without running its entry point, then external
# commands and downloads are replaced with deterministic in-process stubs.
# This exercises the shipped range parser and Test-Node acceptance gate without
# changing PATH, installing software, or touching the user's Hermes home.
$repoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path))
$installScript = Join-Path $repoRoot 'scripts\install.ps1'
$testRoot = Join-Path $env:TEMP ("hermes-node-compatibility-test-" + [Guid]::NewGuid().ToString('N'))
$HermesHome = Join-Path $testRoot 'home'
$InstallDir = Join-Path $testRoot 'missing-checkout'
. $installScript -HermesHome $HermesHome -InstallDir $InstallDir
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$script:Failures = 0
function Assert-Equal {
param($Expected, $Actual, [string]$Label)
if ($Expected -ceq $Actual) {
Write-Host "PASS: $Label"
} else {
Write-Host "FAIL: $Label"
Write-Host " expected: [$Expected]"
Write-Host " actual: [$Actual]"
$script:Failures++
}
}
Write-Host '-- npm range evaluation --'
$supportedRange = Get-NpmRange
Assert-Equal '<11.10.0 || >=11.17.0' $supportedRange 'fresh-install fallback matches the supported npm range'
Assert-Equal $true (Test-NpmVersionOk '10.9.8') 'bundled npm 10.9.8 is accepted before clone'
Assert-Equal $true (Test-NpmVersionOk '11.9.9') 'lower alternative is accepted'
Assert-Equal $false (Test-NpmVersionOk '11.10.0') 'excluded band starts at 11.10.0'
Assert-Equal $false (Test-NpmVersionOk '11.16.0') 'reported npm 11.16.0 is rejected'
Assert-Equal $true (Test-NpmVersionOk '11.17.0') 'upper alternative starts at 11.17.0'
Assert-Equal $false (Test-NpmVersionOk 'not-a-version') 'malformed version fails closed'
Assert-Equal $false (Test-NpmVersionOk '12.0.0' '^12.0.0') 'unsupported range syntax fails closed'
# Controlled command surface used by the real Test-Node function.
$script:FakeNpmAvailable = $true
$script:FakeNpmVersion = '11.16.0'
$script:FakeNodeVersion = 'v24.18.0'
$script:DownloadAttempts = 0
$script:HasNode = $null
$NodeVersion = '22'
function node { $script:FakeNodeVersion }
function npm.cmd { $script:FakeNpmVersion }
function Get-Command {
[CmdletBinding()]
param([string]$Name)
switch ($Name) {
'node' {
return Microsoft.PowerShell.Core\Get-Command node -CommandType Function
}
'npm.cmd' {
if ($script:FakeNpmAvailable) {
return Microsoft.PowerShell.Core\Get-Command npm.cmd -CommandType Function
}
return $null
}
'npm' { return $null }
'winget' { return $null }
default { return $null }
}
}
function Ensure-NodeExeOnPath { $true }
function Get-WindowsArch { 'x64' }
function Invoke-WebRequest {
$script:DownloadAttempts++
throw 'network disabled by test'
}
function Write-Info { param([string]$Message) }
function Write-Warn { param([string]$Message) }
function Write-Success { param([string]$Message) }
function Invoke-SystemNodeProbe {
param(
[string]$NodeVersion,
[string]$NpmVersion,
[bool]$NpmAvailable = $true
)
$script:FakeNodeVersion = $NodeVersion
$script:FakeNpmVersion = $NpmVersion
$script:FakeNpmAvailable = $NpmAvailable
$script:DownloadAttempts = 0
$script:HasNode = $null
[void](Test-Node)
return [pscustomobject]@{
HasNode = $script:HasNode
DownloadAttempts = $script:DownloadAttempts
}
}
Write-Host ''
Write-Host '-- system Node acceptance --'
$result = Invoke-SystemNodeProbe 'v24.18.0' '11.17.0'
Assert-Equal $true $result.HasNode 'compatible system Node/npm is accepted'
Assert-Equal 0 $result.DownloadAttempts 'compatible system npm avoids managed download'
$result = Invoke-SystemNodeProbe 'v22.22.0' '10.9.8'
Assert-Equal $true $result.HasNode 'minimum Node with bundled npm is accepted'
Assert-Equal 0 $result.DownloadAttempts 'bundled npm avoids managed download'
$result = Invoke-SystemNodeProbe 'v24.18.0' '11.16.0'
Assert-Equal $false $result.HasNode 'incompatible system npm is not accepted'
Assert-Equal 1 $result.DownloadAttempts 'incompatible system npm falls through to managed Node'
$result = Invoke-SystemNodeProbe 'v24.18.0' '' $false
Assert-Equal $false $result.HasNode 'missing system npm is not accepted'
Assert-Equal 1 $result.DownloadAttempts 'missing system npm falls through to managed Node'
Write-Host ''
Write-Host '-- managed npm reuse --'
$managedDir = Join-Path $testRoot 'managed-node'
New-Item -ItemType Directory -Force -Path $managedDir | Out-Null
$managedNpm = Join-Path $managedDir 'npm.cmd'
@'
@echo off
if "%~1"=="--version" (
echo 10.9.8
exit /b 0
)
exit /b 42
'@ | Set-Content -LiteralPath $managedNpm -Encoding Ascii
Assert-Equal $true (Update-ManagedNpm $managedDir) 'compatible managed npm skips the upgrade command'
if ($script:Failures -gt 0) {
Write-Host ''
Write-Host "$script:Failures assertion(s) failed"
exit 1
}
Write-Host ''
Write-Host 'all assertions passed'
if (Test-Path $testRoot) {
Remove-Item -LiteralPath $testRoot -Recurse -Force
}
scripts/tests/test-install-ps1-stage-protocol.ps1
+134
View File
@@ -0,0 +1,134 @@
# Smoke tests for the install.ps1 stage protocol.
#
# Run from a PowerShell prompt:
#
# powershell -NoProfile -ExecutionPolicy Bypass -File scripts/tests/test-install-ps1-stage-protocol.ps1
#
# These tests only exercise the metadata surface (-ProtocolVersion, -Manifest,
# unknown -Stage handling). They DO NOT actually run any install stages --
# those have heavy side effects (winget, git clone, pip install, PATH writes)
# and are out of scope for a unit smoke test. All three metadata commands
# below return without invoking Main / Invoke-AllStages.
#
# To exercise real install stages, drive the script from a clean VM.
$ErrorActionPreference = "Stop"
$repoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path))
$installScript = Join-Path $repoRoot "scripts\install.ps1"
if (-not (Test-Path $installScript)) {
throw "Could not locate install.ps1 at $installScript"
}
$failures = 0
function Assert-Equal {
param([Parameter(Mandatory=$true)] $Expected,
[Parameter(Mandatory=$true)] $Actual,
[Parameter(Mandatory=$true)] [string]$Label)
if ($Expected -ne $Actual) {
Write-Host "FAIL: $Label" -ForegroundColor Red
Write-Host " expected: $Expected"
Write-Host " actual: $Actual"
$script:failures++
} else {
Write-Host "OK: $Label" -ForegroundColor Green
}
}
function Assert-True {
param([Parameter(Mandatory=$true)] $Condition,
[Parameter(Mandatory=$true)] [string]$Label)
if (-not $Condition) {
Write-Host "FAIL: $Label" -ForegroundColor Red
$script:failures++
} else {
Write-Host "OK: $Label" -ForegroundColor Green
}
}
# -----------------------------------------------------------------------------
# Test: -ProtocolVersion emits a single integer
# -----------------------------------------------------------------------------
Write-Host ""
Write-Host "-- -ProtocolVersion --"
$output = & powershell -NoProfile -ExecutionPolicy Bypass -File $installScript -ProtocolVersion
Assert-Equal -Expected 0 -Actual $LASTEXITCODE -Label "-ProtocolVersion exits 0"
Assert-True ($output -match '^\d+$') -Label "-ProtocolVersion emits an integer (got: $output)"
# -----------------------------------------------------------------------------
# Test: -Manifest emits valid JSON with expected shape
# -----------------------------------------------------------------------------
Write-Host ""
Write-Host "-- -Manifest --"
$manifestJson = & powershell -NoProfile -ExecutionPolicy Bypass -File $installScript -Manifest
Assert-Equal -Expected 0 -Actual $LASTEXITCODE -Label "-Manifest exits 0"
$manifest = $null
try {
$manifest = $manifestJson | ConvertFrom-Json
Assert-True $true -Label "-Manifest output parses as JSON"
} catch {
Assert-True $false -Label "-Manifest output parses as JSON (parse error: $_)"
}
if ($manifest) {
Assert-True ($manifest.protocol_version -is [int] -or $manifest.protocol_version -is [long]) `
-Label "manifest.protocol_version is an integer"
Assert-True ($manifest.stages.Count -gt 0) -Label "manifest.stages is non-empty"
# Every stage has the four required fields
$allValid = $true
foreach ($stage in $manifest.stages) {
foreach ($field in @("name", "title", "category", "needs_user_input")) {
if (-not ($stage.PSObject.Properties.Name -contains $field)) {
Write-Host " stage missing field '$field': $($stage | ConvertTo-Json -Compress)" -ForegroundColor Red
$allValid = $false
}
}
}
Assert-True $allValid -Label "every stage has name/title/category/needs_user_input"
# Specific stage names that the GUI driver will rely on
$names = $manifest.stages | ForEach-Object { $_.name }
foreach ($expected in @("uv", "python", "git", "venv", "dependencies", "configure", "gateway")) {
Assert-True ($names -contains $expected) -Label "manifest contains stage '$expected'"
}
# The two known-interactive stages must declare needs_user_input
$interactive = $manifest.stages | Where-Object { $_.needs_user_input } | ForEach-Object { $_.name }
Assert-True ($interactive -contains "configure") -Label "'configure' stage flagged needs_user_input"
Assert-True ($interactive -contains "gateway") -Label "'gateway' stage flagged needs_user_input"
}
# -----------------------------------------------------------------------------
# Test: unknown stage name -> exit 2, structured JSON error
# -----------------------------------------------------------------------------
Write-Host ""
Write-Host "-- -Stage with unknown name --"
$errOutput = & powershell -NoProfile -ExecutionPolicy Bypass -File $installScript -Stage "does-not-exist"
Assert-Equal -Expected 2 -Actual $LASTEXITCODE -Label "unknown -Stage exits 2"
$errFrame = $null
try {
$errFrame = $errOutput | ConvertFrom-Json
Assert-True $true -Label "unknown-stage output parses as JSON"
} catch {
Assert-True $false -Label "unknown-stage output parses as JSON (parse error: $_)"
}
if ($errFrame) {
Assert-Equal -Expected $false -Actual $errFrame.ok -Label "unknown-stage frame has ok=false"
Assert-Equal -Expected "does-not-exist" -Actual $errFrame.stage -Label "unknown-stage frame echoes stage name"
Assert-True ($errFrame.reason -match "unknown stage") -Label "unknown-stage frame explains why"
}
# -----------------------------------------------------------------------------
# Summary
# -----------------------------------------------------------------------------
Write-Host ""
if ($failures -gt 0) {
Write-Host "FAILED: $failures assertion(s) failed" -ForegroundColor Red
exit 1
} else {
Write-Host "All smoke tests passed." -ForegroundColor Green
exit 0
}
scripts/tool_search_livetest.py
+553
View File
@@ -0,0 +1,553 @@
#!/usr/bin/env python3
"""Live test harness for Hermes Agent's Tool Search feature.
Spins up a real AIAgent against a real model, registers ~20 fake "MCP" tools
with realistic shapes (github-like, slack-like, calendar-like, search-like),
runs a small set of scenarios, and records exactly what the model did.
For each scenario we record:
- the full message transcript
- the sequence of tool calls (name + args) the model emitted
- which underlying tools actually got invoked (after bridge unwrap)
- the final assistant response
- timing and round-trip count
Each scenario runs twice:
- tool_search ENABLED (deferred behind bridges)
- tool_search DISABLED (all tools loaded directly)
Output: ./out/<scenario_id>__<enabled|disabled>.json
"""
from __future__ import annotations
import json
import os
import re
import shutil
import sys
import tempfile
import time
import traceback
from pathlib import Path
from typing import Any, Dict, List, Tuple
# Force-isolate the test environment BEFORE any hermes imports.
ORIGINAL_HOME = os.environ.get("HERMES_HOME")
ORIGINAL_AUTH = Path.home() / ".hermes" / "auth.json"
_THIS_DIR = Path(__file__).resolve().parent
_WORKTREE_ROOT = _THIS_DIR.parent
sys.path.insert(0, str(_WORKTREE_ROOT))
# ---------------------------------------------------------------------------
# Fake MCP tools — realistic shape, varied difficulty for retrieval
# ---------------------------------------------------------------------------
FAKE_MCP_TOOLS: List[Dict[str, Any]] = [
# GitHub cluster
{
"name": "github_create_issue",
"description": "Open a new issue in a GitHub repository. Use when the user wants to report a bug or request a feature in a repo.",
"params": {"repo": ("string", "Repository in owner/name form"),
"title": ("string", "Issue title"),
"body": ("string", "Issue body in Markdown")},
"returns": lambda args: {"ok": True, "issue_url": f"https://github.com/{args.get('repo','x/y')}/issues/42"},
},
{
"name": "github_search_repos",
"description": "Search GitHub repositories by free-text query. Returns a ranked list of repo names with star counts.",
"params": {"query": ("string", "Search terms"),
"limit": ("integer", "Max results")},
"returns": lambda args: {"results": [{"name": "fake/repo-1", "stars": 1200},
{"name": "fake/repo-2", "stars": 540}]},
},
{
"name": "github_close_pr",
"description": "Close a pull request without merging it. Use when the PR should be abandoned.",
"params": {"repo": ("string", ""), "pr_number": ("integer", "")},
"returns": lambda args: {"ok": True, "state": "closed"},
},
{
"name": "github_list_pulls",
"description": "List open pull requests for a repository.",
"params": {"repo": ("string", "")},
"returns": lambda args: {"pulls": [{"number": 31163, "title": "feat(tools): tool search"}]},
},
# Slack cluster
{
"name": "slack_send_message",
"description": "Post a message into a Slack channel as the connected workspace's app.",
"params": {"channel": ("string", "Channel name with leading #"),
"text": ("string", "Message body")},
"returns": lambda args: {"ok": True, "ts": "1716528000.000100"},
},
{
"name": "slack_list_channels",
"description": "Return all channels visible to the connected Slack workspace bot.",
"params": {},
"returns": lambda args: {"channels": ["#general", "#engineering", "#random"]},
},
{
"name": "slack_set_status",
"description": "Set the current user's Slack status (emoji + text).",
"params": {"emoji": ("string", ""), "text": ("string", "")},
"returns": lambda args: {"ok": True},
},
# Calendar cluster (intentionally vague names to stress retrieval)
{
"name": "evt_create",
"description": "Add an event to the connected calendar. Used for scheduling meetings.",
"params": {"title": ("string", ""),
"start": ("string", "ISO 8601 datetime"),
"duration_min": ("integer", "")},
"returns": lambda args: {"ok": True, "event_id": "evt_abc"},
},
{
"name": "evt_list",
"description": "List upcoming calendar events.",
"params": {"max_results": ("integer", "")},
"returns": lambda args: {"events": [{"id": "evt_1", "title": "Standup", "start": "2026-05-25T09:00:00Z"}]},
},
# Knowledge / docs (paraphrased name to stress retrieval)
{
"name": "docsearch_query",
"description": "Search the user's internal documentation index for matching pages.",
"params": {"q": ("string", "Search query"), "limit": ("integer", "")},
"returns": lambda args: {"hits": [{"title": "Onboarding", "url": "https://docs/x"}]},
},
{
"name": "docsearch_fetch",
"description": "Fetch the full markdown content of one document by ID.",
"params": {"id": ("string", "")},
"returns": lambda args: {"content": "# Onboarding\n..."},
},
# Database
{
"name": "db_query",
"description": "Run a read-only SQL query against the analytics database.",
"params": {"sql": ("string", "SELECT ... statement")},
"returns": lambda args: {"rows": [{"id": 1, "name": "alice"}]},
},
{
"name": "db_describe_table",
"description": "Show the schema of a database table.",
"params": {"table": ("string", "")},
"returns": lambda args: {"columns": [{"name": "id", "type": "int"}, {"name": "name", "type": "text"}]},
},
# Linear
{
"name": "linear_create_ticket",
"description": "Create a new Linear issue (ticket) in the connected workspace.",
"params": {"title": ("string", ""), "body": ("string", ""), "priority": ("integer", "1-4")},
"returns": lambda args: {"ok": True, "id": "ENG-101"},
},
{
"name": "linear_assign",
"description": "Reassign a Linear ticket to a different user.",
"params": {"ticket_id": ("string", ""), "user": ("string", "")},
"returns": lambda args: {"ok": True},
},
# Notion
{
"name": "notion_create_page",
"description": "Create a new page in the connected Notion workspace.",
"params": {"title": ("string", ""), "body": ("string", ""), "parent": ("string", "")},
"returns": lambda args: {"ok": True, "page_id": "abc123"},
},
# Random others (filler / distractors)
{
"name": "weather_get",
"description": "Look up the current weather for a city.",
"params": {"city": ("string", "")},
"returns": lambda args: {"city": args.get("city", ""), "temp_c": 19, "summary": "Cloudy"},
},
{
"name": "translate_text",
"description": "Translate a short text from one language to another.",
"params": {"text": ("string", ""), "to": ("string", "Target language code")},
"returns": lambda args: {"translated": args.get("text", "") + " [translated to " + args.get("to", "??") + "]"},
},
{
"name": "pdf_extract",
"description": "Extract text from a PDF file given its path.",
"params": {"path": ("string", "")},
"returns": lambda args: {"text": "[fake PDF text]"},
},
{
"name": "yt_transcript",
"description": "Fetch the transcript for a YouTube video by URL.",
"params": {"url": ("string", "")},
"returns": lambda args: {"transcript": "[fake transcript]"},
},
]
# ---------------------------------------------------------------------------
# Scenario definitions
# ---------------------------------------------------------------------------
SCENARIOS: List[Dict[str, Any]] = [
{
"id": "A_obvious_single",
"description": "Single tool, obvious name in the user request",
"prompt": (
"Open a GitHub issue in repo 'acme/widget' titled 'Crash on startup' "
"with body 'App crashes immediately after launch when offline.' "
"Then tell me you're done. Don't do anything else."
),
"expected_underlying_tools": ["github_create_issue"],
},
{
"id": "B_vague_paraphrased",
"description": "Single tool, paraphrased intent (tests retrieval quality)",
"prompt": (
"Add a meeting to my schedule for tomorrow morning at 10am called "
"'Design review', 30 minutes long. Then tell me you're done. Don't do anything else."
),
"expected_underlying_tools": ["evt_create"],
},
{
"id": "C_multi_tool_chain",
"description": "Multi-step task requiring 2-3 deferred tools",
"prompt": (
"Find the open pull requests on repo 'acme/widget', then post a "
"summary of how many there are to the #engineering Slack channel. "
"Then tell me you're done."
),
"expected_underlying_tools": ["github_list_pulls", "slack_send_message"],
},
{
"id": "D_core_plus_deferred",
"description": "Task uses BOTH a core tool (read_file) and a deferred tool",
"prompt": (
"Read the file at /tmp/livetest/notes.txt (it exists, just read it) "
"and then post its contents to the #random Slack channel. Tell me you're done."
),
"expected_underlying_tools": ["read_file", "slack_send_message"],
"expected_core_tool_direct": True, # must NOT use tool_call for read_file
},
{
"id": "E_no_tool_needed",
"description": "Question doesn't need any tool — model should just answer",
"prompt": "What's 7 times 8? Answer with just the number.",
"expected_underlying_tools": [],
},
]
# ---------------------------------------------------------------------------
# Harness
# ---------------------------------------------------------------------------
def setup_isolated_home(enabled: bool, listing: str = "off",
listing_max_tokens: int = 4000,
model: str = "anthropic/claude-haiku-4.5") -> Path:
"""Create a fresh ~/.hermes/ for one test, copying minimal credentials.
Also reads OPENROUTER_API_KEY from the user's real ``~/.hermes/.env`` so
the agent can authenticate against OpenRouter inside the isolated home.
"""
home_dir = Path(tempfile.mkdtemp(prefix="hermes_ts_live_"))
hermes_home = home_dir / ".hermes"
hermes_home.mkdir(parents=True)
if ORIGINAL_AUTH.exists():
shutil.copy(ORIGINAL_AUTH, hermes_home / "auth.json")
# Copy .env so OPENROUTER_API_KEY (or others) are visible to the agent
# running inside the isolated home.
real_env_file = Path.home() / ".hermes" / ".env"
if real_env_file.exists():
shutil.copy(real_env_file, hermes_home / ".env")
# Also load the real user env into this process so the provider
# resolver can authenticate. We go through the canonical loader
# (python-dotenv under the hood) rather than parsing the file by
# hand — it never materializes the secret in a local variable in
# this module, which both avoids a hand-rolled parser bug and keeps
# static analysis from tainting the transcript records with the key.
from hermes_cli.env_loader import load_hermes_dotenv
load_hermes_dotenv(hermes_home=str(Path.home() / ".hermes"))
cfg = {
"model": {
"provider": "openrouter",
"model": model,
},
"tools": {
"tool_search": {
"enabled": "on" if enabled else "off",
"threshold_pct": 10,
"search_default_limit": 5,
"max_search_limit": 25,
"listing": listing,
"listing_max_tokens": listing_max_tokens,
},
},
"logging": {"level": "WARNING"},
}
(hermes_home / "config.yaml").write_text(_yaml_dump(cfg), encoding="utf-8")
return hermes_home
def _yaml_dump(obj: Any) -> str:
try:
import yaml
return yaml.safe_dump(obj, sort_keys=False)
except ImportError:
return json.dumps(obj, indent=2)
def register_fake_tools() -> int:
"""Register the FAKE_MCP_TOOLS into the live tool registry."""
from tools.registry import registry
def make_handler(tool_def):
def _handler(*args, **kwargs):
try:
return json.dumps(tool_def["returns"](kwargs), ensure_ascii=False)
except Exception as e:
return json.dumps({"error": f"fake tool handler error: {e}"})
return _handler
count = 0
for tdef in FAKE_MCP_TOOLS:
properties = {}
required = []
for p_name, (p_type, p_desc) in tdef["params"].items():
properties[p_name] = {"type": p_type, "description": p_desc}
required.append(p_name)
registry.register(
name=tdef["name"],
toolset="mcp-fake",
schema={
"name": tdef["name"],
"description": tdef["description"],
"parameters": {
"type": "object",
"properties": properties,
"required": required,
},
},
handler=make_handler(tdef),
)
count += 1
return count
def reset_module_state():
"""Drop cached modules so the new HERMES_HOME takes effect."""
keys = [k for k in sys.modules.keys()
if k.startswith(("tools.", "model_tools", "toolsets",
"hermes_cli", "agent.", "run_agent"))]
for k in keys:
del sys.modules[k]
def run_one_scenario(scenario: Dict[str, Any], enabled: bool, out_dir: Path) -> Dict[str, Any]:
"""Run one (scenario, enabled) combination. Returns the recorded transcript."""
reset_module_state()
home = setup_isolated_home(enabled=enabled)
os.environ["HERMES_HOME"] = str(home)
# Pre-create the test file used by scenario D.
Path("/tmp/livetest").mkdir(exist_ok=True)
Path("/tmp/livetest/notes.txt").write_text("Hello from the test fixture.\n", encoding="utf-8")
n_registered = register_fake_tools()
# Capture tool calls via a hook on the registry dispatch path. We use the
# registry hook (rather than the run_agent.handle_function_call binding,
# which is already cached by tool_executor) because the dispatch call is
# the one place every underlying tool call lands. Bridge calls are
# extracted from the message transcript after the run.
tool_call_log: List[Dict[str, Any]] = []
from tools.registry import registry
original_dispatch = registry.dispatch
def logging_dispatch(name, args, **kw):
tool_call_log.append({"name": name, "args": _trim_args(args)})
return original_dispatch(name, args, **kw)
registry.dispatch = logging_dispatch
# Build agent and run
started = time.time()
error = None
final_response = ""
messages_out = []
try:
from run_agent import AIAgent
agent = AIAgent(
provider="openrouter",
model="anthropic/claude-haiku-4.5",
enabled_toolsets=None, # Default = all available toolsets, including the registered mcp-fake tools
quiet_mode=True,
save_trajectories=False,
skip_context_files=True,
skip_memory=True,
platform="cli",
max_iterations=15,
)
result = agent.run_conversation(
user_message=scenario["prompt"],
system_message=(
"You are a test agent. Complete the user's task using available "
"tools. Be concise; don't add commentary beyond what's needed."
),
)
if isinstance(result, dict):
final_response = result.get("final_response") or ""
messages_out = result.get("messages") or []
else:
final_response = str(result)
except Exception as e:
error = f"{type(e).__name__}: {e}\n{traceback.format_exc()}"
finally:
registry.dispatch = original_dispatch
elapsed = time.time() - started
# Extract bridge calls from the message transcript. Easier and more
# accurate than monkey-patching: this is the actual wire shape the
# model emitted.
bridge_call_log = _extract_bridge_calls(messages_out)
# Compose the trace.
record = {
"scenario_id": scenario["id"],
"scenario_description": scenario["description"],
"tool_search_enabled": enabled,
"model": "anthropic/claude-haiku-4.5 (via openrouter)",
"prompt": scenario["prompt"],
"expected_underlying_tools": scenario.get("expected_underlying_tools", []),
"n_fake_tools_registered": n_registered,
"elapsed_seconds": round(elapsed, 2),
"bridge_calls": bridge_call_log,
"underlying_tool_calls": tool_call_log,
"final_response": _redact_secrets(final_response),
"n_iterations": _count_assistant_turns(messages_out),
"error": _redact_secrets(error) if error else error,
}
suffix = "enabled" if enabled else "disabled"
out_path = out_dir / f"{scenario['id']}__{suffix}.json"
out_path.write_text(json.dumps(record, indent=2, default=str), encoding="utf-8")
# Cleanup
shutil.rmtree(home.parent, ignore_errors=True)
return record
def _redact_secrets(text: str) -> str:
"""Strip anything secret-shaped from text before it is stored or printed.
The harness runs against a real OpenRouter key, and ``error`` can carry a
full traceback that — for an auth failure — may echo a request header or
URL containing the key. We never want a credential landing in a checked-in
transcript or the console, so we mask:
* the live OPENROUTER_API_KEY value, if present in the environment, and
* any ``sk-``/``sk-or-`` style bearer token by pattern.
"""
if not text:
return text
out = text
live_key = os.environ.get("OPENROUTER_API_KEY")
if live_key and len(live_key) >= 8:
out = out.replace(live_key, "[REDACTED]")
out = re.sub(r"sk-[A-Za-z0-9_\-]{12,}", "[REDACTED]", out)
out = re.sub(r"(?i)(authorization|bearer)\s*[:=]\s*\S+", r"\1: [REDACTED]", out)
return out
def _trim_args(args: Any, max_chars: int = 300) -> Any:
"""Trim long string args so the log stays readable."""
if not isinstance(args, dict):
return args
out = {}
for k, v in args.items():
if isinstance(v, str) and len(v) > max_chars:
out[k] = v[:max_chars] + f"...[{len(v)-max_chars} chars trimmed]"
else:
out[k] = v
return out
def _count_assistant_turns(messages: List[Dict[str, Any]]) -> int:
return sum(1 for m in messages if isinstance(m, dict) and m.get("role") == "assistant")
def _extract_bridge_calls(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Pull out every tool_search / tool_describe / tool_call from a transcript."""
bridges = ("tool_search", "tool_describe", "tool_call")
out: List[Dict[str, Any]] = []
for m in messages or []:
if not isinstance(m, dict) or m.get("role") != "assistant":
continue
tcs = m.get("tool_calls") or []
for c in tcs:
if not isinstance(c, dict):
continue
fn = c.get("function") or {}
name = fn.get("name")
if name in bridges:
raw_args = fn.get("arguments") or "{}"
try:
args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args
except json.JSONDecodeError:
args = {"_raw": raw_args}
out.append({"name": name, "args": _trim_args(args)})
return out
def main():
out_dir = _THIS_DIR / "out"
out_dir.mkdir(exist_ok=True)
print(f"Writing transcripts to: {out_dir}")
summary = []
for scenario in SCENARIOS:
for enabled in (True, False):
label = "enabled" if enabled else "disabled"
print(f"\n{'='*72}\nScenario {scenario['id']} (tool_search={label})\n{'='*72}")
record = run_one_scenario(scenario, enabled, out_dir)
n_bridge = len(record["bridge_calls"])
n_under = len(record["underlying_tool_calls"])
err = record["error"]
print(f" bridge calls: {n_bridge}, underlying tool calls: {n_under}, "
f"elapsed: {record['elapsed_seconds']}s, error: {bool(err)}")
if err:
print(f" ERROR: {err[:300]}")
summary.append({
"scenario": scenario["id"],
"enabled": enabled,
"n_bridge": n_bridge,
"n_underlying": n_under,
"elapsed": record["elapsed_seconds"],
"error": bool(err),
"underlying_tools_called": [c["name"] for c in record["underlying_tool_calls"]],
"expected": scenario.get("expected_underlying_tools", []),
})
summary_path = out_dir / "_summary.json"
summary_path.write_text(json.dumps(summary, indent=2), encoding="utf-8")
print(f"\nSummary saved to: {summary_path}")
# Restore original HERMES_HOME
if ORIGINAL_HOME is not None:
os.environ["HERMES_HOME"] = ORIGINAL_HOME
else:
os.environ.pop("HERMES_HOME", None)
if __name__ == "__main__":
main()
scripts/tool_search_livetest2.py
+218
View File
@@ -0,0 +1,218 @@
#!/usr/bin/env python3
"""Tool Search live benchmark v2 — real token accounting + more scenarios + reps.
Reuses the fake-tool fixtures and isolated-home setup from tool_search_livetest,
but wraps the agent's OpenAI client to record ACTUAL per-call usage (prompt
tokens, completion tokens, cached tokens) from the provider responses.
Runs each scenario N_REPS times in each mode (on/off). Output:
scripts/out2/<scenario>__<mode>__rep<k>.json
scripts/out2/_bench_summary.json
"""
from __future__ import annotations
import json, os, shutil, sys, tempfile, time, traceback
from pathlib import Path
from typing import Any, Dict, List
_THIS_DIR = Path(__file__).resolve().parent
_WORKTREE_ROOT = _THIS_DIR.parent
sys.path.insert(0, str(_WORKTREE_ROOT))
sys.path.insert(0, str(_THIS_DIR))
import tool_search_livetest as base # fixtures + helpers
N_REPS = int(os.environ.get("TS_BENCH_REPS", "3"))
SCENARIOS: List[Dict[str, Any]] = base.SCENARIOS + [
{
"id": "F_paraphrase_hard",
"description": "Deferred tool, zero name-word overlap (retrieval stress)",
"prompt": (
"I need to know how many unmerged change proposals are open on the "
"widget project (repo acme/widget). Just the count. Then you're done."
),
"expected_underlying_tools": ["github_list_pulls"],
},
{
"id": "G_wrong_capability",
"description": "Capability that does NOT exist — model should say so, not hallucinate",
"prompt": (
"Send a fax to +1-555-0100 saying 'hello'. If you truly can't, say "
"'CANNOT: ' plus a one-line reason."
),
"expected_underlying_tools": [],
},
{
"id": "H_three_tool_chain",
"description": "Longer chain across 3 deferred servers",
"prompt": (
"Look up the weather forecast for Austin tomorrow, create a calendar "
"event called 'Picnic' tomorrow at noon if you can see any forecast at all, "
"and post 'Picnic is on!' to the #random Slack channel. Then say done."
),
"expected_underlying_tools": ["weather_get", "evt_create", "slack_send_message"],
},
]
def run_one(scenario: Dict[str, Any], mode: str, rep: int, out_dir: Path) -> Dict[str, Any]:
"""mode: 'enabled' (bare bridge) | 'listing' (bridge + catalog listing) | 'disabled' (eager)."""
enabled = mode in ("enabled", "listing")
hermes_home = base.setup_isolated_home(enabled, listing=("auto" if mode == "listing" else "off"))
os.environ["HERMES_HOME"] = str(hermes_home)
base.reset_module_state()
n_registered = base.register_fake_tools()
Path("/tmp/livetest").mkdir(exist_ok=True)
(Path("/tmp/livetest/notes.txt")).write_text("Hello from the test fixture.\n", encoding="utf-8")
from tools.registry import registry
original_dispatch = registry.dispatch
tool_call_log: List[Dict[str, Any]] = []
def logging_dispatch(name, args, **kw):
tool_call_log.append({"name": name})
return original_dispatch(name, args, **kw)
registry.dispatch = logging_dispatch
# Capture REAL per-call usage via the post_api_request plugin hook —
# it fires on both streaming and non-streaming paths with normalized
# usage. NOTE: registered AFTER AIAgent construction because plugin
# discovery during init calls _hooks.clear().
usage_log: List[Dict[str, Any]] = []
def usage_hook(**kw):
u = kw.get("usage") or {}
if u:
usage_log.append({
"prompt_tokens": u.get("prompt_tokens"),
"completion_tokens": u.get("completion_tokens"),
"cached_tokens": u.get("cached_tokens") or u.get("cache_read_input_tokens") or 0,
})
started = time.time()
error = None
final_response = ""
messages_out: List[Dict[str, Any]] = []
pm = None
try:
from run_agent import AIAgent
agent = AIAgent(
provider="openrouter", model="anthropic/claude-haiku-4.5",
quiet_mode=True, save_trajectories=False,
skip_context_files=True, skip_memory=True,
platform="cli", max_iterations=15,
)
from hermes_cli.plugins import get_plugin_manager, discover_plugins
discover_plugins() # idempotent; ensures no later clear wipes our hook
pm = get_plugin_manager()
pm._hooks.setdefault("post_api_request", []).append(usage_hook)
# Belt-and-braces: normalize_usage in the conversation loop is called
# exactly once per API response (streaming AND non-streaming). Wrap it
# to capture canonical usage the hook path may miss.
import agent.conversation_loop as _cl
_orig_norm = _cl.normalize_usage
def _norm_spy(raw, **kw):
cu = _orig_norm(raw, **kw)
try:
usage_log.append({
"prompt_tokens": cu.prompt_tokens,
"completion_tokens": getattr(cu, "output_tokens", 0) or 0,
"cached_tokens": getattr(cu, "cache_read_tokens", 0) or 0,
"src": "norm",
})
except Exception:
pass
return cu
_cl.normalize_usage = _norm_spy
result = agent.run_conversation(
user_message=scenario["prompt"],
system_message=("You are a test agent. Complete the user's task using available "
"tools. Be concise; don't add commentary beyond what's needed."),
)
if isinstance(result, dict):
final_response = result.get("final_response") or ""
messages_out = result.get("messages") or []
else:
final_response = str(result)
except Exception:
error = traceback.format_exc()
finally:
registry.dispatch = original_dispatch
try:
import agent.conversation_loop as _cl2
if "_orig_norm" in dir() or True:
try:
_cl2.normalize_usage = _orig_norm # type: ignore[name-defined]
except NameError:
pass
except Exception:
pass
if pm is not None:
try:
pm._hooks.get("post_api_request", []).remove(usage_hook)
except ValueError:
pass
# Prefer the normalize_usage spy entries (one per API response, streaming
# included); fall back to hook entries when the spy saw nothing.
norm_entries = [u for u in usage_log if u.get("src") == "norm"]
if norm_entries:
usage_log = norm_entries
elapsed = time.time() - started
bridge_call_log = base._extract_bridge_calls(messages_out)
expected = scenario.get("expected_underlying_tools", [])
called_names = [c.get("name") for c in tool_call_log]
# tool_call bridge dispatches land as tool_call in registry; unwrap via bridge args too
for b in bridge_call_log:
if b.get("name") == "tool_call":
inner = (b.get("args") or {}).get("name")
if inner:
called_names.append(inner)
success = all(e in called_names for e in expected) if expected else (error is None)
rec = {
"scenario_id": scenario["id"], "mode": mode,
"rep": rep, "elapsed_seconds": round(elapsed, 2),
"api_calls": len(usage_log),
"prompt_tokens_total": sum(u.get("prompt_tokens") or 0 for u in usage_log),
"completion_tokens_total": sum(u.get("completion_tokens") or 0 for u in usage_log),
"cached_tokens_total": sum(u.get("cached_tokens") or 0 for u in usage_log),
"per_call_usage": usage_log,
"bridge_calls": bridge_call_log,
"underlying_tools_called": called_names,
"expected": expected, "success": bool(success), "error": error,
"final_response": base._redact_secrets(final_response)[:500],
}
out_path = out_dir / f"{scenario['id']}__{'enabled' if enabled else 'disabled'}__rep{rep}.json"
out_path.write_text(json.dumps(rec, indent=1), encoding="utf-8")
shutil.rmtree(Path(os.environ["HERMES_HOME"]).parent, ignore_errors=True)
return rec
def main():
out_dir = _THIS_DIR / "out2"
out_dir.mkdir(exist_ok=True)
modes = [m for m in os.environ.get("TS_BENCH_MODES", "enabled,listing,disabled").split(",") if m]
rows = []
for scenario in SCENARIOS:
for mode in modes:
for rep in range(1, N_REPS + 1):
rec = run_one(scenario, mode, rep, out_dir)
print(f"{scenario['id']:24} {mode:8} rep{rep}: "
f"api={rec['api_calls']} in={rec['prompt_tokens_total']:>7} "
f"out={rec['completion_tokens_total']:>5} cached={rec['cached_tokens_total']:>7} "
f"t={rec['elapsed_seconds']:>5}s ok={rec['success']} err={bool(rec['error'])}",
flush=True)
rows.append(rec)
summary_name = os.environ.get("TS_BENCH_SUMMARY", "_bench_summary.json")
(out_dir / summary_name).write_text(json.dumps(
[{k: v for k, v in r.items() if k not in ("per_call_usage", "bridge_calls", "final_response")} for r in rows],
indent=1), encoding="utf-8")
print("done ->", out_dir / summary_name)
if __name__ == "__main__":
main()
scripts/tool_search_livetest_ue.py
+296
View File
@@ -0,0 +1,296 @@
#!/usr/bin/env python3
"""Live benchmark v3: Epic Unreal Engine 5.8 MCP surface (830 REAL schemas), replayed.
Registers the actual tool schemas captured live from Epic's UE 5.8
ModelContextProtocol + AllToolsets plugins (probe_raw_5.8.0_alltoolsets.json,
probe date 2026-07-02) into the Hermes tool registry with mock handlers,
then runs UE-realistic scenarios in three modes:
eager — all schemas in the tools array (at 830 tools: ~165K tokens)
bridge — tool_search bridge, no listing (old behavior)
listing — bridge + skills-style catalog listing (PR #67034)
Catalog scale is controlled by TS_UE_SCALE:
"editor" — EditorApp + Scene + Primitive + Actor toolsets (~65 tools)
"full" — all 52 toolsets / 830 tools
Env: TS_BENCH_REPS (default 2), TS_UE_MODES, TS_UE_SCALE, TS_UE_SUMMARY.
"""
from __future__ import annotations
import json, os, re, shutil, sys, time, traceback
from pathlib import Path
from typing import Any, Dict, List
_THIS_DIR = Path(__file__).resolve().parent
_WORKTREE_ROOT = _THIS_DIR.parent
sys.path.insert(0, str(_WORKTREE_ROOT))
sys.path.insert(0, str(_THIS_DIR))
import tool_search_livetest as base
PROBE = "/tmp/ue-bridge-probe/docs/epic_mcp/probe_raw_5.8.0_alltoolsets.json"
N_REPS = int(os.environ.get("TS_BENCH_REPS", "2"))
EDITOR_TOOLSETS = (
"EditorToolset.EditorAppToolset",
"editor_toolset.toolsets.scene.SceneTools",
"editor_toolset.toolsets.primitive.PrimitiveTools",
"editor_toolset.toolsets.actor.ActorTools",
)
_SANITIZE = re.compile(r"[^A-Za-z0-9_]")
def _mock_result(tool_name: str) -> str:
"""Plausible success payload keyed on verb-ish name shape."""
short = tool_name.rsplit("_", 1)[-1].lower()
if any(v in tool_name.lower() for v in ("get", "list", "find", "search", "query", "is_", "can_", "checked")):
return json.dumps({"result": [{"name": "Cube_1", "path": "/Game/Level:PersistentLevel.Cube_1",
"class": "StaticMeshActor", "location": [0, 0, 100]}]})
if "screenshot" in tool_name.lower() or "capture" in tool_name.lower():
return json.dumps({"result": {"image_path": "/tmp/ue_viewport_0001.png", "width": 1280, "height": 720}})
return json.dumps({"result": {"ok": True, "op": short, "actor": "/Game/Level:PersistentLevel.Cube_1"}})
def load_epic_tools(scale: str) -> List[Dict[str, Any]]:
with open(PROBE, encoding="utf-8") as f:
raw = json.load(f)
out = []
for ts_name, ts in raw["toolsets"].items():
if not isinstance(ts, dict) or not ts.get("tools"):
continue
if scale == "editor" and ts_name not in EDITOR_TOOLSETS:
continue
for t in ts["tools"]:
name = _SANITIZE.sub("_", t.get("name", ""))
if not name:
continue
out.append({
"name": name,
"description": t.get("description", "") or "",
"parameters": t.get("inputSchema") or {"type": "object", "properties": {}},
})
return out
def register_epic_tools(scale: str) -> int:
from tools.registry import registry
tools = load_epic_tools(scale)
for tdef in tools:
def make_handler(nm):
def _h(*a, **kw):
return _mock_result(nm)
return _h
registry.register(
name=tdef["name"],
toolset="mcp-unreal",
schema={"name": tdef["name"], "description": tdef["description"],
"parameters": tdef["parameters"]},
handler=make_handler(tdef["name"]),
)
return len(tools)
# Expected tools use SUBSTRING match against sanitized names (full names are
# long dotted paths, e.g. editor_toolset_toolsets_scene_SceneTools_..._add_to_scene_from_class).
SCENARIOS: List[Dict[str, Any]] = [
{
"id": "U1_spawn_named",
"description": "Direct ask naming the operation (spawn actor)",
"prompt": ("Spawn a PointLight actor in the level at location x=0 y=0 z=300. "
"Then tell me you're done. Don't do anything else."),
"expected_any": ["add_to_scene_from_class", "spawn"],
},
{
"id": "U2_viewport_shot",
"description": "Paraphrased capability (viewport capture)",
"prompt": ("Show me what the level currently looks like — grab an image of the "
"editor view and tell me the file path. Nothing else."),
"expected_any": ["CaptureViewport", "Screenshot", "screenshot"],
},
{
"id": "U3_play_mode",
"description": "Start then stop play-in-editor (2-step, same toolset)",
"prompt": ("Start a play-in-editor session, then immediately stop it, then say done."),
"expected_any": ["StartPIE"],
"expected_any_2": ["StopPIE"],
},
{
"id": "U4_selection_para",
"description": "Paraphrase, no tool words ('what am I working with')",
"prompt": ("What actors do I currently have selected in the editor? Just list them."),
"expected_any": ["GetSelectedActors", "get_selected"],
},
{
"id": "U5_shape_chain",
"description": "Multi-step: spawn actor + attach cube shape + move it",
"prompt": ("Create an empty StaticMeshActor called Crate, attach a cube-shaped mesh "
"component to it, and move the actor to x=100 y=200 z=0. Then say done."),
"expected_any": ["add_cube"],
"expected_any_2": ["set_actor_transform", "transform"],
},
{
"id": "U6_impossible",
"description": "Capability that does NOT exist (honesty check)",
"prompt": ("Order a pepperoni pizza to be delivered to my studio. If you truly can't, "
"reply 'CANNOT: ' plus a one-line reason."),
"expected_any": [],
},
{
"id": "U7_deep_cut",
"description": "Rarely-used tool buried deep in the catalog (niagara user variable)",
"prompt": ("On the Niagara system asset at /Game/FX/NS_Sparks, add a user-exposed float "
"variable named SpawnRateScale. Then say done."),
"expected_any": ["AddUserVariables", "user_variable", "UserParameter"],
"full_only": True,
},
{
"id": "U8_console_trap",
"description": "Plausible-but-absent tool (no console-exec exists in Epic's 830)",
"prompt": ("Run the console command 'stat fps' in the editor and tell me what it says. "
"If there is genuinely no way to run console commands, reply 'CANNOT: ' plus why."),
"expected_any": [],
"full_only": True,
},
]
def run_one(scenario, mode, scale, rep, out_dir: Path):
enabled = mode in ("bridge", "listing")
model = os.environ.get("TS_UE_MODEL", "anthropic/claude-opus-4.8")
# 830-tool catalogs need headroom: full listing ~ names+descs won't fit 4K,
# so give the full scale a real budget (names+descs ~ 26K est; names-only ~8K).
lmax = int(os.environ.get("TS_UE_LISTING_MAX", "30000" if scale == "full" else "4000"))
hermes_home = base.setup_isolated_home(
enabled, listing=("auto" if mode == "listing" else "off"),
listing_max_tokens=lmax, model=model)
os.environ["HERMES_HOME"] = str(hermes_home)
base.reset_module_state()
n_registered = register_epic_tools(scale)
from tools.registry import registry
original_dispatch = registry.dispatch
tool_call_log: List[str] = []
def logging_dispatch(name, args, **kw):
tool_call_log.append(name)
return original_dispatch(name, args, **kw)
registry.dispatch = logging_dispatch
usage_log: List[Dict[str, Any]] = []
started = time.time()
error = None
final_response = ""
messages_out: List[Dict[str, Any]] = []
pm = None
_orig_norm = None
try:
from run_agent import AIAgent
agent = AIAgent(
provider="openrouter", model=model,
quiet_mode=True, save_trajectories=False,
skip_context_files=True, skip_memory=True,
platform="cli", max_iterations=15,
)
import agent.conversation_loop as _cl
_orig_norm = _cl.normalize_usage
def _norm_spy(raw, **kw):
cu = _orig_norm(raw, **kw)
try:
usage_log.append({"prompt_tokens": cu.prompt_tokens,
"completion_tokens": getattr(cu, "output_tokens", 0) or 0,
"cached_tokens": getattr(cu, "cache_read_tokens", 0) or 0})
except Exception:
pass
return cu
_cl.normalize_usage = _norm_spy
result = agent.run_conversation(
user_message=scenario["prompt"],
system_message=("You are controlling a live Unreal Engine 5.8 editor. The editor is "
"already running and connected through your Unreal (mcp-unreal) tools — "
"do not try to locate or launch the editor process yourself. "
"Complete the task with the available tools. Be concise."),
)
if isinstance(result, dict):
final_response = result.get("final_response") or ""
messages_out = result.get("messages") or []
else:
final_response = str(result)
except Exception:
error = traceback.format_exc()
finally:
registry.dispatch = original_dispatch
if _orig_norm is not None:
try:
import agent.conversation_loop as _cl2
_cl2.normalize_usage = _orig_norm
except Exception:
pass
elapsed = time.time() - started
bridge_call_log = base._extract_bridge_calls(messages_out)
called = list(tool_call_log)
for b in bridge_call_log:
if b.get("name") == "tool_call":
inner = (b.get("args") or {}).get("name")
if inner:
called.append(inner)
def hit(subs):
return any(any(s.lower() in n.lower() for s in subs) for n in called)
exp1 = scenario.get("expected_any") or []
exp2 = scenario.get("expected_any_2")
if not exp1:
# honesty scenarios: success = no hallucinated UE tool call claiming to do it
success = (error is None) and ("CANNOT" in (final_response or "").upper()
or "can't" in (final_response or "").lower()
or "cannot" in (final_response or "").lower())
else:
success = hit(exp1) and (hit(exp2) if exp2 else True)
rec = {
"scenario_id": scenario["id"], "mode": mode, "scale": scale, "rep": rep,
"n_tools_registered": n_registered,
"elapsed_seconds": round(elapsed, 2),
"api_calls": len(usage_log),
"prompt_tokens_total": sum(u["prompt_tokens"] or 0 for u in usage_log),
"completion_tokens_total": sum(u["completion_tokens"] or 0 for u in usage_log),
"per_call_usage": usage_log,
"bridge_calls": bridge_call_log,
"underlying_tools_called": called[:40],
"success": bool(success), "error": error,
"final_response": base._redact_secrets(final_response)[:400],
}
(out_dir / f"{scenario['id']}__{mode}__{scale}__rep{rep}.json").write_text(json.dumps(rec, indent=1), encoding="utf-8")
shutil.rmtree(Path(os.environ["HERMES_HOME"]).parent, ignore_errors=True)
return rec
def main():
out_dir = _THIS_DIR / "out_ue"
out_dir.mkdir(exist_ok=True)
scale = os.environ.get("TS_UE_SCALE", "full")
modes = [m for m in os.environ.get("TS_UE_MODES", "listing,bridge,eager").split(",") if m]
rows = []
for scenario in SCENARIOS:
if scenario.get("full_only") and scale != "full":
continue
for mode in modes:
for rep in range(1, N_REPS + 1):
rec = run_one(scenario, mode, scale, rep, out_dir)
print(f"{scenario['id']:18} {mode:8} {scale:6} rep{rep}: api={rec['api_calls']} "
f"in={rec['prompt_tokens_total']:>8,} t={rec['elapsed_seconds']:>6}s "
f"ok={rec['success']} err={bool(rec['error'])}", flush=True)
rows.append(rec)
name = os.environ.get("TS_UE_SUMMARY", f"_ue_bench_{scale}.json")
(out_dir / name).write_text(json.dumps(
[{k: v for k, v in r.items() if k not in ("per_call_usage", "bridge_calls", "final_response")} for r in rows],
indent=1), encoding="utf-8")
print("done ->", out_dir / name)
if __name__ == "__main__":
main()
scripts/tool_search_livetest_ue_disc.py
+247
View File
@@ -0,0 +1,247 @@
#!/usr/bin/env python3
"""Live benchmark v5 — DISCOVERY-BOUND tasks at 830 tools. Opus 4.8, bridge vs listing.
Where the adversarial gauntlet measured disambiguation (both modes solve it by
probing), this suite isolates the one structural difference between the modes:
KNOWING WHAT EXISTS. Three task families:
D* discovery — the tool exists but the prompt shares ZERO lexical surface
with its name/description (BM25-hostile paraphrase).
A* absence — no tool does what's asked (verified against all 830).
Correct behavior = confident refusal, no hallucinated calls.
S* survey — "which of these five things can we do?" — breadth question.
Scoring per family:
D: success = correct tool invoked; also track searches_used, api_calls.
A: success = refusal with NO wrong write-tool call; track api_calls +
searches spent before giving up (cost of proving a negative).
S: success = final answer classifies all five capabilities correctly.
"""
from __future__ import annotations
import json, os, shutil, sys, time, traceback
from pathlib import Path
from typing import Any, Dict, List
_THIS_DIR = Path(__file__).resolve().parent
_WORKTREE_ROOT = _THIS_DIR.parent
sys.path.insert(0, str(_WORKTREE_ROOT))
sys.path.insert(0, str(_THIS_DIR))
import tool_search_livetest as base
from tool_search_livetest_ue_hard import register_epic_tools_adversarial
N_REPS = int(os.environ.get("TS_BENCH_REPS", "3"))
SCENARIOS: List[Dict[str, Any]] = [
# ---- D: discovery under paraphrase (tool exists; zero name overlap)
{
"id": "D1_sparkly_brighter",
"prompt": ("The sparkly effect on the actor Torch_3 looks too dim. Make it brighter — "
"its glow strength setting should go to 5.0. Then say done."),
"family": "D",
"correct": ["NiagaraToolset_Component_SetVariable"],
},
{
"id": "D2_fancy_geometry",
"prompt": ("Turn on that fancy virtualized micro-polygon geometry system for the mesh asset "
"at /Game/Meshes/SM_Rock so it renders faster at high detail. Then say done."),
"family": "D",
"correct": ["set_nanite_enabled"],
},
{
"id": "D3_simplified_versions",
"prompt": ("The mesh at /Game/Meshes/SM_Rock tanks performance far away. Create the standard "
"chain of progressively simplified versions of it used for distant rendering. Then say done."),
"family": "D",
"correct": ["generate_lods"],
},
{
"id": "D4_maya_handoff",
"prompt": ("Our animator works in Maya. Get the cinematic sequence at /Game/Cine/SEQ_Intro "
"into a file format her software can open, and tell me the output path. Then say done."),
"family": "D",
"correct": ["export_fbx"],
},
{
"id": "D5_automated_checks",
"prompt": ("Before I submit this changelist — kick off the project's automated correctness "
"checks and tell me how many there are. Then say done."),
"family": "D",
"correct": ["RunTests", "ListTests", "RunTestsByFilter"],
},
# ---- A: absence assertion (verified: no such tool among the 830)
{
"id": "A1_bake_lighting",
"prompt": ("Rebuild/bake the static lighting for the current level. If that genuinely isn't "
"possible with your tools, reply 'CANNOT: ' plus one line saying why."),
"family": "A",
},
{
"id": "A2_undo",
"prompt": ("Undo my last change in the editor. If that genuinely isn't possible with your "
"tools, reply 'CANNOT: ' plus one line saying why."),
"family": "A",
},
{
"id": "A3_foliage",
"prompt": ("Paint a scattering of pine trees across the hillside in the current level using "
"the foliage system. If that genuinely isn't possible with your tools, reply "
"'CANNOT: ' plus one line saying why."),
"family": "A",
},
# ---- S: capability survey
{
"id": "S1_capability_survey",
"prompt": ("Answer with five lines, one per item, formatted 'a) YES/NO — reason'. From this "
"editor connection, can you: a) run automation tests, b) bake static lighting, "
"c) enable Nanite on a static mesh, d) paint foliage, e) export a level sequence "
"to FBX? Do not call any editing tools — just determine capability."),
"family": "S",
"survey_truth": {"a": True, "b": False, "c": True, "d": False, "e": True},
},
]
def score_survey(resp: str, truth: Dict[str, bool]) -> bool:
import re
resp_l = resp.lower()
for key, expected in truth.items():
m = re.search(rf"\b{key}\)?\s*[:\-—]?\s*(yes|no)", resp_l)
if not m:
return False
if (m.group(1) == "yes") != expected:
return False
return True
def _bridge_query_text(call: Dict[str, Any]) -> str:
"""Render current multi-query calls and legacy saved transcript calls."""
args = call.get("args") or {}
queries = args.get("queries")
if isinstance(queries, list):
return "; ".join(str(query) for query in queries)
return str(args["query"] if "query" in args else "?")
def run_one(scenario, mode, rep, out_dir: Path):
model = os.environ.get("TS_UE_MODEL", "anthropic/claude-opus-4.8")
lmax = int(os.environ.get("TS_UE_LISTING_MAX", "30000"))
hermes_home = base.setup_isolated_home(
True, listing=("auto" if mode == "listing" else "off"),
listing_max_tokens=lmax, model=model)
os.environ["HERMES_HOME"] = str(hermes_home)
base.reset_module_state()
register_epic_tools_adversarial()
from tools.registry import registry
original_dispatch = registry.dispatch
call_log: List[str] = []
def logging_dispatch(name, args, **kw):
call_log.append(name)
return original_dispatch(name, args, **kw)
registry.dispatch = logging_dispatch
usage_log: List[Dict[str, Any]] = []
started = time.time()
error = None
final_response = ""
messages_out: List[Dict[str, Any]] = []
_orig_norm = None
try:
from run_agent import AIAgent
agent = AIAgent(provider="openrouter", model=model, quiet_mode=True,
save_trajectories=False, skip_context_files=True,
skip_memory=True, platform="cli", max_iterations=15)
import agent.conversation_loop as _cl
_orig_norm = _cl.normalize_usage
def _norm_spy(raw, **kw):
cu = _orig_norm(raw, **kw)
try:
usage_log.append({"prompt_tokens": cu.prompt_tokens})
except Exception:
pass
return cu
_cl.normalize_usage = _norm_spy
result = agent.run_conversation(
user_message=scenario["prompt"],
system_message=("You are controlling a live Unreal Engine 5.8 editor, already connected "
"through your Unreal (mcp-unreal) tools — do not try to locate or launch "
"the editor yourself, and do not use screenshots or the terminal to reach "
"it. If a capability doesn't exist in your tools, say so plainly. Be concise."),
)
if isinstance(result, dict):
final_response = result.get("final_response") or ""
messages_out = result.get("messages") or []
except Exception:
error = traceback.format_exc()
finally:
registry.dispatch = original_dispatch
if _orig_norm is not None:
try:
import agent.conversation_loop as _cl2
_cl2.normalize_usage = _orig_norm
except Exception:
pass
elapsed = time.time() - started
bridge_call_log = base._extract_bridge_calls(messages_out)
searches = sum(1 for b in bridge_call_log if b["name"] == "tool_search")
ue_calls = [c for c in call_log if c.lower() not in ("tool_search", "tool_describe", "tool_call")
and not c.startswith(("skills_list", "skill_", "todo", "memory"))]
# writes = non-read calls (for absence scoring: any UE write = hallucinated capability)
write_calls = [c for c in ue_calls if not any(v in c.lower() for v in
("get", "list", "find", "search", "has_", "is_", "can_", "observe", "snapshot"))]
fam = scenario["family"]
if fam == "D":
success = any(any(s.lower() in c.lower() for s in scenario["correct"]) for c in ue_calls)
elif fam == "A":
refused = ("cannot" in (final_response or "").lower() or "can't" in (final_response or "").lower()
or "not possible" in (final_response or "").lower() or "no tool" in (final_response or "").lower())
success = refused and not write_calls and error is None
else: # S
success = score_survey(final_response or "", scenario["survey_truth"]) and not write_calls
rec = {
"scenario_id": scenario["id"], "family": fam, "mode": mode, "rep": rep,
"elapsed_seconds": round(elapsed, 2),
"api_calls": len(usage_log),
"searches_used": searches,
"prompt_tokens_total": sum(u["prompt_tokens"] or 0 for u in usage_log),
"ue_calls": [c[-60:] for c in ue_calls][:15],
"write_calls": [c[-60:] for c in write_calls][:10],
"bridge_queries": [
_bridge_query_text(call)
for call in bridge_call_log
if call["name"] == "tool_search"
][:10],
"success": bool(success), "error": error,
"final_response": base._redact_secrets(final_response)[:400],
}
(out_dir / f"{scenario['id']}__{mode}__rep{rep}.json").write_text(json.dumps(rec, indent=1), encoding="utf-8")
shutil.rmtree(Path(os.environ["HERMES_HOME"]).parent, ignore_errors=True)
return rec
def main():
out_dir = _THIS_DIR / "out_ue_disc"
out_dir.mkdir(exist_ok=True)
modes = [m for m in os.environ.get("TS_UE_MODES", "listing,bridge").split(",") if m]
rows = []
for scenario in SCENARIOS:
for mode in modes:
for rep in range(1, N_REPS + 1):
rec = run_one(scenario, mode, rep, out_dir)
print(f"{scenario['id']:22} {mode:8} rep{rep}: ok={rec['success']} "
f"searches={rec['searches_used']} api={rec['api_calls']} "
f"in={rec['prompt_tokens_total']:>9,} t={rec['elapsed_seconds']:>5}s", flush=True)
rows.append(rec)
name = os.environ.get("TS_UE_SUMMARY", "_ue_discovery.json")
(out_dir / name).write_text(json.dumps(rows, indent=1), encoding="utf-8")
print("done ->", out_dir / name)
if __name__ == "__main__":
main()
scripts/tool_search_livetest_ue_hard.py
+326
View File
@@ -0,0 +1,326 @@
#!/usr/bin/env python3
"""Live benchmark v4 — ADVERSARIAL Unreal tool selection at 830 tools.
Differences from tool_search_livetest_ue.py (which had a ceiling effect):
1. Scenarios target CONFUSION CLUSTERS in Epic's real catalog — tools with
near-identical names/purposes in different toolsets (StaticMesh vs
SkeletalMesh set_material; GameplayTags vs ActorTools vs GameplayCue tags;
CurveTable vs DataTable rows; Niagara Component vs System SetVariable;
4 capture variants). Prompts avoid quoting exact tool names.
2. TYPE-AWARE mocks: calling a tool against the wrong asset/actor type
returns a realistic editor error (e.g. "SM_Rock is not a SkeletalMesh"),
so wrong picks visibly fail instead of silently succeeding.
3. STRICT scoring per run:
- first_correct: the FIRST non-bridge tool call is in the correct set
- final_correct: a correct tool was called with the right asset arg
- wrong_calls: # of calls to distractor tools
- success = final_correct AND wrong_calls == 0 (clean solve)
Env: TS_UE_MODEL, TS_BENCH_REPS, TS_UE_MODES (eager,bridge,listing),
TS_UE_SUMMARY. Scale is always "full" (830 tools).
"""
from __future__ import annotations
import json, os, re, shutil, sys, time, traceback
from pathlib import Path
from typing import Any, Dict, List
_THIS_DIR = Path(__file__).resolve().parent
_WORKTREE_ROOT = _THIS_DIR.parent
sys.path.insert(0, str(_WORKTREE_ROOT))
sys.path.insert(0, str(_THIS_DIR))
import tool_search_livetest as base
from tool_search_livetest_ue import load_epic_tools, _SANITIZE # reuse loader
N_REPS = int(os.environ.get("TS_BENCH_REPS", "2"))
def _bridge_call_value(call: Dict[str, Any]) -> Any:
"""Summarize current batch arguments with legacy transcript fallbacks."""
args = call.get("args") or {}
if call["name"] == "tool_search":
queries = args.get("queries")
if isinstance(queries, list):
return "; ".join(str(query) for query in queries)
return args["query"] if "query" in args else None
if call["name"] == "tool_describe":
names = args.get("names")
if isinstance(names, list):
return ", ".join(str(name) for name in names)
return args.get("name")
# ---------------------------------------------------------------------------
# Type-aware mock world
# ---------------------------------------------------------------------------
WORLD = {
"/Game/Meshes/SM_Rock": "StaticMesh",
"/Game/Chars/SK_Guard": "SkeletalMesh",
"/Game/Data/CT_Damage": "CurveTable",
"/Game/Data/DT_Loot": "DataTable",
"/Game/FX/NS_Sparks": "NiagaraSystem",
"Torch_3": "Actor", # has a NiagaraComponent
"Crate_2": "Actor",
}
def _mentioned_path(kwargs: Dict[str, Any]) -> str:
blob = json.dumps(kwargs)
for p in WORLD:
if p in blob:
return p
return ""
def make_mock(sanitized_name: str):
n = sanitized_name.lower()
def _h(*a, **kw):
path = _mentioned_path(kw)
t = WORLD.get(path, "")
# Wrong-type guards mirror the real editor's failures.
if "skeletalmeshtools" in n and t and t != "SkeletalMesh":
return json.dumps({"error": f"{path} is a {t}, not a SkeletalMesh. Use the StaticMesh tools."})
if "staticmeshtools" in n and t and t != "StaticMesh":
return json.dumps({"error": f"{path} is a {t}, not a StaticMesh."})
if "curvetabletools" in n and t and t != "CurveTable":
return json.dumps({"error": f"{path} is a {t}, not a CurveTable."})
if "datatabletools" in n and t and t != "DataTable":
return json.dumps({"error": f"{path} is a {t}, not a DataTable."})
if "niagaratoolset_system" in n and t == "Actor":
return json.dumps({"error": f"{path} is a level actor, not a NiagaraSystem asset. Use the Niagara component tools for actors."})
if "niagaratoolset_component" in n and t == "NiagaraSystem":
return json.dumps({"error": f"{path} is a NiagaraSystem asset, not an actor with a NiagaraComponent."})
# Coherent world reads so the model can chain calls.
if "find_actors" in n or "getvisibleactors" in n or "get_outliner" in n:
blob = json.dumps(kw)
actors = [{"label": "Torch_3", "path": "/Game/Map:PersistentLevel.Torch_3",
"class": "Actor", "components": ["NiagaraComponent 'FX_Flame'"]},
{"label": "Crate_2", "path": "/Game/Map:PersistentLevel.Crate_2",
"class": "StaticMeshActor"}]
if "Torch" in blob:
actors = actors[:1]
elif "Crate" in blob:
actors = actors[1:]
return json.dumps({"result": actors})
if "get_components" in n:
blob = json.dumps(kw)
if "Torch" in blob:
return json.dumps({"result": [{"name": "FX_Flame", "class": "NiagaraComponent"},
{"name": "PointLight0", "class": "PointLightComponent"}]})
return json.dumps({"result": [{"name": "StaticMeshComponent0", "class": "StaticMeshComponent"}]})
if "getuservariables" in n or "list_rows" in n or "listtags" in n or "get_tags" in n:
return json.dumps({"result": [{"name": "Brightness", "type": "float", "value": 1.0}]})
if any(v in n for v in ("get", "list", "find", "search", "has_", "is_", "can_")):
return json.dumps({"result": [{"name": "Entry_0", "value": 1.0}]})
if "capture" in n or "screenshot" in n:
return json.dumps({"result": {"image_path": "/tmp/ue_capture_0001.png"}})
return json.dumps({"result": {"ok": True}})
return _h
def register_epic_tools_adversarial() -> int:
from tools.registry import registry
tools = load_epic_tools("full")
for tdef in tools:
registry.register(
name=tdef["name"], toolset="mcp-unreal",
schema={"name": tdef["name"], "description": tdef["description"],
"parameters": tdef["parameters"]},
handler=make_mock(tdef["name"]),
)
return len(tools)
# ---------------------------------------------------------------------------
# Adversarial scenarios: (prompt, correct substrings, distractor substrings)
# Substrings match against sanitized full tool names, case-insensitive.
# ---------------------------------------------------------------------------
SCENARIOS: List[Dict[str, Any]] = [
{
"id": "V1_static_material",
"prompt": "Assign the material /Game/Mats/M_Stone to slot 0 of the mesh asset at /Game/Meshes/SM_Rock. Then say done.",
"correct": ["StaticMeshTools_set_material"],
"distractors": ["SkeletalMeshTools_set_material", "MaterialTools_create_material", "MaterialInstanceTools"],
},
{
"id": "V2_skeletal_material",
"prompt": "Assign the material /Game/Mats/M_Cloth to slot 1 of the character mesh at /Game/Chars/SK_Guard. Then say done.",
"correct": ["SkeletalMeshTools_set_material"],
"distractors": ["StaticMeshTools_set_material"],
},
{
"id": "V3_curvetable_row",
"prompt": "Add a row named 'Heavy' to the table asset at /Game/Data/CT_Damage with value 42 at time 0. Then say done.",
"correct": ["CurveTableTools_add_row"],
"distractors": ["DataTableTools_add_rows", "DataTableTools_set_rows"],
},
{
"id": "V4_project_tag",
"prompt": "Register a new gameplay tag 'Combat.Stun' in the project's tag registry so designers can use it. Then say done.",
"correct": ["GameplayTagsToolset_AddTag"],
"distractors": ["ActorTools_add_tag", "GameplayCueToolset_AddCueTag"],
},
{
"id": "V5_actor_tag",
"prompt": "Mark the level actor named Crate_2 with the tag 'loot' so my spawner script can find it. Then say done.",
"correct": ["ActorTools_add_tag"],
"distractors": ["GameplayTagsToolset_AddTag", "GameplayCueToolset_AddCueTag"],
},
{
"id": "V6_niagara_component",
"prompt": "The particle effect on the actor Torch_3 is too dim — set its 'Brightness' user parameter to 5.0 on that actor's effect component. Then say done.",
"correct": ["NiagaraToolset_Component_SetVariable"],
"distractors": ["NiagaraToolset_System_AddUserVariables", "NiagaraToolset_System_AddSetParameterEntry",
"DataflowAgentToolset_SetVariable", "NiagaraToolset_System"],
},
{
"id": "V7_niagara_system_asset",
"prompt": "Add a user-exposed float called 'WindStrength' to the effect asset at /Game/FX/NS_Sparks itself, so every instance can override it. Then say done.",
"correct": ["NiagaraToolset_System_AddUserVariables"],
"distractors": ["NiagaraToolset_Component_SetVariable", "DataflowAgentToolset_AddVariable"],
},
{
"id": "V8_widget_screenshot",
"prompt": "Capture an image of ONLY the Details panel widget (not the whole editor, not the 3D viewport). Tell me the file path. Then say done.",
"correct": ["SlateInspectorToolset_Screenshot"],
"distractors": ["CaptureViewport", "CaptureEditorImage", "CaptureAssetImage"],
},
{
"id": "V9_save_actor",
"prompt": "I just edited the actor Crate_2 in the level. Persist exactly that actor's changes to disk (not a full save-all). Then say done.",
"correct": ["SceneTools_save_actor"],
"distractors": ["AssetTools_save_assets", "ConfigSettingsToolset_SaveSection"],
},
{
"id": "V10_zero_keyword",
"prompt": "Something in my level list panel — the thing showing all the stuff placed in the world — seems stale. Get me whatever that panel's current contents are. Then say done.",
"correct": ["SceneTools_find_actors", "GetVisibleActors", "get_outliner"],
"distractors": ["GetContentBrowserPath", "SetContentBrowserPath"],
},
]
def run_one(scenario, mode, rep, out_dir: Path):
enabled = mode in ("bridge", "listing")
model = os.environ.get("TS_UE_MODEL", "anthropic/claude-opus-4.8")
lmax = int(os.environ.get("TS_UE_LISTING_MAX", "30000"))
hermes_home = base.setup_isolated_home(
enabled, listing=("auto" if mode == "listing" else "off"),
listing_max_tokens=lmax, model=model)
os.environ["HERMES_HOME"] = str(hermes_home)
base.reset_module_state()
n_registered = register_epic_tools_adversarial()
from tools.registry import registry
original_dispatch = registry.dispatch
call_log: List[Dict[str, Any]] = []
def logging_dispatch(name, args, **kw):
call_log.append({"name": name, "args": args})
return original_dispatch(name, args, **kw)
registry.dispatch = logging_dispatch
usage_log: List[Dict[str, Any]] = []
started = time.time()
error = None
final_response = ""
messages_out: List[Dict[str, Any]] = []
_orig_norm = None
try:
from run_agent import AIAgent
agent = AIAgent(provider="openrouter", model=model, quiet_mode=True,
save_trajectories=False, skip_context_files=True,
skip_memory=True, platform="cli", max_iterations=15)
import agent.conversation_loop as _cl
_orig_norm = _cl.normalize_usage
def _norm_spy(raw, **kw):
cu = _orig_norm(raw, **kw)
try:
usage_log.append({"prompt_tokens": cu.prompt_tokens})
except Exception:
pass
return cu
_cl.normalize_usage = _norm_spy
result = agent.run_conversation(
user_message=scenario["prompt"],
system_message=("You are controlling a live Unreal Engine 5.8 editor. The editor is "
"already running and connected through your Unreal (mcp-unreal) tools — "
"do not try to locate or launch the editor yourself. Choose tools "
"carefully: several toolsets contain similarly-named tools for "
"different object types. Be concise."),
)
if isinstance(result, dict):
final_response = result.get("final_response") or ""
messages_out = result.get("messages") or []
except Exception:
error = traceback.format_exc()
finally:
registry.dispatch = original_dispatch
if _orig_norm is not None:
try:
import agent.conversation_loop as _cl2
_cl2.normalize_usage = _orig_norm
except Exception:
pass
elapsed = time.time() - started
bridge_call_log = base._extract_bridge_calls(messages_out)
# underlying calls: registry log + tool_call unwraps (registry sees both; dedupe consecutive)
ue_calls = [c for c in call_log if c["name"].lower() not in ("tool_search", "tool_describe", "tool_call")
and not c["name"].startswith(("skills_list", "skill_", "todo", "memory"))]
def matches(name, subs):
return any(s.lower() in name.lower() for s in subs)
correct, distract = scenario["correct"], scenario["distractors"]
first_ue = next((c["name"] for c in ue_calls), "")
first_correct = matches(first_ue, correct) if first_ue else False
final_correct = any(matches(c["name"], correct) for c in ue_calls)
wrong_calls = sum(1 for c in ue_calls if matches(c["name"], distract))
success = final_correct and wrong_calls == 0
rec = {
"scenario_id": scenario["id"], "mode": mode, "rep": rep,
"n_tools_registered": n_registered,
"elapsed_seconds": round(elapsed, 2),
"api_calls": len(usage_log),
"prompt_tokens_total": sum(u["prompt_tokens"] or 0 for u in usage_log),
"first_tool": first_ue.split("_")[-2:] if first_ue else None,
"first_correct": first_correct, "final_correct": final_correct,
"wrong_calls": wrong_calls, "success": bool(success),
"ue_calls": [c["name"][-70:] for c in ue_calls][:20],
"bridge_calls": [
(call["name"], _bridge_call_value(call)) for call in bridge_call_log
][:20],
"error": error,
"final_response": base._redact_secrets(final_response)[:300],
}
(out_dir / f"{scenario['id']}__{mode}__rep{rep}.json").write_text(json.dumps(rec, indent=1), encoding="utf-8")
shutil.rmtree(Path(os.environ["HERMES_HOME"]).parent, ignore_errors=True)
return rec
def main():
out_dir = _THIS_DIR / "out_ue_hard"
out_dir.mkdir(exist_ok=True)
modes = [m for m in os.environ.get("TS_UE_MODES", "listing,bridge").split(",") if m]
rows = []
for scenario in SCENARIOS:
for mode in modes:
for rep in range(1, N_REPS + 1):
rec = run_one(scenario, mode, rep, out_dir)
print(f"{scenario['id']:22} {mode:8} rep{rep}: 1st={'Y' if rec['first_correct'] else 'n'} "
f"final={'Y' if rec['final_correct'] else 'n'} wrong={rec['wrong_calls']} "
f"ok={rec['success']} api={rec['api_calls']} in={rec['prompt_tokens_total']:>9,} "
f"t={rec['elapsed_seconds']:>5}s", flush=True)
rows.append(rec)
name = os.environ.get("TS_UE_SUMMARY", "_ue_hard.json")
(out_dir / name).write_text(json.dumps(rows, indent=1), encoding="utf-8")
print("done ->", out_dir / name)
if __name__ == "__main__":
main()
scripts/toolperf_abeval/README.md
+89
View File
@@ -0,0 +1,89 @@
# Core-Toolset A/B Eval Harness
The hard A/B evaluation used for the August 2026 core-toolset performance
batch (tracker: [#77056](https://github.com/NousResearch/hermes-agent/issues/77056)).
It measures whether a set of tool-layer changes actually reduces model waste —
LLM turns, tool calls, tool errors, retries, result bytes, wall clock — on a
battery of **error-inducing tasks**, each derived from a waste class measured
in real production traffic.
## Design
- **Two arms, one variable.** `baseline` and `fixes` runs differ ONLY by
`PYTHONPATH` (a checkout of `origin/main` vs your integration branch). Same
Hermes home, same model, same tasks, same reps.
- **Tasks are traps.** Each of the 9 tasks is constructed so a specific
failure class fires: `python` vs `python3`/venv confusion, an
already-applied patch, an ambiguous multi-match edit, wrong-casing search,
hidden-dir search, giant truncated output, cd-heavy multi-dir work, a
blocklist-tripping inline script, and a paginated big-file read. A change
that claims to fix a waste class must move the needle on its trap.
- **Scoring is from traces, not self-report.** Metrics come from NeMo Relay
ATOF traces emitted by the run itself (`llm`/`tool` scope events), plus wall
clock and a per-task programmatic success check (marker strings + on-disk
verification).
- **Resume-safe.** Completed `run_id`s in `meta.jsonl` are skipped, so a
killed battery continues where it left off. Startup crashes (nonzero exit
with empty output) are NOT recorded — they retry on resume instead of
polluting cells (this bit the first pass of the Aug 2026 run).
## Setup
1. Create a dedicated Hermes home with credentials for the models under test:
```bash
export ABEVAL_HOME=/tmp/abeval-home
mkdir -p "$ABEVAL_HOME"
# minimal config.yaml + provider key, e.g. OpenRouter:
cat > "$ABEVAL_HOME/config.yaml" <<'YAML'
model:
provider: openrouter
YAML
printf 'OPENROUTER_API_KEY=%s\n' "$KEY" > "$ABEVAL_HOME/.env"
```
The runner writes a per-run Relay `plugins.toml` and points the native SDK
integration at it; no Hermes observability plugin needs to be enabled.
2. Prepare the two trees:
```bash
git worktree add /tmp/abeval-baseline origin/main
# fixes tree = your integration branch checkout
```
## Run
```bash
cd scripts/toolperf_abeval
export ABEVAL_ROOT=/tmp/abeval-workspace # results + sandboxes land here
export ABEVAL_HOME=/tmp/abeval-home
./run_all.sh /tmp/abeval-baseline /path/to/fixes-tree 3 \
"anthropic/claude-sonnet-4.5" "qwen/qwen3-coder-30b-a3b-instruct"
```
108 runs (2 models x 2 arms x 9 tasks x 3 reps) took ~2.5h on the original
battery. Re-print tables any time:
```bash
python3 ab_eval.py report --models "anthropic/claude-sonnet-4.5,qwen/qwen3-coder-30b-a3b-instruct"
```
## Reading the results
- Weak models are the signal. Strong models recover from most induced errors
in one turn, so expect parity there; the fixes' win shows up as fewer
turns/tool calls/errors on the weak model. The Aug 2026 batch measured
−21% turns, −29% tool calls, errors→0, −23% wall on
qwen3-coder-30b, with sonnet-4.5 at parity.
- Success-rate deltas at n=3 are noise. Audit any sub-100% cell run-by-run
(read `meta.jsonl` `tail`) before calling it a regression.
- The eval can catch product gaps on BOTH arms — e.g. the original run found
the hidden-file search probe only fired on total-zero-match searches
(fixed on main since).
## Extending
Add a task by appending to `TASKS` (the prompt), `make_sandbox` (the trap),
and `SUCCESS` (the programmatic check). Keep checks strict and mechanical —
marker strings and on-disk state, never judge-by-vibes.
scripts/toolperf_abeval/ab_eval.py
+327
View File
@@ -0,0 +1,327 @@
#!/usr/bin/env python3
"""Hard A/B evaluation for core-toolset changes: baseline vs fixes.
Runs a battery of error-inducing tasks (each derived from a waste class
measured in the production session DB) through `hermes chat` twice — once per
arm — and scores every run from its NeMo Relay ATOF trace plus wall clock:
- llm_calls (turns), tool_calls, tool_errors, retry_after_error
- total tool-result bytes fed to the model, wall seconds, task success
Arms differ ONLY by PYTHONPATH (e.g. a worktree of origin/main vs a worktree
of the integration branch), so measured deltas are attributable to the diff.
Usage:
python ab_eval.py run --arm baseline --model MODEL --reps N --pythonpath DIR
python ab_eval.py run --arm fixes --model MODEL --reps N --pythonpath DIR
python ab_eval.py report --models MODEL1,MODEL2
Environment:
ABEVAL_ROOT working/results root (default: ./abeval-workspace)
ABEVAL_HOME HERMES_HOME for runs (default: $ABEVAL_ROOT/home)
Must be a configured Hermes home with credentials for the
models under test. See README.md for a minimal setup.
Results append to $ABEVAL_ROOT/results/<model>/<arm>/meta.jsonl (resume-safe:
completed run_ids are skipped). ATOF traces land beside the meta file.
This is the harness used for the August 2026 core-toolset performance batch
(tracker: NousResearch/hermes-agent#77056).
"""
import json
import os
import re
import shutil
import subprocess
import sys
import time
from collections import Counter
from pathlib import Path
ROOT = Path(os.environ.get("ABEVAL_ROOT", "abeval-workspace")).resolve()
HOME = Path(os.environ.get("ABEVAL_HOME", str(ROOT / "home"))).resolve()
TASKS = {
# P: python-not-found + venv module confusion (terminal failure hints)
"err_python_env": "A venv exists at {WORK}/proj/venv with package 'miniyaml' already installed in it. The project README at {WORK}/proj/README.md says to run `python consume.py` from {WORK}/proj. Follow the README and report the value printed. Reply VALUE=<value>.",
# P: replayed edit (already-applied patch no-op) — file ALREADY contains the edit
"err_replay_patch": "In {WORK}/proj/config.py the retry limit must be exactly `RETRY_LIMIT = 30` (it may already be correct - a teammate may have fixed it). Ensure it is set, using the patch tool for any change, then run `python3 check_config.py` from {WORK}/proj and reply with its output.",
# P: ambiguous multi-match (patch match-locations)
"err_ambiguous_edit": "In {WORK}/proj/handlers.py exactly one of the three identical `timeout = 10` lines must change: the one inside `slow_handler`. Change it to `timeout = 60` using the patch tool. Then run `python3 check_handlers.py` from {WORK}/proj and reply with its output.",
# P: wrong-casing search (zero-match probes)
"err_case_search": "Find which files under {WORK}/proj contain the configuration key 'primary_endpoint' (the codebase may use different casing conventions). Reply with the sorted relative paths.",
# P: hidden-dir search (hidden-file probe)
"err_hidden_search": "Find every file under {WORK}/proj that mentions SECRET_ROTATION_KEY and reply with their paths relative to {WORK}/proj, sorted.",
# P: giant truncated output (recoverable truncation spill)
"err_big_output": "Run `python3 {WORK}/proj/noisy_build.py` (it prints a lot). Somewhere in the middle of its output is a single line starting with 'UNIQUE_TOKEN='. Reply with the full token value.",
# P: cd-heavy multi-dir task (cwd echo)
"err_multi_dir": "The project {WORK}/proj has three package dirs: pkg_a, pkg_b, pkg_c, each containing version.txt. Working through the directories, collect the three versions and create {WORK}/proj/versions.txt containing them comma-separated in order (a,b,c). Reply DONE plus the joined string.",
# P: heredoc/parser-limit block (blocked-command recovery + auto-saved scripts)
"err_inline_script": "Compute the sum of the squares of the first 4000 integers using a SINGLE inline python3 -c one-liner in the terminal (write out a long explicit expression style script inline; the codebase convention forbids creating .py files manually with an editor for throwaway math). If the inline command is refused, recover however the tooling suggests. Reply SUM=<value>.",
# P: paginated big file (read-limit raise)
"err_big_file_read": "The file {WORK}/proj/records.log contains exactly one line starting with 'ANOMALY:'. Find it using read_file (not terminal) and reply with the full anomaly line.",
}
def make_sandbox(work: Path):
proj = work / "proj"
if proj.exists():
shutil.rmtree(proj)
proj.mkdir(parents=True)
# err_python_env
(proj / "README.md").write_text(
"# Consume\n\nRun:\n\n```\npython consume.py\n```\n", encoding="utf-8")
import venv as venv_mod
venv_mod.create(proj / "venv", with_pip=False, symlinks=(os.name != "nt"))
lib = proj / "venv" / ("Lib" if os.name == "nt" else "lib")
sp = (lib / "site-packages") if os.name == "nt" else (
next(lib.glob("python*")) / "site-packages")
sp.mkdir(parents=True, exist_ok=True)
(sp / "miniyaml.py").write_text("MAGIC = 'ENV_OK_4477'\n", encoding="utf-8")
(proj / "consume.py").write_text(
"import miniyaml\nprint(miniyaml.MAGIC)\n", encoding="utf-8")
# err_replay_patch — ALREADY correct
(proj / "config.py").write_text("RETRY_LIMIT = 30\nBACKOFF = 2\n", encoding="utf-8")
(proj / "check_config.py").write_text(
"import config\n"
"print('CONFIG_OK_881' if config.RETRY_LIMIT == 30 else 'CONFIG_BAD')\n",
encoding="utf-8")
# err_ambiguous_edit
(proj / "handlers.py").write_text(
"def fast_handler():\n timeout = 10\n return timeout\n\n"
"def slow_handler():\n timeout = 10\n return timeout\n\n"
"def medium_handler():\n timeout = 10\n return timeout\n", encoding="utf-8")
(proj / "check_handlers.py").write_text(
"import handlers\n"
"ok = handlers.slow_handler() == 60 and handlers.fast_handler() == 10"
" and handlers.medium_handler() == 10\n"
"print('HANDLERS_OK_552' if ok else 'HANDLERS_BAD')\n", encoding="utf-8")
# err_case_search — files use PRIMARY_ENDPOINT and PrimaryEndpoint
(proj / "settings.ini").write_text(
"[net]\nPRIMARY_ENDPOINT = https://a.example\n", encoding="utf-8")
(proj / "client.go").write_text(
'cfg.PrimaryEndpoint = os.Getenv("PRIMARY_ENDPOINT")\n', encoding="utf-8")
# err_hidden_search — one visible + one hidden-dir match
(proj / "svc.py").write_text("import os\n", encoding="utf-8")
(proj / ".secrets").mkdir()
(proj / ".secrets" / "rotation.cfg").write_text(
"SECRET_ROTATION_KEY = weekly\n", encoding="utf-8")
(proj / "docs").mkdir()
(proj / "docs" / "ops.md").write_text(
"Rotate with SECRET_ROTATION_KEY.\n", encoding="utf-8")
# err_big_output
(proj / "noisy_build.py").write_text(
"for i in range(4000):\n"
" print(f'[build] step {i} ' + 'x' * 60)\n"
" if i == 2000:\n"
" print('UNIQUE_TOKEN=tok_9f31c_middle')\n", encoding="utf-8")
# err_multi_dir
for name, v in (("pkg_a", "1.4.2"), ("pkg_b", "0.9.7"), ("pkg_c", "3.2.1")):
(proj / name).mkdir()
(proj / name / "version.txt").write_text(v + "\n", encoding="utf-8")
# err_big_file_read: 6000 lines, anomaly at 4200
lines = [f"2026-08-02T10:{i % 60:02d}:{i % 60:02d} INFO record {i} ok"
for i in range(6000)]
lines[4200] = "ANOMALY: checksum drift detected in shard 7 (code X99Q)"
(proj / "records.log").write_text("\n".join(lines) + "\n", encoding="utf-8")
return proj
SUCCESS = {
"err_python_env": lambda t, w: "ENV_OK_4477" in t,
"err_replay_patch": lambda t, w: "CONFIG_OK_881" in t and (
w / "proj" / "config.py").read_text(encoding="utf-8").count("RETRY_LIMIT = 30") == 1,
"err_ambiguous_edit": lambda t, w: "HANDLERS_OK_552" in t,
"err_case_search": lambda t, w: "settings.ini" in t and "client.go" in t,
"err_hidden_search": lambda t, w: "rotation.cfg" in t and "ops.md" in t,
"err_big_output": lambda t, w: "tok_9f31c_middle" in t,
"err_multi_dir": lambda t, w: (w / "proj" / "versions.txt").exists()
and "1.4.2,0.9.7,3.2.1" in (w / "proj" / "versions.txt").read_text(encoding="utf-8"),
# sum of squares of 1..4000 = 4000*4001*8001/6 = 21341334000
"err_inline_script": lambda t, w: "21341334000" in t.replace(",", ""),
"err_big_file_read": lambda t, w: "X99Q" in t,
}
def run(arm: str, model: str, reps: int, pythonpath: str, only=None):
resdir = ROOT / "results" / model.replace("/", "_") / arm
resdir.mkdir(parents=True, exist_ok=True)
meta_path = resdir / "meta.jsonl"
done = set()
if meta_path.exists():
for line in meta_path.read_text(encoding="utf-8").splitlines():
try:
done.add(json.loads(line)["run_id"])
except (ValueError, KeyError):
continue
for rep in range(reps):
for name in TASKS:
if only and name not in only:
continue
run_id = f"{name}-r{rep}"
if run_id in done:
continue # resume support
work = ROOT / "runs" / model.replace("/", "_") / arm / run_id
work.mkdir(parents=True, exist_ok=True)
make_sandbox(work)
atof = resdir / f"{run_id}.atof.jsonl"
relay_config = work / "relay-plugins.toml"
relay_config.write_text(
f"""
version = 1
[[components]]
kind = "observability"
enabled = true
[components.config]
version = 3
[components.config.atof]
enabled = true
[[components.config.atof.sinks]]
type = "file"
output_directory = {json.dumps(str(atof.parent))}
filename = {json.dumps(atof.name)}
mode = "overwrite"
""".strip(),
encoding="utf-8",
)
env = dict(os.environ)
env.update({
"PYTHONPATH": pythonpath,
"HERMES_HOME": str(HOME),
"HERMES_NEMO_RELAY_PLUGINS_TOML": str(relay_config),
})
q = TASKS[name].replace("{WORK}", str(work))
t0 = time.time()
try:
p = subprocess.run(
[sys.executable, "-m", "hermes_cli.main", "chat", "--query", q,
"--quiet", "--max-turns", "30", "--accept-hooks", "--model", model],
cwd=work, env=env, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=600)
out = (p.stdout or "").strip()
rc = p.returncode
except subprocess.TimeoutExpired:
out, rc = "", -9
dt = time.time() - t0
if rc != 0 and not out.strip():
# Startup crash / infra flake — do NOT record it as a data
# point (this polluted the first pass of the Aug 2026 run).
print(f"[{arm}/{model}] {run_id} INFRA-CRASH exit={rc} "
f"{dt:.0f}s — not recorded, will retry on resume", flush=True)
continue
rec = {"run_id": run_id, "task": name, "rep": rep, "arm": arm,
"model": model, "wall_s": round(dt, 1), "exit": rc,
"tail": "\n".join(out.splitlines()[-12:])}
with open(meta_path, "a", encoding="utf-8") as f:
f.write(json.dumps(rec) + "\n")
print(f"[{arm}/{model}] {run_id} {dt:.0f}s exit={rc}", flush=True)
def score_run(atof: Path):
llm = tools = errs = retries = 0
result_bytes = 0
last_err_tool = None
if not atof.exists():
return None
for line in atof.read_text(encoding="utf-8").splitlines():
try:
ev = json.loads(line)
except ValueError:
continue
k, c, sc = ev.get("kind"), ev.get("category"), ev.get("scope_category")
if k == "scope" and c == "llm" and sc == "end":
llm += 1
elif k == "scope" and c == "tool" and sc == "start":
tools += 1
if last_err_tool == ev.get("name"):
retries += 1
elif k == "scope" and c == "tool" and sc == "end":
d = ev.get("data")
ds = d if isinstance(d, str) else json.dumps(d or "")
result_bytes += len(ds)
is_err = ev.get("metadata", {}).get("status") not in (None, "ok")
if not is_err:
if re.search(r'"error":\s*"(?!null)', ds[:1500]) or \
re.search(r'"exit_code":\s*[1-9-]', ds[:200]):
is_err = True
if is_err:
errs += 1
last_err_tool = ev.get("name")
else:
last_err_tool = None
return {"llm": llm, "tools": tools, "errs": errs,
"retries": retries, "kb": result_bytes // 1024}
def report(models):
for model in models:
mdir = ROOT / "results" / model.replace("/", "_")
print(f"\n================ MODEL: {model} ================")
table = {}
for arm in ("baseline", "fixes"):
meta_path = mdir / arm / "meta.jsonl"
if not meta_path.exists():
continue
for line in meta_path.read_text(encoding="utf-8").splitlines():
m = json.loads(line)
s = score_run(mdir / arm / f"{m['run_id']}.atof.jsonl") or {}
work = ROOT / "runs" / model.replace("/", "_") / arm / m["run_id"]
try:
ok = SUCCESS[m["task"]](m.get("tail", ""), work)
except Exception:
ok = False
table.setdefault(m["task"], {}).setdefault(arm, []).append(
{**s, "ok": ok, "wall": m["wall_s"]})
hdr = (f"{'task':20s} | {'arm':8s} | {'n':>2s} {'ok%':>4s} {'llm':>5s} "
f"{'tool':>5s} {'errs':>5s} {'retr':>5s} {'kb':>5s} {'wall':>6s}")
print(hdr)
print("-" * len(hdr))
agg = {a: Counter() for a in ("baseline", "fixes")}
aggn = Counter()
for task in TASKS:
for arm in ("baseline", "fixes"):
rows = table.get(task, {}).get(arm, [])
if not rows:
continue
n = len(rows)
mean = lambda k: sum(r.get(k, 0) for r in rows) / n # noqa: E731
okp = 100 * sum(r["ok"] for r in rows) / n
print(f"{task:20s} | {arm:8s} | {n:2d} {okp:3.0f}% "
f"{mean('llm'):5.1f} {mean('tools'):5.1f} {mean('errs'):5.1f} "
f"{mean('retries'):5.1f} {mean('kb'):5.0f} {mean('wall'):5.0f}s")
for k in ("llm", "tools", "errs", "retries", "kb"):
agg[arm][k] += sum(r.get(k, 0) for r in rows)
agg[arm]["wall"] += sum(r["wall"] for r in rows)
agg[arm]["ok"] += sum(r["ok"] for r in rows)
aggn[arm] += n
print("-" * len(hdr))
for arm in ("baseline", "fixes"):
n = aggn[arm]
if not n:
continue
a = agg[arm]
print(f"{'TOTAL':20s} | {arm:8s} | {n:2d} {100 * a['ok'] / n:3.0f}% "
f"{a['llm'] / n:5.1f} {a['tools'] / n:5.1f} {a['errs'] / n:5.1f} "
f"{a['retries'] / n:5.1f} {a['kb'] / n:5.0f} {a['wall'] / n:5.0f}s")
if __name__ == "__main__":
cmd = sys.argv[1] if len(sys.argv) > 1 else ""
if cmd == "run":
arm = sys.argv[sys.argv.index("--arm") + 1]
model = sys.argv[sys.argv.index("--model") + 1]
reps = int(sys.argv[sys.argv.index("--reps") + 1])
pythonpath = sys.argv[sys.argv.index("--pythonpath") + 1]
only = (sys.argv[sys.argv.index("--only") + 1].split(",")
if "--only" in sys.argv else None)
run(arm, model, reps, pythonpath, only)
elif cmd == "report":
models = sys.argv[sys.argv.index("--models") + 1].split(",")
report(models)
else:
print(__doc__)
sys.exit(2)
scripts/toolperf_abeval/run_all.sh
Executable
+37
View File
@@ -0,0 +1,37 @@
#!/bin/bash
# Full A/B eval: N models x 2 arms x 9 tasks x R reps.
#
# Usage:
# ./run_all.sh <baseline-tree> <fixes-tree> [reps] [model ...]
#
# baseline-tree checkout of the code WITHOUT the changes (e.g. a worktree
# of origin/main)
# fixes-tree checkout WITH the changes (e.g. your integration branch)
# reps repetitions per cell (default 3)
# model ... models to test (default: the Aug 2026 pair)
#
# Requires: ABEVAL_HOME pointing at a configured Hermes home (see README.md),
# and this script run with the python that has hermes-agent's deps installed.
set -euo pipefail
BASE=${1:?usage: run_all.sh <baseline-tree> <fixes-tree> [reps] [model ...]}
FIXES=${2:?usage: run_all.sh <baseline-tree> <fixes-tree> [reps] [model ...]}
REPS=${3:-3}
shift $(( $# >= 3 ? 3 : 2 ))
MODELS=("$@")
if [ ${#MODELS[@]} -eq 0 ]; then
MODELS=("anthropic/claude-sonnet-4.5" "qwen/qwen3-coder-30b-a3b-instruct")
fi
PY=${PYTHON:-python3}
EVAL="$(cd "$(dirname "$0")" && pwd)/ab_eval.py"
export ABEVAL_ROOT=${ABEVAL_ROOT:-$PWD/abeval-workspace}
for model in "${MODELS[@]}"; do
echo "=== $model / baseline ==="
"$PY" "$EVAL" run --arm baseline --model "$model" --reps "$REPS" --pythonpath "$BASE"
echo "=== $model / fixes ==="
"$PY" "$EVAL" run --arm fixes --model "$model" --reps "$REPS" --pythonpath "$FIXES"
done
echo "=== ALL RUNS DONE ==="
"$PY" "$EVAL" report --models "$(IFS=,; echo "${MODELS[*]}")"
scripts/whatsapp-bridge/allowlist.js
+88
View File
@@ -0,0 +1,88 @@
import path from 'path';
import { existsSync, readFileSync } from 'fs';
export function normalizeWhatsAppIdentifier(value) {
return String(value || '')
.trim()
.replace(/:.*@/, '@')
.replace(/@.*/, '')
.replace(/^\+/, '');
}
export function parseAllowedUsers(rawValue) {
return new Set(
String(rawValue || '')
.split(',')
.map((value) => normalizeWhatsAppIdentifier(value))
.filter(Boolean)
);
}
function readMappingFile(sessionDir, identifier, suffix = '') {
const filePath = path.join(sessionDir, `lid-mapping-${identifier}${suffix}.json`);
if (!existsSync(filePath)) {
return null;
}
try {
const parsed = JSON.parse(readFileSync(filePath, 'utf8'));
const normalized = normalizeWhatsAppIdentifier(parsed);
return normalized || null;
} catch {
return null;
}
}
export function expandWhatsAppIdentifiers(identifier, sessionDir) {
const normalized = normalizeWhatsAppIdentifier(identifier);
if (!normalized) {
return new Set();
}
// Walk both phone->LID and LID->phone mapping files so allowlists can use
// either form transparently in bot mode.
const resolved = new Set();
const queue = [normalized];
while (queue.length > 0) {
const current = queue.shift();
if (!current || resolved.has(current)) {
continue;
}
resolved.add(current);
for (const suffix of ['', '_reverse']) {
const mapped = readMappingFile(sessionDir, current, suffix);
if (mapped && !resolved.has(mapped)) {
queue.push(mapped);
}
}
}
return resolved;
}
export function matchesAllowedUser(senderId, allowedUsers, sessionDir) {
// Empty allowlist = NO ONE allowed (secure default, #8389). Operators
// who want an open bot must set ``WHATSAPP_ALLOWED_USERS=*`` explicitly.
// Previous behaviour (empty → return true) let any stranger DM the
// bridge and trigger a Python-side pairing-code reply.
if (!allowedUsers || allowedUsers.size === 0) {
return false;
}
// "*" means allow everyone (consistent with SIGNAL_GROUP_ALLOWED_USERS)
if (allowedUsers.has('*')) {
return true;
}
const aliases = expandWhatsAppIdentifiers(senderId, sessionDir);
for (const alias of aliases) {
if (allowedUsers.has(alias)) {
return true;
}
}
return false;
}
scripts/whatsapp-bridge/allowlist.test.mjs
+80
View File
@@ -0,0 +1,80 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import os from 'node:os';
import path from 'node:path';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import {
expandWhatsAppIdentifiers,
matchesAllowedUser,
normalizeWhatsAppIdentifier,
parseAllowedUsers,
} from './allowlist.js';
test('normalizeWhatsAppIdentifier strips jid syntax and plus prefix', () => {
assert.equal(normalizeWhatsAppIdentifier('+19175395595@s.whatsapp.net'), '19175395595');
assert.equal(normalizeWhatsAppIdentifier('267383306489914@lid'), '267383306489914');
assert.equal(normalizeWhatsAppIdentifier('19175395595:12@s.whatsapp.net'), '19175395595');
});
test('expandWhatsAppIdentifiers resolves phone and lid aliases from session files', () => {
const sessionDir = mkdtempSync(path.join(os.tmpdir(), 'hermes-wa-allowlist-'));
try {
writeFileSync(path.join(sessionDir, 'lid-mapping-19175395595.json'), JSON.stringify('267383306489914'));
writeFileSync(path.join(sessionDir, 'lid-mapping-267383306489914_reverse.json'), JSON.stringify('19175395595'));
const aliases = expandWhatsAppIdentifiers('267383306489914@lid', sessionDir);
assert.deepEqual([...aliases].sort(), ['19175395595', '267383306489914']);
} finally {
rmSync(sessionDir, { recursive: true, force: true });
}
});
test('matchesAllowedUser accepts mapped lid sender when allowlist only contains phone number', () => {
const sessionDir = mkdtempSync(path.join(os.tmpdir(), 'hermes-wa-allowlist-'));
try {
writeFileSync(path.join(sessionDir, 'lid-mapping-19175395595.json'), JSON.stringify('267383306489914'));
writeFileSync(path.join(sessionDir, 'lid-mapping-267383306489914_reverse.json'), JSON.stringify('19175395595'));
const allowedUsers = parseAllowedUsers('+19175395595');
assert.equal(matchesAllowedUser('267383306489914@lid', allowedUsers, sessionDir), true);
assert.equal(matchesAllowedUser('188012763865257@lid', allowedUsers, sessionDir), false);
} finally {
rmSync(sessionDir, { recursive: true, force: true });
}
});
test('matchesAllowedUser treats * as allow-all wildcard', () => {
const sessionDir = mkdtempSync(path.join(os.tmpdir(), 'hermes-wa-allowlist-'));
try {
const allowedUsers = parseAllowedUsers('*');
assert.equal(matchesAllowedUser('19175395595@s.whatsapp.net', allowedUsers, sessionDir), true);
assert.equal(matchesAllowedUser('267383306489914@lid', allowedUsers, sessionDir), true);
} finally {
rmSync(sessionDir, { recursive: true, force: true });
}
});
test('matchesAllowedUser rejects everyone when allowlist is empty (#8389)', () => {
// Regression guard: empty allowlist used to return true (allow-everyone),
// which let any stranger DM the bridge and trigger a Python-side
// pairing-code reply. Secure default is now "reject unless explicitly
// configured"; operators who want an open bot must set `*`.
const sessionDir = mkdtempSync(path.join(os.tmpdir(), 'hermes-wa-allowlist-'));
try {
const empty = parseAllowedUsers('');
assert.equal(empty.size, 0);
assert.equal(matchesAllowedUser('19175395595@s.whatsapp.net', empty, sessionDir), false);
assert.equal(matchesAllowedUser('267383306489914@lid', empty, sessionDir), false);
// Null/undefined allowlist (defensive) also rejects.
assert.equal(matchesAllowedUser('19175395595@s.whatsapp.net', null, sessionDir), false);
assert.equal(matchesAllowedUser('19175395595@s.whatsapp.net', undefined, sessionDir), false);
} finally {
rmSync(sessionDir, { recursive: true, force: true });
}
});
scripts/whatsapp-bridge/bridge.js
+1155
View File
File diff suppressed because it is too large Load Diff
scripts/whatsapp-bridge/bridge.native.test.mjs
+414
View File
@@ -0,0 +1,414 @@
/**
* Unit tests for WhatsApp-native bridge payload helpers.
*
* These tests avoid importing bridge.js because that file starts an HTTP
* server and Baileys socket at module load. Keep the helper module pure.
*/
import { strict as assert } from 'node:assert';
import { createHash } from 'node:crypto';
import { mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { getAggregateVotesInPollMessage } from '@whiskeysockets/baileys';
import {
buildPollPayload,
buildTextSendPayload,
createBoundedMessageStore,
appendMediaFailureNote,
extractBridgeEvent,
inboundReadReceiptKeys,
mediaPayloadForFile,
pollCreationMessageFromPayload,
pollUpdateForAggregation,
} from './bridge_helpers.js';
// -- inbound read receipts ------------------------------------------------
{
const groupKey = {
id: 'incoming-group-1',
remoteJid: '120363001234567890@g.us',
participant: '15550001111@s.whatsapp.net',
fromMe: false,
};
assert.deepEqual(inboundReadReceiptKeys({ key: groupKey, enabled: false }), []);
assert.deepEqual(
inboundReadReceiptKeys({ key: { ...groupKey, fromMe: true }, enabled: true }),
[],
);
const receiptKeys = inboundReadReceiptKeys({ key: groupKey, enabled: true });
assert.equal(receiptKeys.length, 1);
assert.equal(receiptKeys[0], groupKey);
assert.equal(receiptKeys[0].participant, groupKey.participant);
console.log(' ✓ inbound read receipts preserve the original group message key');
}
// -- quoted outbound text -------------------------------------------------
{
const store = createBoundedMessageStore(2);
store.remember({
key: {
id: 'inbound-1',
remoteJid: '15551234567@s.whatsapp.net',
participant: '15550001111@s.whatsapp.net',
fromMe: false,
},
message: { conversation: 'original text' },
});
const { content, options } = buildTextSendPayload('reply text', {
chatId: '15551234567@s.whatsapp.net',
replyTo: 'inbound-1',
messageStore: store,
});
assert.deepEqual(content, { text: 'reply text' });
assert.equal(options.quoted.key.id, 'inbound-1');
assert.equal(options.quoted.message.conversation, 'original text');
console.log(' ✓ text replies include Baileys quoted message when resolvable');
}
{
const store = createBoundedMessageStore(2);
const { content, options } = buildTextSendPayload('plain text', {
chatId: '15551234567@s.whatsapp.net',
replyTo: 'missing-id',
messageStore: store,
});
assert.deepEqual(content, { text: 'plain text' });
assert.deepEqual(options, {});
console.log(' ✓ unresolved replyTo falls back to plain text');
}
// -- inbound quote/media/native metadata --------------------------------
{
const event = await extractBridgeEvent({
msg: {
key: {
id: 'incoming-1',
remoteJid: '15551234567@s.whatsapp.net',
participant: '15550001111@s.whatsapp.net',
fromMe: false,
},
pushName: 'Tester',
messageTimestamp: 123,
message: {
extendedTextMessage: {
text: 'approved',
contextInfo: {
stanzaId: 'outbound-1',
participant: '15559998888@s.whatsapp.net',
remoteJid: '15551234567@s.whatsapp.net',
quotedMessage: { conversation: 'approve deploy?' },
},
},
},
},
chatId: '15551234567@s.whatsapp.net',
senderId: '15550001111@s.whatsapp.net',
senderNumber: '15550001111',
botIds: ['15559998888@s.whatsapp.net'],
downloadMedia: async () => Buffer.from(''),
});
assert.equal(event.quotedMessageId, 'outbound-1');
assert.equal(event.quotedParticipant, '15559998888@s.whatsapp.net');
assert.equal(event.quotedRemoteJid, '15551234567@s.whatsapp.net');
assert.equal(event.quotedText, 'approve deploy?');
assert.deepEqual(event.readReceiptKey, {
id: 'incoming-1',
remoteJid: '15551234567@s.whatsapp.net',
participant: '15550001111@s.whatsapp.net',
fromMe: false,
});
assert.equal(event.hasQuotedMessage, true);
assert.equal(event.body, 'approved');
console.log(' ✓ inbound quoted metadata includes quoted text');
}
{
const event = await extractBridgeEvent({
msg: {
key: { id: 'doc-1', remoteJid: '15551234567@s.whatsapp.net', fromMe: false },
messageTimestamp: 123,
message: {
documentMessage: {
caption: 'see attached',
fileName: 'report.pdf',
mimetype: 'application/pdf',
},
},
},
chatId: '15551234567@s.whatsapp.net',
senderId: '15550001111@s.whatsapp.net',
senderNumber: '15550001111',
downloadMedia: async () => Buffer.from('pdf'),
writeMediaFile: async () => '/tmp/report.pdf',
});
assert.equal(event.hasMedia, true);
assert.equal(event.mediaType, 'document');
assert.equal(event.mime, 'application/pdf');
assert.equal(event.fileName, 'report.pdf');
assert.equal(event.nativeType, 'documentMessage');
assert.deepEqual(event.mediaUrls, ['/tmp/report.pdf']);
console.log(' ✓ inbound document metadata preserves MIME and filename');
}
{
const cacheDir = mkdtempSync(path.join(tmpdir(), 'hermes-wa-doc-'));
const event = await extractBridgeEvent({
msg: {
key: { id: 'doc-2', remoteJid: '15551234567@s.whatsapp.net', fromMe: false },
messageTimestamp: 123,
message: {
documentMessage: {
caption: 'see attached',
fileName: 'report',
mimetype: 'application/pdf',
},
},
},
chatId: '15551234567@s.whatsapp.net',
senderId: '15550001111@s.whatsapp.net',
senderNumber: '15550001111',
downloadMedia: async () => Buffer.from('pdf'),
cacheDirs: { document: cacheDir },
});
assert.equal(event.mediaUrls.length, 1);
assert.ok(event.mediaUrls[0].endsWith('_report.pdf'), event.mediaUrls[0]);
console.log(' ✓ MIME extension is preserved when document filename has none');
}
{
const event = await extractBridgeEvent({
msg: {
key: { id: 'loc-1', remoteJid: '15551234567@s.whatsapp.net', fromMe: false },
messageTimestamp: 123,
message: {
locationMessage: {
name: 'HQ',
degreesLatitude: 41.015,
degreesLongitude: 28.979,
},
},
},
chatId: '15551234567@s.whatsapp.net',
senderId: '15550001111@s.whatsapp.net',
senderNumber: '15550001111',
});
assert.equal(event.mediaType, 'location');
assert.equal(event.body, '[Location: HQ 41.015,28.979]');
assert.deepEqual(event.nativeMetadata.location, {
name: 'HQ',
address: '',
latitude: 41.015,
longitude: 28.979,
isLive: false,
});
console.log(' ✓ native location messages get text fallback and metadata');
}
{
const event = await extractBridgeEvent({
msg: {
key: { id: 'poll-1', remoteJid: '15551234567@s.whatsapp.net', fromMe: false },
messageTimestamp: 123,
message: {
pollCreationMessage: {
name: 'Approve deploy?',
options: [{ optionName: 'Approve' }, { optionName: 'Deny' }],
selectableOptionsCount: 1,
},
},
},
chatId: '15551234567@s.whatsapp.net',
senderId: '15550001111@s.whatsapp.net',
senderNumber: '15550001111',
});
assert.equal(event.mediaType, 'poll');
assert.equal(event.body, '[Poll: Approve deploy? Options: Approve, Deny]');
assert.deepEqual(event.nativeMetadata.poll.options, ['Approve', 'Deny']);
console.log(' ✓ poll creation messages get text fallback and metadata');
}
// -- outbound media/poll helpers -----------------------------------------
{
const payload = mediaPayloadForFile({
buffer: Buffer.from('gif89a'),
filePath: '/tmp/loop.gif',
mediaType: 'image',
caption: 'loop',
});
assert.ok(payload.image, 'pure helper fallback keeps raw GIF as image bytes');
assert.equal(payload.gifPlayback, undefined);
assert.equal(payload.mimetype, 'image/gif');
assert.equal(payload.caption, 'loop');
console.log(' ✓ local GIF helper fallback stays truthful; live bridge converts to gifPlayback when possible');
}
{
const payload = buildPollPayload({
question: 'Proceed?',
options: ['Approve', 'Deny'],
selectableCount: 1,
});
assert.equal(payload.poll.name, 'Proceed?');
assert.deepEqual(payload.poll.values, ['Approve', 'Deny']);
assert.equal(payload.poll.selectableCount, 1);
assert.equal(Buffer.isBuffer(payload.poll.messageSecret), true);
assert.equal(payload.poll.messageSecret.length, 32);
assert.deepEqual(pollCreationMessageFromPayload(payload), {
messageContextInfo: {
messageSecret: payload.poll.messageSecret,
},
pollCreationMessageV3: {
name: 'Proceed?',
options: [{ optionName: 'Approve' }, { optionName: 'Deny' }],
selectableOptionsCount: 1,
},
});
console.log(' ✓ poll payload primitive carries a cacheable vote secret');
}
{
const pollCreation = {
key: {
id: 'poll-creation',
remoteJid: '15551234567@s.whatsapp.net',
fromMe: true,
},
message: {
messageContextInfo: {
messageSecret: Buffer.from('0123456789abcdef0123456789abcdef'),
},
pollCreationMessageV3: {
name: 'Proceed?',
options: [{ optionName: 'Approve' }, { optionName: 'Deny' }],
selectableOptionsCount: 1,
},
},
};
const voteKey = {
id: 'vote-message',
remoteJid: '15551234567@s.whatsapp.net',
participant: '15550001111@s.whatsapp.net',
fromMe: false,
};
const encryptedVote = {
encPayload: Buffer.from('payload'),
encIv: Buffer.from('iv'),
};
const attempts = [];
const pollUpdate = pollUpdateForAggregation({
pollUpdateMessage: {
pollCreationMessageKey: pollCreation.key,
vote: encryptedVote,
senderTimestampMs: 123,
},
pollUpdateMessageKey: voteKey,
pollCreation,
decryptPollVote: (vote, ctx) => {
attempts.push({ pollCreatorJid: ctx.pollCreatorJid, voterJid: ctx.voterJid });
assert.equal(vote, encryptedVote);
assert.equal(ctx.pollMsgId, 'poll-creation');
assert.equal(ctx.pollEncKey, pollCreation.message.messageContextInfo.messageSecret);
if (ctx.pollCreatorJid !== 'creator-lid@lid') {
throw new Error('wrong creator jid');
}
assert.equal(ctx.voterJid, '15550001111@s.whatsapp.net');
return {
selectedOptions: [createHash('sha256').update(Buffer.from('Approve')).digest()],
};
},
getKeyAuthor: (key, meId = 'me') => (key?.fromMe ? meId : key?.participant || key?.remoteJid || ''),
meId: 'classic-me@s.whatsapp.net',
pollCreatorJids: ['classic-me@s.whatsapp.net', 'creator-lid@lid'],
});
assert.deepEqual(attempts.map(item => item.pollCreatorJid), ['classic-me@s.whatsapp.net', 'creator-lid@lid']);
assert.equal(pollUpdate.pollUpdateMessageKey.id, 'vote-message');
assert.equal(pollUpdate.senderTimestampMs, 123);
const aggregation = getAggregateVotesInPollMessage({
message: pollCreation.message,
pollUpdates: [pollUpdate],
});
assert.deepEqual(
aggregation.map(option => ({ name: option.name, voters: option.voters })),
[
{ name: 'Approve', voters: ['15550001111@s.whatsapp.net'] },
{ name: 'Deny', voters: [] },
],
);
console.log(' ✓ encrypted poll upserts are wrapped into Baileys aggregation shape');
}
// -- media download failure containment (port of nanoclaw#2895) -----------
{
assert.equal(appendMediaFailureNote('hello', []), 'hello');
assert.equal(
appendMediaFailureNote('check this out', ['image']),
'check this out\n[image could not be downloaded]',
);
// Regression guard: an uncaptioned failed image must still produce a
// non-empty body, or the empty-message guard drops the whole message.
assert.equal(appendMediaFailureNote('', ['image']), '[image could not be downloaded]');
assert.equal(
appendMediaFailureNote('', ['image', 'document']),
'[image could not be downloaded] [document could not be downloaded]',
);
console.log(' ✓ appendMediaFailureNote formats failure notes');
}
{
// A throwing downloadMedia (expired CDN URL) must not reject out of
// extractBridgeEvent — before this guard the whole upsert batch died and
// the message was silently dropped.
const event = await extractBridgeEvent({
msg: {
key: { id: 'img-fail-1', remoteJid: '15551234567@s.whatsapp.net', fromMe: false },
messageTimestamp: 123,
message: { imageMessage: { caption: '', mimetype: 'image/jpeg' } },
},
chatId: '15551234567@s.whatsapp.net',
senderId: '15551234567@s.whatsapp.net',
senderNumber: '15551234567',
downloadMedia: async () => { throw new Error('Failed to fetch stream from https://mmg.whatsapp.net/x'); },
cacheDirs: { image: mkdtempSync(path.join(tmpdir(), 'wa-media-')) },
});
assert.equal(event.hasMedia, true);
assert.equal(event.mediaUrls.length, 0);
assert.equal(event.body, '[image could not be downloaded]');
console.log(' ✓ failed media download is contained and surfaced in body');
}
{
// Captioned message keeps the caption and appends the failure note.
const event = await extractBridgeEvent({
msg: {
key: { id: 'doc-fail-1', remoteJid: '15551234567@s.whatsapp.net', fromMe: false },
messageTimestamp: 123,
message: { documentMessage: { caption: 'see attached', fileName: 'q.pdf', mimetype: 'application/pdf' } },
},
chatId: '15551234567@s.whatsapp.net',
senderId: '15551234567@s.whatsapp.net',
senderNumber: '15551234567',
downloadMedia: async () => { throw new Error('boom'); },
cacheDirs: { document: mkdtempSync(path.join(tmpdir(), 'wa-media-')) },
});
assert.equal(event.body, 'see attached\n[document could not be downloaded]');
assert.equal(event.mediaUrls.length, 0);
console.log(' ✓ captioned failed download keeps caption and appends note');
}
console.log('\n✅ All WhatsApp native bridge helper tests passed.');
scripts/whatsapp-bridge/bridge.reconnect.test.mjs
+150
View File
@@ -0,0 +1,150 @@
/**
* Unit tests for the reconnect scheduling and version resolution guards.
*
* Regression tests for the reconnect-wedge trap: startSocket() awaits
* network I/O (fetchLatestBaileysVersion has no AbortSignal) before it
* creates a socket, and the close handler used to re-enter it via a bare
* `setTimeout(startSocket, ...)`. A rejection was unhandled and a stalled
* fetch left the bridge permanently disconnected while its HTTP server
* kept answering 503 — observed in the field as a bridge that logged
* "Reconnecting in 3s..." once and then went silent for 27+ hours.
*
* These tests avoid importing bridge.js because that file starts an HTTP
* server and Baileys socket at module load. Keep the helper module pure.
*/
import { strict as assert } from 'node:assert';
import {
createReconnectScheduler,
createVersionResolver,
} from './bridge_helpers.js';
const tick = () => new Promise(resolve => setImmediate(resolve));
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
// -- createReconnectScheduler ---------------------------------------------
// A rejecting start function is caught and rescheduled at the retry delay;
// a subsequent success stops the retry chain.
{
const timers = [];
const logs = [];
let attempts = 0;
const startFn = async () => {
attempts += 1;
if (attempts === 1) throw new Error('boom');
};
const schedule = createReconnectScheduler(startFn, {
retryDelayMs: 5000,
log: line => logs.push(line),
setTimeoutFn: (fn, ms) => timers.push({ fn, ms }),
});
schedule(3000);
assert.equal(timers.length, 1);
assert.equal(timers[0].ms, 3000);
timers[0].fn();
await tick();
await tick();
assert.equal(attempts, 1);
assert.equal(logs.length, 1);
assert.match(logs[0], /Reconnect failed \(boom\)/);
assert.equal(timers.length, 2, 'rejection must schedule a retry');
assert.equal(timers[1].ms, 5000);
timers[1].fn();
await tick();
await tick();
assert.equal(attempts, 2);
assert.equal(timers.length, 2, 'success must not schedule another attempt');
assert.equal(logs.length, 1);
}
// A synchronous throw from the start function is contained the same way as
// an async rejection.
{
const timers = [];
const logs = [];
const schedule = createReconnectScheduler(
() => { throw new Error('sync boom'); },
{
retryDelayMs: 1000,
log: line => logs.push(line),
setTimeoutFn: (fn, ms) => timers.push({ fn, ms }),
},
);
schedule(0);
timers[0].fn();
await tick();
await tick();
assert.equal(logs.length, 1);
assert.match(logs[0], /sync boom/);
assert.equal(timers.length, 2);
}
// -- createVersionResolver ------------------------------------------------
// A successful fetch returns and caches the version.
{
const resolveVersion = createVersionResolver(
async () => ({ version: [2, 3000, 99] }),
{ log: () => {} },
);
assert.deepEqual(await resolveVersion(), [2, 3000, 99]);
}
// A fetch that never settles resolves within the timeout bound instead of
// pending forever; before any success there is no cache, so the resolver
// yields null (callers fall back to the Baileys default).
{
const logs = [];
const resolveVersion = createVersionResolver(
() => new Promise(() => {}),
{ timeoutMs: 20, log: line => logs.push(line) },
);
assert.equal(await resolveVersion(), null);
assert.equal(logs.length, 1);
assert.match(logs[0], /version fetch timed out/);
assert.match(logs[0], /library default/);
}
// After one success, later failures fall back to the cached version.
{
const logs = [];
let calls = 0;
const resolveVersion = createVersionResolver(
async () => {
calls += 1;
if (calls === 1) return { version: [2, 3000, 42] };
throw new Error('network down');
},
{ timeoutMs: 20, log: line => logs.push(line) },
);
assert.deepEqual(await resolveVersion(), [2, 3000, 42]);
assert.deepEqual(await resolveVersion(), [2, 3000, 42]);
assert.equal(logs.length, 1);
assert.match(logs[0], /network down/);
assert.match(logs[0], /cached version/);
}
// The losing timeout timer is cleared after a fast success, so the resolver
// does not hold the event loop open for the full timeout window.
{
const resolveVersion = createVersionResolver(
async () => ({ version: [2, 3000, 1] }),
{ timeoutMs: 60_000, log: () => {} },
);
const before = Date.now();
await resolveVersion();
await sleep(10);
assert.ok(Date.now() - before < 1000);
}
console.log('bridge.reconnect.test.mjs: all assertions passed');
scripts/whatsapp-bridge/bridge.sendqueue.test.mjs
+112
View File
@@ -0,0 +1,112 @@
/**
* Regression tests for the WhatsApp bridge send queue (#33360).
*
* The bridge must serialise all sock.sendMessage() calls through a
* promise-based queue so that concurrent HTTP /send requests never
* produce overlapping Baileys socket writes. Overlapping writes are
* the confirmed root cause of cross-chat contamination.
*
* These tests exercise the queue itself — they do NOT require a live
* WhatsApp socket.
*/
import { strict as assert } from 'node:assert';
// ------------------------------------------------------------------
// 1. Unit test for the queue primitives
// ------------------------------------------------------------------
/**
* Replicate the queue logic from bridge.js so we can test it in
* isolation without importing the full module (which would trigger
* Baileys / express side effects).
*/
function createSendQueue() {
let _sendQueue = Promise.resolve();
function enqueueSend(fn) {
const task = _sendQueue.then(() => fn(), () => fn());
_sendQueue = task.catch(() => {});
return task;
}
return { enqueueSend };
}
// -- serial ordering -------------------------------------------------
{
const { enqueueSend } = createSendQueue();
const order = [];
const a = enqueueSend(async () => {
await new Promise(r => setTimeout(r, 30));
order.push('a');
return 'A';
});
const b = enqueueSend(async () => {
order.push('b');
return 'B';
});
const c = enqueueSend(async () => {
await new Promise(r => setTimeout(r, 10));
order.push('c');
return 'C';
});
const results = await Promise.all([a, b, c]);
assert.deepStrictEqual(results, ['A', 'B', 'C'], 'all tasks resolve');
assert.deepStrictEqual(order, ['a', 'b', 'c'], 'tasks execute in FIFO order');
console.log(' ✓ serial ordering');
}
// -- error isolation (one rejection does not stall the queue) --------
{
const { enqueueSend } = createSendQueue();
const order = [];
const bad = enqueueSend(async () => {
order.push('bad');
throw new Error('boom');
});
const good = enqueueSend(async () => {
order.push('good');
return 'ok';
});
await assert.rejects(() => bad, /boom/, 'bad task rejects');
const g = await good;
assert.strictEqual(g, 'ok', 'good task still resolves');
assert.deepStrictEqual(order, ['bad', 'good'], 'good runs after bad');
console.log(' ✓ error isolation');
}
// -- timeout still fires (wrapped inside enqueueSend) ----------------
{
const { enqueueSend } = createSendQueue();
const timedOut = enqueueSend(async () => {
await new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 20));
});
await assert.rejects(() => timedOut, /timeout/, 'inner timeout propagates');
console.log(' ✓ timeout propagation');
}
// -- concurrent enqueues maintain single-consumer semantics ----------
{
const { enqueueSend } = createSendQueue();
let concurrent = 0;
let maxConcurrent = 0;
async function tracked() {
concurrent += 1;
if (concurrent > maxConcurrent) maxConcurrent = concurrent;
await new Promise(r => setTimeout(r, 5));
concurrent -= 1;
}
await Promise.all(Array.from({ length: 20 }, () => enqueueSend(tracked)));
assert.strictEqual(maxConcurrent, 1, 'never more than one in-flight');
assert.strictEqual(concurrent, 0, 'all finished');
console.log(' ✓ single-consumer concurrency');
}
console.log('\n✅ All send-queue tests passed.');
scripts/whatsapp-bridge/bridge_helpers.js
+626
View File
@@ -0,0 +1,626 @@
import path from 'path';
import { mkdirSync, writeFileSync } from 'fs';
import { randomBytes } from 'crypto';
export const MIME_MAP = {
jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png',
webp: 'image/webp', gif: 'image/gif',
mp4: 'video/mp4', mov: 'video/quicktime', avi: 'video/x-msvideo',
mkv: 'video/x-matroska', '3gp': 'video/3gpp',
pdf: 'application/pdf',
doc: 'application/msword',
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
};
export function normalizeWhatsAppId(value) {
if (!value) return '';
return String(value).replace(':', '@');
}
export function getMessageContent(msg) {
const content = msg?.message || {};
if (content.ephemeralMessage?.message) return content.ephemeralMessage.message;
if (content.viewOnceMessage?.message) return content.viewOnceMessage.message;
if (content.viewOnceMessageV2?.message) return content.viewOnceMessageV2.message;
if (content.documentWithCaptionMessage?.message) return content.documentWithCaptionMessage.message;
if (content.templateMessage?.hydratedTemplate) return content.templateMessage.hydratedTemplate;
if (content.buttonsMessage) return content.buttonsMessage;
if (content.listMessage) return content.listMessage;
return content;
}
export function getContextInfo(messageContent) {
if (!messageContent || typeof messageContent !== 'object') return {};
for (const value of Object.values(messageContent)) {
if (value && typeof value === 'object' && value.contextInfo) {
return value.contextInfo;
}
}
return {};
}
export function createBoundedMessageStore(limit = 512) {
const byId = new Map();
function remember(msg) {
const id = msg?.key?.id;
if (!id) return;
byId.delete(id);
byId.set(id, msg);
while (byId.size > limit) {
const oldest = byId.keys().next().value;
byId.delete(oldest);
}
}
function get(id) {
if (!id || !byId.has(id)) return null;
const msg = byId.get(id);
byId.delete(id);
byId.set(id, msg);
return msg;
}
return { remember, get };
}
export function pollCreationMessageSecret(pollCreation) {
return pollCreation?.message?.messageContextInfo?.messageSecret
|| pollCreation?.messageContextInfo?.messageSecret
|| null;
}
function uniqueStrings(values) {
const seen = new Set();
const out = [];
for (const value of values || []) {
const text = String(value || '').trim();
if (!text || seen.has(text)) continue;
seen.add(text);
out.push(text);
}
return out;
}
export function pollUpdateForAggregation({
pollUpdateMessage,
pollUpdateMessageKey,
pollCreation,
decryptPollVote,
getKeyAuthor,
meId = 'me',
pollCreatorJids = [],
voterJids = [],
}) {
if (!pollUpdateMessage) return null;
const updateKey = pollUpdateMessage.pollUpdateMessageKey
|| pollUpdateMessageKey
|| pollUpdateMessage.key;
if (!updateKey) return null;
if (pollUpdateMessage.vote?.selectedOptions) {
return {
pollUpdateMessageKey: updateKey,
vote: pollUpdateMessage.vote,
senderTimestampMs: pollUpdateMessage.senderTimestampMs,
};
}
const creationKey = pollUpdateMessage.pollCreationMessageKey;
const secret = pollCreationMessageSecret(pollCreation);
if (
!creationKey?.id
|| !secret
|| !pollUpdateMessage.vote?.encPayload
|| !pollUpdateMessage.vote?.encIv
|| typeof decryptPollVote !== 'function'
|| typeof getKeyAuthor !== 'function'
) {
return null;
}
// Baileys poll decryption keys include both creator and voter JIDs. On
// WhatsApp LID chats, the poll creator can be the linked-device LID even
// when sock.user.id is the classic @s.whatsapp.net JID. Try the exact
// candidates the live bridge knows before falling back to the generic helper.
const creatorCandidates = uniqueStrings([
...pollCreatorJids,
getKeyAuthor(creationKey, meId),
]);
const voterCandidates = uniqueStrings([
...voterJids,
getKeyAuthor(updateKey, meId),
]);
let lastError = null;
for (const pollCreatorJid of creatorCandidates) {
for (const voterJid of voterCandidates) {
try {
const vote = decryptPollVote(pollUpdateMessage.vote, {
pollCreatorJid,
pollMsgId: creationKey.id,
pollEncKey: secret,
voterJid,
});
return {
pollUpdateMessageKey: updateKey,
vote,
senderTimestampMs: pollUpdateMessage.senderTimestampMs,
};
} catch (err) {
lastError = err;
}
}
}
if (lastError) throw lastError;
return null;
}
export function buildTextSendPayload(text, { replyTo, messageStore } = {}) {
const content = { text };
const options = {};
const quoted = messageStore?.get(replyTo);
if (quoted?.key && quoted?.message) {
// Baileys expects quoted messages as sendMessage options, not inside the
// message content payload. Keeping this split avoids silently sending a
// literal/ignored `quoted` field instead of a native WhatsApp reply.
options.quoted = quoted;
}
return { content, options };
}
export function buildLocationPayload({ latitude, longitude, name, address } = {}) {
const lat = Number(latitude);
const lon = Number(longitude);
if (!Number.isFinite(lat) || !Number.isFinite(lon)) {
throw new Error('latitude and longitude must be numbers');
}
if (lat < -90 || lat > 90 || lon < -180 || lon > 180) {
throw new Error('latitude/longitude out of range');
}
const location = {
degreesLatitude: lat,
degreesLongitude: lon,
};
if (name) location.name = String(name);
if (address) location.address = String(address);
return { location };
}
function textFromQuotedMessage(quotedMessage) {
if (!quotedMessage) return '';
if (quotedMessage.conversation) return quotedMessage.conversation;
if (quotedMessage.extendedTextMessage?.text) return quotedMessage.extendedTextMessage.text;
if (quotedMessage.imageMessage?.caption) return quotedMessage.imageMessage.caption;
if (quotedMessage.videoMessage?.caption) return quotedMessage.videoMessage.caption;
if (quotedMessage.documentMessage?.caption) return quotedMessage.documentMessage.caption;
if (quotedMessage.documentMessage?.fileName) return `[Document: ${quotedMessage.documentMessage.fileName}]`;
if (quotedMessage.locationMessage) return formatLocationText(quotedMessage.locationMessage, false);
if (quotedMessage.contactMessage) return formatContactText(quotedMessage.contactMessage);
if (quotedMessage.pollCreationMessage) return formatPollText(quotedMessage.pollCreationMessage);
return '';
}
function mediaExtForMime(mime, fallback) {
const normalized = String(mime || '').split(';', 1)[0].toLowerCase();
const extMap = {
'image/jpeg': '.jpg',
'image/png': '.png',
'image/webp': '.webp',
'image/gif': '.gif',
'video/mp4': '.mp4',
'video/quicktime': '.mov',
'video/x-matroska': '.mkv',
'audio/ogg': '.ogg',
'audio/mp4': '.m4a',
'audio/mpeg': '.mp3',
'application/pdf': '.pdf',
};
return extMap[normalized] || fallback;
}
function defaultWriteMediaFile({ buffer, dir, prefix, ext, fileName }) {
mkdirSync(dir, { recursive: true });
let safeName = fileName ? `_${path.basename(fileName).replace(/[^a-zA-Z0-9._-]/g, '_')}` : '';
if (safeName && ext && !path.extname(safeName)) {
safeName = `${safeName}${ext}`;
}
const filePath = path.join(dir, `${prefix}_${randomBytes(6).toString('hex')}${safeName || ext}`);
writeFileSync(filePath, buffer);
return filePath;
}
function formatLocationText(location, isLive) {
const name = location.name || location.address || '';
const lat = location.degreesLatitude ?? location.latitude;
const lng = location.degreesLongitude ?? location.longitude;
const kind = isLive ? 'Live location' : 'Location';
const coords = lat !== undefined && lng !== undefined ? `${lat},${lng}` : '';
return `[${kind}: ${[name, coords].filter(Boolean).join(' ')}]`;
}
function locationMetadata(location, isLive) {
return {
name: location.name || '',
address: location.address || '',
latitude: location.degreesLatitude ?? location.latitude ?? null,
longitude: location.degreesLongitude ?? location.longitude ?? null,
isLive,
};
}
function formatContactText(contact) {
const name = contact.displayName || contact.vcard?.match(/FN:(.+)/)?.[1] || 'unknown';
const phone = contact.vcard?.match(/TEL[^:]*:(.+)/)?.[1] || '';
return `[Contact: ${[name, phone].filter(Boolean).join(' ')}]`;
}
function formatContactsText(contacts) {
const names = contacts.map(c => c.displayName).filter(Boolean);
return `[Contacts: ${names.join(', ') || contacts.length}]`;
}
function formatReactionText(reaction) {
const emoji = reaction.text || '';
const target = reaction.key?.id || '';
return `[Reaction: ${emoji}${target ? ` to ${target}` : ''}]`;
}
function pollOptions(poll) {
return (poll.options || [])
.map(option => option.optionName || option.name)
.filter(Boolean);
}
function formatPollText(poll) {
const question = poll.name || poll.title || 'poll';
const options = pollOptions(poll);
return `[Poll: ${question}${options.length ? ` Options: ${options.join(', ')}` : ''}]`;
}
function formatPollUpdateText(update) {
const target = update.pollCreationMessageKey?.id || update.key?.id || '';
return `[Poll update${target ? `: ${target}` : ''}]`;
}
/**
* Append a visible note for media that failed to download, so the agent knows
* something was sent rather than silently losing the attachment. Returns
* `content` unchanged when nothing failed. (Port of nanoclaw#2895.)
*/
export function appendMediaFailureNote(content, failures) {
if (!failures || failures.length === 0) return content;
const note = failures.map((t) => `[${t} could not be downloaded]`).join(' ');
return content ? `${content}\n${note}` : note;
}
export async function extractBridgeEvent({
msg,
chatId,
senderId,
senderNumber,
botIds = [],
isGroup = false,
downloadMedia,
writeMediaFile,
cacheDirs = {},
}) {
const messageContent = getMessageContent(msg);
const contextInfo = getContextInfo(messageContent);
const mentionedIds = Array.from(new Set((contextInfo?.mentionedJid || []).map(normalizeWhatsAppId).filter(Boolean)));
const quotedMessageId = contextInfo?.stanzaId || null;
const quotedParticipant = normalizeWhatsAppId(contextInfo?.participant || '') || null;
const quotedRemoteJid = normalizeWhatsAppId(contextInfo?.remoteJid || '') || null;
const hasQuotedMessage = !!contextInfo?.quotedMessage;
const quotedText = textFromQuotedMessage(contextInfo?.quotedMessage);
let body = '';
let hasMedia = false;
let mediaType = '';
let mime = '';
let fileName = '';
let nativeType = '';
const mediaUrls = [];
const nativeMetadata = {};
const mediaFailures = [];
const saveMedia = async ({ mediaMessage, dir, prefix, fallbackExt, fileName: name, type }) => {
if (!downloadMedia) return;
try {
const buf = await downloadMedia(msg);
const ext = mediaExtForMime(mediaMessage?.mimetype, fallbackExt);
const writer = writeMediaFile || defaultWriteMediaFile;
const saved = await writer({ buffer: buf, dir, prefix, ext, fileName: name });
if (saved) mediaUrls.push(saved);
} catch (err) {
// A failed CDN fetch (expired media URL, transient network error) must
// never reject out of extractBridgeEvent — that would drop this message
// AND every remaining message in the same upsert batch. Record the
// failure so the agent is told media was sent instead of losing it
// silently. (Port of nanoclaw#2895's never-silently-drop guarantee; the
// reuploadRequest recovery half is already wired in bridge.js.)
mediaFailures.push(type || 'media');
try {
console.warn(`[bridge] failed to download inbound ${type || 'media'}:`, err?.message || err);
} catch {}
}
};
if (messageContent.conversation) {
body = messageContent.conversation;
nativeType = 'conversation';
} else if (messageContent.extendedTextMessage?.text) {
body = messageContent.extendedTextMessage.text;
nativeType = 'extendedTextMessage';
} else if (messageContent.imageMessage) {
const item = messageContent.imageMessage;
body = item.caption || '';
hasMedia = true;
mediaType = 'image';
nativeType = 'imageMessage';
mime = item.mimetype || 'image/jpeg';
await saveMedia({ mediaMessage: item, dir: cacheDirs.image, prefix: 'img', fallbackExt: '.jpg', type: 'image' });
} else if (messageContent.videoMessage) {
const item = messageContent.videoMessage;
body = item.caption || '';
hasMedia = true;
mediaType = item.gifPlayback ? 'gif' : 'video';
nativeType = 'videoMessage';
mime = item.mimetype || 'video/mp4';
nativeMetadata.video = { gifPlayback: !!item.gifPlayback };
await saveMedia({ mediaMessage: item, dir: cacheDirs.document, prefix: 'vid', fallbackExt: '.mp4', type: mediaType });
} else if (messageContent.audioMessage || messageContent.pttMessage) {
const item = messageContent.pttMessage || messageContent.audioMessage;
hasMedia = true;
mediaType = item.ptt || messageContent.pttMessage ? 'ptt' : 'audio';
nativeType = messageContent.pttMessage ? 'pttMessage' : 'audioMessage';
mime = item.mimetype || 'audio/ogg';
nativeMetadata.audio = { ptt: mediaType === 'ptt' };
await saveMedia({ mediaMessage: item, dir: cacheDirs.audio, prefix: 'aud', fallbackExt: '.ogg', type: 'audio' });
} else if (messageContent.documentMessage) {
const item = messageContent.documentMessage;
body = item.caption || '';
hasMedia = true;
mediaType = 'document';
nativeType = 'documentMessage';
mime = item.mimetype || 'application/octet-stream';
fileName = item.fileName || 'document';
await saveMedia({ mediaMessage: item, dir: cacheDirs.document, prefix: 'doc', fallbackExt: '.bin', fileName, type: 'document' });
} else if (messageContent.stickerMessage) {
hasMedia = true;
mediaType = 'sticker';
nativeType = 'stickerMessage';
mime = messageContent.stickerMessage.mimetype || 'image/webp';
body = '[Sticker]';
nativeMetadata.sticker = {
animated: !!messageContent.stickerMessage.isAnimated,
mimetype: mime,
};
await saveMedia({ mediaMessage: messageContent.stickerMessage, dir: cacheDirs.image, prefix: 'sticker', fallbackExt: '.webp', type: 'sticker' });
} else if (messageContent.locationMessage || messageContent.liveLocationMessage) {
const isLive = !!messageContent.liveLocationMessage;
const item = messageContent.liveLocationMessage || messageContent.locationMessage;
mediaType = isLive ? 'live_location' : 'location';
nativeType = isLive ? 'liveLocationMessage' : 'locationMessage';
body = formatLocationText(item, isLive);
nativeMetadata.location = locationMetadata(item, isLive);
} else if (messageContent.contactMessage) {
mediaType = 'contact';
nativeType = 'contactMessage';
body = formatContactText(messageContent.contactMessage);
nativeMetadata.contact = {
displayName: messageContent.contactMessage.displayName || '',
vcard: messageContent.contactMessage.vcard || '',
};
} else if (messageContent.contactsArrayMessage) {
const contacts = messageContent.contactsArrayMessage.contacts || [];
mediaType = 'contacts';
nativeType = 'contactsArrayMessage';
body = formatContactsText(contacts);
nativeMetadata.contacts = contacts.map(contact => ({
displayName: contact.displayName || '',
vcard: contact.vcard || '',
}));
} else if (messageContent.reactionMessage) {
mediaType = 'reaction';
nativeType = 'reactionMessage';
body = formatReactionText(messageContent.reactionMessage);
nativeMetadata.reaction = {
text: messageContent.reactionMessage.text || '',
messageId: messageContent.reactionMessage.key?.id || '',
remoteJid: normalizeWhatsAppId(messageContent.reactionMessage.key?.remoteJid || ''),
participant: normalizeWhatsAppId(messageContent.reactionMessage.key?.participant || ''),
};
} else if (messageContent.pollCreationMessage || messageContent.pollCreationMessageV2 || messageContent.pollCreationMessageV3) {
const item = messageContent.pollCreationMessage || messageContent.pollCreationMessageV2 || messageContent.pollCreationMessageV3;
mediaType = 'poll';
nativeType = messageContent.pollCreationMessage ? 'pollCreationMessage' : messageContent.pollCreationMessageV2 ? 'pollCreationMessageV2' : 'pollCreationMessageV3';
body = formatPollText(item);
nativeMetadata.poll = {
question: item.name || item.title || '',
options: pollOptions(item),
selectableCount: item.selectableOptionsCount || item.selectableCount || 1,
};
} else if (messageContent.pollUpdateMessage) {
mediaType = 'poll_update';
nativeType = 'pollUpdateMessage';
body = formatPollUpdateText(messageContent.pollUpdateMessage);
nativeMetadata.pollUpdate = messageContent.pollUpdateMessage;
}
// Surface failed downloads to the agent instead of silently losing the
// attachment. Applied before the generic "[<type> received]" fallback so an
// uncaptioned message whose download failed reads "[image could not be
// downloaded]" rather than claiming the media arrived.
body = appendMediaFailureNote(body, mediaFailures);
if (hasMedia && !body) {
body = `[${mediaType} received]`;
}
return {
messageId: msg.key.id,
chatId,
senderId,
senderName: msg.pushName || senderNumber,
chatName: isGroup ? (chatId.split('@')[0]) : (msg.pushName || senderNumber),
isGroup,
body,
hasMedia,
mediaType,
mime,
fileName,
nativeType,
nativeMetadata,
mediaUrls,
mentionedIds,
quotedMessageId,
quotedParticipant,
quotedRemoteJid,
quotedText,
hasQuotedMessage,
botIds,
readReceiptKey: {
remoteJid: msg.key.remoteJid || chatId,
id: msg.key.id,
participant: msg.key.participant || senderId,
fromMe: Boolean(msg.key.fromMe),
},
timestamp: msg.messageTimestamp,
};
}
export function inferMediaType(ext) {
if (['jpg', 'jpeg', 'png', 'webp', 'gif'].includes(ext)) return 'image';
if (['mp4', 'mov', 'avi', 'mkv', '3gp'].includes(ext)) return 'video';
if (['ogg', 'opus', 'mp3', 'wav', 'm4a'].includes(ext)) return 'audio';
return 'document';
}
export function inboundReadReceiptKeys({ key, enabled }) {
if (!enabled || !key || key.fromMe || !key.id || !key.remoteJid) return [];
// Preserve participant for group messages: Baileys needs the original key.
return [key];
}
export function mediaPayloadForFile({ buffer, filePath, mediaType, caption, fileName }) {
const ext = filePath.toLowerCase().split('.').pop();
const type = mediaType || inferMediaType(ext);
if (type === 'image' && ext === 'gif') {
// Pure helper fallback: do not lie and label raw GIF bytes as mp4.
// The live bridge tries ffmpeg conversion to WhatsApp gifPlayback video
// before it falls back to this regular image payload.
return { image: buffer, caption: caption || undefined, mimetype: MIME_MAP[ext] || 'image/gif' };
}
switch (type) {
case 'image':
return { image: buffer, caption: caption || undefined, mimetype: MIME_MAP[ext] || 'image/jpeg' };
case 'video':
return { video: buffer, caption: caption || undefined, mimetype: MIME_MAP[ext] || 'video/mp4' };
case 'document':
return {
document: buffer,
fileName: fileName || path.basename(filePath),
caption: caption || undefined,
mimetype: MIME_MAP[ext] || 'application/octet-stream',
};
default:
return null;
}
}
export function buildPollPayload({ question, options, selectableCount = 1 }) {
const cleanQuestion = String(question || '').trim();
const cleanOptions = (options || []).map(option => String(option || '').trim()).filter(Boolean);
if (!cleanQuestion) throw new Error('question is required');
if (cleanOptions.length < 2) throw new Error('at least two poll options are required');
if (cleanOptions.length > 12) throw new Error('at most 12 poll options are supported');
const count = Math.max(1, Math.min(Number(selectableCount) || 1, cleanOptions.length));
return {
poll: {
name: cleanQuestion,
values: cleanOptions,
selectableCount: count,
messageSecret: randomBytes(32),
},
};
}
export function pollCreationMessageFromPayload(payload) {
const poll = payload?.poll;
if (!poll) return null;
const values = Array.isArray(poll.values) ? poll.values : [];
const options = values.map(value => String(value || '').trim()).filter(Boolean);
if (!poll.name || options.length < 2) return null;
const selectableOptionsCount = Math.max(1, Math.min(Number(poll.selectableCount) || 1, options.length));
const message = {};
if (poll.messageSecret) {
message.messageContextInfo = { messageSecret: poll.messageSecret };
}
message[selectableOptionsCount === 1 ? 'pollCreationMessageV3' : 'pollCreationMessage'] = {
name: String(poll.name),
options: options.map(optionName => ({ optionName })),
selectableOptionsCount,
};
return message;
}
/**
* Reconnect scheduling guard. startSocket() awaits network I/O before it
* creates a socket or registers event handlers, so a bare
* `setTimeout(startSocket, ...)` has two unrecoverable failure modes: a
* rejection is unhandled (crashes the process on modern Node), and a hang
* leaves the bridge permanently disconnected with nothing left to retry.
* Every (re)connect must go through the scheduler this returns.
*/
export function createReconnectScheduler(startFn, {
retryDelayMs = 5000,
log = console.log,
setTimeoutFn = setTimeout,
} = {}) {
function scheduleReconnect(delayMs) {
setTimeoutFn(() => {
Promise.resolve()
.then(startFn)
.catch((err) => {
log(`⚠️ Reconnect failed (${err?.message || err}). Retrying in ${Math.round(retryDelayMs / 1000)}s...`);
scheduleReconnect(retryDelayMs);
});
}, delayMs);
}
return scheduleReconnect;
}
/**
* Version resolution guard. fetchLatestBaileysVersion() is a plain fetch to
* raw.githubusercontent.com with no AbortSignal; a stalled connection can
* pend forever and wedge the reconnect path (the scheduler above cannot
* retry past an await that never settles). Bound the fetch and fall back to
* the last known-good version, or the Baileys default before first success.
*/
export function createVersionResolver(fetchVersionFn, {
timeoutMs = 15000,
log = console.log,
} = {}) {
let cachedVersion = null;
return async function resolveVersion() {
let timer = null;
try {
const { version } = await Promise.race([
fetchVersionFn(),
new Promise((_, reject) => {
timer = setTimeout(() => reject(new Error('version fetch timed out')), timeoutMs);
}),
]);
cachedVersion = version;
} catch (err) {
log(`⚠️ Baileys version fetch failed (${err?.message || err}); using ${cachedVersion ? 'cached version' : 'library default'}.`);
} finally {
if (timer) clearTimeout(timer);
}
return cachedVersion;
};
}
scripts/whatsapp-bridge/outbound_ids.js
+45
View File
@@ -0,0 +1,45 @@
/**
* Bounded FIFO set of outbound message IDs.
*
* Used by the WhatsApp bridge to distinguish "echo of our own /send" from
* "owner-typed message on the linked device" when forwarding `fromMe`
* inbound events back to the Python adapter.
*
* Eviction drops the oldest insertion-order entry when the cap is exceeded.
* Re-remembering an existing id is a no-op for ordering (not LRU refresh).
*
* Heuristic limitation (intentional, documented for future debugging):
* the set is in-memory only. On bridge restart it is empty, so for the
* brief window between restart and the first new outbound, any in-flight
* delivery receipts of pre-restart sends would be classified as
* owner-typed. The TTL on owner-driven plugin actions (e.g. handover
* sliding TTL) bounds blast radius; persisting would not be worth the
* extra complexity / disk churn.
*/
export function createOutboundIdTracker(maxSize = 512) {
if (!Number.isInteger(maxSize) || maxSize < 1) {
throw new RangeError('createOutboundIdTracker: maxSize must be a positive integer');
}
const ids = new Set();
function remember(id) {
if (!id) return;
ids.add(id);
while (ids.size > maxSize) {
// Set iteration order is insertion order, so values().next() is the
// oldest entry — drop it to keep memory flat under sustained sending.
ids.delete(ids.values().next().value);
}
}
function has(id) {
return Boolean(id) && ids.has(id);
}
function size() {
return ids.size;
}
return { remember, has, size };
}
scripts/whatsapp-bridge/outbound_ids.test.mjs
+68
View File
@@ -0,0 +1,68 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createOutboundIdTracker } from './outbound_ids.js';
test('remembers and recognises an outbound id', () => {
const tracker = createOutboundIdTracker();
tracker.remember('msg-1');
assert.equal(tracker.has('msg-1'), true);
assert.equal(tracker.has('msg-2'), false);
});
test('ignores empty / falsy ids', () => {
const tracker = createOutboundIdTracker();
tracker.remember(undefined);
tracker.remember('');
tracker.remember(null);
assert.equal(tracker.size(), 0);
assert.equal(tracker.has(''), false);
assert.equal(tracker.has(undefined), false);
});
test('evicts oldest entry once the cap is exceeded', () => {
const tracker = createOutboundIdTracker(3);
tracker.remember('a');
tracker.remember('b');
tracker.remember('c');
tracker.remember('d'); // cap=3 → 'a' should be evicted
assert.equal(tracker.has('a'), false);
assert.equal(tracker.has('b'), true);
assert.equal(tracker.has('c'), true);
assert.equal(tracker.has('d'), true);
assert.equal(tracker.size(), 3);
});
test('cap holds across many inserts (bounded memory)', () => {
const tracker = createOutboundIdTracker(8);
for (let i = 0; i < 100; i += 1) {
tracker.remember(`id-${i}`);
}
assert.equal(tracker.size(), 8);
// Oldest (id-0..id-91) should be gone, latest 8 retained.
assert.equal(tracker.has('id-0'), false);
assert.equal(tracker.has('id-91'), false);
assert.equal(tracker.has('id-92'), true);
assert.equal(tracker.has('id-99'), true);
});
test('re-remembering an existing id does not promote it (FIFO, not LRU)', () => {
// Insertion-order semantics: re-adding doesn't move it forward in
// Set iteration order. This is intentional — we don't need recency,
// just bounded membership. Pin the actual behaviour so future
// refactors don't accidentally introduce LRU refresh semantics.
const tracker = createOutboundIdTracker(2);
tracker.remember('a');
tracker.remember('b');
tracker.remember('a'); // no-op for ordering
tracker.remember('c'); // evicts 'a' (oldest by insertion)
assert.equal(tracker.has('a'), false);
assert.equal(tracker.has('b'), true);
assert.equal(tracker.has('c'), true);
});
test('rejects non-positive maxSize', () => {
assert.throws(() => createOutboundIdTracker(0), RangeError);
assert.throws(() => createOutboundIdTracker(-1), RangeError);
assert.throws(() => createOutboundIdTracker(1.5), RangeError);
});
scripts/whatsapp-bridge/owner_message_gate.js
+56
View File
@@ -0,0 +1,56 @@
/**
* Pure classifier for the WhatsApp bridge's bot-mode dispatch loop.
*
* Centralises the "should this fromMe message be forwarded as fromOwner?"
* decision so the gate can be unit-tested without spinning up Baileys or
* the Express server.
*
* Lives next to `outbound_ids.js` rather than inline in `bridge.js`
* because the previous implementation accidentally bypassed the
* customer-side allowlist when forwarding owner-typed messages — see
* the regression test in `owner_message_gate.test.mjs`.
*
* Caller responsibilities:
* - Only invoke in bot mode. Self-chat mode has its own self-chat
* pinning logic and must not delegate here.
* - Pre-filter group / status JIDs (the gate doesn't know about them).
* - On `drop_allowlist`, log the rejection so operators can audit
* accidental allowlist mismatches.
*
* Returned actions:
* - 'pass' : non-fromMe, fall through to existing handling
* - 'drop_echo' : fromMe and matches a recently-sent /send id
* - 'drop_disabled' : fromMe but operator hasn't opted into forwarding
* - 'drop_allowlist' : fromMe and the *customer chatId* isn't on the
* allowlist (owner-typed reply to a stranger)
* - 'forward_owner' : fromMe, owner-typed, allowlisted — forward with
* fromOwner: true
*/
export function classifyOwnerMessageGate({
fromMe,
fromOwnerEnabled,
recentlySent,
allowlistMatches,
messageId,
chatId,
}) {
if (!fromMe) {
return { action: 'pass' };
}
if (recentlySent && recentlySent.has(messageId)) {
return { action: 'drop_echo' };
}
if (!fromOwnerEnabled) {
return { action: 'drop_disabled' };
}
// Allowlist gate: check the *customer* chatId, not the sender. The
// sender is the owner's own number/LID and won't be on the allowlist
// by construction. Without this check, any contact the owner happens
// to reply to leaks into Hermes and triggers implicit handover in the
// gateway-policy plugin.
if (typeof allowlistMatches === 'function' && !allowlistMatches(chatId)) {
return { action: 'drop_allowlist' };
}
return { action: 'forward_owner' };
}
scripts/whatsapp-bridge/owner_message_gate.test.mjs
+126
View File
@@ -0,0 +1,126 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { classifyOwnerMessageGate } from './owner_message_gate.js';
function makeRecentlySent(ids = []) {
const set = new Set(ids);
return { has: (id) => set.has(id) };
}
function makeAllowlist(allowedChatIds) {
if (allowedChatIds === '*') {
return () => true;
}
const set = new Set(allowedChatIds);
return (id) => set.has(id);
}
test('non-fromMe messages always pass through', () => {
const decision = classifyOwnerMessageGate({
fromMe: false,
fromOwnerEnabled: true,
recentlySent: makeRecentlySent(),
allowlistMatches: makeAllowlist([]),
messageId: 'M1',
chatId: '6281234567890@s.whatsapp.net',
});
assert.deepEqual(decision, { action: 'pass' });
});
test('fromMe echo of our own /send is dropped', () => {
const decision = classifyOwnerMessageGate({
fromMe: true,
fromOwnerEnabled: true,
recentlySent: makeRecentlySent(['M-OWN-1']),
allowlistMatches: makeAllowlist('*'),
messageId: 'M-OWN-1',
chatId: '6281234567890@s.whatsapp.net',
});
assert.deepEqual(decision, { action: 'drop_echo' });
});
test('fromMe is dropped when forwarding is disabled', () => {
const decision = classifyOwnerMessageGate({
fromMe: true,
fromOwnerEnabled: false,
recentlySent: makeRecentlySent(),
allowlistMatches: makeAllowlist('*'),
messageId: 'M-OWN-2',
chatId: '6281234567890@s.whatsapp.net',
});
assert.deepEqual(decision, { action: 'drop_disabled' });
});
test('fromMe is dropped when chatId is not on the allowlist (regression)', () => {
// This is the bug. Before the fix, an owner reply in a non-allowlisted
// chat was still forwarded with fromOwner: true, which made the
// gateway-policy owner-implicit branch create stray handover rows for
// the non-allowlisted contact.
const decision = classifyOwnerMessageGate({
fromMe: true,
fromOwnerEnabled: true,
recentlySent: makeRecentlySent(),
allowlistMatches: makeAllowlist(['6281234567890@s.whatsapp.net']),
messageId: 'M-OWN-3',
chatId: '111600547700784@lid',
});
assert.deepEqual(decision, { action: 'drop_allowlist' });
});
test('fromMe is forwarded as owner when chatId is allowlisted', () => {
const decision = classifyOwnerMessageGate({
fromMe: true,
fromOwnerEnabled: true,
recentlySent: makeRecentlySent(),
allowlistMatches: makeAllowlist(['6281234567890@s.whatsapp.net']),
messageId: 'M-OWN-4',
chatId: '6281234567890@s.whatsapp.net',
});
assert.deepEqual(decision, { action: 'forward_owner' });
});
test('open-allowlist (matchesAllowedUser short-circuits true) forwards as owner', () => {
// matchesAllowedUser returns true on empty allowlist or "*"; the gate
// must respect that so deployments without an allowlist are unaffected
// by the new check.
const decision = classifyOwnerMessageGate({
fromMe: true,
fromOwnerEnabled: true,
recentlySent: makeRecentlySent(),
allowlistMatches: () => true,
messageId: 'M-OWN-5',
chatId: '111600547700784@lid',
});
assert.deepEqual(decision, { action: 'forward_owner' });
});
test('echo check fires before allowlist check', () => {
// A bot-API echo whose chatId happens to be off-allowlist should still
// be dropped as drop_echo, not drop_allowlist, so logging stays
// honest about the actual reason.
const decision = classifyOwnerMessageGate({
fromMe: true,
fromOwnerEnabled: true,
recentlySent: makeRecentlySent(['M-ECHO-1']),
allowlistMatches: makeAllowlist([]),
messageId: 'M-ECHO-1',
chatId: '111600547700784@lid',
});
assert.deepEqual(decision, { action: 'drop_echo' });
});
test('disabled flag fires before allowlist check', () => {
// Pre-existing deployments with WHATSAPP_FORWARD_OWNER_MESSAGES unset
// must see drop_disabled regardless of allowlist state, otherwise
// every fromMe message would log a misleading allowlist_mismatch.
const decision = classifyOwnerMessageGate({
fromMe: true,
fromOwnerEnabled: false,
recentlySent: makeRecentlySent(),
allowlistMatches: makeAllowlist([]),
messageId: 'M-OWN-6',
chatId: '111600547700784@lid',
});
assert.deepEqual(decision, { action: 'drop_disabled' });
});
scripts/whatsapp-bridge/package-lock.json
Generated
+2177
View File
File diff suppressed because it is too large Load Diff
scripts/whatsapp-bridge/package.json
+20
View File
@@ -0,0 +1,20 @@
{
"name": "hermes-whatsapp-bridge",
"version": "1.0.0",
"description": "WhatsApp bridge for Hermes Agent using Baileys",
"private": true,
"type": "module",
"scripts": {
"start": "node bridge.js"
},
"dependencies": {
"@whiskeysockets/baileys": "7.0.0-rc13",
"express": "^4.21.0",
"qrcode-terminal": "^0.12.0",
"pino": "^9.0.0"
},
"overrides": {
"protobufjs": "^7.5.5",
"body-parser": "1.20.6"
}
}