Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/ci/test_assemble_review_comment.py

@@ -0,0 +1,357 @@
+"""Tests for scripts/ci/assemble_review_comment.py.
+
+The assembler collects status from every CI sub-workflow into ReviewItems
+classified by severity (error / action_required / warning / info / debug), then
+renders them into a single PR comment body.
+
+Status data comes from two sources:
+  1. --review-statuses-json: JSON array of {source, results: [...]} objects
+     from workflow_call jobs. Each result has kind/title/summary/detail/
+     how_to_fix/link. The assembler flattens all results into ReviewItems.
+  2. --needs-json: {job_name: result} from all-checks-pass. Failed jobs not
+     claimed by any status become synthesized ❌ Error items.
+
+Layout rules tested here:
+  - group headers: ## ❌ Job failures, ## ⚠️ Action required, ## ⚠️ Warnings
+  - each item is a ### section under its group header
+  - errors + action_required always visible
+  - warnings shown only when present
+  - info above the fold; debug in a collapsible <details> block
+  - sections separated by ---
+  - how_to_fix rendered at bottom of action_required items
+  - empty → clean banner
+  - jobs with declared statuses excluded from failed-jobs list
+  - per-job URLs used for failed job links when available
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "assemble_review_comment.py"
+_spec = importlib.util.spec_from_file_location("assemble_review_comment", _PATH)
+if _spec is None or _spec.loader is None:
+    raise ImportError("Failed to load assemble_review_comment.py")
+_mod = importlib.util.module_from_spec(_spec)
+sys.modules["assemble_review_comment"] = _mod
+_spec.loader.exec_module(_mod)
+
+MARKER = _mod.MARKER
+ReviewItem = _mod.ReviewItem
+
+
+def _status(source: str, results: list[dict]) -> str:
+    """Helper: build a review_statuses JSON string with one source entry."""
+    return json.dumps([{"source": source, "results": results}])
+
+
+# ─── collect_from_statuses ──────────────────────────────────────────
+
+
+def test_statuses_empty_json():
+    items, sources = _mod.collect_from_statuses("")
+    assert items == []
+    assert sources == set()
+
+
+def test_statuses_bad_json():
+    items, sources = _mod.collect_from_statuses("not json")
+    assert items == []
+    assert sources == set()
+
+
+
+
+def test_statuses_info():
+    statuses = _status("review-label-gate", [{
+        "kind": "info",
+        "title": "CI-sensitive file review",
+        "summary": "Label present.",
+    }])
+    items, sources = _mod.collect_from_statuses(statuses)
+    assert len(items) == 1
+    assert items[0].severity == "info"
+    assert sources == {"review-label-gate"}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# ─── collect_failed_jobs ─────────────────────────────────────────────
+
+
+def test_failed_jobs_empty_needs():
+    assert _mod.collect_failed_jobs("", "https://run") == []
+
+
+
+
+
+
+
+
+def test_failed_jobs_excluded_by_source():
+    """Jobs whose name contains a declared source are excluded."""
+    needs = json.dumps({
+        "Review label gate / Review label gate": "failure",
+        "tests": "failure",
+    })
+    items = _mod.collect_failed_jobs(needs, "https://run", exclude_sources={"review-label-gate"})
+    assert len(items) == 1
+    assert items[0].title == "tests"
+
+
+
+
+
+
+
+
+# ─── render_comment ───────────────────────────────────────────────────
+
+
+
+
+
+
+
+
+def test_render_group_header_for_errors():
+    """Errors appear under a '## ❌ Job failures' group header."""
+    items = [
+        ReviewItem(severity="error", title="tests", summary="Job **tests** failed.", link="https://run"),
+        ReviewItem(severity="error", title="lint", summary="Job **lint** failed.", link="https://run"),
+    ]
+    body = _mod.render_comment(items)
+    assert "## ❌ Job failures" in body
+    assert "### tests" in body
+    assert "### lint" in body
+    assert body.index("## ❌ Job failures") < body.index("### tests")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# ─── render_comment (pending jobs) ────────────────────────────────────
+
+
+def test_render_pending_only_shows_header_with_clock():
+    """Pending jobs only — header has 'still waiting', footer lists jobs, no sections."""
+    body = _mod.render_comment([], pending_jobs=["ci-timings"])
+    assert body.startswith(MARKER)
+    assert "૮ >ﻌ< ა" in body
+    assert "Still running" in body
+    assert "`ci-timings`" in body
+    assert "##" not in body
+
+
+def test_render_pending_notif():
+    items = [ReviewItem(severity="info", title="lockfile", summary="No changes.")]
+    body = _mod.render_comment(items, pending_jobs=["ci-timings"])
+    assert "૮ >ﻌ< ა" in body
+    assert "<sub>Still running 1 job: `ci-timings`</sub>" in body
+
+
+# ─── render_comment (waiting for jobs to start) ───────────────────────
+
+
+def test_waiting_with_no_items_shows_waiting_not_all_good():
+    """A run with no jobs yet must not render the final 'all good!' banner."""
+    body = _mod.render_comment([], waiting=True)
+    assert "all good" not in body
+    assert "waiting for jobs to start" in body
+
+
+def test_waiting_with_items_but_no_pending_keeps_a_live_footer():
+    """Between job waves: results exist, nothing pending, run not done."""
+    items = [ReviewItem(severity="info", title="lockfile", summary="No changes.")]
+    body = _mod.render_comment(items, waiting=True)
+    assert "waiting for more jobs to start" in body
+    assert "### lockfile" in body
+
+
+def test_not_waiting_and_no_items_still_renders_all_good():
+    body = _mod.render_comment([])
+    assert "all good!" in body
+
+
+def test_assemble_passes_waiting_through():
+    body = _mod.assemble(waiting=True)
+    assert "waiting for jobs to start" in body
+    assert "all good" not in body
+
+
+
+
+
+
+# ─── assemble (integration) ──────────────────────────────────────────
+
+
+
+
+
+
+
+
+def test_assemble_review_status_detail_renders_sensitive_file_links():
+    statuses = _status("review-label-gate", [{
+        "kind": "action_required",
+        "title": "CI-sensitive file review",
+        "summary": "Changes detected.",
+        "detail": "**Sensitive files:**\n- [`ci.yml`](https://example.test/ci.yml)",
+    }])
+    body = _mod.assemble(review_statuses_json=statuses)
+    assert "**Sensitive files:**" in body
+    assert "[`ci.yml`](https://example.test/ci.yml)" in body
+
+
+def test_assemble_info_keeps_screenshot_details_visible_below_its_summary():
+    statuses = _status("playwright e2e", [{
+        "kind": "info",
+        "title": "Desktop E2E screenshots",
+        "summary": "1 screenshot captured; 0 visual diffs.",
+        "detail": "<details>\n<summary>1 captured screenshot</summary>\n\n- [`proof.png`](https://example.test/artifact)\n\n</details>",
+    }])
+    body = _mod.assemble(review_statuses_json=statuses)
+    assert "## ℹ️ Info" in body
+    assert "1 screenshot captured; 0 visual diffs." in body
+    assert "<summary>1 captured screenshot</summary>" in body
+    assert "[`proof.png`](https://example.test/artifact)" in body
+
+
+
+
+
+
+def test_assemble_with_timings_status():
+    """Timings status from the nested format renders as debug or warning."""
+    statuses = _status("ci-timings", [{
+        "kind": "debug",
+        "title": "CI timings",
+        "summary": "Wall time 3m (no baseline yet).",
+        "detail": "",
+        "link": "https://report",
+    }])
+    body = _mod.assemble(review_statuses_json=statuses)
+    assert "<details>" in body
+    assert "### CI timings" in body
+    assert "Wall time 3m" in body
+    assert "## ❌" not in body
+    assert "## ⚠️" not in body
+
+
+def test_assemble_with_lockfile_status():
+    """Lockfile no-changes status renders as visible info."""
+    statuses = _status("lockfile-diff", [{
+        "kind": "info",
+        "title": "package-lock.json",
+        "summary": "No lockfile changes — locked versions match the target branch.",
+    }])
+    body = _mod.assemble(review_statuses_json=statuses)
+    assert "## ℹ️ Info" in body
+    assert "### package-lock.json" in body
+    assert "No lockfile changes" in body
+
+
+
+
+# ─── _attach_job_urls ────────────────────────────────────────────────
+
+
+def test_attach_job_urls_fills_missing_links():
+    """Items without a link get one from job_urls via source matching."""
+    items = [
+        ReviewItem(severity="info", title="Supply chain scan",
+                   summary="No risks.", source="supply chain"),
+        ReviewItem(severity="warning", title="CI timings",
+                   summary="Slower.", source="ci timings",
+                   link="https://report"),  # already has a link
+    ]
+    job_urls = {
+        "Supply Chain Audit / Scan PR for critical supply chain risks": "https://run/1/job/2",
+    }
+    _mod._attach_job_urls(items, job_urls, "https://fallback")
+    # First item gets the per-job URL as job_url (link untouched)
+    assert items[0].job_url == "https://run/1/job/2"
+    assert items[0].link == ""  # no emitted link
+    # Second item keeps its existing link, job_url is set separately
+    assert items[1].link == "https://report"
+    assert items[1].job_url == "https://fallback"  # fell back to run_url
+
+
+
+
+
+
+
+
+def test_render_commit_info_below_header():
+    """Commit info is rendered below the header, above the content."""
+    body = _mod.render_comment(
+        [ReviewItem(severity="error", title="tests", summary="failed.")],
+        commit_info="<sub>running on [abc1234](https://commit-url) — fix: thing</sub>",
+    )
+    assert "# ૮ >ﻌ< ა ci review" in body
+    assert "running on [abc1234](https://commit-url)" in body
+    assert "fix: thing" in body
+    # Commit info appears before the content
+    assert body.index("abc1234") < body.index("## ❌")
+
+
+
+
+def test_assemble_passes_commit_info():
+    """assemble() passes commit_info through to render_comment."""
+    body = _mod.assemble(commit_info="<sub>running on abc1234</sub>")
+    assert "running on abc1234" in body
+    assert "all good!" in body
+
+
+def test_render_both_emitted_link_and_job_url():
+    """An item with both an emitted link and a job_url shows both."""
+    item = ReviewItem(
+        severity="warning",
+        title="CI timings",
+        summary="Slower.",
+        link="https://artifact/report.html",
+        link_label="View report",
+        source="ci timings",
+        job_url="https://github.com/run/1/job/5",
+    )
+    body = _mod.render_comment([item])
+    assert "[View report](https://artifact/report.html)" in body
+    assert "[View job](https://github.com/run/1/job/5)" in body
+    # Both links on the same line, separated by ·
+    assert " · " in body
+
+

tests/ci/test_classify_changes.py

@@ -0,0 +1,414 @@
+"""Tests for scripts/ci/classify_changes.py.
+
+Check some common patterns of file modifications and the CI lanes they should run.
+We should always fail open. We may run a lane we didn't need, never skip one a
+change could have broken.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import io
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "classify_changes.py"
+_spec = importlib.util.spec_from_file_location("classify_changes", _PATH)
+if _spec is None or _spec.loader is None:
+    raise ImportError("Failed to load classify_changes.py")
+_mod = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_mod)
+classify = _mod.classify
+ci_review_files = _mod.ci_review_files
+pull_request_changed_files = _mod.pull_request_changed_files
+main = _mod.main
+
+DEFAULT = {
+    "python": True,
+    "python_prod": True,
+    "frontend": True,
+    "docker": True,
+    "docker_meta": True,
+    "nix": True,
+    "site": True,
+    "scan": True,
+    "deps": True,
+    "uv_lock": True,
+    "npm_lock": True,
+    "installer": True,
+    "desktop_updater": True,
+    "rust": True,
+    "mcp_catalog": False,
+    "ci_review": True,
+}
+
+
+def _lanes(python=False, frontend=False, site=False, scan=False, deps=False, uv_lock=False, npm_lock=False, installer=False, desktop_updater=False, rust=False, mcp_catalog=False, docker_meta=False, ci_review=False, python_prod=None, nix=None, docker=None) -> dict[str, bool]:
+    # python_prod tracks python except for tests-only diffs; default it to
+    # python so the majority of cases don't need to spell it out.
+    #
+    # docker and nix are derived: both build the product, so both ride on
+    # python_prod and frontend. The image ships the built web assets, and the
+    # flake bundles the compiled ui-tui. Pass either explicitly to override.
+    _python_prod = python if python_prod is None else python_prod
+    _product = _python_prod or frontend
+    return {
+        "python": python,
+        "python_prod": _python_prod,
+        "docker": (docker_meta or _product) if docker is None else docker,
+        "nix": _product if nix is None else nix,
+        "frontend": frontend,
+        "docker_meta": docker_meta,
+        "site": site,
+        "scan": scan,
+        "deps": deps,
+        "uv_lock": uv_lock,
+        "npm_lock": npm_lock,
+        "installer": installer,
+        "desktop_updater": desktop_updater,
+        "rust": rust,
+        "mcp_catalog": mcp_catalog,
+        "ci_review": ci_review,
+    }
+
+
+CASES = {
+    "docs-only → nothing heavy": (["README.md", "docs/guide.md"], _lanes()),
+    "python source → python": (["run_agent.py"], _lanes(python=True, scan=True)),
+    # pyproject.toml declares the pytest markers the OS lanes select on, so it
+    # also re-arms the desktop_updater integration tests (fail-open).
+    "dep manifest → python": (["pyproject.toml"], _lanes(python=True, scan=True, deps=True, uv_lock=True, desktop_updater=True)),
+    "uv.lock → python": (["uv.lock"], _lanes(python=True, uv_lock=True)),
+    "ts package → frontend": (["apps/desktop/src/app.tsx"], _lanes(frontend=True)),
+    "ui-tui → frontend": (["ui-tui/src/entry.ts"], _lanes(frontend=True)),
+    # Lockfile bump shifts every TS package's tree, but not the Python suite.
+    "root lockfile → frontend, not python": (["package-lock.json"], _lanes(frontend=True, npm_lock=True)),
+    "nested lockfile → npm_lock": (["website/package-lock.json"], _lanes(site=True, npm_lock=True)),
+    # A website file the Python suite cannot read stays site-only.
+    "website config → site": (["website/docusaurus.config.ts"], _lanes(site=True)),
+    # uv lock --check re-resolves against PyPI, so it must stay off for any
+    # diff that can't desync the lockfile — a registry blip on a docs PR
+    # otherwise shows up as a blocking "uv.lock out of sync" red X.
+    "docs → no uv_lock": (
+        ["website/docs/developer-guide/plugins/index.md"],
+        _lanes(python=True, site=True),
+    ),
+    "frontend → no uv_lock": (["apps/desktop/src/store/profile.ts"], _lanes(frontend=True)),
+    # The published CIMD document is asserted about by the Python suite, so a
+    # lone edit there must not skip the lane that would catch a bad edit.
+    "cimd document → python + site": (
+        ["website/static/oauth/client-metadata.json"],
+        _lanes(python=True, site=True),
+    ),
+    # A new docs page must reach llms.txt, and the generator that puts it there
+    # has its own tests. Skipping Python on either is how the index drifted to
+    # 53% coverage while every PR stayed green.
+    "docs page → python + site": (
+        ["website/docs/user-guide/bot-mode.md"],
+        _lanes(python=True, site=True),
+    ),
+    "docs generator → python + site": (
+        ["website/scripts/generate-llms-txt.py"],
+        _lanes(python=True, scan=True, site=True),
+    ),
+    # SKILL.md reads like docs, but the skill-doc tests read skills/, so a
+    # skill edit must still run Python.
+    "skill md → python + site": (["skills/github/SKILL.md"], _lanes(python=True, site=True)),
+    "dockerfile → docker meta": (["Dockerfile"], _lanes(docker_meta=True)),
+    # Only the flake reads these, so they run nix alone. No Python test opens
+    # them, unlike pyproject.toml and uv.lock below.
+    "nix module → nix only": (["nix/homeManagerModules.nix"], _lanes(nix=True)),
+    "flake.nix → nix only": (["flake.nix"], _lanes(nix=True)),
+    "flake.lock → nix only": (["flake.lock"], _lanes(nix=True)),
+    # A flake-only file must not mask a Python change beside it.
+    "nix + python → both": (["nix/checks.nix", "agent/x.py"], _lanes(python=True, scan=True)),
+    # Nine checks run the built binary, so product Python is a nix input even
+    # when the diff touches no file under nix/.
+    "product python → nix": (["hermes_cli/config.py"], _lanes(python=True, scan=True)),
+    # tests/ is not packaged, so the built binary cannot change.
+    "tests-only → no nix": (
+        ["tests/agent/test_foo.py"],
+        _lanes(python=True, python_prod=False, scan=True),
+    ),
+    # Prose cannot change the closure or the binary.
+    "docs-only → no nix": (["README.md"], _lanes()),
+    # install.ps1 is a shell script Python never imports, but it's also not
+    # provably prose, so python stays on (fail-open) alongside the Windows lane.
+    "install.ps1 → installer": (["scripts/install.ps1"], _lanes(python=True, installer=True)),
+    "installer test → installer": (
+        ["scripts/tests/test-install-ps1-longpath.ps1"],
+        _lanes(python=True, installer=True),
+    ),
+    "python source alone → no installer lane": (["run_agent.py"], _lanes(python=True, scan=True)),
+    # The Windows desktop-update hand-off is a PowerShell integration surface:
+    # its tests spawn the real script and poll its loopback server. They run
+    # when the script, the Electron side that launches it, or their own test
+    # files change — not on every hermes_state.py PR.
+    "windows.ps1 → desktop_updater": (
+        ["scripts/desktop-update/windows.ps1"],
+        _lanes(python=True, desktop_updater=True),
+    ),
+    "desktop-update test → desktop_updater": (
+        ["tests/test_desktop_update_windows_progress.py"],
+        _lanes(python=True, python_prod=False, scan=True, desktop_updater=True),
+    ),
+    "updater-process.ts → desktop_updater": (
+        ["apps/desktop/electron/updater-process.ts"],
+        _lanes(frontend=True, desktop_updater=True),
+    ),
+    "python source alone → no desktop_updater lane": (["hermes_state.py"], _lanes(python=True, scan=True)),
+    # `.rs` lives under apps/, so it matches `frontend` too. That lane builds
+    # TypeScript and cannot notice a Rust error — before `rust` existed it was
+    # the ONLY lane a Rust change ran, and the crate's tests never executed.
+    "rust source → rust": (
+        ["apps/bootstrap-installer/src-tauri/src/powershell.rs"],
+        _lanes(frontend=True, rust=True),
+    ),
+    "cargo lockfile → rust": (
+        ["apps/bootstrap-installer/src-tauri/Cargo.lock"],
+        _lanes(frontend=True, rust=True),
+    ),
+    # Non-.rs files in the crate still change what cargo builds.
+    "tauri config → rust": (
+        ["apps/bootstrap-installer/src-tauri/tauri.conf.json"],
+        _lanes(frontend=True, rust=True),
+    ),
+    "ts source alone → no rust lane": (
+        ["apps/bootstrap-installer/src/main.tsx"],
+        _lanes(frontend=True),
+    ),
+    # Unknown top-level file keeps Python on rather than risk a silent skip.
+    "unknown toplevel → python": (["Makefile"], _lanes(python=True)),
+    "mixed docs+python → python": (["README.md", "agent/x.py"], _lanes(python=True, scan=True)),
+    "mixed docs+frontend → frontend": (["README.md", "apps/x.tsx"], _lanes(frontend=True)),
+    # tests-only diffs: pytest lanes stay ON, product jobs (Desktop E2E,
+    # Docker) gate on python_prod and skip.
+    "tests-only → python without python_prod": (
+        ["tests/agent/test_foo.py"],
+        _lanes(python=True, python_prod=False, scan=True),
+    ),
+    # conftest.py owns the _OS_MARKS skip logic, so it re-arms the
+    # desktop_updater integration tests too (fail-open).
+    "conftest → python + desktop_updater": (
+        ["tests/conftest.py"],
+        _lanes(python=True, python_prod=False, scan=True, desktop_updater=True),
+    ),
+    "tests + prod source → both lanes": (
+        ["tests/agent/test_foo.py", "agent/x.py"],
+        _lanes(python=True, scan=True),
+    ),
+    # Runner infrastructure is NOT tests-only — a bad runner edit can mask
+    # real failures, so it keeps the conservative full lane set.
+    "test runner script → python_prod stays on": (
+        ["scripts/run_tests_parallel.py"],
+        _lanes(python=True, scan=True),
+    ),
+    # Supply-chain lanes
+    ".pth file → scan": (["evil.pth"], _lanes(python=True, scan=True)),
+    "setup.py → scan": (["setup.py"], _lanes(python=True, scan=True)),
+    "mcp catalog manifest → mcp_catalog": (
+        ["optional-mcps/foo/manifest.yaml"],
+        _lanes(python=True, mcp_catalog=True),
+    ),
+    "mcp_catalog.py → mcp_catalog": (
+        ["hermes_cli/mcp_catalog.py"],
+        _lanes(python=True, scan=True, mcp_catalog=True),
+    ),
+    # CI-sensitive files require explicit review label.
+    "eslint config → ci_review": (
+        ["apps/desktop/eslint.config.mjs"],
+        _lanes(frontend=True, ci_review=True),
+    ),
+    "shared eslint config → ci_review": (
+        ["eslint.config.shared.mjs"],
+        _lanes(python=True, ci_review=True),
+    ),
+    "ui-tui eslint config → ci_review": (
+        ["ui-tui/eslint.config.mjs"],
+        _lanes(frontend=True, ci_review=True),
+    ),
+    "web eslint config → ci_review": (
+        ["web/eslint.config.js"],
+        _lanes(frontend=True, ci_review=True),
+    ),
+    "shared package eslint config → ci_review": (
+        ["apps/shared/eslint.config.mjs"],
+        _lanes(frontend=True, ci_review=True),
+    ),
+    "bootstrap-installer eslint config → ci_review": (
+        ["apps/bootstrap-installer/eslint.config.mjs"],
+        _lanes(frontend=True, ci_review=True),
+    ),
+    "prettier config → ci_review": (
+        [".prettierrc"],
+        _lanes(python=True, ci_review=True),
+    ),
+    "workflow yml → ci_review (also fail-open all)": (
+        [".github/workflows/typecheck.yml"],
+        DEFAULT,
+    ),
+    "composite action → ci_review (also fail-open all)": (
+        [".github/actions/retry/action.yml"],
+        DEFAULT,
+    ),
+    # Normal desktop source doesn't trigger ci_review.
+    "desktop src → no ci_review": (
+        ["apps/desktop/src/app.tsx"],
+        _lanes(frontend=True),
+    ),
+    # Fail open: CI-config / empty / blank diffs run everything.
+    ".github change → all": ([".github/workflows/tests.yml"], DEFAULT),
+    "action change → all": ([".github/actions/detect-changes/action.yml"], DEFAULT),
+    "empty diff → all": ([], DEFAULT),
+    "blank lines → all": (["", "  "], DEFAULT),
+}
+
+
+@pytest.mark.parametrize("files,expected", CASES.values(), ids=CASES.keys())
+def test_classify(files, expected):
+    assert classify(files) == expected
+
+
+_REPO = Path(__file__).resolve().parents[2]
+
+
+def _yaml(rel: str) -> dict:
+    yaml = pytest.importorskip("yaml")
+    return yaml.safe_load((_REPO / rel).read_text(encoding="utf-8"))
+
+
+def test_every_lane_reaches_the_composite_action():
+    """The action is the one surface every consumer reads, so it must carry all
+    of them — ci.yaml, nix.yml and docker.yml each re-export a different subset.
+    """
+    lanes = set(classify(["run_agent.py"]))
+    action_outputs = set(_yaml(".github/actions/detect-changes/action.yml")["outputs"])
+    assert lanes - action_outputs == set(), "lane(s) missing from the composite action's outputs"
+
+
+def test_ci_jobs_only_gate_on_detect_outputs_that_detect_actually_declares():
+    """An ``if`` that reads an undeclared output resolves to the empty string.
+
+    The lane then reports "skipping" on every PR, forever, and nothing goes red
+    — there is no error for referencing an output a job never declared. That is
+    exactly how the ``rust`` lane shipped dead: the classifier emitted it and
+    the composite action re-exported it, but ci.yaml's ``detect`` job did not,
+    so ``needs.detect.outputs.rust`` was never anything but "".
+    """
+    ci = _yaml(".github/workflows/ci.yaml")
+    declared = set(ci["jobs"]["detect"]["outputs"])
+
+    referenced: set[str] = set()
+    for job in ci["jobs"].values():
+        for expr in _iter_if_expressions(job):
+            referenced.update(re.findall(r"needs\.detect\.outputs\.(\w+)", expr))
+
+    assert referenced, "found no detect-gated jobs — the walk is broken, not the wiring"
+    assert referenced - declared == set(), "job(s) gate on an output detect never declares"
+
+
+def _iter_if_expressions(job: object):
+    """Yield every ``if:`` string in a job, including inside its steps."""
+    if not isinstance(job, dict):
+        return
+    if isinstance(cond := job.get("if"), str):
+        yield cond
+    for step in job.get("steps", []) or []:
+        if isinstance(step, dict) and isinstance(cond := step.get("if"), str):
+            yield cond
+
+
+def test_ci_review_files_returns_only_sensitive_paths_sorted_and_unique():
+    assert ci_review_files([
+        "apps/desktop/src/app.tsx",
+        ".github/workflows/ci.yml",
+        "apps/desktop/eslint.config.mjs",
+        ".github/workflows/ci.yml",
+    ]) == [
+        ".github/workflows/ci.yml",
+        "apps/desktop/eslint.config.mjs",
+    ]
+
+
+def _write_event(tmp_path, number: int | None = 88442) -> Path:
+    payload = {"pull_request": {"number": number}} if number is not None else {}
+    path = tmp_path / "event.json"
+    path.write_text(json.dumps(payload), encoding="utf-8")
+    return path
+
+
+def test_pull_request_changed_files_skips_non_pr_events(monkeypatch):
+    monkeypatch.setenv("EVENT_NAME", "push")
+    monkeypatch.setenv("REPO", "NousResearch/hermes-agent")
+    assert pull_request_changed_files() == []
+
+
+def test_pull_request_changed_files_skips_without_pr_number(tmp_path, monkeypatch):
+    monkeypatch.setenv("EVENT_NAME", "pull_request")
+    monkeypatch.setenv("REPO", "NousResearch/hermes-agent")
+    monkeypatch.setenv("GITHUB_EVENT_PATH", str(_write_event(tmp_path, number=None)))
+    assert pull_request_changed_files() == []
+
+
+def test_pull_request_changed_files_parses_gh_output(tmp_path, monkeypatch):
+    monkeypatch.setenv("EVENT_NAME", "pull_request")
+    monkeypatch.setenv("REPO", "NousResearch/hermes-agent")
+    monkeypatch.setenv("GITHUB_EVENT_PATH", str(_write_event(tmp_path)))
+
+    def fake_run(*args, **kwargs):
+        return subprocess.CompletedProcess(
+            args[0],
+            0,
+            stdout="scripts/install.sh\ntests/test_install_sh_node_deps_workspaces.py\n",
+            stderr="",
+        )
+
+    monkeypatch.setattr(_mod.subprocess, "run", fake_run)
+    assert pull_request_changed_files() == [
+        "scripts/install.sh",
+        "tests/test_install_sh_node_deps_workspaces.py",
+    ]
+
+
+def test_pull_request_changed_files_returns_empty_when_gh_fails(tmp_path, monkeypatch):
+    monkeypatch.setenv("EVENT_NAME", "pull_request")
+    monkeypatch.setenv("REPO", "NousResearch/hermes-agent")
+    monkeypatch.setenv("GITHUB_EVENT_PATH", str(_write_event(tmp_path)))
+
+    def fake_run(*args, **kwargs):
+        return subprocess.CompletedProcess(args[0], 1, stdout="", stderr="gh: Not Found")
+
+    monkeypatch.setattr(_mod.subprocess, "run", fake_run)
+    assert pull_request_changed_files() == []
+
+
+def test_main_recovers_pr_files_instead_of_fail_open_ci_review(monkeypatch, capsys):
+    """A fork compare 404 must not demand ci-reviewed for a CLI-only install."""
+    monkeypatch.setattr(
+        _mod,
+        "pull_request_changed_files",
+        lambda: ["scripts/install.sh", "tests/test_install_sh_node_deps_workspaces.py"],
+    )
+    monkeypatch.setattr(sys, "stdin", io.StringIO("\n"))
+    monkeypatch.delenv("GITHUB_OUTPUT", raising=False)
+
+    assert main() == 0
+    out = capsys.readouterr().out
+    assert "ci_review=false" in out
+    assert "python=true" in out
+    assert "python_prod=true" in out
+
+
+def test_main_still_fail_opens_when_recovery_is_empty(monkeypatch, capsys):
+    monkeypatch.setattr(_mod, "pull_request_changed_files", lambda: [])
+    monkeypatch.setattr(sys, "stdin", io.StringIO(""))
+    monkeypatch.delenv("GITHUB_OUTPUT", raising=False)
+
+    assert main() == 0
+    out = capsys.readouterr().out
+    assert "ci_review=true" in out

tests/ci/test_e2e_screenshot_status.py

@@ -0,0 +1,57 @@
+"""Tests for scripts/ci/e2e_screenshot_status.py."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "e2e_screenshot_status.py"
+_spec = importlib.util.spec_from_file_location("e2e_screenshot_status", _PATH)
+if _spec is None or _spec.loader is None:
+    raise ImportError("Failed to load e2e_screenshot_status.py")
+_mod = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_mod)
+
+
+def test_status_selects_only_new_explicit_screenshots_and_all_diffs(tmp_path):
+    for name in (
+        "explicit-proof.png",
+        "test-finished-1.png",
+        "visual-actual.png",
+        "visual-expected.png",
+        "visual-diff.png",
+    ):
+        (tmp_path / name).write_bytes(b"png")
+
+    base_manifest = tmp_path / "main-manifest.json"
+    base_manifest.write_text('{"screenshot_names":["already-on-main.png"]}', encoding="utf-8")
+    (tmp_path / "already-on-main.png").write_bytes(b"png")
+
+    selection = _mod.select_evidence(tmp_path, base_manifest)
+    status = _mod.build_status(selection, "https://github.test/artifacts/1")
+
+    result = status[0]["results"][0]
+    assert result["kind"] == "info"
+    assert result["summary"] == "1 new screenshot vs main; 1 visual diff."
+    assert _mod.EVIDENCE_START in result["detail"]
+    assert "already-on-main.png" not in result["detail"]
+    assert result["link"] == "https://github.test/artifacts/1"
+
+
+def test_cli_output_ends_with_newline_for_github_output_delimiter(tmp_path, monkeypatch):
+    output = tmp_path / "review-status.json"
+    manifest = tmp_path / "main-manifest.json"
+    evidence_dir = tmp_path / "evidence"
+    monkeypatch.setattr(sys, "argv", [
+        "e2e_screenshot_status.py",
+        "--results-dir", str(tmp_path),
+        "--manifest-output", str(manifest),
+        "--evidence-dir", str(evidence_dir),
+        "--output", str(output),
+    ])
+
+    assert _mod.main() == 0
+    assert output.read_text(encoding="utf-8") == "[]\n"
+    assert manifest.read_text(encoding="utf-8") == '{"screenshot_names": [], "version": 1}\n'
+    assert evidence_dir.joinpath("e2e-evidence.json").is_file()

tests/ci/test_emit_review_status.py

@@ -0,0 +1,60 @@
+"""Tests for scripts/ci/emit_review_status.py."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "emit_review_status.py"
+_spec = importlib.util.spec_from_file_location("emit_review_status", _PATH)
+if _spec is None or _spec.loader is None:
+    raise ImportError("Failed to load emit_review_status.py")
+_mod = importlib.util.module_from_spec(_spec)
+sys.modules["emit_review_status"] = _mod
+_spec.loader.exec_module(_mod)
+
+
+def test_ci_review_status_links_to_each_sensitive_file_change():
+    results = _mod.build_results(
+        ci_review=True,
+        mcp_catalog=False,
+        supply_chain=False,
+        label_present=False,
+        ci_review_files='[".github/workflows/ci.yml", "apps/desktop/eslint.config.mjs"]',
+        repo_url="https://github.com/nousresearch/hermes-agent",
+        base_sha="base456",
+        head_sha="abc123",
+    )
+
+    assert results[0]["detail"] == (
+        "**Sensitive files changed:**\n"
+        "- [`.github/workflows/ci.yml`](https://github.com/nousresearch/hermes-agent/compare/base456...abc123#diff-b803fcb7f17ed9235f1e5cb1fcd2f5d3b2838429d4368ae4c57ce4436577f03f)\n"
+        "- [`apps/desktop/eslint.config.mjs`](https://github.com/nousresearch/hermes-agent/compare/base456...abc123#diff-a45471520795db6e46840d1ba2a82c1f8a2841039bd60fb50624488c5f192438)"
+    )
+
+
+def test_approved_ci_review_is_visible_info():
+    results = _mod.build_results(
+        ci_review=True,
+        mcp_catalog=False,
+        supply_chain=False,
+        label_present=True,
+        ci_review_files='[".github/workflows/ci.yml"]',
+        repo_url="https://github.com/nousresearch/hermes-agent",
+        base_sha="base456",
+        head_sha="abc123",
+    )
+
+    assert results == [{
+        "kind": "info",
+        "title": "CI-sensitive file review",
+        "summary": (
+            "PR touches sensitive files, but the `ci-reviewed` label has been "
+            "added, approving them."
+        ),
+        "detail": (
+            "**Sensitive files changed:**\n"
+            "- [`.github/workflows/ci.yml`](https://github.com/nousresearch/hermes-agent/compare/base456...abc123#diff-b803fcb7f17ed9235f1e5cb1fcd2f5d3b2838429d4368ae4c57ce4436577f03f)"
+        ),
+    }]

tests/ci/test_list_os_marked_tests.py

@@ -0,0 +1,136 @@
+"""Tests for ``scripts/ci/list_os_marked_tests.py``.
+
+The helper decides which files the macOS / Windows CI lanes import. Its
+failure modes matter more than its happy path: if it silently returned an
+empty list, the OS lane would run zero tests and still report green — the
+exact silent-coverage-loss the lanes exist to prevent. So the contracts under
+test are "finds real markers", "refuses to emit nothing", and "rejects an
+unknown marker".
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = REPO_ROOT / "scripts" / "ci" / "list_os_marked_tests.py"
+
+
+def _run(*args: str) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        [sys.executable, str(SCRIPT), *args],
+        capture_output=True,
+        text=True,
+        timeout=120,
+        cwd=REPO_ROOT,
+    )
+
+
+def _write(root: Path, relpath: str, body: str) -> Path:
+    path = root / relpath
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(body, encoding="utf-8")
+    return path
+
+
+@pytest.mark.parametrize("marker", ["linux_only", "macos_only", "windows_only"])
+def test_finds_decorator_and_pytestmark_forms(tmp_path, marker):
+    """Both the decorator form and module-level ``pytestmark`` are detected."""
+    _write(
+        tmp_path,
+        "test_decorated.py",
+        f"import pytest\n\n\n@pytest.mark.{marker}\ndef test_x():\n    pass\n",
+    )
+    _write(
+        tmp_path,
+        "nested/test_module_level.py",
+        f"import pytest\n\npytestmark = pytest.mark.{marker}\n\n\ndef test_y():\n    pass\n",
+    )
+    # A file with no marker at all must not be selected.
+    _write(tmp_path, "test_plain.py", "def test_z():\n    pass\n")
+
+    result = _run(marker, str(tmp_path))
+
+    assert result.returncode == 0, result.stderr
+    listed = result.stdout.split()
+    assert any(p.endswith("test_decorated.py") for p in listed)
+    assert any(p.endswith("test_module_level.py") for p in listed)
+    assert not any(p.endswith("test_plain.py") for p in listed)
+
+
+def test_marker_matched_as_whole_word(tmp_path):
+    """``macos_only`` must not match a longer identifier that contains it."""
+    _write(
+        tmp_path,
+        "test_lookalike.py",
+        "import pytest\n\n\n@pytest.mark.macos_only_extra\ndef test_x():\n    pass\n",
+    )
+
+    result = _run("macos_only", str(tmp_path))
+
+    # No genuine match: the helper must fail rather than emit nothing.
+    assert result.returncode == 1
+    assert "macos_only" in result.stderr
+
+
+def test_exits_nonzero_when_no_file_carries_the_marker(tmp_path):
+    """The load-bearing guard: an empty result is an error, never a silent pass."""
+    _write(tmp_path, "test_plain.py", "def test_z():\n    pass\n")
+
+    result = _run("windows_only", str(tmp_path))
+
+    assert result.returncode == 1
+    assert result.stdout.strip() == ""
+    assert "renamed or dropped" in result.stderr
+
+
+def test_rejects_unknown_marker(tmp_path):
+    result = _run("bsd_only", str(tmp_path))
+
+    assert result.returncode == 2
+    assert "unknown marker" in result.stderr
+
+
+def test_rejects_missing_root():
+    result = _run("macos_only", "/nonexistent/path/for/this/test")
+
+    assert result.returncode == 2
+    assert "no such directory" in result.stderr
+
+
+def test_emits_repo_relative_posix_paths():
+    """Output feeds a bash command line on the Windows runner, so separators
+    must be POSIX and paths repo-relative.
+
+    Asserted against the real ``tests/`` tree, which is the only case CI
+    exercises — an out-of-repo root can't be made repo-relative and is
+    emitted absolute instead.
+    """
+    result = _run("windows_only")
+
+    assert result.returncode == 0, result.stderr
+    listed = result.stdout.split()
+    assert listed
+    for line in listed:
+        assert "\\" not in line
+        assert not Path(line).is_absolute()
+
+
+def test_real_tree_selects_files_for_every_marker():
+    """Against the actual ``tests/`` tree each marker resolves to real files.
+
+    This is the invariant the CI lanes depend on — not a snapshot of which
+    files those are, only that each marker is in use and every listed path
+    exists.
+    """
+    for marker in ("linux_only", "macos_only", "windows_only"):
+        result = _run(marker)
+        assert result.returncode == 0, f"{marker}: {result.stderr}"
+        listed = result.stdout.split()
+        assert listed, f"{marker} selected no files"
+        for rel in listed:
+            assert (REPO_ROOT / rel).is_file(), f"{marker} listed missing {rel}"

tests/ci/test_live_comment.py

@@ -0,0 +1,159 @@
+"""Tests for scripts/ci/live_comment.py run selection.
+
+The poller now reports on a run it is not part of, and merges jobs from
+sibling runs of the same commit (the Docker image build, which left ci.yml
+to stop holding the CI run open). ``select_watched_runs`` decides which
+sibling runs count.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+import pytest
+
+_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "live_comment.py"
+_spec = importlib.util.spec_from_file_location("live_comment", _PATH)
+if _spec is None or _spec.loader is None:
+    raise ImportError("Failed to load live_comment.py")
+_mod = importlib.util.module_from_spec(_spec)
+sys.modules["live_comment"] = _mod
+_spec.loader.exec_module(_mod)
+
+select_watched_runs = _mod.select_watched_runs
+classify_jobs = _mod.classify_jobs
+
+DOCKER = "Docker Build, Test, and Publish"
+
+
+def _run(run_id: int, name: str, created_at: str) -> dict:
+    return {"id": run_id, "name": name, "created_at": created_at}
+
+
+def test_selects_only_named_workflows():
+    runs = [
+        _run(1, DOCKER, "2026-08-08T10:00:00Z"),
+        _run(2, "Deploy site", "2026-08-08T10:00:00Z"),
+        _run(3, "CI", "2026-08-08T10:00:00Z"),
+    ]
+    selected = select_watched_runs(runs, [DOCKER])
+    assert [r["id"] for r in selected] == [1]
+
+
+def test_keeps_newest_attempt_per_workflow():
+    """A rerun makes a second run for the same commit; the old one is stale."""
+    runs = [
+        _run(1, DOCKER, "2026-08-08T10:00:00Z"),
+        _run(2, DOCKER, "2026-08-08T11:30:00Z"),
+    ]
+    selected = select_watched_runs(runs, [DOCKER])
+    assert [r["id"] for r in selected] == [2]
+
+
+def test_excludes_the_ci_run_itself():
+    runs = [_run(7, "CI", "2026-08-08T10:00:00Z")]
+    assert select_watched_runs(runs, ["CI"], exclude_run_id="7") == []
+    assert len(select_watched_runs(runs, ["CI"], exclude_run_id="8")) == 1
+
+
+def test_no_watch_names_selects_nothing():
+    runs = [_run(1, DOCKER, "2026-08-08T10:00:00Z")]
+    assert select_watched_runs(runs, []) == []
+    assert select_watched_runs(runs, [""]) == []
+
+
+def test_watched_run_jobs_carry_the_workflow_name_into_the_comment():
+    """A watched run's jobs must stay distinguishable from CI's own jobs."""
+    jobs = [
+        {"name": "build (amd64)", "status": "completed", "conclusion": "failure",
+         "html_url": "https://example/1", "_workflow_name": DOCKER},
+        {"name": "Python tests", "status": "completed", "conclusion": "success",
+         "html_url": "https://example/2"},
+    ]
+    completed, pending, job_urls = classify_jobs(jobs)
+    assert completed[f"{DOCKER} / build (amd64)"] == "failure"
+    assert completed["Python tests"] == "success"
+    assert pending == []
+    assert job_urls[f"{DOCKER} / build (amd64)"] == "https://example/1"
+
+
+def test_parse_watch_workflows_keeps_commas_inside_a_name():
+    """Workflow names contain commas, so the list is newline-separated."""
+    assert _mod.parse_watch_workflows("Docker Build, Test, and Publish\n") == [
+        "Docker Build, Test, and Publish"
+    ]
+    assert _mod.parse_watch_workflows("A\nB\n\n  C  \n") == ["A", "B", "C"]
+    assert _mod.parse_watch_workflows("") == []
+
+
+def test_workflow_watch_list_names_a_workflow_that_exists():
+    """The names the workflow passes must match real workflow ``name:`` values.
+
+    A name that matches nothing makes the poller silently drop that run
+    from the comment, which no unit test on its own would notice.
+    """
+    yaml = pytest.importorskip("yaml")
+    root = Path(__file__).resolve().parents[2]
+    caller = yaml.safe_load(
+        (root / ".github/workflows/ci-review-comment.yml").read_text(encoding="utf-8")
+    )
+    step = next(
+        s for s in caller["jobs"]["comment"]["steps"]
+        if "WATCH_WORKFLOWS" in (s.get("env") or {})
+    )
+    watched = _mod.parse_watch_workflows(step["env"]["WATCH_WORKFLOWS"])
+    assert watched, "the poller is watching nothing"
+
+    known = set()
+    for path in (root / ".github/workflows").glob("*.yml"):
+        doc = yaml.safe_load(path.read_text(encoding="utf-8"))
+        if isinstance(doc, dict) and isinstance(doc.get("name"), str):
+            known.add(doc["name"])
+
+    assert set(watched) <= known, f"unknown workflow names: {set(watched) - known}"
+
+
+def test_poller_never_watches_its_own_workflow():
+    """The poller's own run must never gate completion.
+
+    ``runs_all_completed`` waits until every relevant run is completed.
+    The poller's run is in progress for as long as it polls, so watching
+    itself would make the loop wait for itself and only ever exit on
+    timeout.
+    """
+    yaml = pytest.importorskip("yaml")
+    root = Path(__file__).resolve().parents[2]
+    doc = yaml.safe_load(
+        (root / ".github/workflows/ci-review-comment.yml").read_text(encoding="utf-8")
+    )
+    own_name = doc["name"]
+    step = next(
+        s for s in doc["jobs"]["comment"]["steps"]
+        if "WATCH_WORKFLOWS" in (s.get("env") or {})
+    )
+    watched = _mod.parse_watch_workflows(step["env"]["WATCH_WORKFLOWS"])
+    assert own_name not in watched
+
+
+# ─── runs_all_completed ───────────────────────────────────────────────
+
+
+def test_runs_all_completed_true_only_when_every_run_finished():
+    done = {"status": "completed"}
+    running = {"status": "in_progress"}
+    queued = {"status": "queued"}
+    assert _mod.runs_all_completed([done])
+    assert _mod.runs_all_completed([done, done])
+    assert not _mod.runs_all_completed([done, running])
+    assert not _mod.runs_all_completed([queued])
+
+
+def test_runs_all_completed_empty_list_is_not_done():
+    """No run info at all must not read as 'everything passed'."""
+    assert not _mod.runs_all_completed([])
+
+
+def test_runs_all_completed_missing_status_is_not_done():
+    assert not _mod.runs_all_completed([{}])

tests/ci/test_lockfile_diff.py

@@ -0,0 +1,93 @@
+"""Tests for scripts/ci/lockfile_diff.py.
+
+The differ's job is semantic comparison: reordering and integrity-hash
+churn in the lockfile text must produce an empty diff, while actual
+version movement must show up as added/removed/updated regardless of
+where in the file it appears.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+
+_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "lockfile_diff.py"
+_spec = importlib.util.spec_from_file_location("lockfile_diff", _PATH)
+if _spec is None or _spec.loader is None:
+    raise ImportError("Failed to load lockfile_diff.py")
+_mod = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_mod)
+
+
+def _lock(packages: dict[str, dict]) -> str:
+    return json.dumps(
+        {
+            "name": "hermes",
+            "lockfileVersion": 3,
+            "packages": {"": {"name": "hermes"}, **packages},
+        }
+    )
+
+
+BASE = _lock(
+    {
+        "node_modules/react": {"version": "18.2.0", "integrity": "sha512-aaa"},
+        "node_modules/left-pad": {"version": "1.3.0", "integrity": "sha512-bbb"},
+        "node_modules/foo/node_modules/react": {"version": "17.0.2"},
+    }
+)
+
+
+
+
+
+
+def test_add_remove_update_all_detected():
+    head = _lock(
+        {
+            "node_modules/react": {"version": "18.3.1"},  # updated
+            "node_modules/is-even": {"version": "1.0.0"},  # added
+            "node_modules/foo/node_modules/react": {"version": "17.0.2"},  # unchanged
+            # left-pad removed
+        }
+    )
+    d = _mod.diff_locks(_mod.parse_lockfile(BASE), _mod.parse_lockfile(head))
+    assert d["added"] == [("node_modules/is-even", "1.0.0")]
+    assert d["removed"] == [("node_modules/left-pad", "1.3.0")]
+    assert d["updated"] == [("node_modules/react", "18.2.0", "18.3.1")]
+
+
+def test_nested_dedup_is_distinct_entry():
+    # The same package at two nesting levels must be tracked separately —
+    # bumping only the nested copy must not look like a top-level change.
+    head = _lock(
+        {
+            "node_modules/react": {"version": "18.2.0"},
+            "node_modules/left-pad": {"version": "1.3.0"},
+            "node_modules/foo/node_modules/react": {"version": "17.0.3"},
+        }
+    )
+    d = _mod.diff_locks(_mod.parse_lockfile(BASE), _mod.parse_lockfile(head))
+    assert d["updated"] == [("node_modules/foo/node_modules/react", "17.0.2", "17.0.3")]
+
+
+def test_render_markdown_contains_versions_and_nested_display():
+    d = _mod.diff_locks(
+        _mod.parse_lockfile(BASE),
+        _mod.parse_lockfile(_lock({"node_modules/react": {"version": "19.0.0"}})),
+    )
+    md = _mod.render_markdown({"apps/desktop/package-lock.json": d})
+    # Fragment starts directly with the per-lockfile subsection header.
+    assert md.startswith("#### `apps/desktop/package-lock.json`")
+    assert "`18.2.0`" in md and "`19.0.0`" in md
+    # nested display name keeps the parent chain visible
+    assert "nested under foo" in md
+
+
+def test_render_markdown_omits_unchanged_lockfiles():
+    changed = _mod.diff_locks({}, {"node_modules/x": "1.0.0"})
+    unchanged = _mod.diff_locks({}, {})
+    md = _mod.render_markdown({"a/package-lock.json": changed, "b/package-lock.json": unchanged})
+    assert "a/package-lock.json" in md
+    assert "b/package-lock.json" not in md

tests/ci/test_publish_e2e_evidence.py

@@ -0,0 +1,217 @@
+"""Tests for scripts/ci/publish_e2e_evidence.py."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+import pytest
+
+_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "publish_e2e_evidence.py"
+_spec = importlib.util.spec_from_file_location("publish_e2e_evidence", _PATH)
+if _spec is None or _spec.loader is None:
+    raise ImportError("Failed to load publish_e2e_evidence.py")
+_mod = importlib.util.module_from_spec(_spec)
+sys.modules["publish_e2e_evidence"] = _mod
+_spec.loader.exec_module(_mod)
+
+
+def _png(width: int = 4, height: int = 3) -> bytes:
+    return _mod.PNG_SIGNATURE + b"\x00\x00\x00\rIHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big")
+
+
+def test_load_evidence_validates_manifest_and_pngs(tmp_path):
+    (tmp_path / "shot.png").write_bytes(_png())
+    (tmp_path / "diff.png").write_bytes(_png())
+    (tmp_path / "actual.png").write_bytes(_png())
+    (tmp_path / "expected.png").write_bytes(_png())
+    (tmp_path / "e2e-evidence.json").write_text(
+        """{
+          "version": 1,
+          "screenshots": [{"name": "main-view.png", "file": "shot.png"}],
+          "diffs": [{"name": "main-view", "diff": "diff.png", "actual": "actual.png", "expected": "expected.png"}]
+        }""",
+        encoding="utf-8",
+    )
+
+    files, payloads = _mod.load_evidence(tmp_path)
+
+    assert [item.label for item in files] == [
+        "new screenshot: main-view.png",
+        "visual diff: main-view",
+        "visual actual: main-view",
+        "visual expected: main-view",
+    ]
+    assert set(payloads) == {"shot.png", "diff.png", "actual.png", "expected.png"}
+
+
+def test_load_evidence_rejects_path_escape_and_non_png(tmp_path):
+    (tmp_path / "e2e-evidence.json").write_text(
+        '{"version":1,"screenshots":[{"name":"bad","file":"../secret.png"}],"diffs":[]}',
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match="unsafe filename"):
+        _mod.load_evidence(tmp_path)
+
+    (tmp_path / "e2e-evidence.json").write_text(
+        '{"version":1,"screenshots":[{"name":"bad","file":"not-png.png"}],"diffs":[]}',
+        encoding="utf-8",
+    )
+    (tmp_path / "not-png.png").write_bytes(b"not a png")
+
+    with pytest.raises(ValueError, match="not a PNG"):
+        _mod.load_evidence(tmp_path)
+
+
+
+
+def test_upload_evidence_accepts_only_attachment_urls(tmp_path, monkeypatch):
+    shot = tmp_path / "shot.png"
+    shot.write_bytes(_png())
+    calls = []
+
+    def fake_run(args, **kwargs):
+        calls.append((args, kwargs))
+        return _mod.subprocess.CompletedProcess(
+            args,
+            0,
+            stdout="![shot.png](https://github.com/user-attachments/assets/12345678-1234-1234-1234-123456789abc)\n",
+        )
+
+    monkeypatch.setattr(_mod.subprocess, "run", fake_run)
+
+    result = _mod.upload_evidence(
+        [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
+        tmp_path,
+        "NousResearch/hermes-agent",
+        "bot-session-token",
+    )
+
+    assert result == {"shot.png": "https://github.com/user-attachments/assets/12345678-1234-1234-1234-123456789abc"}
+    assert calls[0][0] == ["gh", "image", "--repo", "NousResearch/hermes-agent", str(shot)]
+    assert calls[0][1]["env"]["GH_SESSION_TOKEN"] == "bot-session-token"
+
+
+
+
+def test_upload_evidence_reports_gh_image_error(tmp_path, monkeypatch, capsys):
+    shot = tmp_path / "shot.png"
+    shot.write_bytes(_png())
+
+    def fake_run(args, **kwargs):
+        raise _mod.subprocess.CalledProcessError(
+            1,
+            args,
+            output="upload output",
+            stderr="upload error",
+        )
+
+    monkeypatch.setattr(_mod.subprocess, "run", fake_run)
+
+    with pytest.raises(RuntimeError, match="Failed to upload shot.png.*upload error"):
+        _mod.upload_evidence(
+            [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
+            tmp_path,
+            "NousResearch/hermes-agent",
+            "bot-session-token",
+        )
+
+    captured = capsys.readouterr()
+    assert "Failed to upload shot.png" in captured.err
+    assert "upload output" in captured.err
+    assert "upload error" in captured.err
+
+
+def test_publish_marks_evidence_upload_failure_in_pr_comment(tmp_path, monkeypatch):
+    comment = {
+        "id": 123,
+        "body": "before\n<!-- hermes-e2e-evidence:start -->\npending\n<!-- hermes-e2e-evidence:end -->\nafter",
+    }
+    updates = []
+
+    monkeypatch.setattr(
+        _mod,
+        "load_evidence",
+        lambda evidence_dir: (
+            [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
+            {},
+        ),
+    )
+    monkeypatch.setattr(_mod, "_wait_for_review_comment", lambda *args: comment)
+    monkeypatch.setattr(
+        _mod,
+        "upload_evidence",
+        lambda *args: (_ for _ in ()).throw(
+            RuntimeError("Failed to upload shot.png: bad <response>")
+        ),
+    )
+    monkeypatch.setattr(
+        _mod,
+        "_api_request",
+        lambda url, token, method, payload: updates.append((
+            url,
+            token,
+            method,
+            payload,
+        )),
+    )
+
+    with pytest.raises(RuntimeError, match="Failed to upload shot.png"):
+        _mod.publish(
+            "github-token",
+            "NousResearch/hermes-agent",
+            tmp_path,
+            "69868",
+            "image-token",
+        )
+
+    assert updates == [
+        (
+            "https://api.github.com/repos/NousResearch/hermes-agent/issues/comments/123",
+            "github-token",
+            "PATCH",
+            {
+                "body": "before\n<!-- hermes-e2e-evidence:start -->\n<sub>inline evidence upload failed.</sub>\n\n<pre>Failed to upload shot.png: bad &lt;response&gt;</pre>\n<!-- hermes-e2e-evidence:end -->\nafter"
+            },
+        )
+    ]
+
+
+def test_publish_skips_when_no_review_comment_exists(tmp_path, monkeypatch, capsys):
+    monkeypatch.setattr(
+        _mod,
+        "load_evidence",
+        lambda evidence_dir: (
+            [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
+            {},
+        ),
+    )
+    monkeypatch.setattr(_mod, "_wait_for_review_comment", lambda *args: None)
+    monkeypatch.setattr(
+        _mod,
+        "upload_evidence",
+        lambda *args: (_ for _ in ()).throw(AssertionError("must not upload")),
+    )
+
+    assert _mod.publish(
+        "github-token",
+        "NousResearch/hermes-agent",
+        tmp_path,
+        "83202",
+        "image-token",
+    ) is False
+    assert "no CI review comment" in capsys.readouterr().out
+
+
+def test_find_review_comment_requires_the_evidence_marker():
+    pending = "<!-- hermes-ci-review-bot -->\n<!-- hermes-e2e-evidence:start -->\npending\n<!-- hermes-e2e-evidence:end -->"
+
+    assert _mod._find_review_comment([{"body": "<!-- hermes-ci-review-bot --> no evidence"}]) is None
+    assert _mod._find_review_comment([{"body": pending, "id": 123}]) == {"body": pending, "id": 123}
+
+
+def test_replace_evidence_marker_requires_exactly_one_marker():
+    with pytest.raises(ValueError, match="does not contain one"):
+        _mod.replace_evidence_marker("no marker", "evidence")

tests/ci/test_timings_report.py

@@ -0,0 +1,125 @@
+"""Tests for scripts/ci/timings_report.py — generate_review_status().
+
+The review status is a JSON array in the unified nested format consumed
+by the review comment assembler. It classifies the CI timings result as
+info/warning (never error — timings is an observability job, not a gate)
+and provides a one-line summary plus optional per-job delta detail.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+from datetime import datetime, timezone
+from pathlib import Path
+
+_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "timings_report.py"
+_spec = importlib.util.spec_from_file_location("timings_report", _PATH)
+if _spec is None or _spec.loader is None:
+    raise ImportError("Failed to load timings_report.py")
+_mod = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_mod)
+
+_T0 = datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
+
+
+def _ts(seconds: float) -> str:
+    """ISO timestamp `seconds` after T0."""
+    dt = _T0.timestamp() + seconds
+    return datetime.fromtimestamp(dt, tz=timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _job(name: str, dur_s: float, start_s: float = 0.0, conclusion: str = "success") -> dict:
+    """Build a normalized job dict with realistic timestamps for wall-time math."""
+    return {
+        "name": name,
+        "duration_s": dur_s,
+        "conclusion": conclusion,
+        "started_at": _ts(start_s),
+        "completed_at": _ts(start_s + dur_s),
+        "wait_s": 0.0,
+    }
+
+
+def _timings(jobs: list[dict]) -> dict:
+    return {"run_id": "123", "head_sha": "abc", "created_at": "", "jobs": jobs}
+
+
+def _result(statuses: list[dict]) -> dict:
+    """Extract the single result dict from the nested format."""
+    assert len(statuses) == 1
+    assert statuses[0]["source"] == "ci timing"
+    results = statuses[0]["results"]
+    assert len(results) == 1
+    return results[0]
+
+
+def test_no_baseline_is_debug():
+    t = _timings([_job("tests", 60.0)])
+    result = _result(_mod.generate_review_status(t, None))
+    assert result["kind"] == "debug"
+    assert "no baseline" in result["summary"].lower()
+    assert "link" not in result  # no report_url → no link field
+
+
+def test_no_regression_is_debug():
+    cur = _timings([_job("tests", 60.0)])
+    bl = _timings([_job("tests", 60.0)])
+    result = _result(_mod.generate_review_status(cur, bl))
+    assert result["kind"] == "debug"
+    assert "+0.0%" in result["summary"]
+
+
+def test_small_regression_is_debug():
+    cur = _timings([_job("tests", 65.0)])
+    bl = _timings([_job("tests", 60.0)])
+    result = _result(_mod.generate_review_status(cur, bl))
+    # +8.3% — well under the 25% warning threshold
+    assert result["kind"] == "debug"
+
+
+def test_large_regression_is_warning():
+    cur = _timings([_job("tests", 80.0)])
+    bl = _timings([_job("tests", 60.0)])
+    result = _result(_mod.generate_review_status(cur, bl))
+    # +33% — above the 25% threshold
+    assert result["kind"] == "warning"
+    assert "+33" in result["summary"]
+
+
+
+
+def test_detail_shows_top_deltas():
+    cur = _timings([_job("slow-job", 120.0), _job("fast-job", 30.0, start_s=120.0)])
+    bl = _timings([_job("slow-job", 60.0), _job("fast-job", 60.0, start_s=60.0)])
+    result = _result(_mod.generate_review_status(cur, bl))
+    assert "slow-job" in result["detail"]
+    assert "fast-job" in result["detail"]
+    # Sorted by abs delta — slow-job (+60) before fast-job (-30)
+    assert result["detail"].index("slow-job") < result["detail"].index("fast-job")
+
+
+
+
+def test_report_url_passed_through():
+    t = _timings([_job("tests", 60.0)])
+    result = _result(_mod.generate_review_status(t, None, report_url="https://artifact/123"))
+    assert result["link"] == "https://artifact/123"
+    assert result["link_label"] == "View report"
+
+
+
+
+def test_nested_format_structure():
+    """The return value is a list with one {source, results: [...]} entry."""
+    t = _timings([_job("tests", 60.0)])
+    statuses = _mod.generate_review_status(t, None)
+    assert isinstance(statuses, list)
+    assert len(statuses) == 1
+    assert statuses[0]["source"] == "ci timing"
+    assert isinstance(statuses[0]["results"], list)
+    assert len(statuses[0]["results"]) == 1
+    r = statuses[0]["results"][0]
+    assert r["kind"] == "debug"
+    assert r["title"] == "CI timings"
+    assert "summary" in r
+    assert "detail" in r