"""Tests for scripts/ci/publish_e2e_evidence.py.""" from __future__ import annotations import importlib.util import sys from pathlib import Path import pytest _PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "publish_e2e_evidence.py" _spec = importlib.util.spec_from_file_location("publish_e2e_evidence", _PATH) if _spec is None or _spec.loader is None: raise ImportError("Failed to load publish_e2e_evidence.py") _mod = importlib.util.module_from_spec(_spec) sys.modules["publish_e2e_evidence"] = _mod _spec.loader.exec_module(_mod) def _png(width: int = 4, height: int = 3) -> bytes: return _mod.PNG_SIGNATURE + b"\x00\x00\x00\rIHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") def test_load_evidence_validates_manifest_and_pngs(tmp_path): (tmp_path / "shot.png").write_bytes(_png()) (tmp_path / "diff.png").write_bytes(_png()) (tmp_path / "actual.png").write_bytes(_png()) (tmp_path / "expected.png").write_bytes(_png()) (tmp_path / "e2e-evidence.json").write_text( """{ "version": 1, "screenshots": [{"name": "main-view.png", "file": "shot.png"}], "diffs": [{"name": "main-view", "diff": "diff.png", "actual": "actual.png", "expected": "expected.png"}] }""", encoding="utf-8", ) files, payloads = _mod.load_evidence(tmp_path) assert [item.label for item in files] == [ "new screenshot: main-view.png", "visual diff: main-view", "visual actual: main-view", "visual expected: main-view", ] assert set(payloads) == {"shot.png", "diff.png", "actual.png", "expected.png"} def test_load_evidence_rejects_path_escape_and_non_png(tmp_path): (tmp_path / "e2e-evidence.json").write_text( '{"version":1,"screenshots":[{"name":"bad","file":"../secret.png"}],"diffs":[]}', encoding="utf-8", ) with pytest.raises(ValueError, match="unsafe filename"): _mod.load_evidence(tmp_path) (tmp_path / "e2e-evidence.json").write_text( '{"version":1,"screenshots":[{"name":"bad","file":"not-png.png"}],"diffs":[]}', encoding="utf-8", ) (tmp_path / "not-png.png").write_bytes(b"not a png") with pytest.raises(ValueError, match="not a PNG"): _mod.load_evidence(tmp_path) def test_upload_evidence_accepts_only_attachment_urls(tmp_path, monkeypatch): shot = tmp_path / "shot.png" shot.write_bytes(_png()) calls = [] def fake_run(args, **kwargs): calls.append((args, kwargs)) return _mod.subprocess.CompletedProcess( args, 0, stdout="![shot.png](https://github.com/user-attachments/assets/12345678-1234-1234-1234-123456789abc)\n", ) monkeypatch.setattr(_mod.subprocess, "run", fake_run) result = _mod.upload_evidence( [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")], tmp_path, "NousResearch/hermes-agent", "bot-session-token", ) assert result == {"shot.png": "https://github.com/user-attachments/assets/12345678-1234-1234-1234-123456789abc"} assert calls[0][0] == ["gh", "image", "--repo", "NousResearch/hermes-agent", str(shot)] assert calls[0][1]["env"]["GH_SESSION_TOKEN"] == "bot-session-token" def test_upload_evidence_reports_gh_image_error(tmp_path, monkeypatch, capsys): shot = tmp_path / "shot.png" shot.write_bytes(_png()) def fake_run(args, **kwargs): raise _mod.subprocess.CalledProcessError( 1, args, output="upload output", stderr="upload error", ) monkeypatch.setattr(_mod.subprocess, "run", fake_run) with pytest.raises(RuntimeError, match="Failed to upload shot.png.*upload error"): _mod.upload_evidence( [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")], tmp_path, "NousResearch/hermes-agent", "bot-session-token", ) captured = capsys.readouterr() assert "Failed to upload shot.png" in captured.err assert "upload output" in captured.err assert "upload error" in captured.err def test_publish_marks_evidence_upload_failure_in_pr_comment(tmp_path, monkeypatch): comment = { "id": 123, "body": "before\n\npending\n\nafter", } updates = [] monkeypatch.setattr( _mod, "load_evidence", lambda evidence_dir: ( [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")], {}, ), ) monkeypatch.setattr(_mod, "_wait_for_review_comment", lambda *args: comment) monkeypatch.setattr( _mod, "upload_evidence", lambda *args: (_ for _ in ()).throw( RuntimeError("Failed to upload shot.png: bad ") ), ) monkeypatch.setattr( _mod, "_api_request", lambda url, token, method, payload: updates.append(( url, token, method, payload, )), ) with pytest.raises(RuntimeError, match="Failed to upload shot.png"): _mod.publish( "github-token", "NousResearch/hermes-agent", tmp_path, "69868", "image-token", ) assert updates == [ ( "https://api.github.com/repos/NousResearch/hermes-agent/issues/comments/123", "github-token", "PATCH", { "body": "before\n\ninline evidence upload failed.\n\n
Failed to upload shot.png: bad <response>
\n\nafter" }, ) ] def test_publish_skips_when_no_review_comment_exists(tmp_path, monkeypatch, capsys): monkeypatch.setattr( _mod, "load_evidence", lambda evidence_dir: ( [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")], {}, ), ) monkeypatch.setattr(_mod, "_wait_for_review_comment", lambda *args: None) monkeypatch.setattr( _mod, "upload_evidence", lambda *args: (_ for _ in ()).throw(AssertionError("must not upload")), ) assert _mod.publish( "github-token", "NousResearch/hermes-agent", tmp_path, "83202", "image-token", ) is False assert "no CI review comment" in capsys.readouterr().out def test_find_review_comment_requires_the_evidence_marker(): pending = "\n\npending\n" assert _mod._find_review_comment([{"body": " no evidence"}]) is None assert _mod._find_review_comment([{"body": pending, "id": 123}]) == {"body": pending, "id": 123} def test_replace_evidence_marker_requires_exactly_one_marker(): with pytest.raises(ValueError, match="does not contain one"): _mod.replace_evidence_marker("no marker", "evidence")