Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/ci/test_publish_e2e_evidence.py

@@ -0,0 +1,217 @@
+"""Tests for scripts/ci/publish_e2e_evidence.py."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+import pytest
+
+_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "publish_e2e_evidence.py"
+_spec = importlib.util.spec_from_file_location("publish_e2e_evidence", _PATH)
+if _spec is None or _spec.loader is None:
+    raise ImportError("Failed to load publish_e2e_evidence.py")
+_mod = importlib.util.module_from_spec(_spec)
+sys.modules["publish_e2e_evidence"] = _mod
+_spec.loader.exec_module(_mod)
+
+
+def _png(width: int = 4, height: int = 3) -> bytes:
+    return _mod.PNG_SIGNATURE + b"\x00\x00\x00\rIHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big")
+
+
+def test_load_evidence_validates_manifest_and_pngs(tmp_path):
+    (tmp_path / "shot.png").write_bytes(_png())
+    (tmp_path / "diff.png").write_bytes(_png())
+    (tmp_path / "actual.png").write_bytes(_png())
+    (tmp_path / "expected.png").write_bytes(_png())
+    (tmp_path / "e2e-evidence.json").write_text(
+        """{
+          "version": 1,
+          "screenshots": [{"name": "main-view.png", "file": "shot.png"}],
+          "diffs": [{"name": "main-view", "diff": "diff.png", "actual": "actual.png", "expected": "expected.png"}]
+        }""",
+        encoding="utf-8",
+    )
+
+    files, payloads = _mod.load_evidence(tmp_path)
+
+    assert [item.label for item in files] == [
+        "new screenshot: main-view.png",
+        "visual diff: main-view",
+        "visual actual: main-view",
+        "visual expected: main-view",
+    ]
+    assert set(payloads) == {"shot.png", "diff.png", "actual.png", "expected.png"}
+
+
+def test_load_evidence_rejects_path_escape_and_non_png(tmp_path):
+    (tmp_path / "e2e-evidence.json").write_text(
+        '{"version":1,"screenshots":[{"name":"bad","file":"../secret.png"}],"diffs":[]}',
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match="unsafe filename"):
+        _mod.load_evidence(tmp_path)
+
+    (tmp_path / "e2e-evidence.json").write_text(
+        '{"version":1,"screenshots":[{"name":"bad","file":"not-png.png"}],"diffs":[]}',
+        encoding="utf-8",
+    )
+    (tmp_path / "not-png.png").write_bytes(b"not a png")
+
+    with pytest.raises(ValueError, match="not a PNG"):
+        _mod.load_evidence(tmp_path)
+
+
+
+
+def test_upload_evidence_accepts_only_attachment_urls(tmp_path, monkeypatch):
+    shot = tmp_path / "shot.png"
+    shot.write_bytes(_png())
+    calls = []
+
+    def fake_run(args, **kwargs):
+        calls.append((args, kwargs))
+        return _mod.subprocess.CompletedProcess(
+            args,
+            0,
+            stdout="![shot.png](https://github.com/user-attachments/assets/12345678-1234-1234-1234-123456789abc)\n",
+        )
+
+    monkeypatch.setattr(_mod.subprocess, "run", fake_run)
+
+    result = _mod.upload_evidence(
+        [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
+        tmp_path,
+        "NousResearch/hermes-agent",
+        "bot-session-token",
+    )
+
+    assert result == {"shot.png": "https://github.com/user-attachments/assets/12345678-1234-1234-1234-123456789abc"}
+    assert calls[0][0] == ["gh", "image", "--repo", "NousResearch/hermes-agent", str(shot)]
+    assert calls[0][1]["env"]["GH_SESSION_TOKEN"] == "bot-session-token"
+
+
+
+
+def test_upload_evidence_reports_gh_image_error(tmp_path, monkeypatch, capsys):
+    shot = tmp_path / "shot.png"
+    shot.write_bytes(_png())
+
+    def fake_run(args, **kwargs):
+        raise _mod.subprocess.CalledProcessError(
+            1,
+            args,
+            output="upload output",
+            stderr="upload error",
+        )
+
+    monkeypatch.setattr(_mod.subprocess, "run", fake_run)
+
+    with pytest.raises(RuntimeError, match="Failed to upload shot.png.*upload error"):
+        _mod.upload_evidence(
+            [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
+            tmp_path,
+            "NousResearch/hermes-agent",
+            "bot-session-token",
+        )
+
+    captured = capsys.readouterr()
+    assert "Failed to upload shot.png" in captured.err
+    assert "upload output" in captured.err
+    assert "upload error" in captured.err
+
+
+def test_publish_marks_evidence_upload_failure_in_pr_comment(tmp_path, monkeypatch):
+    comment = {
+        "id": 123,
+        "body": "before\n<!-- hermes-e2e-evidence:start -->\npending\n<!-- hermes-e2e-evidence:end -->\nafter",
+    }
+    updates = []
+
+    monkeypatch.setattr(
+        _mod,
+        "load_evidence",
+        lambda evidence_dir: (
+            [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
+            {},
+        ),
+    )
+    monkeypatch.setattr(_mod, "_wait_for_review_comment", lambda *args: comment)
+    monkeypatch.setattr(
+        _mod,
+        "upload_evidence",
+        lambda *args: (_ for _ in ()).throw(
+            RuntimeError("Failed to upload shot.png: bad <response>")
+        ),
+    )
+    monkeypatch.setattr(
+        _mod,
+        "_api_request",
+        lambda url, token, method, payload: updates.append((
+            url,
+            token,
+            method,
+            payload,
+        )),
+    )
+
+    with pytest.raises(RuntimeError, match="Failed to upload shot.png"):
+        _mod.publish(
+            "github-token",
+            "NousResearch/hermes-agent",
+            tmp_path,
+            "69868",
+            "image-token",
+        )
+
+    assert updates == [
+        (
+            "https://api.github.com/repos/NousResearch/hermes-agent/issues/comments/123",
+            "github-token",
+            "PATCH",
+            {
+                "body": "before\n<!-- hermes-e2e-evidence:start -->\n<sub>inline evidence upload failed.</sub>\n\n<pre>Failed to upload shot.png: bad &lt;response&gt;</pre>\n<!-- hermes-e2e-evidence:end -->\nafter"
+            },
+        )
+    ]
+
+
+def test_publish_skips_when_no_review_comment_exists(tmp_path, monkeypatch, capsys):
+    monkeypatch.setattr(
+        _mod,
+        "load_evidence",
+        lambda evidence_dir: (
+            [_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
+            {},
+        ),
+    )
+    monkeypatch.setattr(_mod, "_wait_for_review_comment", lambda *args: None)
+    monkeypatch.setattr(
+        _mod,
+        "upload_evidence",
+        lambda *args: (_ for _ in ()).throw(AssertionError("must not upload")),
+    )
+
+    assert _mod.publish(
+        "github-token",
+        "NousResearch/hermes-agent",
+        tmp_path,
+        "83202",
+        "image-token",
+    ) is False
+    assert "no CI review comment" in capsys.readouterr().out
+
+
+def test_find_review_comment_requires_the_evidence_marker():
+    pending = "<!-- hermes-ci-review-bot -->\n<!-- hermes-e2e-evidence:start -->\npending\n<!-- hermes-e2e-evidence:end -->"
+
+    assert _mod._find_review_comment([{"body": "<!-- hermes-ci-review-bot --> no evidence"}]) is None
+    assert _mod._find_review_comment([{"body": pending, "id": 123}]) == {"body": pending, "id": 123}
+
+
+def test_replace_evidence_marker_requires_exactly_one_marker():
+    with pytest.raises(ValueError, match="does not contain one"):
+        _mod.replace_evidence_marker("no marker", "evidence")