Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
# Crash/resume persistence conformance cells
|
||||
|
||||
Phase 1 of the machine-checked conformance suite proposed in #80921, following
|
||||
the contract framing of "Resume Means Resume" (arXiv:2608.03836): each cell is
|
||||
a deterministic, LLM-free probe of one persistence contract clause, run against
|
||||
the **real `SessionDB`** with a real `SIGKILL` delivered to a separate OS
|
||||
process mid-write.
|
||||
|
||||
## Cells
|
||||
|
||||
| cell | contract clause | origin |
|
||||
|---|---|---|
|
||||
| 1 — `test_cell1_prefix_durability` | acknowledged appends survive a hard crash; contiguous prefix; deterministic recovery | adapted from the tracking issue's spot-probe (29.5K-message original, scaled to a ≥200-append kill window with identical assertions) |
|
||||
| 2 — `test_cell2_consume_once` | a parked handoff is claimed by exactly one of N racing processes | adapted from the tracking issue's spot-probe (8-process file-barrier race) |
|
||||
| 3 — `test_cell3_rotation_atomicity` | a compression rotation is visible entirely or not at all — never a compression-ended parent without a continuation (the #80337 orphan shape; recovery for the legacy population merged in #80487) | new in this suite |
|
||||
| 4 — fork determinism on edit/rewind | recovery yields exactly the chosen prefix after a fork | **stub** — interlocked with the rewind/archive redesign (#82956–#82959) |
|
||||
| 5 — delivery-outbox effect exactly-once | crash between provider send and durable record must not double-deliver on catch-up | **stub** — needs a fake-transport seam; cron delivery scope in flight (#83197/#83557) |
|
||||
|
||||
## Method
|
||||
|
||||
- Real `SessionDB(db_path=...)` in an isolated `tmp_path`; no mocks on the
|
||||
persistence layer.
|
||||
- Crashes are real `SIGKILL`s to a separate interpreter, asserted to be
|
||||
**alive at kill time** (a clean early exit cannot masquerade as a crash
|
||||
test); acknowledgement journals tolerate a torn final line (the kill can
|
||||
interrupt the journal write itself).
|
||||
- Every wait is deadline-bounded; coordination uses file barriers, never
|
||||
sleeps-for-correctness.
|
||||
- Journal-mode matrix (cells 1 and 3): the resolver's default, explicit
|
||||
`DELETE`, and explicit `WAL` — each leg steers the child's own resolver
|
||||
via an isolated `HERMES_HOME` config, then **audits the on-disk mode after
|
||||
the run** and skips when the environment didn't honor the request (e.g.
|
||||
the resolver's WAL-reset downgrade gate, the tracking issue's 3.50.4
|
||||
caveat). A leg that ran in a different mode never counts as evidence for
|
||||
the advertised one. Cell 2 runs on the resolver's default only (the
|
||||
consume-once property is journal-mode-independent: it rests on a single
|
||||
predicated UPDATE).
|
||||
|
||||
## Semantics
|
||||
|
||||
These are **conformance** cells: they are expected GREEN on main (cells 1–2
|
||||
reproduce the tracking issue's passing probes; cell 3 pins the atomicity the
|
||||
#80337 forensics established). A failing cell is a **fire**: report it on
|
||||
#80921 with the cell's evidence — do not silence it, and do not attach a fix
|
||||
to this suite.
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Shared harness for the crash/resume persistence conformance cells (#80921).
|
||||
|
||||
Method (per the spot-probes in the tracking issue): real ``SessionDB``
|
||||
against an isolated temp database, real ``SIGKILL`` delivered to a separate
|
||||
OS process mid-write, deterministic and LLM-free. Every wait has a hard
|
||||
deadline so a wedged child can never hang the suite; coordination uses file
|
||||
barriers, never bare sleeps.
|
||||
|
||||
Journal-mode policy mirrors the issue's caveat: cells run on the mode the
|
||||
repo's own ``resolve_journal_mode()`` selects for this interpreter/filesystem
|
||||
(recorded per cell), plus an explicit ``DELETE`` run; an explicit ``WAL`` run
|
||||
is attempted and skipped when the resolver's downgrade gates trip (e.g. the
|
||||
WAL-reset interpreter bug), so WAL semantics are probed exactly where they
|
||||
are actually deployable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
# Generous deadlines: xdist-loaded CI boxes stall; correctness never depends
|
||||
# on these being tight, they only bound a hung child.
|
||||
CHILD_DEADLINE = 60.0
|
||||
POLL_INTERVAL = 0.02
|
||||
|
||||
|
||||
def spawn_child(script_body: str, *, cwd: Path | None = None, env: dict | None = None) -> subprocess.Popen:
|
||||
"""Run ``script_body`` in a fresh interpreter with the repo importable."""
|
||||
child_env = dict(os.environ)
|
||||
# Prepend, don't clobber: CI images may rely on an inherited PYTHONPATH
|
||||
# for dependencies — losing it would fail child imports while the parent
|
||||
# collects fine, surfacing only as an opaque wait_for deadline.
|
||||
inherited = child_env.get("PYTHONPATH")
|
||||
child_env["PYTHONPATH"] = (
|
||||
f"{REPO_ROOT}{os.pathsep}{inherited}" if inherited else str(REPO_ROOT)
|
||||
)
|
||||
child_env["PYTHONUNBUFFERED"] = "1"
|
||||
if env:
|
||||
child_env.update(env)
|
||||
return subprocess.Popen(
|
||||
[sys.executable, "-c", script_body],
|
||||
cwd=str(cwd or REPO_ROOT),
|
||||
env=child_env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
|
||||
def wait_for(
|
||||
predicate,
|
||||
*,
|
||||
deadline: float = CHILD_DEADLINE,
|
||||
what: str = "condition",
|
||||
child: "subprocess.Popen | None" = None,
|
||||
) -> None:
|
||||
"""Poll ``predicate`` until true or fail loudly at the deadline.
|
||||
|
||||
When ``child`` is given and it exits before the predicate turns true,
|
||||
fail IMMEDIATELY with its captured stderr — a crashed writer must be an
|
||||
instant diagnostic, not a 60s opaque deadline.
|
||||
"""
|
||||
end = time.monotonic() + deadline
|
||||
while time.monotonic() < end:
|
||||
if predicate():
|
||||
return
|
||||
if child is not None and child.poll() is not None:
|
||||
err = b""
|
||||
if child.stderr is not None:
|
||||
err = child.stderr.read() or b""
|
||||
raise AssertionError(
|
||||
f"child exited early (rc={child.returncode}) while waiting "
|
||||
f"for {what}; stderr:\n{err.decode(errors='replace')[-2000:]}"
|
||||
)
|
||||
time.sleep(POLL_INTERVAL)
|
||||
raise AssertionError(f"deadline ({deadline}s) waiting for {what}")
|
||||
|
||||
|
||||
def kill9_and_reap(proc: subprocess.Popen, *, deadline: float = CHILD_DEADLINE) -> None:
|
||||
"""SIGKILL ``proc`` and reap it within ``deadline``."""
|
||||
try:
|
||||
proc.kill() # SIGKILL on POSIX
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
proc.wait(timeout=deadline)
|
||||
|
||||
|
||||
def reap(proc: subprocess.Popen, *, deadline: float = CHILD_DEADLINE) -> tuple[int, str, str]:
|
||||
"""Wait for a child to exit on its own; kill + fail if it doesn't."""
|
||||
try:
|
||||
out, err = proc.communicate(timeout=deadline)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
out, err = proc.communicate(timeout=10)
|
||||
raise AssertionError(
|
||||
f"child did not exit within {deadline}s; stderr:\n"
|
||||
f"{err.decode(errors='replace')[-2000:]}"
|
||||
)
|
||||
return proc.returncode, out.decode(errors="replace"), err.decode(errors="replace")
|
||||
|
||||
|
||||
def on_disk_journal_mode(db_path: Path) -> str:
|
||||
"""Record the journal mode actually in effect for a cell result."""
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
row = conn.execute("PRAGMA journal_mode").fetchone()
|
||||
return str(row[0]) if row else "unknown"
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def integrity_ok(db_path: Path) -> bool:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
row = conn.execute("PRAGMA integrity_check").fetchone()
|
||||
return bool(row) and str(row[0]).lower() == "ok"
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def make_hermes_home(base: Path, journal_mode: str) -> Path:
|
||||
"""Create an isolated HERMES_HOME whose config pins ``database.journal_mode``.
|
||||
|
||||
The journal-mode matrix legs must steer the CHILD's own resolver:
|
||||
pre-seeding the DB file alone is not enough, because ``SessionDB.__init__``
|
||||
runs ``apply_wal_with_fallback()`` which upgrades a non-WAL file to WAL
|
||||
whenever the configured mode (default ``wal``) says so — on healthy
|
||||
SQLite the DELETE leg would silently run in WAL.
|
||||
"""
|
||||
home = base / f"hermes-home-{journal_mode}"
|
||||
home.mkdir(parents=True, exist_ok=True)
|
||||
(home / "config.yaml").write_text(
|
||||
f"database:\n journal_mode: {journal_mode}\n", encoding="utf-8"
|
||||
)
|
||||
return home
|
||||
|
||||
|
||||
def effective_mode_or_skip(db_path: Path, requested_mode: str | None) -> str:
|
||||
"""Record the on-disk journal mode; skip if a requested leg wasn't honored.
|
||||
|
||||
A leg that ran in a different mode than advertised must never count as
|
||||
green evidence for the advertised mode.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
mode = on_disk_journal_mode(db_path)
|
||||
if requested_mode is not None and mode.upper() != requested_mode.upper():
|
||||
pytest.skip(
|
||||
f"journal_mode={requested_mode} not honored by this environment "
|
||||
f"(effective {mode}) — matrix leg not probeable here"
|
||||
)
|
||||
return mode
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Cell 1 — prefix continuation + recovery determinism under real SIGKILL.
|
||||
|
||||
Contract clause (arXiv:2608.03836): every append the store *acknowledged*
|
||||
survives a hard crash; recovery yields a contiguous prefix (no holes, no
|
||||
duplicates); two independent recovery passes see the identical transcript.
|
||||
|
||||
Adapted from the spot-probe in the tracking issue (#80921), which ran ~29.5K
|
||||
messages to a SIGKILL with zero lost-after-return. Scaled down for CI: the
|
||||
parent kills the writer once >= 200 acknowledged appends are journaled — the
|
||||
property assertions are identical, only the exposure window is shorter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conformance.persistence._harness import (
|
||||
effective_mode_or_skip,
|
||||
integrity_ok,
|
||||
kill9_and_reap,
|
||||
make_hermes_home,
|
||||
spawn_child,
|
||||
wait_for,
|
||||
)
|
||||
|
||||
WRITER = r"""
|
||||
import json, sys
|
||||
from pathlib import Path
|
||||
from hermes_state import SessionDB
|
||||
|
||||
db_path = Path({db_path!r})
|
||||
journal = Path({journal!r})
|
||||
db = SessionDB(db_path=db_path)
|
||||
db.create_session("cell1", source="conformance")
|
||||
i = 0
|
||||
with journal.open("a", buffering=1) as j:
|
||||
while True:
|
||||
rowid = db.append_message("cell1", "user", content=f"m{{i}}")
|
||||
# fsync-journal the acknowledged index AFTER append returns —
|
||||
# exactly the probe's definition of "claimed durable".
|
||||
j.write(json.dumps({{"i": i, "rowid": rowid}}) + "\n")
|
||||
j.flush()
|
||||
i += 1
|
||||
"""
|
||||
|
||||
|
||||
def _acknowledged(journal: Path) -> list[dict]:
|
||||
if not journal.exists():
|
||||
return []
|
||||
out = []
|
||||
for line in journal.read_text().splitlines():
|
||||
try:
|
||||
out.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
# A torn final line is expected under SIGKILL: the write to the
|
||||
# journal itself was interrupted. That index was never fully
|
||||
# acknowledged to the harness, so it is out of scope.
|
||||
continue
|
||||
return out
|
||||
|
||||
|
||||
def _recover(db_path: Path) -> list[tuple]:
|
||||
"""One independent recovery pass in a fresh connection/process context."""
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
return conn.execute(
|
||||
"SELECT id, content FROM messages WHERE session_id='cell1' ORDER BY id"
|
||||
).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requested_mode", [None, "DELETE", "WAL"])
|
||||
def test_acknowledged_appends_survive_sigkill(tmp_path, requested_mode):
|
||||
db_path = tmp_path / "state.db"
|
||||
journal = tmp_path / "acked.jsonl"
|
||||
|
||||
child_env = {}
|
||||
if requested_mode is not None:
|
||||
# Steer the CHILD's own resolver via an isolated HERMES_HOME config:
|
||||
# pre-seeding the file alone is not enough — SessionDB.__init__ runs
|
||||
# apply_wal_with_fallback(), which upgrades a non-WAL file to WAL
|
||||
# whenever the configured mode says so (the resolver's downgrade
|
||||
# gates may still refuse; effective_mode_or_skip audits post-run).
|
||||
child_env["HERMES_HOME"] = str(
|
||||
make_hermes_home(tmp_path, requested_mode.lower())
|
||||
)
|
||||
|
||||
child = spawn_child(
|
||||
WRITER.format(db_path=str(db_path), journal=str(journal)), env=child_env
|
||||
)
|
||||
try:
|
||||
wait_for(
|
||||
lambda: len(_acknowledged(journal)) >= 200,
|
||||
what=">=200 acknowledged appends",
|
||||
child=child,
|
||||
)
|
||||
# The kill must interrupt a LIVE writer — a child that already
|
||||
# exited would turn this into a clean-shutdown test.
|
||||
assert child.poll() is None, (
|
||||
f"writer exited early (rc={child.returncode}): "
|
||||
f"{(child.stderr.read() if child.stderr else b'').decode(errors='replace')[-1500:]}"
|
||||
)
|
||||
finally:
|
||||
kill9_and_reap(child)
|
||||
|
||||
# A leg that ran in a different mode than requested is no evidence for
|
||||
# the requested mode — skip it rather than silently double-counting WAL.
|
||||
mode = effective_mode_or_skip(db_path, requested_mode)
|
||||
acked = _acknowledged(journal)
|
||||
assert len(acked) >= 200, (
|
||||
f"[journal_mode={mode}] only {len(acked)} acknowledged appends "
|
||||
"journaled before the kill — harness window too small"
|
||||
)
|
||||
|
||||
pass1 = _recover(db_path)
|
||||
pass2 = _recover(db_path)
|
||||
|
||||
# Zero lost-after-return: every acknowledged index is present.
|
||||
recovered_contents = {row[1] for row in pass1}
|
||||
lost = [a for a in acked if f"m{a['i']}" not in recovered_contents]
|
||||
assert not lost, (
|
||||
f"[journal_mode={mode}] {len(lost)} acknowledged appends lost after "
|
||||
f"SIGKILL (first: {lost[:3]})"
|
||||
)
|
||||
|
||||
# Contiguous prefix: indices 0..N-1 with no holes and no duplicates.
|
||||
indices = sorted(int(row[1][1:]) for row in pass1)
|
||||
assert indices == list(range(len(indices))), (
|
||||
f"[journal_mode={mode}] recovered transcript is not a contiguous "
|
||||
"prefix (holes or duplicates present)"
|
||||
)
|
||||
|
||||
# Recovery determinism: two independent passes identical.
|
||||
assert pass1 == pass2, f"[journal_mode={mode}] recovery passes diverge"
|
||||
|
||||
assert integrity_ok(db_path), f"[journal_mode={mode}] integrity_check failed"
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Cell 2 — consume-once under cross-process concurrent delivery.
|
||||
|
||||
Contract clause (arXiv:2608.03836): a parked handoff is claimed by exactly
|
||||
one consumer, no matter how many independent processes race for it. This is
|
||||
the cell the paper found failing at saturation 1.0 in 36/40 cells across
|
||||
deployed frameworks; Hermes' ``claim_handoff`` ships the paper's own repair
|
||||
shape (single UPDATE with a state predicate, rowcount-checked), which the
|
||||
tracking-issue probe confirmed. This cell pins it permanently.
|
||||
|
||||
8 independent OS processes are released from a file barrier simultaneously;
|
||||
each calls ``SessionDB.claim_handoff()`` on the same pending session.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tests.conformance.persistence._harness import (
|
||||
on_disk_journal_mode,
|
||||
reap,
|
||||
spawn_child,
|
||||
wait_for,
|
||||
)
|
||||
|
||||
CLAIMANT = r"""
|
||||
import sys, time
|
||||
from pathlib import Path
|
||||
from hermes_state import SessionDB
|
||||
|
||||
db_path = Path({db_path!r})
|
||||
barrier = Path({barrier!r})
|
||||
ready = Path({ready!r})
|
||||
|
||||
ready.touch()
|
||||
deadline = time.monotonic() + 55
|
||||
while not barrier.exists():
|
||||
if time.monotonic() > deadline:
|
||||
sys.exit(3)
|
||||
time.sleep(0.005)
|
||||
|
||||
db = SessionDB(db_path=db_path)
|
||||
won = db.claim_handoff("cell2")
|
||||
# Disjoint codes: 0=won, 10=lost. An unhandled exception exits 1, which must
|
||||
# NEVER be confusable with a clean "lost the claim" — a run where one
|
||||
# claimant wins and seven CRASH is not a consume-once proof.
|
||||
sys.exit(0 if won else 10)
|
||||
"""
|
||||
|
||||
N_CLAIMANTS = 8
|
||||
|
||||
|
||||
def test_exactly_one_claimant_wins(tmp_path):
|
||||
from hermes_state import SessionDB
|
||||
|
||||
db_path = tmp_path / "state.db"
|
||||
barrier = tmp_path / "go"
|
||||
|
||||
db = SessionDB(db_path=db_path)
|
||||
db.create_session("cell2", source="conformance")
|
||||
assert db.request_handoff("cell2", "telegram") is not False
|
||||
|
||||
children = []
|
||||
ready_files = []
|
||||
for i in range(N_CLAIMANTS):
|
||||
ready = tmp_path / f"ready-{i}"
|
||||
ready_files.append(ready)
|
||||
children.append(
|
||||
spawn_child(
|
||||
CLAIMANT.format(
|
||||
db_path=str(db_path), barrier=str(barrier), ready=str(ready)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# Barrier: release only when every process is up and polling.
|
||||
wait_for(
|
||||
lambda: all(r.exists() for r in ready_files),
|
||||
what="all claimants at the barrier",
|
||||
)
|
||||
barrier.touch()
|
||||
|
||||
results = [reap(c) for c in children]
|
||||
mode = on_disk_journal_mode(db_path)
|
||||
|
||||
codes = [rc for rc, _, _ in results]
|
||||
assert all(rc in (0, 10) for rc in codes), (
|
||||
f"[journal_mode={mode}] claimant crashed/timed out: {codes}; "
|
||||
f"stderr: {[e[-300:] for _, _, e in results if e]}"
|
||||
)
|
||||
winners = codes.count(0)
|
||||
assert winners == 1, (
|
||||
f"[journal_mode={mode}] consume-once violated: {winners} of "
|
||||
f"{N_CLAIMANTS} concurrent claimants won"
|
||||
)
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Cell 3 — crash mid-compression-rotation: the atomic-publication contract.
|
||||
|
||||
Contract clause: a compression rotation (close parent + publish continuation
|
||||
child + write compacted handoff) is visible **entirely or not at all**. A hard
|
||||
crash at any point must never yield an orphan — a parent ended with
|
||||
``end_reason='compression'`` and no continuation child — because that is the
|
||||
state every reader treats as unreachable (the P1 in #80337; the recovery
|
||||
merged in #80487 exists to drain the pre-atomicity population of exactly
|
||||
these rows).
|
||||
|
||||
The forensics on #80337 established the contract holds architecturally on
|
||||
current main (``publish_compression_child`` runs parent-close + child-insert
|
||||
+ handoff in one transaction on one handle). This cell pins it empirically:
|
||||
a writer process performs rotations in a tight loop through the REAL leased
|
||||
path (``try_acquire_compression_lock`` → ``publish_compression_child``) and
|
||||
is SIGKILLed mid-loop; recovery then audits every rotation for atomicity.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conformance.persistence._harness import (
|
||||
effective_mode_or_skip,
|
||||
integrity_ok,
|
||||
kill9_and_reap,
|
||||
make_hermes_home,
|
||||
spawn_child,
|
||||
wait_for,
|
||||
)
|
||||
|
||||
ROTATOR = r"""
|
||||
import json, sys
|
||||
from pathlib import Path
|
||||
from hermes_state import SessionDB
|
||||
|
||||
db_path = Path({db_path!r})
|
||||
journal = Path({journal!r})
|
||||
db = SessionDB(db_path=db_path)
|
||||
|
||||
i = 0
|
||||
with journal.open("a", buffering=1) as j:
|
||||
while True:
|
||||
parent = f"parent-{{i}}"
|
||||
child = f"child-{{i}}"
|
||||
db.create_session(parent, source="conformance")
|
||||
db.append_message(parent, "user", content="pre-rotation turn")
|
||||
holder = f"cell3:pid={{__import__('os').getpid()}}"
|
||||
assert db.try_acquire_compression_lock(parent, holder)
|
||||
db.publish_compression_child(
|
||||
parent_session_id=parent,
|
||||
child_session_id=child,
|
||||
source="conformance",
|
||||
messages=[{{"role": "user", "content": "compacted handoff"}}],
|
||||
compression_lock_holder=holder,
|
||||
)
|
||||
j.write(json.dumps({{"i": i}}) + "\n")
|
||||
j.flush()
|
||||
i += 1
|
||||
"""
|
||||
|
||||
|
||||
def _completed_rotations(journal: Path) -> int:
|
||||
if not journal.exists():
|
||||
return 0
|
||||
n = 0
|
||||
for line in journal.read_text().splitlines():
|
||||
try:
|
||||
json.loads(line)
|
||||
n += 1
|
||||
except json.JSONDecodeError:
|
||||
continue # torn final line under SIGKILL — unacknowledged
|
||||
return n
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requested_mode", [None, "DELETE", "WAL"])
|
||||
def test_rotation_is_atomic_under_sigkill(tmp_path, requested_mode):
|
||||
db_path = tmp_path / "state.db"
|
||||
journal = tmp_path / "rotations.jsonl"
|
||||
|
||||
child_env = {}
|
||||
if requested_mode is not None:
|
||||
# Steer the CHILD's own resolver via an isolated HERMES_HOME config:
|
||||
# pre-seeding the file alone is not enough — SessionDB.__init__ runs
|
||||
# apply_wal_with_fallback(), which upgrades a non-WAL file to WAL
|
||||
# whenever the configured mode says so (the resolver's downgrade
|
||||
# gates may still refuse; effective_mode_or_skip audits post-run).
|
||||
child_env["HERMES_HOME"] = str(
|
||||
make_hermes_home(tmp_path, requested_mode.lower())
|
||||
)
|
||||
|
||||
child = spawn_child(
|
||||
ROTATOR.format(db_path=str(db_path), journal=str(journal)), env=child_env
|
||||
)
|
||||
try:
|
||||
wait_for(
|
||||
lambda: _completed_rotations(journal) >= 50,
|
||||
what=">=50 completed rotations",
|
||||
child=child,
|
||||
)
|
||||
assert child.poll() is None, (
|
||||
f"rotator exited early (rc={child.returncode}): "
|
||||
f"{(child.stderr.read() if child.stderr else b'').decode(errors='replace')[-1500:]}"
|
||||
)
|
||||
finally:
|
||||
kill9_and_reap(child)
|
||||
|
||||
# A leg that ran in a different mode than requested is no evidence for
|
||||
# the requested mode — skip it rather than silently double-counting WAL.
|
||||
mode = effective_mode_or_skip(db_path, requested_mode)
|
||||
acked = _completed_rotations(journal)
|
||||
assert acked >= 50, (
|
||||
f"[journal_mode={mode}] only {acked} rotations acknowledged before "
|
||||
"the kill — harness window too small"
|
||||
)
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
# THE contract: no compression-ended parent may lack a child row.
|
||||
orphans = conn.execute(
|
||||
"""
|
||||
SELECT p.id FROM sessions p
|
||||
WHERE p.end_reason = 'compression'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM sessions c WHERE c.parent_session_id = p.id
|
||||
)
|
||||
"""
|
||||
).fetchall()
|
||||
assert not orphans, (
|
||||
f"[journal_mode={mode}] atomicity violated: {len(orphans)} "
|
||||
f"compression-ended parent(s) with no continuation "
|
||||
f"({[o[0] for o in orphans[:5]]}) — the #80337 orphan shape"
|
||||
)
|
||||
|
||||
# Completeness the other way: every acknowledged rotation is fully
|
||||
# visible (parent ended AND child present).
|
||||
for i in range(acked):
|
||||
parent_row = conn.execute(
|
||||
"SELECT end_reason FROM sessions WHERE id = ?",
|
||||
(f"parent-{i}",),
|
||||
).fetchone()
|
||||
child_row = conn.execute(
|
||||
"SELECT id FROM sessions WHERE id = ?", (f"child-{i}",)
|
||||
).fetchone()
|
||||
assert parent_row and parent_row[0] == "compression" and child_row, (
|
||||
f"[journal_mode={mode}] acknowledged rotation {i} not fully "
|
||||
f"visible after recovery (parent={parent_row}, child={child_row})"
|
||||
)
|
||||
|
||||
# The interrupted trailing rotation (if any) must be all-or-nothing:
|
||||
# either invisible or complete — checked by the orphan query above.
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
assert integrity_ok(db_path), f"[journal_mode={mode}] integrity_check failed"
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Wave-2 cell stubs — contract clauses named, implementations interlocked.
|
||||
|
||||
These are deliberate ``pytest.skip`` placeholders, not TODOs: each names the
|
||||
exact contract clause it will pin and the reason it is not implemented in the
|
||||
skeleton PR. See the tracking issue (#80921) for the full ~39-cell matrix.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_cell4_fork_determinism_on_rewind_stub():
|
||||
"""Cell 4 — fork determinism on the edit/rewind paths.
|
||||
|
||||
Contract clause: after an edit/rewind fork, recovery yields exactly the
|
||||
chosen prefix (no resurrection of superseded turns, no archived-copy
|
||||
duplication), deterministically across independent recovery passes.
|
||||
|
||||
Deliberately deferred: the rewind/archive semantics this cell would pin
|
||||
are being actively redesigned in #82956–#82959 (durable row-id
|
||||
addressing, archive-mode dedup, /retry archive_drop plumbing). Writing
|
||||
the cell against today's behavior would test a moving target; it lands
|
||||
as that cluster's conformance check once the contract settles.
|
||||
"""
|
||||
pytest.skip(
|
||||
"wave 2: interlocked with the rewind/archive redesign "
|
||||
"(#82956-#82959) — cell lands as that cluster's conformance check"
|
||||
)
|
||||
|
||||
|
||||
def test_cell5_delivery_outbox_exactly_once_stub():
|
||||
"""Cell 5 — effect exactly-once across the delivery outbox boundary.
|
||||
|
||||
Contract clause: a crash between provider send and durable record must
|
||||
not double-deliver on catch-up (cron ticker restart, gateway reboot) —
|
||||
the cell the paper (arXiv:2608.03836) probes as 'effect exactly-once',
|
||||
distinct from cell 2's consume-once (claim vs side-effect).
|
||||
|
||||
Deliberately deferred: requires a deterministic fake-transport seam for
|
||||
the delivery path (the send must be observable without a live adapter);
|
||||
the cron delivery-scope interplay is also in flight (#83197/#83557).
|
||||
"""
|
||||
pytest.skip(
|
||||
"wave 2: needs a deterministic fake-transport seam; cron delivery "
|
||||
"scope fixes in flight (#83197/#83557)"
|
||||
)
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Conformance vector generator tests (Phase 5 oracle workstream).
|
||||
|
||||
Behavior contracts, not change-detectors: determinism, corpus invariants,
|
||||
oracle-import health (the generator calls format_message UNBOUND — these
|
||||
tests fail loudly if a refactor gives those renderers instance state), and
|
||||
vector-file shape. Native outputs themselves are NOT snapshotted here — the
|
||||
connector's conformance runner is the consumer that asserts them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
|
||||
from generate_conformance_vectors import ( # noqa: E402
|
||||
ADVERSARIAL,
|
||||
GRID,
|
||||
SCAR,
|
||||
_oracles,
|
||||
corpus,
|
||||
generate,
|
||||
)
|
||||
|
||||
PLATFORMS = ("discord", "slack", "telegram", "whatsapp")
|
||||
|
||||
|
||||
def test_corpus_ids_unique_and_categorized():
|
||||
rows = corpus()
|
||||
ids = [r["id"] for r in rows]
|
||||
assert len(ids) == len(set(ids)), "vector ids must be unique (runner keys on them)"
|
||||
assert {r["category"] for r in rows} == {"grid", "scar", "adversarial"}
|
||||
assert len(GRID) >= 20 and len(SCAR) >= 10 and len(ADVERSARIAL) >= 8
|
||||
|
||||
|
||||
def test_oracles_importable_and_self_free():
|
||||
"""The generator invokes format_message unbound (self=None) — verify every
|
||||
oracle actually renders under that calling convention. A refactor that
|
||||
adds instance state to a renderer must update the generator too."""
|
||||
oracles = _oracles()
|
||||
assert set(oracles) == set(PLATFORMS)
|
||||
for platform, render in oracles.items():
|
||||
out = render("**bold** and `code`")
|
||||
assert isinstance(out, str) and out, platform
|
||||
|
||||
|
||||
|
||||
|
||||
def test_vector_files_shape(tmp_path):
|
||||
generate(tmp_path)
|
||||
for platform in PLATFORMS:
|
||||
doc = json.loads((tmp_path / f"{platform}.json").read_text(encoding="utf-8"))
|
||||
assert doc["platform"] == platform
|
||||
assert doc["oracle"]["repo"] == "NousResearch/hermes-agent"
|
||||
assert re.match(r"^[0-9a-f]{40}$|^unknown$", doc["oracle"]["commit"])
|
||||
assert doc["oracle"]["generator_version"] >= 1
|
||||
ids = set()
|
||||
for v in doc["vectors"]:
|
||||
assert v["expect"] in ("parity", "semantic", "divergent"), v["id"]
|
||||
assert isinstance(v["native_output"], str)
|
||||
if v["expect"] == "divergent":
|
||||
assert v.get("note"), f"{platform}/{v['id']}: divergent needs a note"
|
||||
ids.add(v["id"])
|
||||
assert len(ids) == len(doc["vectors"])
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def test_committed_vectors_match_regeneration(tmp_path):
|
||||
"""The committed tests/conformance/vectors/ must reproduce from the
|
||||
current oracle — the same lockstep discipline as openapi.json (drift
|
||||
means someone changed a renderer without regenerating)."""
|
||||
committed_dir = REPO_ROOT / "tests" / "conformance" / "vectors"
|
||||
if not committed_dir.exists():
|
||||
pytest.skip("no committed vectors in this checkout")
|
||||
generate(tmp_path)
|
||||
for platform in PLATFORMS:
|
||||
fresh = json.loads((tmp_path / f"{platform}.json").read_text(encoding="utf-8"))
|
||||
committed = json.loads((committed_dir / f"{platform}.json").read_text(encoding="utf-8"))
|
||||
# Compare everything except the commit stamp (committed file may be
|
||||
# one commit behind HEAD in a dirty working tree).
|
||||
fresh["oracle"].pop("commit")
|
||||
committed["oracle"].pop("commit")
|
||||
assert fresh == committed, (
|
||||
f"{platform} vectors drifted — run "
|
||||
"`python scripts/generate_conformance_vectors.py` and commit"
|
||||
)
|
||||
@@ -0,0 +1,322 @@
|
||||
{
|
||||
"$comment": "GENERATED — do not hand-edit. Regenerate with hermes-agent scripts/generate_conformance_vectors.py; the native renderers are the oracle (executable spec).",
|
||||
"oracle": {
|
||||
"repo": "NousResearch/hermes-agent",
|
||||
"commit": "40eebc7d70f3d8e95c429c8aa482f31ba6c867f6",
|
||||
"generator": "scripts/generate_conformance_vectors.py",
|
||||
"generator_version": 1
|
||||
},
|
||||
"platform": "discord",
|
||||
"vectors": [
|
||||
{
|
||||
"id": "plain-text",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Just a plain sentence.",
|
||||
"native_output": "Just a plain sentence."
|
||||
},
|
||||
{
|
||||
"id": "bold",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "This is **bold** text.",
|
||||
"native_output": "This is **bold** text."
|
||||
},
|
||||
{
|
||||
"id": "italic",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "This is *italic* text.",
|
||||
"native_output": "This is *italic* text."
|
||||
},
|
||||
{
|
||||
"id": "bold-italic",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Mix of **bold** and *italic* in one line.",
|
||||
"native_output": "Mix of **bold** and *italic* in one line."
|
||||
},
|
||||
{
|
||||
"id": "strikethrough",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "This is ~~struck~~ text.",
|
||||
"native_output": "This is ~~struck~~ text."
|
||||
},
|
||||
{
|
||||
"id": "inline-code",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Run `pip install hermes` to start.",
|
||||
"native_output": "Run `pip install hermes` to start."
|
||||
},
|
||||
{
|
||||
"id": "fenced-code",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "```\nprint('hello')\n```",
|
||||
"native_output": "```\nprint('hello')\n```"
|
||||
},
|
||||
{
|
||||
"id": "fenced-code-lang",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "```python\ndef f(x):\n return x * 2\n```",
|
||||
"native_output": "```python\ndef f(x):\n return x * 2\n```"
|
||||
},
|
||||
{
|
||||
"id": "link",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "See [the docs](https://example.com/docs) for more.",
|
||||
"native_output": "See [the docs](https://example.com/docs) for more."
|
||||
},
|
||||
{
|
||||
"id": "link-parens-url",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "See [spec](https://example.com/a_(b)) here.",
|
||||
"native_output": "See [spec](https://example.com/a_(b)) here."
|
||||
},
|
||||
{
|
||||
"id": "header-h1",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "# Big Title\nBody follows.",
|
||||
"native_output": "# Big Title\nBody follows."
|
||||
},
|
||||
{
|
||||
"id": "header-h2",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "## Section\nBody follows.",
|
||||
"native_output": "## Section\nBody follows."
|
||||
},
|
||||
{
|
||||
"id": "header-h3",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "### Sub-section\nBody follows.",
|
||||
"native_output": "### Sub-section\nBody follows."
|
||||
},
|
||||
{
|
||||
"id": "ul-list",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "- first\n- second\n- third",
|
||||
"native_output": "- first\n- second\n- third"
|
||||
},
|
||||
{
|
||||
"id": "ol-list",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "1. first\n2. second\n3. third",
|
||||
"native_output": "1. first\n2. second\n3. third"
|
||||
},
|
||||
{
|
||||
"id": "nested-list",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "- outer\n - inner one\n - inner two\n- outer two",
|
||||
"native_output": "- outer\n - inner one\n - inner two\n- outer two"
|
||||
},
|
||||
{
|
||||
"id": "blockquote",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "> quoted wisdom\nregular line",
|
||||
"native_output": "> quoted wisdom\nregular line"
|
||||
},
|
||||
{
|
||||
"id": "hrule",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "above\n\n---\n\nbelow",
|
||||
"native_output": "above\n\n---\n\nbelow"
|
||||
},
|
||||
{
|
||||
"id": "table-simple",
|
||||
"category": "grid",
|
||||
"expect": "divergent",
|
||||
"input": "| name | value |\n|------|-------|\n| a | 1 |\n| b | 2 |",
|
||||
"native_output": "**a**\n• value: 1\n\n**b**\n• value: 2",
|
||||
"note": "native converts GFM tables to bullet groups; connector passes raw markdown through (port deferred — parity report Phase 4/oracle section)"
|
||||
},
|
||||
{
|
||||
"id": "emoji",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Done ✅ with 🎉 emoji 👀 test.",
|
||||
"native_output": "Done ✅ with 🎉 emoji 👀 test."
|
||||
},
|
||||
{
|
||||
"id": "cjk",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "中文测试：**粗体** 和 `代码` 混排。",
|
||||
"native_output": "中文测试：**粗体** 和 `代码` 混排。"
|
||||
},
|
||||
{
|
||||
"id": "bare-url",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Visit https://example.com/path?q=1&r=2 today.",
|
||||
"native_output": "Visit https://example.com/path?q=1&r=2 today."
|
||||
},
|
||||
{
|
||||
"id": "mixed-document",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "## Report\n\nStatus: **green**. Details in `runbook.md`.\n\n- item *one*\n- item **two**\n\n```sh\nmake deploy\n```\n\nSee [dashboard](https://grafana.example.com/d/x).",
|
||||
"native_output": "## Report\n\nStatus: **green**. Details in `runbook.md`.\n\n- item *one*\n- item **two**\n\n```sh\nmake deploy\n```\n\nSee [dashboard](https://grafana.example.com/d/x)."
|
||||
},
|
||||
{
|
||||
"id": "mdv2-reserved-chars",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "Price is 3.50 (was 4.00) — save ~12%! #deal +tax = win.",
|
||||
"native_output": "Price is 3.50 (was 4.00) — save ~12%! #deal +tax = win."
|
||||
},
|
||||
{
|
||||
"id": "mdv2-underscores",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "snake_case_name and file_name.py in prose.",
|
||||
"native_output": "snake_case_name and file_name.py in prose."
|
||||
},
|
||||
{
|
||||
"id": "mdv2-brackets",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "Array[0] and dict{key} and (parens) live here.",
|
||||
"native_output": "Array[0] and dict{key} and (parens) live here."
|
||||
},
|
||||
{
|
||||
"id": "slack-bold-conversion",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "**important** word",
|
||||
"native_output": "**important** word"
|
||||
},
|
||||
{
|
||||
"id": "slack-link-conversion",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "[click here](https://example.com)",
|
||||
"native_output": "[click here](https://example.com)"
|
||||
},
|
||||
{
|
||||
"id": "slack-broadcast-mention",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "Hey <!everyone> and <!channel> and <!here>!",
|
||||
"native_output": "Hey <!everyone> and <!channel> and <!here>!"
|
||||
},
|
||||
{
|
||||
"id": "fence-lang-tag-slack",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "```text\nliteral first line issue\n```",
|
||||
"native_output": "```text\nliteral first line issue\n```"
|
||||
},
|
||||
{
|
||||
"id": "backslash-in-code",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "`C:\\Users\\ben\\file.txt` and ```\npath = \"a\\\\b\"\n```",
|
||||
"native_output": "`C:\\Users\\ben\\file.txt` and ```\npath = \"a\\\\b\"\n```"
|
||||
},
|
||||
{
|
||||
"id": "backtick-in-fence",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "```\nuse `inline` inside fence\n```",
|
||||
"native_output": "```\nuse `inline` inside fence\n```"
|
||||
},
|
||||
{
|
||||
"id": "header-with-bold",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "## The **Real** Deal",
|
||||
"native_output": "## The **Real** Deal"
|
||||
},
|
||||
{
|
||||
"id": "table-cjk",
|
||||
"category": "scar",
|
||||
"expect": "divergent",
|
||||
"input": "| 名前 | 値 |\n|------|----|\n| 中文 | 42 |\n| b | 2 |",
|
||||
"native_output": "**中文**\n• 値: 42\n\n**b**\n• 値: 2",
|
||||
"note": "same as table-simple"
|
||||
},
|
||||
{
|
||||
"id": "link-display-escapes",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "[v2.0 (beta)](https://example.com/v2)",
|
||||
"native_output": "[v2.0 (beta)](https://example.com/v2)"
|
||||
},
|
||||
{
|
||||
"id": "media-tag",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "Here you go\nMEDIA:/tmp/output.png\ndone",
|
||||
"native_output": "Here you go\nMEDIA:/tmp/output.png\ndone"
|
||||
},
|
||||
{
|
||||
"id": "unclosed-fence",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "```python\nprint('never closed')",
|
||||
"native_output": "```python\nprint('never closed')"
|
||||
},
|
||||
{
|
||||
"id": "pathological-nesting",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "**bold *italic ~~struck `code` struck~~ italic* bold**",
|
||||
"native_output": "**bold *italic ~~struck `code` struck~~ italic* bold**"
|
||||
},
|
||||
{
|
||||
"id": "placeholder-injection",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "sneaky \u0000PH0\u0000 token and \u0000SL1\u0000 too",
|
||||
"native_output": "sneaky \u0000PH0\u0000 token and \u0000SL1\u0000 too"
|
||||
},
|
||||
{
|
||||
"id": "triple-markers",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "***what is this*** and ____that____",
|
||||
"native_output": "***what is this*** and ____that____"
|
||||
},
|
||||
{
|
||||
"id": "empty-string",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "",
|
||||
"native_output": ""
|
||||
},
|
||||
{
|
||||
"id": "whitespace-only",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": " \n\t\n ",
|
||||
"native_output": " \n\t\n "
|
||||
},
|
||||
{
|
||||
"id": "long-line",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word ",
|
||||
"native_output": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word "
|
||||
},
|
||||
{
|
||||
"id": "many-fences",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "```\na\n```\nmid\n```\nb\n```\nend ```inline``` tail",
|
||||
"native_output": "```\na\n```\nmid\n```\nb\n```\nend ```inline``` tail"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
{
|
||||
"$comment": "GENERATED — do not hand-edit. Regenerate with hermes-agent scripts/generate_conformance_vectors.py; the native renderers are the oracle (executable spec).",
|
||||
"oracle": {
|
||||
"repo": "NousResearch/hermes-agent",
|
||||
"commit": "40eebc7d70f3d8e95c429c8aa482f31ba6c867f6",
|
||||
"generator": "scripts/generate_conformance_vectors.py",
|
||||
"generator_version": 1
|
||||
},
|
||||
"platform": "slack",
|
||||
"vectors": [
|
||||
{
|
||||
"id": "plain-text",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Just a plain sentence.",
|
||||
"native_output": "Just a plain sentence."
|
||||
},
|
||||
{
|
||||
"id": "bold",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "This is **bold** text.",
|
||||
"native_output": "This is *bold* text."
|
||||
},
|
||||
{
|
||||
"id": "italic",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "This is *italic* text.",
|
||||
"native_output": "This is _italic_ text."
|
||||
},
|
||||
{
|
||||
"id": "bold-italic",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Mix of **bold** and *italic* in one line.",
|
||||
"native_output": "Mix of *bold* and _italic_ in one line."
|
||||
},
|
||||
{
|
||||
"id": "strikethrough",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "This is ~~struck~~ text.",
|
||||
"native_output": "This is ~struck~ text."
|
||||
},
|
||||
{
|
||||
"id": "inline-code",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Run `pip install hermes` to start.",
|
||||
"native_output": "Run `pip install hermes` to start."
|
||||
},
|
||||
{
|
||||
"id": "fenced-code",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "```\nprint('hello')\n```",
|
||||
"native_output": "```\nprint('hello')\n```"
|
||||
},
|
||||
{
|
||||
"id": "fenced-code-lang",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "```python\ndef f(x):\n return x * 2\n```",
|
||||
"native_output": "```\ndef f(x):\n return x * 2\n```"
|
||||
},
|
||||
{
|
||||
"id": "link",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "See [the docs](https://example.com/docs) for more.",
|
||||
"native_output": "See <https://example.com/docs|the docs> for more."
|
||||
},
|
||||
{
|
||||
"id": "link-parens-url",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "See [spec](https://example.com/a_(b)) here.",
|
||||
"native_output": "See <https://example.com/a_(b)|spec> here."
|
||||
},
|
||||
{
|
||||
"id": "header-h1",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "# Big Title\nBody follows.",
|
||||
"native_output": "*Big Title*\nBody follows."
|
||||
},
|
||||
{
|
||||
"id": "header-h2",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "## Section\nBody follows.",
|
||||
"native_output": "*Section*\nBody follows."
|
||||
},
|
||||
{
|
||||
"id": "header-h3",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "### Sub-section\nBody follows.",
|
||||
"native_output": "*Sub-section*\nBody follows."
|
||||
},
|
||||
{
|
||||
"id": "ul-list",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "- first\n- second\n- third",
|
||||
"native_output": "- first\n- second\n- third"
|
||||
},
|
||||
{
|
||||
"id": "ol-list",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "1. first\n2. second\n3. third",
|
||||
"native_output": "1. first\n2. second\n3. third"
|
||||
},
|
||||
{
|
||||
"id": "nested-list",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "- outer\n - inner one\n - inner two\n- outer two",
|
||||
"native_output": "- outer\n - inner one\n - inner two\n- outer two"
|
||||
},
|
||||
{
|
||||
"id": "blockquote",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "> quoted wisdom\nregular line",
|
||||
"native_output": "> quoted wisdom\nregular line"
|
||||
},
|
||||
{
|
||||
"id": "hrule",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "above\n\n---\n\nbelow",
|
||||
"native_output": "above\n\n---\n\nbelow"
|
||||
},
|
||||
{
|
||||
"id": "table-simple",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "| name | value |\n|------|-------|\n| a | 1 |\n| b | 2 |",
|
||||
"native_output": "```\n| name | value |\n| ---- | ----- |\n| a | 1 |\n| b | 2 |\n```"
|
||||
},
|
||||
{
|
||||
"id": "emoji",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Done ✅ with 🎉 emoji 👀 test.",
|
||||
"native_output": "Done ✅ with 🎉 emoji 👀 test."
|
||||
},
|
||||
{
|
||||
"id": "cjk",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "中文测试：**粗体** 和 `代码` 混排。",
|
||||
"native_output": "中文测试：*粗体* 和 `代码` 混排。"
|
||||
},
|
||||
{
|
||||
"id": "bare-url",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Visit https://example.com/path?q=1&r=2 today.",
|
||||
"native_output": "Visit https://example.com/path?q=1&r=2 today."
|
||||
},
|
||||
{
|
||||
"id": "mixed-document",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "## Report\n\nStatus: **green**. Details in `runbook.md`.\n\n- item *one*\n- item **two**\n\n```sh\nmake deploy\n```\n\nSee [dashboard](https://grafana.example.com/d/x).",
|
||||
"native_output": "*Report*\n\nStatus: *green*. Details in `runbook.md`.\n\n- item _one_\n- item *two*\n\n```\nmake deploy\n```\n\nSee <https://grafana.example.com/d/x|dashboard>."
|
||||
},
|
||||
{
|
||||
"id": "mdv2-reserved-chars",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "Price is 3.50 (was 4.00) — save ~12%! #deal +tax = win.",
|
||||
"native_output": "Price is 3.50 (was 4.00) — save ~12%! #deal +tax = win."
|
||||
},
|
||||
{
|
||||
"id": "mdv2-underscores",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "snake_case_name and file_name.py in prose.",
|
||||
"native_output": "snake_case_name and file_name.py in prose."
|
||||
},
|
||||
{
|
||||
"id": "mdv2-brackets",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "Array[0] and dict{key} and (parens) live here.",
|
||||
"native_output": "Array[0] and dict{key} and (parens) live here."
|
||||
},
|
||||
{
|
||||
"id": "slack-bold-conversion",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "**important** word",
|
||||
"native_output": "*important* word"
|
||||
},
|
||||
{
|
||||
"id": "slack-link-conversion",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "[click here](https://example.com)",
|
||||
"native_output": "<https://example.com|click here>"
|
||||
},
|
||||
{
|
||||
"id": "slack-broadcast-mention",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "Hey <!everyone> and <!channel> and <!here>!",
|
||||
"native_output": "Hey <!everyone> and <!channel> and <!here>!"
|
||||
},
|
||||
{
|
||||
"id": "fence-lang-tag-slack",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "```text\nliteral first line issue\n```",
|
||||
"native_output": "```\nliteral first line issue\n```"
|
||||
},
|
||||
{
|
||||
"id": "backslash-in-code",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "`C:\\Users\\ben\\file.txt` and ```\npath = \"a\\\\b\"\n```",
|
||||
"native_output": "`C:\\Users\\ben\\file.txt` and ```\npath = \"a\\\\b\"\n```"
|
||||
},
|
||||
{
|
||||
"id": "backtick-in-fence",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "```\nuse `inline` inside fence\n```",
|
||||
"native_output": "```\nuse `inline` inside fence\n```"
|
||||
},
|
||||
{
|
||||
"id": "header-with-bold",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "## The **Real** Deal",
|
||||
"native_output": "*The Real Deal*"
|
||||
},
|
||||
{
|
||||
"id": "table-cjk",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "| 名前 | 値 |\n|------|----|\n| 中文 | 42 |\n| b | 2 |",
|
||||
"native_output": "```\n| 名前 | 値 |\n| ---- | --- |\n| 中文 | 42 |\n| b | 2 |\n```"
|
||||
},
|
||||
{
|
||||
"id": "link-display-escapes",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "[v2.0 (beta)](https://example.com/v2)",
|
||||
"native_output": "<https://example.com/v2|v2.0 (beta)>"
|
||||
},
|
||||
{
|
||||
"id": "media-tag",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "Here you go\nMEDIA:/tmp/output.png\ndone",
|
||||
"native_output": "Here you go\nMEDIA:/tmp/output.png\ndone"
|
||||
},
|
||||
{
|
||||
"id": "unclosed-fence",
|
||||
"category": "adversarial",
|
||||
"expect": "divergent",
|
||||
"input": "```python\nprint('never closed')",
|
||||
"native_output": "```python\nprint('never closed')",
|
||||
"note": "unterminated fence handling differs; both degrade without dropping content"
|
||||
},
|
||||
{
|
||||
"id": "pathological-nesting",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "**bold *italic ~~struck `code` struck~~ italic* bold**",
|
||||
"native_output": "*bold *italic ~~struck `code` struck~~ italic* bold*"
|
||||
},
|
||||
{
|
||||
"id": "placeholder-injection",
|
||||
"category": "adversarial",
|
||||
"expect": "divergent",
|
||||
"input": "sneaky \u0000PH0\u0000 token and \u0000SL1\u0000 too",
|
||||
"native_output": "sneaky \u0000PH0\u0000 token and \u0000SL1\u0000 too",
|
||||
"note": "\\x00SL tokens are the native renderer's own placeholder alphabet; connector uses a different scheme"
|
||||
},
|
||||
{
|
||||
"id": "triple-markers",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "***what is this*** and ____that____",
|
||||
"native_output": "*_what is this_* and ____that____"
|
||||
},
|
||||
{
|
||||
"id": "empty-string",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "",
|
||||
"native_output": ""
|
||||
},
|
||||
{
|
||||
"id": "whitespace-only",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": " \n\t\n ",
|
||||
"native_output": " \n\t\n "
|
||||
},
|
||||
{
|
||||
"id": "long-line",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word ",
|
||||
"native_output": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word "
|
||||
},
|
||||
{
|
||||
"id": "many-fences",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "```\na\n```\nmid\n```\nb\n```\nend ```inline``` tail",
|
||||
"native_output": "```\na\n```\nmid\n```\nb\n```\nend ```inline``` tail"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
{
|
||||
"$comment": "GENERATED — do not hand-edit. Regenerate with hermes-agent scripts/generate_conformance_vectors.py; the native renderers are the oracle (executable spec).",
|
||||
"oracle": {
|
||||
"repo": "NousResearch/hermes-agent",
|
||||
"commit": "40eebc7d70f3d8e95c429c8aa482f31ba6c867f6",
|
||||
"generator": "scripts/generate_conformance_vectors.py",
|
||||
"generator_version": 1
|
||||
},
|
||||
"platform": "telegram",
|
||||
"vectors": [
|
||||
{
|
||||
"id": "plain-text",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "Just a plain sentence.",
|
||||
"native_output": "Just a plain sentence\\."
|
||||
},
|
||||
{
|
||||
"id": "bold",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "This is **bold** text.",
|
||||
"native_output": "This is *bold* text\\."
|
||||
},
|
||||
{
|
||||
"id": "italic",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "This is *italic* text.",
|
||||
"native_output": "This is _italic_ text\\."
|
||||
},
|
||||
{
|
||||
"id": "bold-italic",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "Mix of **bold** and *italic* in one line.",
|
||||
"native_output": "Mix of *bold* and _italic_ in one line\\."
|
||||
},
|
||||
{
|
||||
"id": "strikethrough",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "This is ~~struck~~ text.",
|
||||
"native_output": "This is ~struck~ text\\."
|
||||
},
|
||||
{
|
||||
"id": "inline-code",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "Run `pip install hermes` to start.",
|
||||
"native_output": "Run `pip install hermes` to start\\."
|
||||
},
|
||||
{
|
||||
"id": "fenced-code",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "```\nprint('hello')\n```",
|
||||
"native_output": "```\nprint('hello')\n```"
|
||||
},
|
||||
{
|
||||
"id": "fenced-code-lang",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "```python\ndef f(x):\n return x * 2\n```",
|
||||
"native_output": "```python\ndef f(x):\n return x * 2\n```"
|
||||
},
|
||||
{
|
||||
"id": "link",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "See [the docs](https://example.com/docs) for more.",
|
||||
"native_output": "See [the docs](https://example.com/docs) for more\\."
|
||||
},
|
||||
{
|
||||
"id": "link-parens-url",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "See [spec](https://example.com/a_(b)) here.",
|
||||
"native_output": "See [spec](https://example.com/a_\\(b\\)) here\\."
|
||||
},
|
||||
{
|
||||
"id": "header-h1",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "# Big Title\nBody follows.",
|
||||
"native_output": "*Big Title*\nBody follows\\."
|
||||
},
|
||||
{
|
||||
"id": "header-h2",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "## Section\nBody follows.",
|
||||
"native_output": "*Section*\nBody follows\\."
|
||||
},
|
||||
{
|
||||
"id": "header-h3",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "### Sub-section\nBody follows.",
|
||||
"native_output": "*Sub\\-section*\nBody follows\\."
|
||||
},
|
||||
{
|
||||
"id": "ul-list",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "- first\n- second\n- third",
|
||||
"native_output": "\\- first\n\\- second\n\\- third"
|
||||
},
|
||||
{
|
||||
"id": "ol-list",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "1. first\n2. second\n3. third",
|
||||
"native_output": "1\\. first\n2\\. second\n3\\. third"
|
||||
},
|
||||
{
|
||||
"id": "nested-list",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "- outer\n - inner one\n - inner two\n- outer two",
|
||||
"native_output": "\\- outer\n \\- inner one\n \\- inner two\n\\- outer two"
|
||||
},
|
||||
{
|
||||
"id": "blockquote",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "> quoted wisdom\nregular line",
|
||||
"native_output": "> quoted wisdom\nregular line"
|
||||
},
|
||||
{
|
||||
"id": "hrule",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "above\n\n---\n\nbelow",
|
||||
"native_output": "above\n\n\\-\\-\\-\n\nbelow"
|
||||
},
|
||||
{
|
||||
"id": "table-simple",
|
||||
"category": "grid",
|
||||
"expect": "divergent",
|
||||
"input": "| name | value |\n|------|-------|\n| a | 1 |\n| b | 2 |",
|
||||
"native_output": "*a*\n• value: 1\n\n*b*\n• value: 2",
|
||||
"note": "native wraps tables into row groups; connector renders <pre> — content preserved, layout differs"
|
||||
},
|
||||
{
|
||||
"id": "emoji",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "Done ✅ with 🎉 emoji 👀 test.",
|
||||
"native_output": "Done ✅ with 🎉 emoji 👀 test\\."
|
||||
},
|
||||
{
|
||||
"id": "cjk",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "中文测试：**粗体** 和 `代码` 混排。",
|
||||
"native_output": "中文测试：*粗体* 和 `代码` 混排。"
|
||||
},
|
||||
{
|
||||
"id": "bare-url",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "Visit https://example.com/path?q=1&r=2 today.",
|
||||
"native_output": "Visit https://example\\.com/path?q\\=1&r\\=2 today\\."
|
||||
},
|
||||
{
|
||||
"id": "mixed-document",
|
||||
"category": "grid",
|
||||
"expect": "semantic",
|
||||
"input": "## Report\n\nStatus: **green**. Details in `runbook.md`.\n\n- item *one*\n- item **two**\n\n```sh\nmake deploy\n```\n\nSee [dashboard](https://grafana.example.com/d/x).",
|
||||
"native_output": "*Report*\n\nStatus: *green*\\. Details in `runbook.md`\\.\n\n\\- item _one_\n\\- item *two*\n\n```sh\nmake deploy\n```\n\nSee [dashboard](https://grafana.example.com/d/x)\\."
|
||||
},
|
||||
{
|
||||
"id": "mdv2-reserved-chars",
|
||||
"category": "scar",
|
||||
"expect": "semantic",
|
||||
"input": "Price is 3.50 (was 4.00) — save ~12%! #deal +tax = win.",
|
||||
"native_output": "Price is 3\\.50 \\(was 4\\.00\\) — save \\~12%\\! \\#deal \\+tax \\= win\\."
|
||||
},
|
||||
{
|
||||
"id": "mdv2-underscores",
|
||||
"category": "scar",
|
||||
"expect": "semantic",
|
||||
"input": "snake_case_name and file_name.py in prose.",
|
||||
"native_output": "snake\\_case\\_name and file\\_name\\.py in prose\\."
|
||||
},
|
||||
{
|
||||
"id": "mdv2-brackets",
|
||||
"category": "scar",
|
||||
"expect": "semantic",
|
||||
"input": "Array[0] and dict{key} and (parens) live here.",
|
||||
"native_output": "Array\\[0\\] and dict\\{key\\} and \\(parens\\) live here\\."
|
||||
},
|
||||
{
|
||||
"id": "slack-bold-conversion",
|
||||
"category": "scar",
|
||||
"expect": "semantic",
|
||||
"input": "**important** word",
|
||||
"native_output": "*important* word"
|
||||
},
|
||||
{
|
||||
"id": "slack-link-conversion",
|
||||
"category": "scar",
|
||||
"expect": "semantic",
|
||||
"input": "[click here](https://example.com)",
|
||||
"native_output": "[click here](https://example.com)"
|
||||
},
|
||||
{
|
||||
"id": "slack-broadcast-mention",
|
||||
"category": "scar",
|
||||
"expect": "semantic",
|
||||
"input": "Hey <!everyone> and <!channel> and <!here>!",
|
||||
"native_output": "Hey <\\!everyone\\> and <\\!channel\\> and <\\!here\\>\\!"
|
||||
},
|
||||
{
|
||||
"id": "fence-lang-tag-slack",
|
||||
"category": "scar",
|
||||
"expect": "semantic",
|
||||
"input": "```text\nliteral first line issue\n```",
|
||||
"native_output": "```text\nliteral first line issue\n```"
|
||||
},
|
||||
{
|
||||
"id": "backslash-in-code",
|
||||
"category": "scar",
|
||||
"expect": "semantic",
|
||||
"input": "`C:\\Users\\ben\\file.txt` and ```\npath = \"a\\\\b\"\n```",
|
||||
"native_output": "`C:\\\\Users\\\\ben\\\\file.txt` and ```\npath = \"a\\\\\\\\b\"\n```"
|
||||
},
|
||||
{
|
||||
"id": "backtick-in-fence",
|
||||
"category": "scar",
|
||||
"expect": "semantic",
|
||||
"input": "```\nuse `inline` inside fence\n```",
|
||||
"native_output": "```\nuse \\`inline\\` inside fence\n```"
|
||||
},
|
||||
{
|
||||
"id": "header-with-bold",
|
||||
"category": "scar",
|
||||
"expect": "semantic",
|
||||
"input": "## The **Real** Deal",
|
||||
"native_output": "*The Real Deal*"
|
||||
},
|
||||
{
|
||||
"id": "table-cjk",
|
||||
"category": "scar",
|
||||
"expect": "divergent",
|
||||
"input": "| 名前 | 値 |\n|------|----|\n| 中文 | 42 |\n| b | 2 |",
|
||||
"native_output": "*中文*\n• 値: 42\n\n*b*\n• 値: 2",
|
||||
"note": "same as table-simple (CJK width alignment is native-only)"
|
||||
},
|
||||
{
|
||||
"id": "link-display-escapes",
|
||||
"category": "scar",
|
||||
"expect": "semantic",
|
||||
"input": "[v2.0 (beta)](https://example.com/v2)",
|
||||
"native_output": "[v2\\.0 \\(beta\\)](https://example.com/v2)"
|
||||
},
|
||||
{
|
||||
"id": "media-tag",
|
||||
"category": "adversarial",
|
||||
"expect": "semantic",
|
||||
"input": "Here you go\nMEDIA:/tmp/output.png\ndone",
|
||||
"native_output": "Here you go\nMEDIA:/tmp/output\\.png\ndone"
|
||||
},
|
||||
{
|
||||
"id": "unclosed-fence",
|
||||
"category": "adversarial",
|
||||
"expect": "divergent",
|
||||
"input": "```python\nprint('never closed')",
|
||||
"native_output": "\\`\\`\\`python\nprint\\('never closed'\\)",
|
||||
"note": "unterminated fence: native escapes as prose, connector HTML may close the block — degraded either way, never a 400"
|
||||
},
|
||||
{
|
||||
"id": "pathological-nesting",
|
||||
"category": "adversarial",
|
||||
"expect": "semantic",
|
||||
"input": "**bold *italic ~~struck `code` struck~~ italic* bold**",
|
||||
"native_output": "*bold \\*italic \\~\\~struck `code` struck\\~\\~ italic\\* bold*"
|
||||
},
|
||||
{
|
||||
"id": "placeholder-injection",
|
||||
"category": "adversarial",
|
||||
"expect": "divergent",
|
||||
"input": "sneaky \u0000PH0\u0000 token and \u0000SL1\u0000 too",
|
||||
"native_output": "sneaky \u0000PH0\u0000 token and \u0000SL1\u0000 too",
|
||||
"note": "NUL placeholder tokens are renderer-internal; each side neutralizes its own pattern"
|
||||
},
|
||||
{
|
||||
"id": "triple-markers",
|
||||
"category": "adversarial",
|
||||
"expect": "semantic",
|
||||
"input": "***what is this*** and ____that____",
|
||||
"native_output": "*\\*what is this*\\* and \\_\\_\\_\\_that\\_\\_\\_\\_"
|
||||
},
|
||||
{
|
||||
"id": "empty-string",
|
||||
"category": "adversarial",
|
||||
"expect": "semantic",
|
||||
"input": "",
|
||||
"native_output": ""
|
||||
},
|
||||
{
|
||||
"id": "whitespace-only",
|
||||
"category": "adversarial",
|
||||
"expect": "divergent",
|
||||
"input": " \n\t\n ",
|
||||
"native_output": " \n\t\n ",
|
||||
"note": "native collapses to empty-ish prose, connector HTML preserves — cosmetic"
|
||||
},
|
||||
{
|
||||
"id": "long-line",
|
||||
"category": "adversarial",
|
||||
"expect": "semantic",
|
||||
"input": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word ",
|
||||
"native_output": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word "
|
||||
},
|
||||
{
|
||||
"id": "many-fences",
|
||||
"category": "adversarial",
|
||||
"expect": "semantic",
|
||||
"input": "```\na\n```\nmid\n```\nb\n```\nend ```inline``` tail",
|
||||
"native_output": "```\na\n```\nmid\n```\nb\n```\nend ```inline``` tail"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
{
|
||||
"$comment": "GENERATED — do not hand-edit. Regenerate with hermes-agent scripts/generate_conformance_vectors.py; the native renderers are the oracle (executable spec).",
|
||||
"oracle": {
|
||||
"repo": "NousResearch/hermes-agent",
|
||||
"commit": "40eebc7d70f3d8e95c429c8aa482f31ba6c867f6",
|
||||
"generator": "scripts/generate_conformance_vectors.py",
|
||||
"generator_version": 1
|
||||
},
|
||||
"platform": "whatsapp",
|
||||
"vectors": [
|
||||
{
|
||||
"id": "plain-text",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Just a plain sentence.",
|
||||
"native_output": "Just a plain sentence."
|
||||
},
|
||||
{
|
||||
"id": "bold",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "This is **bold** text.",
|
||||
"native_output": "This is *bold* text."
|
||||
},
|
||||
{
|
||||
"id": "italic",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "This is *italic* text.",
|
||||
"native_output": "This is _italic_ text."
|
||||
},
|
||||
{
|
||||
"id": "bold-italic",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Mix of **bold** and *italic* in one line.",
|
||||
"native_output": "Mix of *bold* and _italic_ in one line."
|
||||
},
|
||||
{
|
||||
"id": "strikethrough",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "This is ~~struck~~ text.",
|
||||
"native_output": "This is ~struck~ text."
|
||||
},
|
||||
{
|
||||
"id": "inline-code",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Run `pip install hermes` to start.",
|
||||
"native_output": "Run `pip install hermes` to start."
|
||||
},
|
||||
{
|
||||
"id": "fenced-code",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "```\nprint('hello')\n```",
|
||||
"native_output": "```\nprint('hello')\n```"
|
||||
},
|
||||
{
|
||||
"id": "fenced-code-lang",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "```python\ndef f(x):\n return x * 2\n```",
|
||||
"native_output": "```python\ndef f(x):\n return x * 2\n```"
|
||||
},
|
||||
{
|
||||
"id": "link",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "See [the docs](https://example.com/docs) for more.",
|
||||
"native_output": "See the docs (https://example.com/docs) for more."
|
||||
},
|
||||
{
|
||||
"id": "link-parens-url",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "See [spec](https://example.com/a_(b)) here.",
|
||||
"native_output": "See spec (https://example.com/a_(b)) here."
|
||||
},
|
||||
{
|
||||
"id": "header-h1",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "# Big Title\nBody follows.",
|
||||
"native_output": "*Big Title*\nBody follows."
|
||||
},
|
||||
{
|
||||
"id": "header-h2",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "## Section\nBody follows.",
|
||||
"native_output": "*Section*\nBody follows."
|
||||
},
|
||||
{
|
||||
"id": "header-h3",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "### Sub-section\nBody follows.",
|
||||
"native_output": "*Sub-section*\nBody follows."
|
||||
},
|
||||
{
|
||||
"id": "ul-list",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "- first\n- second\n- third",
|
||||
"native_output": "- first\n- second\n- third"
|
||||
},
|
||||
{
|
||||
"id": "ol-list",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "1. first\n2. second\n3. third",
|
||||
"native_output": "1. first\n2. second\n3. third"
|
||||
},
|
||||
{
|
||||
"id": "nested-list",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "- outer\n - inner one\n - inner two\n- outer two",
|
||||
"native_output": "- outer\n - inner one\n - inner two\n- outer two"
|
||||
},
|
||||
{
|
||||
"id": "blockquote",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "> quoted wisdom\nregular line",
|
||||
"native_output": "> quoted wisdom\nregular line"
|
||||
},
|
||||
{
|
||||
"id": "hrule",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "above\n\n---\n\nbelow",
|
||||
"native_output": "above\n\n---\n\nbelow"
|
||||
},
|
||||
{
|
||||
"id": "table-simple",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "| name | value |\n|------|-------|\n| a | 1 |\n| b | 2 |",
|
||||
"native_output": "| name | value |\n|------|-------|\n| a | 1 |\n| b | 2 |"
|
||||
},
|
||||
{
|
||||
"id": "emoji",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Done ✅ with 🎉 emoji 👀 test.",
|
||||
"native_output": "Done ✅ with 🎉 emoji 👀 test."
|
||||
},
|
||||
{
|
||||
"id": "cjk",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "中文测试：**粗体** 和 `代码` 混排。",
|
||||
"native_output": "中文测试：*粗体* 和 `代码` 混排。"
|
||||
},
|
||||
{
|
||||
"id": "bare-url",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "Visit https://example.com/path?q=1&r=2 today.",
|
||||
"native_output": "Visit https://example.com/path?q=1&r=2 today."
|
||||
},
|
||||
{
|
||||
"id": "mixed-document",
|
||||
"category": "grid",
|
||||
"expect": "parity",
|
||||
"input": "## Report\n\nStatus: **green**. Details in `runbook.md`.\n\n- item *one*\n- item **two**\n\n```sh\nmake deploy\n```\n\nSee [dashboard](https://grafana.example.com/d/x).",
|
||||
"native_output": "*Report*\n\nStatus: *green*. Details in `runbook.md`.\n\n- item _one_\n- item *two*\n\n```sh\nmake deploy\n```\n\nSee dashboard (https://grafana.example.com/d/x)."
|
||||
},
|
||||
{
|
||||
"id": "mdv2-reserved-chars",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "Price is 3.50 (was 4.00) — save ~12%! #deal +tax = win.",
|
||||
"native_output": "Price is 3.50 (was 4.00) — save ~12%! #deal +tax = win."
|
||||
},
|
||||
{
|
||||
"id": "mdv2-underscores",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "snake_case_name and file_name.py in prose.",
|
||||
"native_output": "snake_case_name and file_name.py in prose."
|
||||
},
|
||||
{
|
||||
"id": "mdv2-brackets",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "Array[0] and dict{key} and (parens) live here.",
|
||||
"native_output": "Array[0] and dict{key} and (parens) live here."
|
||||
},
|
||||
{
|
||||
"id": "slack-bold-conversion",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "**important** word",
|
||||
"native_output": "*important* word"
|
||||
},
|
||||
{
|
||||
"id": "slack-link-conversion",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "[click here](https://example.com)",
|
||||
"native_output": "click here (https://example.com)"
|
||||
},
|
||||
{
|
||||
"id": "slack-broadcast-mention",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "Hey <!everyone> and <!channel> and <!here>!",
|
||||
"native_output": "Hey <!everyone> and <!channel> and <!here>!"
|
||||
},
|
||||
{
|
||||
"id": "fence-lang-tag-slack",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "```text\nliteral first line issue\n```",
|
||||
"native_output": "```text\nliteral first line issue\n```"
|
||||
},
|
||||
{
|
||||
"id": "backslash-in-code",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "`C:\\Users\\ben\\file.txt` and ```\npath = \"a\\\\b\"\n```",
|
||||
"native_output": "`C:\\Users\\ben\\file.txt` and ```\npath = \"a\\\\b\"\n```"
|
||||
},
|
||||
{
|
||||
"id": "backtick-in-fence",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "```\nuse `inline` inside fence\n```",
|
||||
"native_output": "```\nuse `inline` inside fence\n```"
|
||||
},
|
||||
{
|
||||
"id": "header-with-bold",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "## The **Real** Deal",
|
||||
"native_output": "*The *Real* Deal*"
|
||||
},
|
||||
{
|
||||
"id": "table-cjk",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "| 名前 | 値 |\n|------|----|\n| 中文 | 42 |\n| b | 2 |",
|
||||
"native_output": "| 名前 | 値 |\n|------|----|\n| 中文 | 42 |\n| b | 2 |"
|
||||
},
|
||||
{
|
||||
"id": "link-display-escapes",
|
||||
"category": "scar",
|
||||
"expect": "parity",
|
||||
"input": "[v2.0 (beta)](https://example.com/v2)",
|
||||
"native_output": "v2.0 (beta) (https://example.com/v2)"
|
||||
},
|
||||
{
|
||||
"id": "media-tag",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "Here you go\nMEDIA:/tmp/output.png\ndone",
|
||||
"native_output": "Here you go\nMEDIA:/tmp/output.png\ndone"
|
||||
},
|
||||
{
|
||||
"id": "unclosed-fence",
|
||||
"category": "adversarial",
|
||||
"expect": "divergent",
|
||||
"input": "```python\nprint('never closed')",
|
||||
"native_output": "```python\nprint('never closed')",
|
||||
"note": "unterminated fence handling differs; both degrade without dropping content"
|
||||
},
|
||||
{
|
||||
"id": "pathological-nesting",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "**bold *italic ~~struck `code` struck~~ italic* bold**",
|
||||
"native_output": "*bold _italic ~struck `code` struck~ italic_ bold*"
|
||||
},
|
||||
{
|
||||
"id": "placeholder-injection",
|
||||
"category": "adversarial",
|
||||
"expect": "divergent",
|
||||
"input": "sneaky \u0000PH0\u0000 token and \u0000SL1\u0000 too",
|
||||
"native_output": "sneaky \u0000PH0\u0000 token and \u0000SL1\u0000 too",
|
||||
"note": "placeholder alphabets are renderer-internal"
|
||||
},
|
||||
{
|
||||
"id": "triple-markers",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "***what is this*** and ____that____",
|
||||
"native_output": "**what is this** and *__that*__"
|
||||
},
|
||||
{
|
||||
"id": "empty-string",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "",
|
||||
"native_output": ""
|
||||
},
|
||||
{
|
||||
"id": "whitespace-only",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": " \n\t\n ",
|
||||
"native_output": " \n\t\n "
|
||||
},
|
||||
{
|
||||
"id": "long-line",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word ",
|
||||
"native_output": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word "
|
||||
},
|
||||
{
|
||||
"id": "many-fences",
|
||||
"category": "adversarial",
|
||||
"expect": "parity",
|
||||
"input": "```\na\n```\nmid\n```\nb\n```\nend ```inline``` tail",
|
||||
"native_output": "```\na\n```\nmid\n```\nb\n```\nend ```inline``` tail"
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user