#!/usr/bin/env python3 """Build a self-contained HTML report comparing compaction runs. Usage: build_report.py Expects runs/_.json pairs from run_compaction.py. """ import html import json import sys from pathlib import Path RUNS = Path(sys.argv[1]) OUT = sys.argv[2] pairs = {} for f in sorted(RUNS.glob("*.json")): co, sid = f.stem.split("_", 1) if co == "main": co, sid = "main-co", f.stem[len("main-co_"):] elif co == "pr": co, sid = "pr-co", f.stem[len("pr-co_"):] data = json.loads(f.read_text(encoding="utf-8")) pairs.setdefault(sid, {})[co] = data E = html.escape def msg_class(m): role = m.get("role", "?") c = m.get("content") or "" if isinstance(c, str): if "[CONTEXT COMPACTION" in c or "[CONTEXT SUMMARY" in c: return "summary" if "SKILL_PRUNED" in c: return "skillpruned" if "SKILL POLICY DIGEST" in c or "SKILL_POLICY_DIGEST" in c: return "digest" if "preserved across context compression" in c: return "todosnap" return role def render_msg(m, idx): role = m.get("role", "?") c = m.get("content") if not isinstance(c, str): c = json.dumps(c, default=str)[:2000] tool = m.get("tool_name") or "" tcs = m.get("tool_calls") or [] tc_names = ", ".join( (t.get("function", {}) or {}).get("name", "?") for t in tcs if isinstance(t, dict) ) cls = msg_class(m) nchars = len(c) label = role if tool: label += f" · {tool}" if tc_names: label += f" → {tc_names}" preview = c[:180].replace("\n", " ") full = c if nchars <= 20000 else c[:20000] + f"\n…[{nchars-20000:,} more chars]" return ( f'
#{idx}' f'{E(label)}' f'{nchars:,}ch' f'{E(preview)}' f"
{E(full)}
" ) def render_column(title, data, key): meta = data["meta"] msgs = data[key] body = "".join(render_msg(m, i) for i, m in enumerate(msgs)) return ( f'

{E(title)}

' f'
{meta[key.replace("before","before_msgs").replace("after","after_msgs")] if False else len(msgs)} msgs · ' f'~{(meta["before_tokens_est"] if key=="before" else meta["after_tokens_est"]):,} tok
' f'
{body}
' ) def survival_stats(before, after): after_texts = set() for m in after: c = m.get("content") if isinstance(c, str) and c: after_texts.add(c[:400]) kept = sum(1 for m in before if isinstance(m.get("content"), str) and (m.get("content") or "")[:400] in after_texts) return kept sections = [] toc = [] for sid, versions in pairs.items(): if "main-co" not in versions or "pr-co" not in versions: continue main_d, pr_d = versions["main-co"], versions["pr-co"] title = main_d["meta"].get("title") or sid mm, pm = main_d["meta"], pr_d["meta"] def count_markers(msgs, needle): return sum((m.get("content") or "").count(needle) for m in msgs if isinstance(m.get("content"), str)) rows = [] def stat(name, mv, pv): cls = "diff" if mv != pv else "" rows.append(f"{E(name)}{E(str(mv))}{E(str(pv))}") stat("Messages after", mm["after_msgs"], pm["after_msgs"]) stat("Est. tokens after", f"{mm['after_tokens_est']:,}", f"{pm['after_tokens_est']:,}") stat("Reduction", f"{100-100*mm['after_tokens_est']//max(1,mm['before_tokens_est'])}%", f"{100-100*pm['after_tokens_est']//max(1,pm['before_tokens_est'])}%") stat("Compress time", f"{mm['elapsed_s']}s", f"{pm['elapsed_s']}s") stat("SKILL_PRUNED markers", count_markers(main_d["after"], "SKILL_PRUNED"), count_markers(pr_d["after"], "SKILL_PRUNED")) stat("Policy digest blocks", count_markers(main_d["after"], "SKILL POLICY DIGEST") + count_markers(main_d["after"], "SKILL_POLICY_DIGEST"), count_markers(pr_d["after"], "SKILL POLICY DIGEST") + count_markers(pr_d["after"], "SKILL_POLICY_DIGEST")) stat("Todo snapshot present", "yes" if count_markers(main_d["after"], "preserved across context compression") else "no", "yes" if count_markers(pr_d["after"], "preserved across context compression") else "no") stat("Kept-verbatim msgs", survival_stats(main_d["before"], main_d["after"]), survival_stats(pr_d["before"], pr_d["after"])) stat("Summary error", mm.get("summary_error") or "—", pm.get("summary_error") or "—") todo_html = "" for label, d in (("main", mm), ("PR #87090", pm)): tb = d.get("todo_injection_block") if tb: todo_html += f"

Todo injection block — {E(label)}

{E(tb)}
" anchor = f"s-{sid}" toc.append(f'{E(title)} ({sid})') sections.append(f"""

{E(title)} {sid}

{"".join(rows)}
mainPR #87090
{todo_html}
{render_column("BEFORE (original transcript)", main_d, "before")} {render_column("AFTER — main", main_d, "after")} {render_column("AFTER — PR #87090", pr_d, "after")}
""") page = f""" Compaction comparison — main vs PR #87090

Compaction comparison — current main (7619564fb) vs PR #87090 (41fd511f6)

Real sessions from state.db (copy), replayed through each checkout's ContextCompressor with force=True. Real LLM summaries. Click any row to expand the full message.

compaction summary SKILL_PRUNED marker policy digest todo snapshot user
{"".join(sections)} """ Path(OUT).write_text(page, encoding="utf-8") print(f"wrote {OUT} ({len(page):,} bytes, {len(sections)} sessions)")