Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a self-contained HTML report comparing compaction runs.
|
||||
|
||||
Usage: build_report.py <runs_dir> <out_html>
|
||||
Expects runs/<checkout>_<session>.json pairs from run_compaction.py.
|
||||
"""
|
||||
import html
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
RUNS = Path(sys.argv[1])
|
||||
OUT = sys.argv[2]
|
||||
|
||||
pairs = {}
|
||||
for f in sorted(RUNS.glob("*.json")):
|
||||
co, sid = f.stem.split("_", 1)
|
||||
if co == "main":
|
||||
co, sid = "main-co", f.stem[len("main-co_"):]
|
||||
elif co == "pr":
|
||||
co, sid = "pr-co", f.stem[len("pr-co_"):]
|
||||
data = json.loads(f.read_text(encoding="utf-8"))
|
||||
pairs.setdefault(sid, {})[co] = data
|
||||
|
||||
E = html.escape
|
||||
|
||||
def msg_class(m):
|
||||
role = m.get("role", "?")
|
||||
c = m.get("content") or ""
|
||||
if isinstance(c, str):
|
||||
if "[CONTEXT COMPACTION" in c or "[CONTEXT SUMMARY" in c:
|
||||
return "summary"
|
||||
if "SKILL_PRUNED" in c:
|
||||
return "skillpruned"
|
||||
if "SKILL POLICY DIGEST" in c or "SKILL_POLICY_DIGEST" in c:
|
||||
return "digest"
|
||||
if "preserved across context compression" in c:
|
||||
return "todosnap"
|
||||
return role
|
||||
|
||||
def render_msg(m, idx):
|
||||
role = m.get("role", "?")
|
||||
c = m.get("content")
|
||||
if not isinstance(c, str):
|
||||
c = json.dumps(c, default=str)[:2000]
|
||||
tool = m.get("tool_name") or ""
|
||||
tcs = m.get("tool_calls") or []
|
||||
tc_names = ", ".join(
|
||||
(t.get("function", {}) or {}).get("name", "?") for t in tcs if isinstance(t, dict)
|
||||
)
|
||||
cls = msg_class(m)
|
||||
nchars = len(c)
|
||||
label = role
|
||||
if tool:
|
||||
label += f" · {tool}"
|
||||
if tc_names:
|
||||
label += f" → {tc_names}"
|
||||
preview = c[:180].replace("\n", " ")
|
||||
full = c if nchars <= 20000 else c[:20000] + f"\n…[{nchars-20000:,} more chars]"
|
||||
return (
|
||||
f'<details class="msg {cls}"><summary><span class="idx">#{idx}</span>'
|
||||
f'<span class="role">{E(label)}</span>'
|
||||
f'<span class="chars">{nchars:,}ch</span>'
|
||||
f'<span class="preview">{E(preview)}</span></summary>'
|
||||
f"<pre>{E(full)}</pre></details>"
|
||||
)
|
||||
|
||||
def render_column(title, data, key):
|
||||
meta = data["meta"]
|
||||
msgs = data[key]
|
||||
body = "".join(render_msg(m, i) for i, m in enumerate(msgs))
|
||||
return (
|
||||
f'<div class="col"><div class="colhead"><h3>{E(title)}</h3>'
|
||||
f'<div class="stats">{meta[key.replace("before","before_msgs").replace("after","after_msgs")] if False else len(msgs)} msgs · '
|
||||
f'~{(meta["before_tokens_est"] if key=="before" else meta["after_tokens_est"]):,} tok</div></div>'
|
||||
f'<div class="msgs">{body}</div></div>'
|
||||
)
|
||||
|
||||
def survival_stats(before, after):
|
||||
after_texts = set()
|
||||
for m in after:
|
||||
c = m.get("content")
|
||||
if isinstance(c, str) and c:
|
||||
after_texts.add(c[:400])
|
||||
kept = sum(1 for m in before if isinstance(m.get("content"), str) and (m.get("content") or "")[:400] in after_texts)
|
||||
return kept
|
||||
|
||||
sections = []
|
||||
toc = []
|
||||
for sid, versions in pairs.items():
|
||||
if "main-co" not in versions or "pr-co" not in versions:
|
||||
continue
|
||||
main_d, pr_d = versions["main-co"], versions["pr-co"]
|
||||
title = main_d["meta"].get("title") or sid
|
||||
mm, pm = main_d["meta"], pr_d["meta"]
|
||||
|
||||
def count_markers(msgs, needle):
|
||||
return sum((m.get("content") or "").count(needle) for m in msgs if isinstance(m.get("content"), str))
|
||||
|
||||
rows = []
|
||||
def stat(name, mv, pv):
|
||||
cls = "diff" if mv != pv else ""
|
||||
rows.append(f"<tr class='{cls}'><td>{E(name)}</td><td>{E(str(mv))}</td><td>{E(str(pv))}</td></tr>")
|
||||
|
||||
stat("Messages after", mm["after_msgs"], pm["after_msgs"])
|
||||
stat("Est. tokens after", f"{mm['after_tokens_est']:,}", f"{pm['after_tokens_est']:,}")
|
||||
stat("Reduction", f"{100-100*mm['after_tokens_est']//max(1,mm['before_tokens_est'])}%", f"{100-100*pm['after_tokens_est']//max(1,pm['before_tokens_est'])}%")
|
||||
stat("Compress time", f"{mm['elapsed_s']}s", f"{pm['elapsed_s']}s")
|
||||
stat("SKILL_PRUNED markers", count_markers(main_d["after"], "SKILL_PRUNED"), count_markers(pr_d["after"], "SKILL_PRUNED"))
|
||||
stat("Policy digest blocks", count_markers(main_d["after"], "SKILL POLICY DIGEST") + count_markers(main_d["after"], "SKILL_POLICY_DIGEST"), count_markers(pr_d["after"], "SKILL POLICY DIGEST") + count_markers(pr_d["after"], "SKILL_POLICY_DIGEST"))
|
||||
stat("Todo snapshot present", "yes" if count_markers(main_d["after"], "preserved across context compression") else "no", "yes" if count_markers(pr_d["after"], "preserved across context compression") else "no")
|
||||
stat("Kept-verbatim msgs", survival_stats(main_d["before"], main_d["after"]), survival_stats(pr_d["before"], pr_d["after"]))
|
||||
stat("Summary error", mm.get("summary_error") or "—", pm.get("summary_error") or "—")
|
||||
|
||||
todo_html = ""
|
||||
for label, d in (("main", mm), ("PR #87090", pm)):
|
||||
tb = d.get("todo_injection_block")
|
||||
if tb:
|
||||
todo_html += f"<h4>Todo injection block — {E(label)}</h4><pre class='todoblock'>{E(tb)}</pre>"
|
||||
|
||||
anchor = f"s-{sid}"
|
||||
toc.append(f'<a href="#{anchor}">{E(title)} <span class="dim">({sid})</span></a>')
|
||||
sections.append(f"""
|
||||
<section id="{anchor}">
|
||||
<h2>{E(title)} <span class="dim">{sid}</span></h2>
|
||||
<table class="stats-table"><tr><th></th><th>main</th><th>PR #87090</th></tr>{"".join(rows)}</table>
|
||||
{todo_html}
|
||||
<div class="cols">
|
||||
{render_column("BEFORE (original transcript)", main_d, "before")}
|
||||
{render_column("AFTER — main", main_d, "after")}
|
||||
{render_column("AFTER — PR #87090", pr_d, "after")}
|
||||
</div>
|
||||
</section>""")
|
||||
|
||||
page = f"""<!DOCTYPE html>
|
||||
<html><head><meta charset="utf-8"><title>Compaction comparison — main vs PR #87090</title>
|
||||
<style>
|
||||
:root {{ color-scheme: dark; }}
|
||||
body {{ background:#0d1117; color:#c9d1d9; font:14px/1.45 -apple-system,Segoe UI,sans-serif; margin:0; padding:24px; }}
|
||||
h1 {{ font-size:22px; }} h2 {{ font-size:18px; border-bottom:1px solid #30363d; padding-bottom:6px; margin-top:48px; }}
|
||||
.dim {{ color:#8b949e; font-weight:normal; font-size:12px; }}
|
||||
nav a {{ display:block; color:#58a6ff; margin:2px 0; text-decoration:none; }}
|
||||
.legend span {{ display:inline-block; padding:2px 10px; margin-right:8px; border-radius:4px; font-size:12px; }}
|
||||
.stats-table {{ border-collapse:collapse; margin:12px 0; }}
|
||||
.stats-table td, .stats-table th {{ border:1px solid #30363d; padding:4px 12px; text-align:left; font-size:13px; }}
|
||||
.stats-table tr.diff td {{ background:#1c2a1c; }}
|
||||
.cols {{ display:grid; grid-template-columns:1fr 1fr 1fr; gap:10px; }}
|
||||
.col {{ min-width:0; }}
|
||||
.colhead {{ position:sticky; top:0; background:#161b22; padding:8px; border:1px solid #30363d; border-radius:6px 6px 0 0; z-index:2; }}
|
||||
.colhead h3 {{ margin:0; font-size:13px; }} .colhead .stats {{ color:#8b949e; font-size:12px; }}
|
||||
.msgs {{ border:1px solid #30363d; border-top:none; max-height:80vh; overflow-y:auto; }}
|
||||
.msg {{ border-bottom:1px solid #21262d; }}
|
||||
.msg summary {{ cursor:pointer; padding:3px 6px; display:flex; gap:6px; align-items:baseline; white-space:nowrap; overflow:hidden; }}
|
||||
.msg summary::-webkit-details-marker {{ display:none; }}
|
||||
.idx {{ color:#484f58; font-size:11px; min-width:34px; }}
|
||||
.role {{ font-size:11px; font-weight:600; min-width:110px; overflow:hidden; text-overflow:ellipsis; }}
|
||||
.chars {{ color:#8b949e; font-size:11px; min-width:52px; }}
|
||||
.preview {{ color:#8b949e; font-size:11px; overflow:hidden; text-overflow:ellipsis; flex:1; }}
|
||||
.msg pre {{ white-space:pre-wrap; word-break:break-word; font-size:11px; background:#161b22; margin:0; padding:8px; max-height:400px; overflow-y:auto; }}
|
||||
.msg.user summary {{ background:#0d2137; }} .msg.user .role {{ color:#58a6ff; }}
|
||||
.msg.assistant .role {{ color:#d2a8ff; }}
|
||||
.msg.tool .role {{ color:#7ee787; }}
|
||||
.msg.system summary {{ background:#21262d; }} .msg.system .role {{ color:#8b949e; }}
|
||||
.msg.summary summary {{ background:#3d2e00; }} .msg.summary .role {{ color:#e3b341; }}
|
||||
.msg.skillpruned summary {{ background:#3d1418; }} .msg.skillpruned .role {{ color:#ff7b72; }}
|
||||
.msg.digest summary {{ background:#1b3d2e; }} .msg.digest .role {{ color:#56d364; }}
|
||||
.msg.todosnap summary {{ background:#2d1b3d; }} .msg.todosnap .role {{ color:#d2a8ff; }}
|
||||
.todoblock {{ background:#1b1230; border:1px solid #6e40c9; padding:10px; white-space:pre-wrap; font-size:12px; }}
|
||||
</style></head><body>
|
||||
<h1>Compaction comparison — current main (7619564fb) vs PR #87090 (41fd511f6)</h1>
|
||||
<p class="dim">Real sessions from state.db (copy), replayed through each checkout's ContextCompressor with force=True. Real LLM summaries. Click any row to expand the full message.</p>
|
||||
<div class="legend">
|
||||
<span style="background:#3d2e00;color:#e3b341">compaction summary</span>
|
||||
<span style="background:#3d1418;color:#ff7b72">SKILL_PRUNED marker</span>
|
||||
<span style="background:#1b3d2e;color:#56d364">policy digest</span>
|
||||
<span style="background:#2d1b3d;color:#d2a8ff">todo snapshot</span>
|
||||
<span style="background:#0d2137;color:#58a6ff">user</span>
|
||||
</div>
|
||||
<nav>{"".join(toc)}</nav>
|
||||
{"".join(sections)}
|
||||
</body></html>"""
|
||||
|
||||
Path(OUT).write_text(page, encoding="utf-8")
|
||||
print(f"wrote {OUT} ({len(page):,} bytes, {len(sections)} sessions)")
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the codex CLI as an eval arm on the same transcripts + question banks.
|
||||
|
||||
Per transcript:
|
||||
1. Split the 500K-token prefix into ~150KB chunk files in a work dir.
|
||||
2. `codex exec` reads every file (2-3 sentence summary each) — the read
|
||||
volume exceeds codex's 258K window, so its auto-compaction fires
|
||||
naturally (verified via token_count drops / compacted events in the
|
||||
rollout jsonl).
|
||||
3. `codex exec resume --last` asks the SAME 15 exam questions; answers are
|
||||
judged by the same LLM judge against the same golds.
|
||||
|
||||
Usage: codex_arm.py <lineage_json> <questions_json> <workdir> <out_json>
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[0] / "main-co"))
|
||||
|
||||
LINEAGE = sys.argv[1]
|
||||
QUESTIONS = sys.argv[2]
|
||||
WORKDIR = Path(sys.argv[3])
|
||||
OUT = sys.argv[4]
|
||||
|
||||
JUDGE_PROMPT = """Score this answer against the gold answer. Reply with STRICT JSON: {{"score": 2|1|0, "why": "..."}}.
|
||||
2 = factually matches gold (wording may differ)
|
||||
1 = partially correct or hedged-but-right
|
||||
0 = wrong, or refuses/says it doesn't know with a wrong/no guess
|
||||
|
||||
QUESTION: {question}
|
||||
GOLD: {gold}
|
||||
ANSWER: {answer}"""
|
||||
|
||||
|
||||
def prepare_chunks() -> int:
|
||||
from evals.compaction.fixtures import load_transcript
|
||||
|
||||
WORKDIR.mkdir(parents=True, exist_ok=True)
|
||||
msgs = load_transcript(LINEAGE, cap_tokens=500_000)
|
||||
chunk, size, idx = [], 0, 0
|
||||
for m in msgs:
|
||||
c = m.get("content") or ""
|
||||
if not isinstance(c, str) or not c:
|
||||
continue
|
||||
chunk.append(f"--- {m['role']} ---\n{c}\n")
|
||||
size += len(c)
|
||||
if size > 150_000:
|
||||
(WORKDIR / f"transcript_{idx:02d}.txt").write_text(
|
||||
"\n".join(chunk), encoding="utf-8")
|
||||
chunk, size = [], 0
|
||||
idx += 1
|
||||
if chunk:
|
||||
(WORKDIR / f"transcript_{idx:02d}.txt").write_text(
|
||||
"\n".join(chunk), encoding="utf-8")
|
||||
idx += 1
|
||||
return idx
|
||||
|
||||
|
||||
def newest_rollout() -> str:
|
||||
files = sorted(
|
||||
glob.glob(os.path.expanduser("~/.codex/sessions/*/*/*/rollout-*.jsonl")),
|
||||
key=os.path.getmtime,
|
||||
)
|
||||
return files[-1] if files else ""
|
||||
|
||||
|
||||
def rollout_session_id(path: str) -> str:
|
||||
for line in open(path, encoding="utf-8", errors="replace"):
|
||||
try:
|
||||
d = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if d.get("type") == "session_meta":
|
||||
return d.get("payload", {}).get("session_id", "")
|
||||
return ""
|
||||
|
||||
|
||||
def last_agent_message(path: str) -> str:
|
||||
msgs = []
|
||||
for line in open(path, encoding="utf-8", errors="replace"):
|
||||
try:
|
||||
d = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
p = d.get("payload", {})
|
||||
if p.get("type") == "agent_message":
|
||||
msgs.append(p.get("message", ""))
|
||||
return msgs[-1] if msgs else ""
|
||||
|
||||
|
||||
def rollout_stats(path: str) -> dict:
|
||||
compacted = 0
|
||||
peak = 0
|
||||
for line in open(path, encoding="utf-8", errors="replace"):
|
||||
try:
|
||||
d = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
p = d.get("payload", {})
|
||||
if d.get("type") == "compacted" or p.get("type") == "compacted":
|
||||
compacted += 1
|
||||
if p.get("type") == "token_count" and p.get("info"):
|
||||
last = p["info"].get("last_token_usage") or {}
|
||||
ctx = last.get("input_tokens", 0) + last.get("cached_input_tokens", 0)
|
||||
peak = max(peak, ctx)
|
||||
return {"compaction_events": compacted, "peak_context_tokens": peak}
|
||||
|
||||
|
||||
def codex(args: list, prompt: str, timeout: int = 3600) -> str:
|
||||
proc = subprocess.run(
|
||||
["codex", "exec", *args, "--skip-git-repo-check", prompt],
|
||||
cwd=str(WORKDIR), capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
return proc.stdout + proc.stderr
|
||||
|
||||
|
||||
def judge(question: str, gold: str, answer: str) -> dict:
|
||||
from agent.auxiliary_client import call_llm
|
||||
|
||||
resp = call_llm(
|
||||
messages=[{"role": "user", "content": JUDGE_PROMPT.format(
|
||||
question=question, gold=gold, answer=answer)}],
|
||||
task="compression", max_tokens=300,
|
||||
)
|
||||
text = resp.choices[0].message.content if hasattr(resp, "choices") else str(resp)
|
||||
m = re.search(r"\{.*\}", text, re.S)
|
||||
try:
|
||||
return json.loads(m.group(0))
|
||||
except Exception:
|
||||
return {"score": 0, "why": f"judge parse failure: {text[:80]}"}
|
||||
|
||||
|
||||
def main():
|
||||
n = prepare_chunks()
|
||||
print(f"[codex-arm] {WORKDIR.name}: {n} chunk files", flush=True)
|
||||
t0 = time.time()
|
||||
codex(
|
||||
["-s", "read-only"],
|
||||
f"This directory contains transcript_00.txt through transcript_{n-1:02d}.txt. "
|
||||
"Read EVERY file COMPLETELY one at a time using 'cat transcript_NN.txt' "
|
||||
"(full file, do not use head/tail/grep). After each file, write a 2-3 "
|
||||
"sentence summary of what happened in that portion. Do not skip any file.",
|
||||
)
|
||||
rollout = newest_rollout()
|
||||
session_id = rollout_session_id(rollout)
|
||||
stats = rollout_stats(rollout)
|
||||
# Codex auto-compacts at ~90% of its 258K window. If one read pass didn't
|
||||
# trigger it, re-read files in the SAME session until it does (max 3
|
||||
# extra passes) — the comparison requires post-compaction state.
|
||||
passes = 0
|
||||
while stats["compaction_events"] == 0 and passes < 3:
|
||||
passes += 1
|
||||
print(f"[codex-arm] no compaction yet (peak={stats['peak_context_tokens']:,}) — re-read pass {passes}", flush=True)
|
||||
codex(
|
||||
["resume", session_id],
|
||||
"Re-read ALL transcript files again completely with 'cat', one at a "
|
||||
"time, and refine each of your per-file summaries with any details "
|
||||
"you missed. Do not skip any file.",
|
||||
)
|
||||
stats = rollout_stats(rollout)
|
||||
read_s = time.time() - t0
|
||||
print(f"[codex-arm] read phase {read_s:.0f}s, {stats}", flush=True)
|
||||
if stats["compaction_events"] == 0:
|
||||
print("[codex-arm] WARNING: compaction never fired — arm invalid", flush=True)
|
||||
|
||||
questions = json.loads(Path(QUESTIONS).read_text(encoding="utf-8"))
|
||||
qlist = "\n".join(f"{i+1}. {q['q']}" for i, q in enumerate(questions))
|
||||
codex(
|
||||
["resume", session_id],
|
||||
"Based on everything you learned from the transcript files earlier in "
|
||||
"this session, answer the following questions from memory. Do NOT "
|
||||
"re-read any files — answer only from what you currently retain in "
|
||||
"context. If you don't know, say 'UNKNOWN' and give your best guess. "
|
||||
"Reply with a numbered list, one concise answer per question.\n\n" + qlist,
|
||||
)
|
||||
quiz_text = last_agent_message(rollout)
|
||||
print(f"[codex-arm] quiz reply: {len(quiz_text)} chars", flush=True)
|
||||
answers = {}
|
||||
for m in re.finditer(r"(?m)^\s*\**(\d{1,2})[.)]\**\s+(.+?)(?=^\s*\**\d{1,2}[.)]\**\s|\Z)",
|
||||
quiz_text, re.S):
|
||||
answers[int(m.group(1))] = m.group(2).strip()[:600]
|
||||
|
||||
results = []
|
||||
for i, q in enumerate(questions):
|
||||
ans = answers.get(i + 1, "(no answer parsed)")
|
||||
verdict = judge(q["q"], q["gold"], ans)
|
||||
results.append({"q": q["q"], "gold": q["gold"], "answer": ans, **verdict})
|
||||
print(f" Q{i+1}: {verdict['score']}", flush=True)
|
||||
|
||||
scored = [r["score"] for r in results]
|
||||
summary = {
|
||||
"policy": "codex_real",
|
||||
"recall_pct": round(100 * sum(scored) / (2 * len(scored)), 1),
|
||||
"scores": scored,
|
||||
"read_seconds": round(read_s),
|
||||
**stats,
|
||||
"rollout": rollout,
|
||||
}
|
||||
Path(OUT).write_text(json.dumps({"summary": summary, "results": results}, indent=1),
|
||||
encoding="utf-8")
|
||||
print(json.dumps(summary, indent=1), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reconstruct the full uncompacted transcript of a session LINEAGE.
|
||||
|
||||
Rotation children start with a copy of the compressed parent (head + summary +
|
||||
tail). To rebuild the real history: walk the chain root->leaf, append messages
|
||||
not seen before (hash of role+content+tool_calls), and skip synthetic
|
||||
compaction summaries / todo snapshots so we get the organic transcript.
|
||||
|
||||
Usage: reconstruct_lineage.py <state_db_copy> <root_session_id> <out_json>
|
||||
|
||||
ALWAYS run against a COPY of state.db, never the live file.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
DB = sys.argv[1]
|
||||
ROOT = sys.argv[2]
|
||||
OUT = sys.argv[3]
|
||||
|
||||
db = sqlite3.connect(DB)
|
||||
db.row_factory = sqlite3.Row
|
||||
|
||||
# collect the whole descendant tree, chronological by started_at
|
||||
import collections
|
||||
children = collections.defaultdict(list)
|
||||
for r in db.execute(
|
||||
"SELECT id, parent_session_id FROM sessions WHERE parent_session_id IS NOT NULL"
|
||||
):
|
||||
children[r["parent_session_id"]].append(r["id"])
|
||||
chain = []
|
||||
frontier = [ROOT]
|
||||
while frontier:
|
||||
sid = frontier.pop(0)
|
||||
chain.append(sid)
|
||||
frontier.extend(children.get(sid, []))
|
||||
starts = {r["id"]: r["started_at"] or "" for r in db.execute(
|
||||
f"SELECT id, started_at FROM sessions WHERE id IN ({','.join('?'*len(chain))})", chain)}
|
||||
chain.sort(key=lambda s: starts.get(s, ""))
|
||||
print(f"chain: {len(chain)} sessions")
|
||||
|
||||
SYNTH_MARKERS = (
|
||||
"[CONTEXT COMPACTION", "[CONTEXT SUMMARY", "[PRIOR CONTEXT",
|
||||
"preserved across context compression",
|
||||
)
|
||||
|
||||
seen = set()
|
||||
out = []
|
||||
sysprompt = None
|
||||
for sid in chain:
|
||||
if sysprompt is None:
|
||||
row = db.execute(
|
||||
"SELECT s.system_prompt, sp.prompt AS dedup_prompt FROM sessions s "
|
||||
"LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash "
|
||||
"WHERE s.id=?", (sid,)).fetchone()
|
||||
if row:
|
||||
sysprompt = row["system_prompt"] or row["dedup_prompt"] or None
|
||||
for r in db.execute(
|
||||
"SELECT * FROM messages WHERE session_id=? ORDER BY id", (sid,)
|
||||
):
|
||||
c = r["content"] or ""
|
||||
if any(m in c for m in SYNTH_MARKERS):
|
||||
continue # synthetic compaction artifact, not organic history
|
||||
h = hashlib.md5(
|
||||
(r["role"] + "\x00" + c + "\x00" + (r["tool_calls"] or "")).encode(
|
||||
"utf-8", "replace")
|
||||
).hexdigest()
|
||||
if h in seen:
|
||||
continue
|
||||
seen.add(h)
|
||||
m = {"role": r["role"], "content": c}
|
||||
if r["tool_calls"]:
|
||||
try:
|
||||
m["tool_calls"] = json.loads(r["tool_calls"])
|
||||
except Exception:
|
||||
pass
|
||||
if r["tool_call_id"]:
|
||||
m["tool_call_id"] = r["tool_call_id"]
|
||||
if r["tool_name"]:
|
||||
m["tool_name"] = r["tool_name"]
|
||||
out.append(m)
|
||||
|
||||
msgs = [{"role": "system", "content": sysprompt or ""}] + out
|
||||
chars = sum(len(m.get("content") or "") + len(json.dumps(m.get("tool_calls", ""), default=str)) for m in msgs)
|
||||
print(f"reconstructed: {len(msgs)} msgs, {chars:,} chars (~{chars//4:,} tok)")
|
||||
json.dump({"root": ROOT, "chain": chain, "messages": msgs}, open(OUT, "w", encoding="utf-8"), default=str)
|
||||
print(f"wrote {OUT}")
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Replay a ~500K-token prefix of a reconstructed lineage through compaction.
|
||||
|
||||
Usage: run_lineage_compaction.py <checkout> <lineage_json> <out_json> [cap_tokens]
|
||||
|
||||
Takes the chronological prefix of the lineage at the token cap (default 500K =
|
||||
the 50% trigger on a 1M-context model), aligned to a tool-group boundary, and
|
||||
runs ContextCompressor.compress() exactly as the live trigger would.
|
||||
"""
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
CHECKOUT = sys.argv[1]
|
||||
LINEAGE = sys.argv[2]
|
||||
OUT = sys.argv[3]
|
||||
CAP = int(sys.argv[4]) if len(sys.argv) > 4 else 500_000
|
||||
|
||||
sys.path.insert(0, CHECKOUT)
|
||||
|
||||
data = json.load(open(LINEAGE, encoding="utf-8"))
|
||||
msgs = data["messages"]
|
||||
|
||||
def tok(m):
|
||||
t = len(m.get("content") or "") // 4
|
||||
tc = m.get("tool_calls")
|
||||
if tc:
|
||||
t += len(json.dumps(tc, default=str)) // 4
|
||||
return t
|
||||
|
||||
# chronological prefix up to CAP tokens
|
||||
prefix = []
|
||||
total = 0
|
||||
for m in msgs:
|
||||
t = tok(m)
|
||||
if total + t > CAP and len(prefix) > 10:
|
||||
break
|
||||
prefix.append(m)
|
||||
total += t
|
||||
|
||||
# align the end: never end on an assistant msg with tool_calls whose results
|
||||
# were cut off; drop trailing orphans
|
||||
while prefix and prefix[-1].get("tool_calls"):
|
||||
prefix.pop()
|
||||
# also drop trailing tool results with no preceding assistant tool_calls kept
|
||||
# (compress()'s _sanitize_tool_pairs would handle it, but keep input clean)
|
||||
|
||||
before_tokens = sum(tok(m) for m in prefix)
|
||||
print(f"[{Path(CHECKOUT).name}] {Path(LINEAGE).stem}: prefix {len(prefix)} msgs ~{before_tokens:,} tok (cap {CAP:,})")
|
||||
|
||||
from agent.context_compressor import ContextCompressor # noqa: E402
|
||||
|
||||
model = "anthropic/claude-fable-5"
|
||||
comp = ContextCompressor(model=model, quiet_mode=True)
|
||||
before = copy.deepcopy(prefix)
|
||||
t0 = time.time()
|
||||
compressed = comp.compress(prefix, current_tokens=before_tokens, force=True)
|
||||
dt = time.time() - t0
|
||||
after_tokens = sum(tok(m) for m in compressed)
|
||||
print(f" -> {len(compressed)} msgs ~{after_tokens:,} tok in {dt:.1f}s (err={getattr(comp,'_last_summary_error',None)})")
|
||||
|
||||
json.dump({
|
||||
"meta": {
|
||||
"checkout": Path(CHECKOUT).name,
|
||||
"session_id": data["root"],
|
||||
"title": f"lineage {data['root']} ({len(data['chain'])} rotations)",
|
||||
"model": model,
|
||||
"elapsed_s": round(dt, 1),
|
||||
"before_msgs": len(before),
|
||||
"after_msgs": len(compressed),
|
||||
"before_tokens_est": before_tokens,
|
||||
"after_tokens_est": after_tokens,
|
||||
"summary_error": getattr(comp, "_last_summary_error", None),
|
||||
"todo_injection_block": None,
|
||||
},
|
||||
"before": before,
|
||||
"after": compressed,
|
||||
}, open(OUT, "w", encoding="utf-8"), default=str)
|
||||
print(f" wrote {OUT}")
|
||||
Reference in New Issue
Block a user