Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/state/test_write_lock_patience.py

@@ -0,0 +1,150 @@
+"""Write-lock patience for the shared state.db (#74478).
+
+A shared state.db is legitimately held for multi-second stretches by
+sibling Hermes processes (VACUUM after auto-prune, TRUNCATE checkpoint at
+close on a large WAL, a long FTS pass from an older still-running
+install).  The old attempt-counted retry budget (15 x <=150ms jitter)
+gave up in ~1-2s of retrying, so:
+
+- ``append_message`` failed -> the conversation loop aborted the turn as
+  ``session_persistence_failed`` ("No reply: ... session storage could
+  not be written") even though the store was healthy and merely busy;
+- ``SessionDB()`` open failed -> the CLI disabled persistence for the
+  whole run ("Failed to initialize SessionDB ... database is locked").
+
+These tests lock the DB from a second connection for a bounded window and
+assert the three contracts: transcript writes ride out long holds, open
+rides out long holds, and exhausted patience raises an error that names
+the real cause instead of reading like disk damage.
+"""
+
+import sqlite3
+import threading
+import time
+
+import pytest
+
+from hermes_state import SessionDB
+
+
+def _hold_write_lock(db_path, hold_s, started_evt):
+    """Hold the SQLite write lock on *db_path* for *hold_s* seconds."""
+    conn = sqlite3.connect(str(db_path), timeout=1.0, isolation_level=None)
+    try:
+        conn.execute("BEGIN IMMEDIATE")
+        started_evt.set()
+        time.sleep(hold_s)
+        conn.execute("COMMIT")
+    finally:
+        conn.close()
+
+
+@pytest.fixture
+def db(tmp_path):
+    d = SessionDB(db_path=tmp_path / "state.db")
+    yield d
+    d.close()
+
+
+class TestTranscriptWritePatience:
+    def test_append_message_survives_multi_second_lock_hold(self, db, tmp_path):
+        """A transcript append must ride out a lock held well past the old
+        ~1-2s attempt-counted budget instead of aborting the turn."""
+        db.create_session("s1", "cli")
+
+        started = threading.Event()
+        # 3s hold: comfortably beyond the old worst-case retry budget,
+        # comfortably inside _TRANSCRIPT_WRITE_PATIENCE_S.
+        holder = threading.Thread(
+            target=_hold_write_lock, args=(db.db_path, 3.0, started)
+        )
+        holder.start()
+        try:
+            assert started.wait(5.0)
+            msg_id = db.append_message(
+                session_id="s1", role="user", content="survived the lock"
+            )
+        finally:
+            holder.join(timeout=10.0)
+        assert not holder.is_alive()
+        assert isinstance(msg_id, int)
+        msgs = db.get_messages("s1")
+        assert any(m["content"] == "survived the lock" for m in msgs)
+
+    def test_transcript_patience_outlasts_routine_patience(self, db):
+        """append_message must be given MORE patience than routine writes —
+        the invariant that lets background writers give up while the
+        turn-critical append keeps waiting."""
+        assert db._TRANSCRIPT_WRITE_PATIENCE_S > db._WRITE_PATIENCE_S
+        # Both budgets must comfortably exceed the old ~2.25s worst case
+        # (15 attempts x 150ms) that lost races against real maintenance.
+        assert db._WRITE_PATIENCE_S >= 10.0
+        assert db._TRANSCRIPT_WRITE_PATIENCE_S >= 30.0
+
+    def test_exhausted_patience_names_the_real_cause(self, db, monkeypatch):
+        """When patience genuinely runs out, the error must say the lock was
+        held by another process — not read like disk/permission damage."""
+        monkeypatch.setattr(SessionDB, "_WRITE_PATIENCE_S", 0.2)
+
+        started = threading.Event()
+        holder = threading.Thread(
+            target=_hold_write_lock, args=(db.db_path, 2.0, started)
+        )
+        holder.start()
+        try:
+            assert started.wait(5.0)
+            with pytest.raises(sqlite3.OperationalError) as excinfo:
+                db.set_meta("k", "v")  # routine write, short patience
+        finally:
+            holder.join(timeout=10.0)
+        assert not holder.is_alive()
+        text = str(excinfo.value)
+        assert "another Hermes process" in text
+        assert "healthy" in text
+
+    def test_write_succeeds_immediately_when_uncontended(self, db):
+        """Patience must cost nothing when there is no contention."""
+        db.create_session("s2", "cli")
+        t0 = time.monotonic()
+        db.append_message(session_id="s2", role="user", content="fast")
+        assert time.monotonic() - t0 < 5.0  # loose: no patience-length stall
+
+
+class TestOpenLockPatience:
+    def test_open_survives_multi_second_lock_hold(self, tmp_path):
+        """SessionDB() open must wait out a sibling's lock hold instead of
+        disabling persistence for the whole run."""
+        db_path = tmp_path / "state.db"
+        # Create + close so the schema exists (open still runs reconcile DDL
+        # through the same 1s-timeout connection).
+        SessionDB(db_path=db_path).close()
+
+        started = threading.Event()
+        holder = threading.Thread(
+            target=_hold_write_lock, args=(db_path, 3.0, started)
+        )
+        holder.start()
+        try:
+            assert started.wait(5.0)
+            db = SessionDB(db_path=db_path)  # must NOT raise
+        finally:
+            holder.join(timeout=10.0)
+        assert not holder.is_alive()
+        try:
+            db.create_session("s-open", "cli")
+            db.append_message(session_id="s-open", role="user", content="ok")
+            assert len(db.get_messages("s-open")) == 1
+        finally:
+            db.close()
+
+    def test_open_propagates_non_lock_errors_immediately(self, tmp_path):
+        """A non-lock open failure must not sit in the patience loop."""
+        # A directory is not openable as a database file — raises an
+        # OperationalError that is NOT the locked/busy class.
+        bad_path = tmp_path / "state.db"
+        bad_path.mkdir()
+        t0 = time.monotonic()
+        with pytest.raises(sqlite3.Error):
+            SessionDB(db_path=bad_path)
+        # Must fail well before a full patience window (loose bound).
+        assert time.monotonic() - t0 < 15.0