Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/test_98924_readonly_fts_decode_error.py

@@ -0,0 +1,138 @@
+"""Test that read-only SessionDB survives invalid UTF-8 in FTS content (issue #98924)."""
+
+import sqlite3
+from pathlib import Path
+
+import pytest
+
+from hermes_state import SessionDB
+
+
+def _write_invalid_utf8_row(db_path: Path) -> None:
+    """Inject a non-UTF-8 byte into an existing message row via low-level API.
+    
+    Python's high-level sqlite3.Connection will not let us write invalid
+    UTF-8 into a TEXT column — it decodes and re-encodes. We use the
+    connection's lower-level interface to force it through.
+    """
+    # CAST(x'61625F816364' AS TEXT) → 'ab_�cd' with 0x81 at position 3
+    conn = sqlite3.connect(str(db_path))
+    try:
+        conn.execute("PRAGMA foreign_keys = OFF")
+        # Force the bytes through by overwriting an existing row id.
+        conn.execute("UPDATE messages SET content = CAST(x'61625F816364' AS TEXT) WHERE id = 1")
+        conn.commit()
+    finally:
+        conn.close()
+
+
+class TestReadOnlyFTSDecodeError:
+    def test_read_only_open_survives_invalid_utf8_in_fts_content(self, tmp_path):
+        """Non-UTF-8 bytes in messages.content don't kill the read-only open.
+        
+        Regression test for issue #98924: a developer's state.db contained a
+        byte 0x81 in a messages row. External-content FTS5 (v23 layout) reads
+        that column when building a result set. On some Python/SQLite builds
+        the decode failure surfaces as UnicodeDecodeError; on others as
+        OperationalError("Could not decode to UTF-8 column ...").
+        
+        The old code caught only sqlite3.OperationalError and would re-raise
+        any other exception. UnicodeDecodeError (a ValueError, not an
+        sqlite3.Error subclass) therefore bypassed the guard and killed the
+        read-only connection, taking down every read endpoint (GET /api/sessions).
+        
+        The fix catches (sqlite3.OperationalError, UnicodeDecodeError) and
+        treats the decode error the same as "FTS unavailable" — the index is
+        degraded but the connection stays open. Search may return less or fail
+        on that corrupted row, but writes and non-FTS reads keep working.
+        """
+        db_path = tmp_path / "state.db"
+        
+        # Create a fresh DB with v23 external-content FTS.
+        writable = SessionDB(db_path=db_path)
+        writable.create_session("decode-test", source="cli")
+        writable.append_message("decode-test", role="user", content="valid text")
+        writable.close()
+        
+        # Insert a row with invalid UTF-8 through the sqlite3 CLI.
+        _write_invalid_utf8_row(db_path)
+        
+        # Also trigger a rebuild so the invalid bytes are in the FTS index.
+        conn = sqlite3.connect(str(db_path))
+        conn.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')")
+        conn.commit()
+        conn.close()
+        
+        # The shipped regression: SessionDB(read_only=True) should NOT raise.
+        # Before the fix, this raised UnicodeDecodeError from _fts_table_probe.
+        read_only = SessionDB(db_path=db_path, read_only=True)
+        try:
+            # Verify the connection opened — _fts_table_probe didn't kill init.
+            assert read_only._conn is not None
+        finally:
+            read_only.close()
+        
+        # Also verify that the index degraded: _fts_enabled should be None or
+        # False, not True, because the corrupt content prevents probing.
+        read_only2 = SessionDB(db_path=db_path, read_only=True)
+        try:
+            # The index is broken; the store itself must stay accessible.
+            assert read_only2._conn is not None
+        finally:
+            read_only2.close()
+
+
+def _corrupt_schema_with_raw_bytes(db_path: Path) -> None:
+    """Rewrite an FTS vtable's sqlite_master row with invalid UTF-8 bytes.
+
+    This is the fixture that actually reproduces #98924 on main: pysqlite
+    raises a bare UnicodeDecodeError at execute() time when SQLite's own
+    error/schema text carries raw non-UTF-8 file bytes. (Invalid UTF-8 in
+    messages.content alone does NOT make the LIMIT 0 probe raise.)
+    """
+    conn = sqlite3.connect(str(db_path), isolation_level=None)
+    try:
+        conn.execute("PRAGMA writable_schema=ON")
+        badname = b"tbl_\x81\x82"
+        conn.execute(
+            "UPDATE sqlite_master SET name=CAST(? AS TEXT), "
+            "tbl_name=CAST(? AS TEXT), sql=CAST(? AS TEXT) "
+            "WHERE name='messages_fts_trigram'",
+            (badname, badname, b"CREATE GARBAGE \x81\x82"),
+        )
+        ver = conn.execute("PRAGMA schema_version").fetchone()[0]
+        conn.execute(f"PRAGMA schema_version = {ver + 1}")
+        conn.execute("PRAGMA writable_schema=OFF")
+    finally:
+        conn.close()
+
+
+class TestSchemaBytesDecodeError:
+    def test_read_only_open_survives_raw_bytes_in_schema(self, tmp_path):
+        """Genuine on-main repro of #98924: schema-area corruption whose raw
+        bytes reach pysqlite's error-message decode. Before the fix the probe
+        re-raised the resulting UnicodeDecodeError and killed read-only init.
+        """
+        db_path = tmp_path / "state.db"
+        writable = SessionDB(db_path=db_path)
+        writable.create_session("schema-bytes", source="cli")
+        writable.append_message("schema-bytes", role="user", content="hello")
+        writable.close()
+
+        _corrupt_schema_with_raw_bytes(db_path)
+
+        # Precondition: the raw probe really raises UnicodeDecodeError on
+        # this fixture (guards the test against silently stopping to
+        # exercise the bug on future SQLite versions).
+        raw = sqlite3.connect(str(db_path))
+        try:
+            with pytest.raises(UnicodeDecodeError):
+                raw.execute("SELECT * FROM messages_fts LIMIT 0")
+        finally:
+            raw.close()
+
+        read_only = SessionDB(db_path=db_path, read_only=True)
+        try:
+            assert read_only._conn is not None
+        finally:
+            read_only.close()