"""Test that read-only SessionDB survives invalid UTF-8 in FTS content (issue #98924).""" import sqlite3 from pathlib import Path import pytest from hermes_state import SessionDB def _write_invalid_utf8_row(db_path: Path) -> None: """Inject a non-UTF-8 byte into an existing message row via low-level API. Python's high-level sqlite3.Connection will not let us write invalid UTF-8 into a TEXT column — it decodes and re-encodes. We use the connection's lower-level interface to force it through. """ # CAST(x'61625F816364' AS TEXT) → 'ab_�cd' with 0x81 at position 3 conn = sqlite3.connect(str(db_path)) try: conn.execute("PRAGMA foreign_keys = OFF") # Force the bytes through by overwriting an existing row id. conn.execute("UPDATE messages SET content = CAST(x'61625F816364' AS TEXT) WHERE id = 1") conn.commit() finally: conn.close() class TestReadOnlyFTSDecodeError: def test_read_only_open_survives_invalid_utf8_in_fts_content(self, tmp_path): """Non-UTF-8 bytes in messages.content don't kill the read-only open. Regression test for issue #98924: a developer's state.db contained a byte 0x81 in a messages row. External-content FTS5 (v23 layout) reads that column when building a result set. On some Python/SQLite builds the decode failure surfaces as UnicodeDecodeError; on others as OperationalError("Could not decode to UTF-8 column ..."). The old code caught only sqlite3.OperationalError and would re-raise any other exception. UnicodeDecodeError (a ValueError, not an sqlite3.Error subclass) therefore bypassed the guard and killed the read-only connection, taking down every read endpoint (GET /api/sessions). The fix catches (sqlite3.OperationalError, UnicodeDecodeError) and treats the decode error the same as "FTS unavailable" — the index is degraded but the connection stays open. Search may return less or fail on that corrupted row, but writes and non-FTS reads keep working. """ db_path = tmp_path / "state.db" # Create a fresh DB with v23 external-content FTS. writable = SessionDB(db_path=db_path) writable.create_session("decode-test", source="cli") writable.append_message("decode-test", role="user", content="valid text") writable.close() # Insert a row with invalid UTF-8 through the sqlite3 CLI. _write_invalid_utf8_row(db_path) # Also trigger a rebuild so the invalid bytes are in the FTS index. conn = sqlite3.connect(str(db_path)) conn.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')") conn.commit() conn.close() # The shipped regression: SessionDB(read_only=True) should NOT raise. # Before the fix, this raised UnicodeDecodeError from _fts_table_probe. read_only = SessionDB(db_path=db_path, read_only=True) try: # Verify the connection opened — _fts_table_probe didn't kill init. assert read_only._conn is not None finally: read_only.close() # Also verify that the index degraded: _fts_enabled should be None or # False, not True, because the corrupt content prevents probing. read_only2 = SessionDB(db_path=db_path, read_only=True) try: # The index is broken; the store itself must stay accessible. assert read_only2._conn is not None finally: read_only2.close() def _corrupt_schema_with_raw_bytes(db_path: Path) -> None: """Rewrite an FTS vtable's sqlite_master row with invalid UTF-8 bytes. This is the fixture that actually reproduces #98924 on main: pysqlite raises a bare UnicodeDecodeError at execute() time when SQLite's own error/schema text carries raw non-UTF-8 file bytes. (Invalid UTF-8 in messages.content alone does NOT make the LIMIT 0 probe raise.) """ conn = sqlite3.connect(str(db_path), isolation_level=None) try: conn.execute("PRAGMA writable_schema=ON") badname = b"tbl_\x81\x82" conn.execute( "UPDATE sqlite_master SET name=CAST(? AS TEXT), " "tbl_name=CAST(? AS TEXT), sql=CAST(? AS TEXT) " "WHERE name='messages_fts_trigram'", (badname, badname, b"CREATE GARBAGE \x81\x82"), ) ver = conn.execute("PRAGMA schema_version").fetchone()[0] conn.execute(f"PRAGMA schema_version = {ver + 1}") conn.execute("PRAGMA writable_schema=OFF") finally: conn.close() class TestSchemaBytesDecodeError: def test_read_only_open_survives_raw_bytes_in_schema(self, tmp_path): """Genuine on-main repro of #98924: schema-area corruption whose raw bytes reach pysqlite's error-message decode. Before the fix the probe re-raised the resulting UnicodeDecodeError and killed read-only init. """ db_path = tmp_path / "state.db" writable = SessionDB(db_path=db_path) writable.create_session("schema-bytes", source="cli") writable.append_message("schema-bytes", role="user", content="hello") writable.close() _corrupt_schema_with_raw_bytes(db_path) # Precondition: the raw probe really raises UnicodeDecodeError on # this fixture (guards the test against silently stopping to # exercise the bug on future SQLite versions). raw = sqlite3.connect(str(db_path)) try: with pytest.raises(UnicodeDecodeError): raw.execute("SELECT * FROM messages_fts LIMIT 0") finally: raw.close() read_only = SessionDB(db_path=db_path, read_only=True) try: assert read_only._conn is not None finally: read_only.close()