Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/state/test_fts_trigram_cron_exclusion.py

@@ -0,0 +1,254 @@
+"""Cron-source exclusion from the external-content trigram FTS index."""
+
+from __future__ import annotations
+
+import sqlite3
+
+import pytest
+
+from hermes_state import FTS_TRIGRAM_SQL, SCHEMA_VERSION, SessionDB
+
+
+@pytest.fixture
+def db(tmp_path):
+    session_db = SessionDB(db_path=tmp_path / "state.db")
+    if not session_db._trigram_available:
+        session_db.close()
+        pytest.skip("trigram tokenizer unavailable in this SQLite build")
+    yield session_db
+    session_db.close()
+
+
+def _trigram_rowids(db: SessionDB) -> set[int]:
+    return {
+        row[0]
+        for row in db._conn.execute(
+            "SELECT id FROM messages_fts_trigram_docsize ORDER BY id"
+        ).fetchall()
+    }
+
+
+def _install_pre_v27_trigram(db: SessionDB, *, with_tool_calls: bool = False) -> None:
+    """Recreate the pre-cron-exclusion external-content trigram boundary.
+
+    ``with_tool_calls=True`` reproduces the FTS_STORAGE_VERSION 1 vtable
+    (``tool_calls`` projected) that installs upgraded before #88217 carry;
+    the default is the v2 column set with only the view/trigger predicates
+    behind, which is what the in-place v29 migration handles.
+    """
+    cols = "content, tool_name" + (", tool_calls" if with_tool_calls else "")
+    vals = "new.content, new.tool_name" + (", new.tool_calls" if with_tool_calls else "")
+    db._conn.executescript(
+        f"""
+        DROP TRIGGER messages_fts_trigram_insert;
+        DROP TRIGGER messages_fts_trigram_delete;
+        DROP TRIGGER messages_fts_trigram_update;
+        DROP TABLE messages_fts_trigram;
+        DROP VIEW messages_fts_trigram_src;
+        CREATE VIEW messages_fts_trigram_src AS
+            SELECT id, role, content, tool_name, tool_calls
+            FROM messages WHERE role <> 'tool';
+        CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(
+            {cols},
+            content='messages_fts_trigram_src',
+            content_rowid='id',
+            tokenize='trigram'
+        );
+        CREATE TRIGGER messages_fts_trigram_insert AFTER INSERT ON messages
+        WHEN new.role <> 'tool'
+        BEGIN
+            INSERT INTO messages_fts_trigram(rowid, {cols})
+            VALUES (new.id, {vals});
+        END;
+        """
+    )
+
+
+def test_fresh_trigram_indexes_conversations_but_not_cron(db: SessionDB):
+    db.create_session("cli", source="cli")
+    db.create_session("cron", source="cron")
+    cli_id = db.append_message("cli", role="user", content="交付状态正常")
+    cron_id = db.append_message("cron", role="user", content="定时任务状态正常")
+
+    assert _trigram_rowids(db) == {cli_id}
+    assert cron_id not in _trigram_rowids(db)
+    assert db._conn.execute(
+        "SELECT id FROM messages_fts_docsize WHERE id = ?", (cron_id,)
+    ).fetchone() is not None
+
+
+def test_cron_remains_searchable_via_standard_fts_and_explicit_cjk_fallback(
+    db: SessionDB,
+):
+    db.create_session("cron", source="cron")
+    db.append_message(
+        "cron", role="assistant", content="quarterly archive 大别山项目 complete"
+    )
+
+    assert [row["session_id"] for row in db.search_messages("quarterly")] == [
+        "cron"
+    ]
+    assert [
+        row["session_id"]
+        for row in db.search_messages("大别山项目", source_filter=["cron"])
+    ] == ["cron"]
+
+
+def test_deferred_rebuild_does_not_reintroduce_cron(db: SessionDB):
+    db.create_session("cli", source="cli")
+    db.create_session("cron", source="cron")
+    cli_id = db.append_message("cli", role="assistant", content="交互会话内容")
+    db.append_message("cron", role="assistant", content="定时会话内容")
+
+    with db._lock:
+        db._reset_fts_index_to_empty(db._conn)
+        db._seed_fts_rebuild_markers(db._conn, force=True)
+        db._conn.commit()
+    while db.fts_rebuild_step():
+        pass
+
+    assert _trigram_rowids(db) == {cli_id}
+
+
+def test_existing_external_layout_rebuilds_trigram_on_upgrade(tmp_path):
+    db_path = tmp_path / "state.db"
+    old = SessionDB(db_path=db_path)
+    if not old._trigram_available:
+        old.close()
+        pytest.skip("trigram tokenizer unavailable in this SQLite build")
+    # The virtual table keeps referring to the view by name, so this recreates
+    # the exact old external-content boundary without reading source text.
+    _install_pre_v27_trigram(old)
+    old.create_session("cli", source="cli")
+    old.create_session("cron", source="cron")
+    cli_id = old.append_message("cli", role="user", content="交互迁移内容")
+    cron_id = old.append_message("cron", role="user", content="定时迁移内容")
+    assert _trigram_rowids(old) == {cli_id, cron_id}
+    old._conn.execute("UPDATE schema_version SET version = ?", (SCHEMA_VERSION - 1,))
+    old._conn.commit()
+    old.close()
+
+    migrated = SessionDB(db_path=db_path)
+    try:
+        assert _trigram_rowids(migrated) == {cli_id}
+        view_sql = migrated._conn.execute(
+            "SELECT sql FROM sqlite_master "
+            "WHERE type = 'view' AND name = 'messages_fts_trigram_src'"
+        ).fetchone()[0]
+        assert "sessions" in view_sql
+        assert "cron" in view_sql
+        migrated._conn.execute(
+            "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('integrity-check')"
+        )
+    finally:
+        migrated.close()
+
+
+def test_install_already_at_v28_still_gets_the_cron_exclusion_migration(tmp_path):
+    """The migration gate must fire for installs that were on main's v28.
+
+    The original PR gated on ``current_version < 27``; main had meanwhile
+    reached SCHEMA_VERSION 28 via column-reconciliation bumps, so a v28
+    database would have skipped the rebuild and kept cron rows in the trigram
+    index forever. Pin the gate against the version main actually shipped.
+    """
+    db_path = tmp_path / "state.db"
+    old = SessionDB(db_path=db_path)
+    if not old._trigram_available:
+        old.close()
+        pytest.skip("trigram tokenizer unavailable in this SQLite build")
+    _install_pre_v27_trigram(old)
+    old.create_session("cli", source="cli")
+    old.create_session("cron", source="cron")
+    cli_id = old.append_message("cli", role="user", content="交互迁移内容")
+    cron_id = old.append_message("cron", role="user", content="定时迁移内容")
+    assert _trigram_rowids(old) == {cli_id, cron_id}
+    old._conn.execute("UPDATE schema_version SET version = 28")
+    old._conn.commit()
+    old.close()
+
+    migrated = SessionDB(db_path=db_path)
+    try:
+        assert _trigram_rowids(migrated) == {cli_id}, (
+            "a v28 database kept cron rows in the trigram index: the migration gate did not fire"
+        )
+    finally:
+        migrated.close()
+
+
+def test_v1_tool_calls_layout_is_left_for_optimize_storage(tmp_path):
+    """A FTS_STORAGE_VERSION 1 trigram vtable (``tool_calls`` projected) must
+    survive the v29 startup migration untouched and be finished by the opt-in
+    ``optimize_fts_storage`` path — not half-migrated into a view/vtable
+    column mismatch (which used to fail the rebuild with
+    ``no such column: T.tool_calls``)."""
+    db_path = tmp_path / "state.db"
+    old = SessionDB(db_path=db_path)
+    if not old._trigram_available:
+        old.close()
+        pytest.skip("trigram tokenizer unavailable in this SQLite build")
+    _install_pre_v27_trigram(old, with_tool_calls=True)
+    old.create_session("cli", source="cli")
+    old.create_session("cron", source="cron")
+    cli_id = old.append_message("cli", role="user", content="交互迁移内容")
+    cron_id = old.append_message("cron", role="user", content="定时迁移内容")
+    assert _trigram_rowids(old) == {cli_id, cron_id}
+    old._conn.execute("UPDATE schema_version SET version = 28")
+    old._conn.commit()
+    old.close()
+
+    migrated = SessionDB(db_path=db_path)  # must not raise
+    try:
+        # Startup left the v1 layout alone (cron row still there) …
+        assert _trigram_rowids(migrated) == {cli_id, cron_id}
+        assert migrated.fts_optimize_available() is True
+        # … and the opt-in path completes the transition: v2 columns,
+        # cron-filtered view, cron row purged.
+        migrated.optimize_fts_storage()
+        cols = [r[1] for r in migrated._conn.execute("PRAGMA table_info(messages_fts_trigram)")]
+        assert "tool_calls" not in cols
+        assert _trigram_rowids(migrated) == {cli_id}
+    finally:
+        migrated.close()
+
+
+def test_partial_upgrade_view_does_not_skip_historical_rebuild(tmp_path):
+    db_path = tmp_path / "state.db"
+    old = SessionDB(db_path=db_path)
+    if not old._trigram_available:
+        old.close()
+        pytest.skip("trigram tokenizer unavailable in this SQLite build")
+    _install_pre_v27_trigram(old)
+    old.create_session("cron", source="cron")
+    cron_id = old.append_message("cron", role="assistant", content="迁移中断内容")
+    assert _trigram_rowids(old) == {cron_id}
+
+    # Simulate a crash after new DDL landed but before the rebuild/schema stamp.
+    for name in (
+        "messages_fts_trigram_insert",
+        "messages_fts_trigram_delete",
+        "messages_fts_trigram_update",
+    ):
+        old._conn.execute(f"DROP TRIGGER IF EXISTS {name}")
+    old._conn.execute("DROP VIEW messages_fts_trigram_src")
+    old._conn.executescript(FTS_TRIGRAM_SQL)
+    old._conn.execute("UPDATE schema_version SET version = ?", (SCHEMA_VERSION - 1,))
+    old._conn.commit()
+    old.close()
+
+    migrated = SessionDB(db_path=db_path)
+    try:
+        assert _trigram_rowids(migrated) == set()
+    finally:
+        migrated.close()
+
+
+def test_delete_of_unindexed_cron_row_keeps_trigram_consistent(db: SessionDB):
+    db.create_session("cron", source="cron")
+    cron_id = db.append_message("cron", role="user", content="不会进入索引")
+    assert cron_id not in _trigram_rowids(db)
+
+    db._conn.execute("DELETE FROM messages WHERE id = ?", (cron_id,))
+    db._conn.execute(
+        "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('integrity-check')"
+    )