Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/test_fts_cjk_bigram.py

@@ -0,0 +1,244 @@
+"""Tests for the messages_fts_cjk CJK-bigram index (salvaged from PR #65544).
+
+Builds the loadable tokenizer from native/fts5_cjk/fts5_cjk.c on the fly;
+skips when no C toolchain / extension loading is available.
+"""
+
+import shutil
+import sqlite3
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from hermes_state import FTS_CJK_STALE_KEY, SessionDB
+
+REPO = Path(__file__).resolve().parent.parent
+SRC = REPO / "native" / "fts5_cjk" / "fts5_cjk.c"
+VENDOR = REPO / "native" / "fts5_cjk" / "vendor"
+
+
+@pytest.fixture(scope="session")
+def cjk_so(tmp_path_factory):
+    if shutil.which("gcc") is None or not SRC.exists():
+        pytest.skip("no C toolchain / tokenizer source")
+    out = tmp_path_factory.mktemp("fts5cjk") / "libfts5_cjk.so"
+    try:
+        subprocess.run(
+            ["gcc", "-shared", "-fPIC", "-O2", f"-I{VENDOR}", str(SRC),
+             "-o", str(out)],
+            check=True, capture_output=True, text=True,
+        )
+    except subprocess.CalledProcessError as e:
+        pytest.skip(f"tokenizer build failed: {e.stderr[:200]}")
+    # Loadability probe (extension loading may be disabled in this build).
+    probe = sqlite3.connect(":memory:")
+    try:
+        probe.enable_load_extension(True)
+        probe.load_extension(str(out))
+    except Exception as e:
+        pytest.skip(f"extension loading unavailable: {e}")
+    finally:
+        probe.close()
+    return out
+
+
+@pytest.fixture()
+def db(cjk_so, tmp_path, monkeypatch):
+    monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
+    d = SessionDB(db_path=tmp_path / "state.db")
+    assert d._fts_cjk_loaded, "tokenizer must load on the writer connection"
+    assert d._fts_cjk_available, "fresh DB must be born with the cjk index"
+    d.create_session(session_id="s1", source="cli", model="m")
+    d.append_message("s1", role="user", content="웅기가 shared default 프로필을 요청했다")
+    d.append_message("s1", role="assistant", content="일본 MCP 후보 우선순위 정리했습니다")
+    d.append_message("s1", role="user", content="graphiti daemon looks healthy")
+    d.append_message("s1", role="tool", content="일본 tool output blob", tool_name="terminal")
+    yield d
+    d.close()
+
+
+def test_two_char_korean_hits_cjk_index(db):
+    rows = db.search_messages("웅기", limit=10)
+    assert rows and "웅기" in rows[0]["snippet"]
+    rows = db.search_messages("일본", limit=10)
+    assert rows
+
+
+def test_mixed_and_ascii_queries(db):
+    assert db.search_messages("graphiti", limit=10)
+    assert db.search_messages('"shared default" AND 웅기', limit=10)
+    assert db.search_messages("우선순위", limit=10)
+
+
+
+
+def test_lone_single_cjk_char_routes_like(db):
+    # 1-char CJK terms keep LIKE substring semantics (bigram index only
+    # holds unigrams for isolated chars). "가" appears inside 웅기가.
+    assert db._describe_search_path("가") == "like_scan"
+    rows = db.search_messages("가", limit=10)
+    assert rows, "LIKE fallback must still find substring matches"
+
+
+
+
+
+
+
+
+def test_config_toggle_disables_cjk(cjk_so, tmp_path, monkeypatch):
+    monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
+    monkeypatch.setenv("HERMES_CJK_FTS", "0")
+    d = SessionDB(db_path=tmp_path / "state.db")
+    try:
+        assert not d._fts_cjk_loaded
+        assert not d._fts_cjk_available
+        # No cjk objects created at all.
+        with d._lock:
+            row = d._conn.execute(
+                "SELECT 1 FROM sqlite_master WHERE name = 'messages_fts_cjk'"
+            ).fetchone()
+        assert row is None
+    finally:
+        d.close()
+
+
+
+
+
+
+def test_existing_v23_db_gains_cjk_via_optimize(cjk_so, tmp_path, monkeypatch):
+    """A v23 DB created BEFORE the extension existed: next capable open
+    creates the index with backfill markers; optimize-storage backfills."""
+    monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(tmp_path / "absent.so"))
+    db_path = tmp_path / "state.db"
+    d1 = SessionDB(db_path=db_path)
+    d1.create_session(session_id="s1", source="cli", model="m")
+    for i in range(10):
+        d1.append_message("s1", role="user", content=f"기존 메시지 {i}")
+    d1.close()
+
+    monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
+    d2 = SessionDB(db_path=db_path)
+    assert d2._fts_cjk_loaded
+    # Backfill pending — index not served yet, old rows not indexed.
+    assert not d2._fts_cjk_available
+    st = d2.fts_cjk_rebuild_status()
+    assert st is not None and st["pending"]
+    assert d2.fts_optimize_available()
+    # NEW rows are indexed live by the id-gated triggers even mid-backfill.
+    d2.append_message("s1", role="user", content="새로운 메시지")
+    # Search answers via legacy routes meanwhile.
+    assert d2.search_messages("기존", limit=10)
+
+    result = d2.optimize_fts_storage(vacuum=False)
+    assert result["ok"]
+    assert d2._fts_cjk_available
+    assert d2.fts_cjk_rebuild_status() is None
+    assert d2._describe_search_path("기존") == "fts_cjk"
+    rows = d2.search_messages("기존", limit=20)
+    assert len(rows) == 10
+    assert d2.search_messages("새로운", limit=10)
+    d2.close()
+
+
+def test_legacy_v22_optimize_lands_on_cjk(cjk_so, tmp_path, monkeypatch):
+    """A legacy inline-FTS (pre-v23) DB optimized on a tokenizer-capable
+    host comes out with BOTH the v23 external-content layout AND a complete
+    cjk index in the same run."""
+    import time as _time
+
+    from hermes_state import SCHEMA_SQL
+
+    monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
+    db_path = tmp_path / "state.db"
+
+    # Hand-build a genuine legacy inline DB (single-column messages_fts).
+    conn = sqlite3.connect(str(db_path))
+    conn.executescript(SCHEMA_SQL)
+    conn.executescript("""
+        DROP TABLE IF EXISTS messages_fts;
+        DROP TABLE IF EXISTS messages_fts_trigram;
+        DROP VIEW IF EXISTS messages_fts_trigram_src;
+        CREATE VIRTUAL TABLE messages_fts USING fts5(content);
+        CREATE TRIGGER messages_fts_insert AFTER INSERT ON messages BEGIN
+            INSERT INTO messages_fts(rowid, content) VALUES (new.id, COALESCE(new.content,''));
+        END;
+    """)
+    conn.execute("DELETE FROM schema_version")
+    conn.execute("INSERT INTO schema_version (version) VALUES (10)")
+    conn.execute(
+        "INSERT INTO sessions (id, source, started_at) VALUES ('s1', 'cli', ?)",
+        (_time.time(),),
+    )
+    for role, content in (
+        ("user", "레거시 일본 메시지"),
+        ("assistant", "legacy english reply"),
+        ("tool", "레거시 tool output"),
+    ):
+        conn.execute(
+            "INSERT INTO messages (session_id, timestamp, role, content) "
+            "VALUES ('s1', ?, ?, ?)",
+            (_time.time(), role, content),
+        )
+    conn.commit()
+    conn.close()
+
+    d = SessionDB(db_path=db_path)
+    try:
+        assert d.fts_optimize_available()
+        # Legacy DB: cjk index deliberately not created at open (the legacy
+        # branch of _init_schema doesn't touch v23 surfaces).
+        assert not d._fts_cjk_available
+
+        result = d.optimize_fts_storage(vacuum=False)
+        assert result["ok"]
+        assert d._fts_cjk_available
+        assert d.fts_cjk_rebuild_status() is None
+        assert d._describe_search_path("일본") == "fts_cjk"
+        assert d.search_messages("일본", limit=10)
+        assert d.search_messages("legacy english", limit=10)
+        with d._lock:
+            idx = d._conn.execute(
+                "SELECT COUNT(*) FROM messages_fts_cjk"
+            ).fetchone()[0]
+            non_tool = d._conn.execute(
+                "SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
+            ).fetchone()[0]
+        assert idx == non_tool
+    finally:
+        d.close()
+
+
+def test_pure_latin_embedded_in_cjk_recovered_via_cjk_index(db):
+    """#54242 residual: a pure-Latin query for a token embedded in CJK text
+    (no whitespace) misses on unicode61; with the cjk index available the
+    zero-result fallback recovers it as an exact ranked token match."""
+    db.append_message("s1", role="user", content="修改youer服务端的계획")
+    rows = db.search_messages("youer", limit=10)
+    assert rows and "youer" in rows[0]["snippet"]
+    # Short tokens (<3 chars, no trigram) are also recoverable via cjk.
+    db.append_message("s1", role="user", content="에러코드ab확인")
+    rows = db.search_messages("ab", limit=10)
+    assert rows
+
+
+def test_fresh_db_index_counts_exclude_tool_rows(db):
+    with db._lock:
+        idx = db._conn.execute(
+            "SELECT COUNT(*) FROM messages_fts_cjk"
+        ).fetchone()[0]
+        non_tool = db._conn.execute(
+            "SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
+        ).fetchone()[0]
+    assert idx == non_tool
+
+
+def test_integrity_after_lifecycle(db):
+    db.append_message("s1", role="user", content="무결성 검사")
+    with db._lock:
+        db._conn.execute(
+            "INSERT INTO messages_fts_cjk(messages_fts_cjk) "
+            "VALUES('integrity-check')"
+        )