297 lines
10 KiB
Python
297 lines
10 KiB
Python
"""Tests for gateway/hosted_room_replicas.py — replica ingest, promotion, and
|
|
stale-authority demotion for hosted Group Chat rooms."""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
import gateway.hosted_room_replicas as replicas
|
|
import gateway.hosted_rooms as rooms
|
|
|
|
USER = {"kind": "user", "id": "tek"}
|
|
MEMBERS = [{"kind": "bot", "id": "planner"}, {"kind": "bot", "id": "coder"}]
|
|
|
|
AUTH_A = "install:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
|
AUTH_B = "install:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
|
|
|
|
|
def _authority_db(tmp_path, name="authority.db"):
|
|
return tmp_path / name
|
|
|
|
|
|
def _replica_db(tmp_path, name="replica.db"):
|
|
return tmp_path / name
|
|
|
|
|
|
def _seed_room(db, *, gateway_id=AUTH_A, n_events=3, room_id="room-1"):
|
|
rooms.create_room(
|
|
db,
|
|
room_id=room_id,
|
|
name="Field Room",
|
|
members=MEMBERS,
|
|
authority_gateway_id=gateway_id,
|
|
)
|
|
for index in range(n_events):
|
|
rooms.append_event(
|
|
db,
|
|
room_id=room_id,
|
|
event_id=f"e{index}",
|
|
kind="message.user",
|
|
actor=USER,
|
|
payload={"text": f"msg {index} 😀"},
|
|
authority_gateway_id=gateway_id,
|
|
authority_epoch=1,
|
|
)
|
|
return rooms.read_events(db, room_id=room_id, since_seq=0, limit=100)
|
|
|
|
|
|
def test_ingest_page_persists_events_and_lineage(tmp_path):
|
|
page = _seed_room(_authority_db(tmp_path))
|
|
rdb = _replica_db(tmp_path)
|
|
result = replicas.ingest_page(
|
|
rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page
|
|
)
|
|
assert result["ingested"] == 3
|
|
assert result["stored_seq"] == 3
|
|
assert result["caught_up"] is True
|
|
state = replicas.replica_state(rdb, room_id="room-1")
|
|
assert state["last_seq"] == 3
|
|
assert state["authority"] == page["authority"]
|
|
assert state["members"] == MEMBERS
|
|
|
|
|
|
def test_ingest_page_is_idempotent(tmp_path):
|
|
page = _seed_room(_authority_db(tmp_path))
|
|
rdb = _replica_db(tmp_path)
|
|
replicas.ingest_page(
|
|
rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page
|
|
)
|
|
again = replicas.ingest_page(
|
|
rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page
|
|
)
|
|
assert again["ingested"] == 0
|
|
assert again["stored_seq"] == 3
|
|
|
|
|
|
def test_ingest_rejects_sequence_gap(tmp_path):
|
|
adb = _authority_db(tmp_path)
|
|
_seed_room(adb, n_events=5)
|
|
later = rooms.read_events(adb, room_id="room-1", since_seq=2, limit=100)
|
|
rdb = _replica_db(tmp_path)
|
|
with pytest.raises(replicas.ReplicaGapError):
|
|
replicas.ingest_page(
|
|
rdb,
|
|
room_id="room-1",
|
|
room_name="Field Room",
|
|
members=MEMBERS,
|
|
page=later,
|
|
)
|
|
|
|
|
|
def test_ingest_rejects_epoch_regression(tmp_path):
|
|
page = _seed_room(_authority_db(tmp_path))
|
|
rdb = _replica_db(tmp_path)
|
|
newer = json.loads(json.dumps(page))
|
|
newer["authority"]["epoch"] = 3
|
|
replicas.ingest_page(
|
|
rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=newer
|
|
)
|
|
stale = json.loads(json.dumps(page))
|
|
stale["authority"]["epoch"] = 2
|
|
with pytest.raises(replicas.ReplicaEpochRegressionError):
|
|
replicas.ingest_page(
|
|
rdb,
|
|
room_id="room-1",
|
|
room_name="Field Room",
|
|
members=MEMBERS,
|
|
page=stale,
|
|
)
|
|
|
|
|
|
def test_ingest_requires_authority_stamp(tmp_path):
|
|
page = _seed_room(_authority_db(tmp_path))
|
|
page.pop("authority")
|
|
with pytest.raises(replicas.ReplicaError):
|
|
replicas.ingest_page(
|
|
_replica_db(tmp_path),
|
|
room_id="room-1",
|
|
room_name="Field Room",
|
|
members=MEMBERS,
|
|
page=page,
|
|
)
|
|
|
|
|
|
def test_promote_replica_continues_room_at_next_epoch(tmp_path, monkeypatch):
|
|
page = _seed_room(_authority_db(tmp_path))
|
|
rdb = _replica_db(tmp_path)
|
|
replicas.ingest_page(
|
|
rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page
|
|
)
|
|
monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_B)
|
|
|
|
promoted = replicas.promote_replica(rdb, room_id="room-1")
|
|
assert promoted["authority_gateway_id"] == AUTH_B
|
|
assert promoted["authority_epoch"] == 2
|
|
assert promoted["previous_gateway_id"] == AUTH_A
|
|
assert promoted["claim_seq"] == 4
|
|
|
|
# The room is now locally authoritative with the full history + claim.
|
|
replay = rooms.read_events(rdb, room_id="room-1", since_seq=0, limit=100)
|
|
assert [e["seq"] for e in replay["events"]] == [1, 2, 3, 4]
|
|
claim = replay["events"][-1]
|
|
assert claim["kind"] == "authority.claimed"
|
|
assert claim["payload"]["previous_gateway_id"] == AUTH_A
|
|
assert claim["payload"]["authority_epoch"] == 2
|
|
assert replay["authority"] == {"gateway_id": AUTH_B, "epoch": 2}
|
|
|
|
# New work continues under the new epoch.
|
|
rooms.append_event(
|
|
rdb,
|
|
room_id="room-1",
|
|
event_id="post-takeover",
|
|
kind="message.user",
|
|
actor=USER,
|
|
payload={"text": "continuing"},
|
|
authority_gateway_id=AUTH_B,
|
|
authority_epoch=2,
|
|
)
|
|
|
|
# The old authority's identity/epoch is fenced out.
|
|
with pytest.raises(rooms.HostedRoomError):
|
|
rooms.append_event(
|
|
rdb,
|
|
room_id="room-1",
|
|
event_id="stale-write",
|
|
kind="message.user",
|
|
actor=USER,
|
|
payload={"text": "stale"},
|
|
authority_gateway_id=AUTH_A,
|
|
authority_epoch=1,
|
|
)
|
|
|
|
# Replica bookkeeping is consumed by promotion.
|
|
with pytest.raises(replicas.ReplicaError):
|
|
replicas.replica_state(rdb, room_id="room-1")
|
|
|
|
|
|
def test_promote_refuses_when_room_exists_locally(tmp_path, monkeypatch):
|
|
db = _authority_db(tmp_path)
|
|
page = _seed_room(db)
|
|
# Same DB also holds a replica row for the same id — conflict must win.
|
|
replicas.ingest_page(
|
|
db, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page
|
|
)
|
|
monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_B)
|
|
with pytest.raises(rooms.RoomConflictError):
|
|
replicas.promote_replica(db, room_id="room-1")
|
|
|
|
|
|
def test_promote_refuses_when_already_authority(tmp_path, monkeypatch):
|
|
page = _seed_room(_authority_db(tmp_path))
|
|
rdb = _replica_db(tmp_path)
|
|
replicas.ingest_page(
|
|
rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page
|
|
)
|
|
monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_A)
|
|
with pytest.raises(replicas.ReplicaError):
|
|
replicas.promote_replica(rdb, room_id="room-1")
|
|
|
|
|
|
def test_demote_fences_stale_local_authority(tmp_path, monkeypatch):
|
|
adb = _authority_db(tmp_path)
|
|
_seed_room(adb)
|
|
monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_A)
|
|
|
|
result = replicas.demote_room(
|
|
adb, room_id="room-1", observed_gateway_id=AUTH_B, observed_epoch=2
|
|
)
|
|
assert result["idempotent"] is False
|
|
assert result["authority_gateway_id"] == AUTH_B
|
|
assert result["authority_epoch"] == 2
|
|
|
|
replay = rooms.read_events(adb, room_id="room-1", since_seq=0, limit=100)
|
|
lost = replay["events"][-1]
|
|
assert lost["kind"] == "authority.lost"
|
|
assert lost["payload"]["authority_gateway_id"] == AUTH_B
|
|
assert replay["authority"] == {"gateway_id": AUTH_B, "epoch": 2}
|
|
|
|
# Local sends at the stale identity/epoch are now rejected.
|
|
with pytest.raises(rooms.HostedRoomError):
|
|
rooms.append_event(
|
|
adb,
|
|
room_id="room-1",
|
|
event_id="after-demote",
|
|
kind="message.user",
|
|
actor=USER,
|
|
payload={"text": "stale"},
|
|
authority_gateway_id=AUTH_A,
|
|
authority_epoch=1,
|
|
)
|
|
|
|
# Repeating the same observation is idempotent.
|
|
again = replicas.demote_room(
|
|
adb, room_id="room-1", observed_gateway_id=AUTH_B, observed_epoch=2
|
|
)
|
|
assert again["idempotent"] is True
|
|
|
|
|
|
def test_demote_rejects_non_superseding_epoch(tmp_path, monkeypatch):
|
|
adb = _authority_db(tmp_path)
|
|
_seed_room(adb)
|
|
monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_A)
|
|
with pytest.raises(replicas.ReplicaEpochRegressionError):
|
|
replicas.demote_room(
|
|
adb, room_id="room-1", observed_gateway_id=AUTH_B, observed_epoch=1
|
|
)
|
|
|
|
|
|
def test_full_failover_round_trip(tmp_path, monkeypatch):
|
|
"""Authority A hosts, replica B follows, A dies, B promotes, A returns
|
|
and is fenced + demoted; the room's history survives intact throughout."""
|
|
adb = _authority_db(tmp_path)
|
|
rdb = _replica_db(tmp_path)
|
|
page = _seed_room(adb, n_events=4)
|
|
replicas.ingest_page(
|
|
rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page
|
|
)
|
|
|
|
# A "dies"; B takes over.
|
|
monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_B)
|
|
promoted = replicas.promote_replica(rdb, room_id="room-1")
|
|
rooms.append_event(
|
|
rdb,
|
|
room_id="room-1",
|
|
event_id="b-work",
|
|
kind="message.user",
|
|
actor=USER,
|
|
payload={"text": "work continues on B"},
|
|
authority_gateway_id=AUTH_B,
|
|
authority_epoch=promoted["authority_epoch"],
|
|
)
|
|
|
|
# A comes back, observes B's claim, and fences itself.
|
|
monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_A)
|
|
replicas.demote_room(
|
|
adb,
|
|
room_id="room-1",
|
|
observed_gateway_id=AUTH_B,
|
|
observed_epoch=promoted["authority_epoch"],
|
|
)
|
|
with pytest.raises(rooms.HostedRoomError):
|
|
rooms.append_event(
|
|
adb,
|
|
room_id="room-1",
|
|
event_id="a-stale",
|
|
kind="message.user",
|
|
actor=USER,
|
|
payload={"text": "split brain attempt"},
|
|
authority_gateway_id=AUTH_A,
|
|
authority_epoch=1,
|
|
)
|
|
|
|
# B's room holds the complete history: 4 original + claim + new work.
|
|
replay = rooms.read_events(rdb, room_id="room-1", since_seq=0, limit=100)
|
|
kinds = [e["kind"] for e in replay["events"]]
|
|
assert kinds == ["message.user"] * 4 + ["authority.claimed", "message.user"]
|
|
assert replay["authority"]["gateway_id"] == AUTH_B
|