"""Tests for the store-level CAS fire claim (Phase 4C). `claim_job_for_fire` gives multi-machine at-most-once semantics when an external scheduler (Chronos) fires a job: across N gateway replicas, exactly ONE wins the claim for a given fire. Single-machine deployments always win (unaffected). These exercise the real store against a temp HERMES_HOME (no mocks) per the E2E-over-mocks discipline for file-touching code. """ import threading import time import pytest @pytest.fixture def temp_home(tmp_path, monkeypatch): """Isolated HERMES_HOME so jobs.json doesn't touch the real store.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path)) # cron.jobs caches no home at import; get_hermes_home() reads the env live. yield tmp_path def test_claim_succeeds_once_then_blocks(temp_home): """First claim for a fire wins; a second claim for the same fire loses, and next_run_at is advanced (a re-delivery for the old time can't re-fire).""" from cron.jobs import create_job, claim_job_for_fire, get_job job = create_job(prompt="x", schedule="every 5m", name="t") jid = job["id"] before = get_job(jid)["next_run_at"] assert claim_job_for_fire(jid) is True assert claim_job_for_fire(jid) is False assert get_job(jid)["next_run_at"] != before def test_claim_oneshot_cannot_be_double_claimed(temp_home): """A one-shot can't be double-claimed (the fresh claim blocks the retry).""" from cron.jobs import create_job, claim_job_for_fire job = create_job(prompt="x", schedule="in 30m", name="o") assert claim_job_for_fire(job["id"]) is True assert claim_job_for_fire(job["id"]) is False def test_claim_unknown_job_returns_false(temp_home): from cron.jobs import claim_job_for_fire assert claim_job_for_fire("nope-does-not-exist") is False def test_claim_paused_job_returns_false(temp_home): """A paused job can't be claimed.""" from cron.jobs import create_job, claim_job_for_fire, pause_job job = create_job(prompt="x", schedule="every 5m", name="p") pause_job(job["id"]) assert claim_job_for_fire(job["id"]) is False def test_forced_claim_atomically_resumes_paused_job(temp_home): """Explicit manual fire may resume a paused job without exposing a due intermediate state to the ticker.""" from cron.jobs import create_job, claim_job_for_fire, get_job, pause_job job = create_job(prompt="x", schedule="every 5m", name="manual") pause_job(job["id"]) assert claim_job_for_fire(job["id"], force=True) is True claimed = get_job(job["id"]) assert claimed["enabled"] is True assert claimed["state"] == "scheduled" assert claimed["paused_at"] is None assert claimed["paused_reason"] is None assert claimed["fire_claim"] is not None def test_stale_claim_is_reclaimable(temp_home, monkeypatch): """A claim older than the TTL is overwritten — the fire isn't stuck forever if the winning machine crashed before mark_job_run cleared the claim.""" from cron.jobs import create_job, claim_job_for_fire job = create_job(prompt="x", schedule="every 5m", name="s") jid = job["id"] assert claim_job_for_fire(jid) is True # With a 0s TTL, the existing claim is always considered stale. assert claim_job_for_fire(jid, claim_ttl_seconds=0) is True def test_mark_job_run_clears_claim(temp_home): """After a recurring job completes, its claim is cleared so the next fire can be claimed again.""" from cron.jobs import create_job, claim_job_for_fire, mark_job_run, get_job job = create_job(prompt="x", schedule="every 5m", name="c") jid = job["id"] assert claim_job_for_fire(jid) is True assert get_job(jid).get("fire_claim") is not None mark_job_run(jid, success=True) assert get_job(jid).get("fire_claim") is None # …and the re-armed recurring job is claimable again. assert claim_job_for_fire(jid) is True def test_fire_claim_heartbeat_refreshes_only_expected_owner(temp_home, monkeypatch): from datetime import datetime, timedelta import cron.jobs as jobs job = jobs.create_job(prompt="x", schedule="every 5m", name="heartbeat") assert jobs.claim_job_for_fire(job["id"]) is True claimed = jobs.get_job(job["id"])["fire_claim"] claimed_at = datetime.fromisoformat(claimed["at"]) monkeypatch.setattr( jobs, "_hermes_now", lambda: claimed_at + timedelta(seconds=30), ) assert jobs.heartbeat_fire_claim( job["id"], expected_owner=claimed["by"], ) is True refreshed = jobs.get_job(job["id"])["fire_claim"] assert refreshed["at"] != claimed["at"] assert refreshed["by"] == claimed["by"] assert jobs.heartbeat_fire_claim( job["id"], expected_owner="replacement-owner", ) is False def test_reclaimed_fire_uses_new_owner_token(temp_home, monkeypatch): from datetime import datetime, timedelta import cron.jobs as jobs job = jobs.create_job(prompt="x", schedule="every 5m", name="reclaim") assert jobs.claim_job_for_fire(job["id"]) is True original = dict(jobs.get_job(job["id"])["fire_claim"]) original_at = datetime.fromisoformat(original["at"]) monkeypatch.setattr( jobs, "_hermes_now", lambda: original_at + timedelta(seconds=301), ) assert jobs.claim_job_for_fire(job["id"]) is True replacement = dict(jobs.get_job(job["id"])["fire_claim"]) assert replacement["by"] != original["by"] assert jobs.heartbeat_fire_claim( job["id"], expected_owner=original["by"], ) is False assert jobs.get_job(job["id"])["fire_claim"] == replacement def test_stale_fire_owner_cannot_mark_replacement_run(temp_home): import cron.jobs as jobs job = jobs.create_job(prompt="x", schedule="every 5m", name="fenced") assert jobs.claim_job_for_fire(job["id"]) is True original = dict(jobs.get_job(job["id"])["fire_claim"]) records = jobs.load_jobs() records[0]["fire_claim"] = {"at": original["at"], "by": "replacement"} jobs.save_jobs(records) assert jobs.mark_job_run( job["id"], success=True, expected_fire_owner=original["by"], ) is False persisted = jobs.get_job(job["id"]) assert persisted["fire_claim"]["by"] == "replacement" assert persisted.get("last_run_at") is None def test_fire_claim_fence_serializes_terminal_revocation(temp_home): """A side effect authorized by owner linearizes before terminal revocation.""" from cron.jobs import ( claim_job_for_fire, create_job, fire_claim_fence, mark_job_run, ) job = create_job(prompt="x", schedule="every 5m", name="fenced-side-effect") claimed = claim_job_for_fire(job["id"], return_job=True) assert isinstance(claimed, dict) owner = claimed["fire_claim"]["by"] terminal_done = threading.Event() def finish_run(): mark_job_run(job["id"], True, expected_fire_owner=owner) terminal_done.set() with fire_claim_fence(job["id"], expected_owner=owner) as owns_claim: assert owns_claim is True thread = threading.Thread(target=finish_run) thread.start() time.sleep(0.05) assert terminal_done.is_set() is False thread.join(timeout=1) assert terminal_done.is_set() is True def test_fire_claim_fence_rejects_stale_owner(temp_home): from cron.jobs import claim_job_for_fire, create_job, fire_claim_fence job = create_job(prompt="x", schedule="every 5m", name="stale-fence") claim_job_for_fire(job["id"]) with fire_claim_fence(job["id"], expected_owner="stale") as owns_claim: assert owns_claim is False def test_same_process_fire_fence_refuses_second_claim_after_timeout(temp_home, monkeypatch): """A wedged local holder must not indefinitely block another claimant.""" import cron.jobs as jobs job = jobs.create_job(prompt="x", schedule="every 5m", name="local-fence-timeout") monkeypatch.setattr(jobs, "_JOBS_LOCK_TIMEOUT_SECONDS", 0.1) completed = threading.Event() result = {} def second_claimant(): result["claimed"] = jobs.claim_job_for_fire(job["id"]) completed.set() with jobs._fire_job_lock(job["id"]) as acquired: assert acquired is True thread = threading.Thread(target=second_claimant) thread.start() assert completed.wait(timeout=2), "same-process claimant waited past the fire-fence timeout" assert result["claimed"] is False thread.join(timeout=2) assert thread.is_alive() is False assert jobs.claim_job_for_fire(job["id"]) is True def test_same_thread_fire_fence_reentrancy_preserves_ownership(temp_home): """Nested same-thread callers retain the existing fire fence.""" import cron.jobs as jobs job = jobs.create_job(prompt="x", schedule="every 5m", name="local-fence-reentrant") completed = threading.Event() result = {} def reentrant_claimant(): with jobs._fire_job_lock(job["id"]) as outer_acquired: result["outer"] = outer_acquired with jobs._fire_job_lock(job["id"]) as inner_acquired: result["inner"] = inner_acquired completed.set() thread = threading.Thread(target=reentrant_claimant, daemon=True) thread.start() assert completed.wait(timeout=2), "same-thread nested fire fence did not return" assert result == {"outer": True, "inner": True} thread.join(timeout=2) assert thread.is_alive() is False