Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/cron/test_misfire_catchup.py

@@ -0,0 +1,163 @@
+"""Tests for fire_overdue_jobs — misfire catch-up for external cron providers.
+
+External providers (Chronos) deliver scheduled fires over HTTP; when the
+loopback hop is down at fire time and the scheduler's retry budget exhausts,
+the job's next_run_at stays parked in the past and nothing ever runs it
+(external providers have no local tick loop). fire_overdue_jobs, called from
+gateway housekeeping, claims and fires those jobs after a grace window.
+"""
+
+import threading
+import time
+from datetime import timedelta
+
+import pytest
+
+from cron.jobs import _hermes_now, create_job, get_job, load_jobs, save_jobs
+from cron.scheduler_provider import (
+    CronScheduler,
+    InProcessCronScheduler,
+    fire_overdue_jobs,
+)
+
+
+@pytest.fixture()
+def tmp_cron_dir(tmp_path, monkeypatch):
+    """Redirect cron storage to a temp directory."""
+    monkeypatch.setattr("cron.jobs.CRON_DIR", tmp_path / "cron")
+    monkeypatch.setattr("cron.jobs.JOBS_FILE", tmp_path / "cron" / "jobs.json")
+    monkeypatch.setattr("cron.jobs.OUTPUT_DIR", tmp_path / "cron" / "output")
+    return tmp_path
+
+
+class RecordingProvider(CronScheduler):
+    """External-provider stand-in: real base-class claim_fire (store CAS),
+    recorded fire_claimed instead of the blocking run_one_job."""
+
+    def __init__(self):
+        self.fired = []
+        self._done = threading.Event()
+
+    @property
+    def name(self):
+        return "recording"
+
+    def start(self, stop_event, **kw):  # pragma: no cover - unused
+        return None
+
+    def fire_claimed(self, claimed_job, *, adapters=None, loop=None,
+                     cancel_event=None):
+        self.fired.append(claimed_job["id"])
+        self._done.set()
+        return True
+
+    def wait_fired(self, timeout=5.0):
+        return self._done.wait(timeout)
+
+
+def _park_in_past(job_id, minutes):
+    """Rewind a job's next_run_at into the past (simulates missed fires)."""
+    jobs = load_jobs()
+    for j in jobs:
+        if j["id"] == job_id:
+            j["next_run_at"] = (
+                _hermes_now() - timedelta(minutes=minutes)
+            ).isoformat()
+    save_jobs(jobs)
+
+
+class TestFireOverdueJobs:
+    def test_noop_for_builtin_provider(self, tmp_cron_dir):
+        """The in-process ticker self-heals past-due jobs — the sweep must
+        never double-dispatch under it."""
+        job = create_job(prompt="p", schedule="every 1h")
+        _park_in_past(job["id"], minutes=60)
+        assert fire_overdue_jobs(InProcessCronScheduler()) == 0
+
+    def test_fires_job_past_grace(self, tmp_cron_dir):
+        job = create_job(prompt="p", schedule="every 1h")
+        _park_in_past(job["id"], minutes=30)
+        provider = RecordingProvider()
+        assert fire_overdue_jobs(provider) == 1
+        assert provider.wait_fired()
+        assert provider.fired == [job["id"]]
+
+    def test_respects_grace_window(self, tmp_cron_dir):
+        """A job only a few minutes overdue is still the external
+        scheduler's to retry — the backstop must not race it."""
+        job = create_job(prompt="p", schedule="every 1h")
+        _park_in_past(job["id"], minutes=5)  # < default 10 min grace
+        provider = RecordingProvider()
+        assert fire_overdue_jobs(provider) == 0
+        assert provider.fired == []
+
+    def test_grace_zero_disables(self, tmp_cron_dir, monkeypatch):
+        monkeypatch.setattr(
+            "cron.scheduler_provider._misfire_grace_minutes", lambda: 0.0
+        )
+        job = create_job(prompt="p", schedule="every 1h")
+        _park_in_past(job["id"], minutes=600)
+        assert fire_overdue_jobs(RecordingProvider()) == 0
+
+    def test_future_job_not_fired(self, tmp_cron_dir):
+        create_job(prompt="p", schedule="every 1h")  # next_run_at in future
+        provider = RecordingProvider()
+        assert fire_overdue_jobs(provider) == 0
+
+    def test_paused_job_not_fired(self, tmp_cron_dir):
+        from cron.jobs import pause_job
+
+        job = create_job(prompt="p", schedule="every 1h")
+        _park_in_past(job["id"], minutes=30)
+        pause_job(job["id"])
+        assert fire_overdue_jobs(RecordingProvider()) == 0
+
+    def test_fresh_external_claim_wins(self, tmp_cron_dir):
+        """A concurrent external fire holds the store claim — the sweep's
+        claim_fire loses the CAS and must not dispatch (at-most-once)."""
+        from cron.jobs import claim_job_for_fire
+
+        job = create_job(prompt="p", schedule="every 1h")
+        _park_in_past(job["id"], minutes=30)
+        assert claim_job_for_fire(job["id"]) is True  # external fire claims
+
+        # The external claim also advanced next_run_at (recurring bump), so
+        # re-park it to isolate the claim-CAS as the thing that blocks us.
+        _park_in_past(job["id"], minutes=30)
+
+        provider = RecordingProvider()
+        assert fire_overdue_jobs(provider) == 0
+        assert provider.fired == []
+
+    def test_claim_advances_next_run_no_refire(self, tmp_cron_dir):
+        """After a catch-up fire, the recurring job's next_run_at moved to
+        the future — the next sweep pass must not fire it again."""
+        job = create_job(prompt="p", schedule="every 1h")
+        _park_in_past(job["id"], minutes=30)
+        provider = RecordingProvider()
+        assert fire_overdue_jobs(provider) == 1
+        assert provider.wait_fired()
+
+        stamped = get_job(job["id"])
+        assert stamped["next_run_at"] > _hermes_now().isoformat()
+
+        provider2 = RecordingProvider()
+        assert fire_overdue_jobs(provider2) == 0
+
+    def test_dispatch_is_nonblocking(self, tmp_cron_dir):
+        """fire_claimed runs off-thread — a slow job must not stall the
+        sweep (housekeeping loop) for the length of an agent run."""
+
+        class SlowProvider(RecordingProvider):
+            def fire_claimed(self, claimed_job, **kw):
+                time.sleep(3.0)
+                return super().fire_claimed(claimed_job, **kw)
+
+        job = create_job(prompt="p", schedule="every 1h")
+        _park_in_past(job["id"], minutes=30)
+        provider = SlowProvider()
+        start = time.monotonic()
+        assert fire_overdue_jobs(provider) == 1
+        assert time.monotonic() - start < 1.0  # returned before the run
+        assert provider.wait_fired(timeout=10)
+        assert provider.fired == [job["id"]]