"""Characterization tests for the cron trigger before/after the provider refactor. These lock the CURRENT in-process-ticker contract (Phase 0 of the pluggable CronScheduler plan, .hermes/plans/cron-scheduler-provider-interface.md). They must pass unchanged on `main` now, and after every subsequent phase of the refactor — they are the regression harness that proves the built-in firing behavior is byte-for-byte preserved when the ticker is moved behind the CronScheduler provider interface. No production code is exercised beyond the two ticker entry points: - gateway/run.py::_start_cron_ticker (production gateway ticker) - hermes_cli/web_server.py::_start_desktop_cron_ticker (desktop fallback) Both call `cron.scheduler.tick(...)` on a loop and exit when their stop_event is set. We patch `cron.scheduler.tick` (both tickers import it locally as `cron_tick`, so the module-attribute patch is observed) and assert the loop drives it and stops promptly. """ import threading import time from unittest.mock import patch def _wait_until(predicate, timeout=10.0, interval=0.005): """Block until ``predicate()`` is truthy or ``timeout`` elapses. Returns the predicate's final value. Used instead of a fixed ``time.sleep`` before asserting that a background ticker thread has called tick()/heartbeat() at least N times — under loaded CI the worker thread may not be scheduled within a short fixed sleep, which made these tests flake (``assert 0 >= 1`` / ``provider never called tick()``). """ deadline = time.monotonic() + timeout while time.monotonic() < deadline: value = predicate() if value: return value time.sleep(interval) return predicate() def test_ticker_calls_tick_at_least_once_then_stops(): """The gateway in-process ticker loop calls cron.scheduler.tick repeatedly and exits promptly once the stop_event is set.""" from gateway.run import _start_cron_ticker calls = [] stop = threading.Event() def fake_tick(*args, **kwargs): calls.append(kwargs) return 0 with patch("cron.scheduler.tick", side_effect=fake_tick): # interval=0 keeps the loop tight; stop after the first observed tick. t = threading.Thread( target=_start_cron_ticker, args=(stop,), kwargs={"interval": 0}, daemon=True, ) t.start() assert _wait_until(lambda: len(calls) >= 1), "ticker never called tick()" stop.set() t.join(timeout=5) assert not t.is_alive(), "ticker did not exit after stop_event was set" assert len(calls) >= 1, "ticker never called tick()" # Contract: the ticker invokes tick with sync=False (fire-and-forget from # the background thread, never the synchronous CLI path). assert calls[0].get("sync") is False def test_desktop_ticker_calls_tick_then_stops(): """The desktop dashboard ticker loop calls cron.scheduler.tick and exits once the stop_event is set. Desktop has no live adapters, so it ticks with no adapters/loop.""" from hermes_cli.web_server import _start_desktop_cron_ticker calls = [] stop = threading.Event() def fake_tick(*args, **kwargs): calls.append(kwargs) return 0 with patch("cron.scheduler.tick", side_effect=fake_tick): t = threading.Thread( target=_start_desktop_cron_ticker, args=(stop,), kwargs={"interval": 0}, daemon=True, ) t.start() assert _wait_until(lambda: len(calls) >= 1), "desktop ticker never called tick()" stop.set() t.join(timeout=5) assert not t.is_alive(), "desktop ticker did not exit after stop_event was set" assert len(calls) >= 1, "desktop ticker never called tick()" assert calls[0].get("sync") is False # ── Phase 1: CronScheduler ABC + InProcessCronScheduler ────────────────────── def test_cronscheduler_is_abstract(): """name + start are abstract — the bare ABC can't be instantiated.""" import pytest from cron.scheduler_provider import CronScheduler with pytest.raises(TypeError): CronScheduler() def test_abc_growth_stays_additive(): """The provider interface stays source-compatible with existing plugins. ``start`` must be the only required implementation hook: future optional behavior belongs in non-abstract default methods so custom plugins do not break on import after an upgrade. """ from cron.scheduler_provider import CronScheduler abstract = set(getattr(CronScheduler, "__abstractmethods__", set())) assert abstract == {"name", "start"}, ( f"CronScheduler abstractmethods changed to {abstract}; growth must be " "additive (optional methods with defaults), not new abstract methods." ) def test_force_fire_capability_detects_legacy_override(): from cron.scheduler_provider import CronScheduler class Current(CronScheduler): @property def name(self): return "current" def start(self, stop_event, **kw): pass class Legacy(Current): def fire_due( # type: ignore[invalid-method-override] self, job_id, *, adapters=None, loop=None ): return True class PositionalOnly(Current): def fire_due( # type: ignore[invalid-method-override] self, job_id, force=False, / ): return True class KeywordSink(Current): def fire_due(self, job_id, **kwargs): return True assert Current().supports_force_fire is True assert Legacy().supports_force_fire is False assert PositionalOnly().supports_force_fire is False assert KeywordSink().supports_force_fire is True def test_inprocess_provider_ticks_and_stops(): """The built-in provider drives cron.scheduler.tick(sync=False) on a loop and exits promptly when stop_event is set — same contract as the raw ticker characterized above.""" from cron.scheduler_provider import InProcessCronScheduler calls = [] stop = threading.Event() prov = InProcessCronScheduler() assert prov.name == "builtin" with patch("cron.scheduler.tick", side_effect=lambda *a, **k: calls.append(k) or 0): t = threading.Thread( target=prov.start, args=(stop,), kwargs={"interval": 0}, daemon=True ) t.start() # Wait for the loop to actually call tick() at least once rather than # sleeping a fixed window — under loaded CI the worker thread may not be # scheduled within a short sleep, which made this flake (assert 0 >= 1). assert _wait_until(lambda: len(calls) >= 1), "provider never called tick()" stop.set() t.join(timeout=5) assert not t.is_alive(), "provider did not exit after stop_event was set" assert len(calls) >= 1, "provider never called tick()" assert calls[0].get("sync") is False # ── Phase 2: config key, discovery, resolver ───────────────────────────────── def test_default_config_cron_provider_is_empty(): """The new cron.provider key defaults to empty (= built-in).""" from hermes_cli.config import DEFAULT_CONFIG assert DEFAULT_CONFIG["cron"]["provider"] == "" def test_discover_cron_schedulers_returns_list(): """Discovery returns bundled non-default providers. The built-in is core, not discovered here. """ from plugins.cron_providers import discover_cron_schedulers result = discover_cron_schedulers() assert isinstance(result, list) assert any(name == "chronos" for name, _desc, _available in result) def test_load_unknown_cron_scheduler_returns_none(): from plugins.cron_providers import load_cron_scheduler assert load_cron_scheduler("does-not-exist-xyz") is None def test_cron_provider_package_does_not_shadow_core_cron_package(monkeypatch): """Putting plugins/ first on sys.path must not hide the core cron package.""" from importlib.machinery import PathFinder from pathlib import Path repo_root = Path(__file__).resolve().parents[2] monkeypatch.syspath_prepend(str(repo_root)) monkeypatch.syspath_prepend(str(repo_root / "plugins")) cron_spec = PathFinder.find_spec("cron") assert cron_spec is not None assert Path(cron_spec.origin).resolve() == repo_root / "cron" / "__init__.py" jobs_spec = PathFinder.find_spec("cron.jobs", [str(repo_root / "cron")]) assert jobs_spec is not None assert Path(jobs_spec.origin).resolve() == repo_root / "cron" / "jobs.py" def test_resolve_defaults_to_builtin(monkeypatch): """Empty cron.provider → built-in.""" import hermes_cli.config as cfg from cron import scheduler_provider as sp monkeypatch.setattr(cfg, "load_config", lambda: {"cron": {"provider": ""}}) prov = sp.resolve_cron_scheduler() assert prov.name == "builtin" def test_resolve_no_cron_section_falls_back_to_builtin(monkeypatch): """Config with no cron section at all → built-in (cfg_get returns default).""" import hermes_cli.config as cfg from cron import scheduler_provider as sp monkeypatch.setattr(cfg, "load_config", lambda: {}) prov = sp.resolve_cron_scheduler() assert prov.name == "builtin" def test_resolve_unknown_provider_falls_back_to_builtin(monkeypatch): """A named provider that doesn't exist → built-in (cron never dies).""" import hermes_cli.config as cfg from cron import scheduler_provider as sp monkeypatch.setattr(cfg, "load_config", lambda: {"cron": {"provider": "nope-not-real"}}) prov = sp.resolve_cron_scheduler() assert prov.name == "builtin" def test_resolve_unavailable_provider_falls_back(monkeypatch): """A provider that loads but reports is_available()==False → built-in.""" import hermes_cli.config as cfg import plugins.cron_providers as pc from cron import scheduler_provider as sp from cron.scheduler_provider import CronScheduler class Unavailable(CronScheduler): @property def name(self): return "unavailable" def is_available(self): return False def start(self, stop_event, **kw): pass monkeypatch.setattr(cfg, "load_config", lambda: {"cron": {"provider": "unavailable"}}) monkeypatch.setattr(pc, "load_cron_scheduler", lambda n: Unavailable()) prov = sp.resolve_cron_scheduler() assert prov.name == "builtin" def test_resolve_available_provider_is_used(monkeypatch): """A provider that loads and is available is returned (not the fallback).""" import hermes_cli.config as cfg import plugins.cron_providers as pc from cron import scheduler_provider as sp from cron.scheduler_provider import CronScheduler class Fake(CronScheduler): @property def name(self): return "fake" def is_available(self): return True def start(self, stop_event, **kw): pass monkeypatch.setattr(cfg, "load_config", lambda: {"cron": {"provider": "fake"}}) monkeypatch.setattr(pc, "load_cron_scheduler", lambda n: Fake()) prov = sp.resolve_cron_scheduler() assert prov.name == "fake" def test_external_provider_falls_back_to_builtin_under_multiplex(): from cron.scheduler_provider import ( CronScheduler, InProcessCronScheduler, scheduler_for_profile_mode, ) class External(CronScheduler): @property def name(self): return "external" def start(self, stop_event, **kwargs): return None external = External() assert scheduler_for_profile_mode(external, multiplex_profiles=False) is external assert isinstance( scheduler_for_profile_mode(external, multiplex_profiles=True), InProcessCronScheduler, ) # ── Phase 4B: additive hooks (on_jobs_changed / fire_due / reconcile) ──────── def test_hooks_did_not_change_required_surface(): """The additive hooks must NOT become abstractmethods — the Phase-1 guard still holds (required surface is exactly name + start).""" from cron.scheduler_provider import CronScheduler assert set(CronScheduler.__abstractmethods__) == {"name", "start"} def test_builtin_inherits_hook_defaults(): """The built-in inherits no-op defaults for the new hooks (it never needs to override them).""" from cron.scheduler_provider import InProcessCronScheduler p = InProcessCronScheduler() assert p.on_jobs_changed() is None assert p.reconcile() is None # built-in does not override fire_due; it simply isn't called for built-in. assert hasattr(p, "fire_due") def test_fire_due_default_claims_then_runs(monkeypatch): """The default fire_due runs the exact owner-bearing CAS snapshot.""" import cron.jobs as jobs import cron.scheduler as sched from cron.scheduler_provider import InProcessCronScheduler ran = [] claims = [] monkeypatch.setattr( jobs, "claim_job_for_fire", lambda jid, **kw: claims.append((jid, kw)) or {"id": jid, "name": "t", "fire_claim": {"by": "exact-owner"}}, raising=False, ) monkeypatch.setattr( sched, "run_one_job", lambda job, **kw: ran.append((job["id"], job["fire_claim"]["by"])) or True, ) assert InProcessCronScheduler().fire_due("j1") is True assert claims == [("j1", {"return_job": True})] assert ran == [("j1", "exact-owner")] def test_claim_fire_persists_attempt_before_fire_claimed(monkeypatch): import cron.executions as executions import cron.jobs as jobs import cron.scheduler as sched from cron.scheduler_provider import InProcessCronScheduler events = [] monkeypatch.setattr( jobs, "claim_job_for_fire", lambda jid, **kwargs: events.append("claim") or {"id": jid, "fire_claim": {"by": "owner"}}, ) monkeypatch.setattr( executions, "create_execution", lambda jid, source: events.append("ledger") or {"id": "exec-1"}, ) monkeypatch.setattr( sched, "run_one_job", lambda job, **kwargs: events.append(("run", job["execution_id"])) or True, ) provider = InProcessCronScheduler() claimed = provider.claim_fire("j1") assert events == ["ledger", "claim"] assert claimed is not None assert claimed["execution_id"] == "exec-1" assert provider.fire_claimed(claimed) is True assert events == ["ledger", "claim", ("run", "exec-1")] def test_fire_due_forwards_manual_force_to_store_claim(monkeypatch): import cron.jobs as jobs import cron.scheduler as sched from cron.scheduler_provider import InProcessCronScheduler claims = [] monkeypatch.setattr( jobs, "claim_job_for_fire", lambda jid, **kw: claims.append((jid, kw)) or {"id": jid, "name": "t", "fire_claim": {"by": "manual-owner"}}, ) monkeypatch.setattr(sched, "run_one_job", lambda job, **kw: True) assert InProcessCronScheduler().fire_due("j1", force=True) is True assert claims == [("j1", {"force": True, "return_job": True})] def test_fire_due_lost_claim_does_not_run(monkeypatch): """If the CAS claim is lost (another machine/retry won), fire_due returns False and never runs the job.""" import cron.jobs as jobs import cron.scheduler as sched from cron.scheduler_provider import InProcessCronScheduler ran = [] monkeypatch.setattr( jobs, "claim_job_for_fire", lambda jid, **kw: False, raising=False, ) monkeypatch.setattr(sched, "run_one_job", lambda job, **kw: ran.append(job["id"]) or True) assert InProcessCronScheduler().fire_due("j1") is False assert ran == [] def test_fire_due_missing_job_does_not_run(monkeypatch): """If the job vanished before atomic claim, fire_due does not run it.""" import cron.jobs as jobs import cron.scheduler as sched from cron.scheduler_provider import InProcessCronScheduler ran = [] monkeypatch.setattr( jobs, "claim_job_for_fire", lambda jid, **kw: False, raising=False, ) monkeypatch.setattr(sched, "run_one_job", lambda job, **kw: ran.append(job["id"]) or True) assert InProcessCronScheduler().fire_due("gone") is False assert ran == [] # ── F2a: ticker liveness — survival, heartbeat, honest status (#32612, #32895) ── def test_failing_tick_records_liveness_but_not_success(): """A tick that raises bumps the liveness heartbeat but NOT the success marker — so status can distinguish 'alive but failing' from 'firing'.""" from cron.scheduler_provider import InProcessCronScheduler beats = [] stop = threading.Event() prov = InProcessCronScheduler() with patch("cron.scheduler.tick", side_effect=RuntimeError("every tick fails")), \ patch("cron.jobs.record_ticker_heartbeat", side_effect=lambda success=False: beats.append(success)): t = threading.Thread(target=prov.start, args=(stop,), kwargs={"interval": 0}, daemon=True) t.start() # Wait for the pre-loop beat + at least one post-tick beat (was flaky # with a fixed 0.2s sleep under loaded CI). assert _wait_until(lambda: len(beats) >= 2), "ticker did not record heartbeats" stop.set() t.join(timeout=5) # every post-tick beat must be success=False (ticks always failed) assert len(beats) >= 2 assert all(b is False for b in beats), "a failing tick wrongly bumped the success marker" def test_heartbeat_roundtrip_and_age(tmp_path, monkeypatch): """record_ticker_heartbeat writes fresh timestamps atomically; the age getters read them back as small positive ages.""" import cron.jobs as jobs cron_dir = tmp_path / "cron" monkeypatch.setattr(jobs, "CRON_DIR", cron_dir) monkeypatch.setattr(jobs, "OUTPUT_DIR", cron_dir / "output") monkeypatch.setattr(jobs, "TICKER_HEARTBEAT_FILE", cron_dir / "ticker_heartbeat") monkeypatch.setattr(jobs, "TICKER_SUCCESS_FILE", cron_dir / "ticker_last_success") # No files yet -> unknown (None), NOT "dead" assert jobs.get_ticker_heartbeat_age() is None assert jobs.get_ticker_success_age() is None # liveness-only: heartbeat set, success still unknown jobs.record_ticker_heartbeat(success=False) hb = jobs.get_ticker_heartbeat_age() assert hb is not None and 0.0 <= hb < 5.0 assert jobs.get_ticker_success_age() is None # success: both set jobs.record_ticker_heartbeat(success=True) ok = jobs.get_ticker_success_age() assert ok is not None and 0.0 <= ok < 5.0 # ── F8: runtime backstop — never resolve a stored pair that exfiltrates a key ── class TestGuardJobCredentialExfil: """run_job() must fail closed before provider resolution when a job's stored provider/base_url pair would ship a named provider's stored credential to an off-host endpoint — covering jobs persisted before the create/update guard or written directly to the store (F8 stored-job path; CWE-200/CWE-522).""" def test_named_registry_provider_offhost_is_blocked(self): import pytest from cron.scheduler import _guard_job_credential_exfil job = {"id": "j1", "provider": "anthropic", "base_url": "https://evil.example/v1"} with pytest.raises(RuntimeError) as exc: _guard_job_credential_exfil(job) assert "blocked for safety" in str(exc.value) def test_bare_custom_is_allowed(self): from cron.scheduler import _guard_job_credential_exfil job = {"id": "j4", "provider": "custom", "base_url": "https://anything.example/v1"} assert _guard_job_credential_exfil(job) is None def test_no_base_url_is_allowed(self): from cron.scheduler import _guard_job_credential_exfil assert _guard_job_credential_exfil({"id": "j5", "provider": "anthropic"}) is None assert _guard_job_credential_exfil({"id": "j6"}) is None def test_validator_exception_with_base_url_fails_closed(self, monkeypatch): # If the validator/import unexpectedly raises, this last-resort backstop # must NOT allow a base_url-bearing job through to provider resolution # (it cannot prove the stored pair is safe). Regression for the # fail-open `except Exception: err = None` path. import pytest import tools.cronjob_tools as ct from cron.scheduler import _guard_job_credential_exfil def _boom(provider, base_url): raise RuntimeError("validator blew up") monkeypatch.setattr(ct, "_validate_cron_base_url", _boom) job = {"id": "j7", "provider": "custom:legit", "base_url": "https://evil.example/v1"} with pytest.raises(RuntimeError) as exc: _guard_job_credential_exfil(job) assert "blocked for safety" in str(exc.value) # ── Multiplex profiles: cron per secondary profile (issue #69377) ───────── def test_multiplex_ticker_ticks_each_profile_once(tmp_path, monkeypatch): """The multiplex cron scheduler calls tick() once per profile home, scoped via use_cron_store, so secondary-profile jobs actually fire instead of languishing in an unticked store.""" from cron.scheduler_provider import InProcessCronScheduler # Set up two profile directories. p1 = tmp_path / "default" p2 = tmp_path / "home-ops" for d in (p1, p2): (d / "cron").mkdir(parents=True) profile_homes = [("default", p1), ("home-ops", p2)] # Count tick() calls — should be called once per profile per iteration. tick_count: list[int] = [] def _tracking_tick(*args, **kwargs): tick_count.append(1) return 0 stop = threading.Event() prov = InProcessCronScheduler() with patch("cron.scheduler.tick", side_effect=_tracking_tick), \ patch("cron.jobs.record_ticker_heartbeat", lambda **kw: None): t = threading.Thread( target=prov.start, args=(stop,), kwargs={"interval": 0, "profile_homes": profile_homes}, daemon=True, ) t.start() # Wait for at least len(profile_homes) tick calls (one full cycle). deadline = time.monotonic() + 10 while len(tick_count) < len(profile_homes) and time.monotonic() < deadline: time.sleep(0.005) # Give one more cycle to ensure it keeps ticking. deadline = time.monotonic() + 3 while len(tick_count) < len(profile_homes) * 2 and time.monotonic() < deadline: time.sleep(0.005) stop.set() t.join(timeout=5) assert not t.is_alive() # The ticker called tick() at least once per profile per iteration. # With 2 profiles and multiple iterations, we should have seen at least 2 calls. assert len(tick_count) >= len(profile_homes), \ f"Expected >= {len(profile_homes)} tick calls, got {len(tick_count)}" def test_multiplex_ticker_skips_deleted_profile_from_startup_snapshot(tmp_path): """A stale profile_homes entry must not recreate a deleted profile.""" import cron.jobs as jobs from cron.scheduler_provider import InProcessCronScheduler default_home = tmp_path / "default" (default_home / "cron").mkdir(parents=True) profiles_dir = tmp_path / "profiles" profiles_dir.mkdir() deleted_home = profiles_dir / "deleted" profile_homes = [("default", default_home), ("deleted", deleted_home)] ticked_homes = [] stop = threading.Event() def _tracking_tick(*args, **kwargs): ticked_homes.append(jobs._current_cron_store().cron_dir.parent) stop.set() return 0 provider = InProcessCronScheduler() with patch("cron.scheduler.tick", side_effect=_tracking_tick): thread = threading.Thread( target=provider.start, args=(stop,), kwargs={"interval": 0, "profile_homes": profile_homes}, daemon=True, ) thread.start() thread.join(timeout=5) assert not thread.is_alive() assert ticked_homes == [default_home.resolve()] assert not deleted_home.exists() def test_existing_profile_homes_filters_deleted(tmp_path): """The existence filter keeps live homes and drops deleted ones, whether entries are (name, path) tuples or bare paths.""" from cron.scheduler_provider import _existing_profile_homes live = tmp_path / "live" deleted = tmp_path / "deleted" live.mkdir(parents=True) # deleted intentionally not created as_tuples = _existing_profile_homes([("live", live), ("deleted", deleted)]) assert [p[0] for p in as_tuples] == ["live"] as_paths = _existing_profile_homes([live, deleted]) assert [p for p in as_paths] == [live] def _run_multiplex_capture(tmp_path, *, profile_adapters, shared_adapters): """Run the multiplex ticker one full cycle and return the ``adapters`` object passed to ``tick()`` for the default profile and the secondary profile (in ``profile_homes`` order: default first, secondary second).""" from cron.scheduler_provider import InProcessCronScheduler p_default = tmp_path / "default" p_sec = tmp_path / "home-ops" for d in (p_default, p_sec): (d / "cron").mkdir(parents=True) profile_homes = [("default", p_default), ("home-ops", p_sec)] captured: list = [] # adapters seen per tick call; order follows profile_homes def _capturing_tick(*args, **kwargs): captured.append(kwargs.get("adapters")) return 0 stop = threading.Event() prov = InProcessCronScheduler() with patch("cron.scheduler.tick", side_effect=_capturing_tick), \ patch("cron.jobs.record_ticker_heartbeat", lambda **kw: None): t = threading.Thread( target=prov.start, args=(stop,), kwargs={ "interval": 0, "profile_homes": profile_homes, "adapters": shared_adapters, "profile_adapters": profile_adapters, "default_profile": "default", }, daemon=True, ) t.start() deadline = time.monotonic() + 10 while len(captured) < 2 and time.monotonic() < deadline: time.sleep(0.005) stop.set() t.join(timeout=5) assert not t.is_alive() assert len(captured) >= 2, f"expected >= 2 tick calls, got {len(captured)}" return captured[0], captured[1] # (default, secondary) of the first cycle def test_multiplex_default_profile_uses_shared_adapters(tmp_path): """The default profile's cron is delivered via the shared ``adapters`` set (which belongs to the default profile).""" shared = {"kind": "shared"} default_ad, _ = _run_multiplex_capture( tmp_path, profile_adapters={"home-ops": {"kind": "secondary"}}, shared_adapters=shared, ) assert default_ad is shared def test_multiplex_connected_secondary_uses_its_own_adapters(tmp_path): """A connected secondary is delivered via ITS OWN adapters, not the shared default-profile set.""" shared = {"kind": "shared"} sec = {"kind": "secondary"} default_ad, sec_ad = _run_multiplex_capture( tmp_path, profile_adapters={"home-ops": sec}, shared_adapters=shared, ) assert default_ad is shared assert sec_ad is sec def test_multiplex_empty_secondary_does_not_fall_back_to_shared(tmp_path): """A secondary whose adapter map is present-but-empty (its bot has not connected yet) must NOT fall back to the default profile's shared adapters — otherwise its cron output ships through the wrong bot. It receives an empty adapter set and simply does not deliver this tick.""" shared = {"kind": "shared"} default_ad, sec_ad = _run_multiplex_capture( tmp_path, profile_adapters={"home-ops": {}}, shared_adapters=shared, ) assert default_ad is shared assert sec_ad is not shared assert not sec_ad # empty → no delivery, not the default bot def test_multiplex_missing_secondary_does_not_fall_back_to_shared(tmp_path): """A secondary absent from profile_adapters entirely (its adapter map has not been created yet) also must not fall back to the shared adapters.""" shared = {"kind": "shared"} default_ad, sec_ad = _run_multiplex_capture( tmp_path, profile_adapters={}, shared_adapters=shared, ) assert default_ad is shared assert sec_ad is not shared assert not sec_ad def test_multiplex_ticker_isolates_profile_failures(tmp_path): """A failing profile's tick must not skip healthy siblings in the same cycle, nor darken their status (#74878).""" from cron.jobs import get_ticker_last_error, record_ticker_error, use_cron_store from cron.scheduler_provider import InProcessCronScheduler from hermes_constants import get_hermes_home failing_home = tmp_path / "failing" healthy_home = tmp_path / "healthy" for home in (failing_home, healthy_home): (home / "cron").mkdir(parents=True) with use_cron_store(home): record_ticker_error("RuntimeError: stale failure") stop = threading.Event() tick_homes: list[str] = [] def _tick(*args, **kwargs): home = str(get_hermes_home()) tick_homes.append(home) if home == str(failing_home): raise RuntimeError("profile-local failure") stop.set() return 0 provider = InProcessCronScheduler() with patch("cron.scheduler.tick", side_effect=_tick): thread = threading.Thread( target=provider.start, args=(stop,), kwargs={ "interval": 0, "profile_homes": [("failing", failing_home), ("healthy", healthy_home)], }, daemon=True, ) thread.start() thread.join(timeout=5) stop.set() thread.join(timeout=5) assert not thread.is_alive() assert str(healthy_home) in tick_homes, "healthy sibling was skipped" assert not (failing_home / "cron" / "ticker_last_success").exists() assert (healthy_home / "cron" / "ticker_last_success").exists() with use_cron_store(failing_home): assert get_ticker_last_error() == "RuntimeError: profile-local failure" with use_cron_store(healthy_home): assert get_ticker_last_error() is None def test_multiplex_recovery_isolates_profile_failures(tmp_path): """A startup-recovery error in one profile's ledger must not kill the ticker thread before it ever ticks (#74878).""" import sqlite3 from cron.scheduler_provider import InProcessCronScheduler from hermes_constants import get_hermes_home failing_home = tmp_path / "failing" healthy_home = tmp_path / "healthy" for home in (failing_home, healthy_home): (home / "cron").mkdir(parents=True) stop = threading.Event() recovery_homes: list[str] = [] tick_homes: list[str] = [] def _recover(): home = str(get_hermes_home()) recovery_homes.append(home) if home == str(failing_home): raise sqlite3.OperationalError("unable to open database file") return 0 def _tick(*args, **kwargs): tick_homes.append(str(get_hermes_home())) if len(tick_homes) >= 2: stop.set() return 0 provider = InProcessCronScheduler() with ( patch.object(provider, "recover_interrupted", side_effect=_recover), patch("cron.scheduler.tick", side_effect=_tick), ): thread = threading.Thread( target=provider.start, args=(stop,), kwargs={ "interval": 0, "profile_homes": [("failing", failing_home), ("healthy", healthy_home)], }, daemon=True, ) thread.start() thread.join(timeout=5) stop.set() thread.join(timeout=5) assert not thread.is_alive() assert recovery_homes == [str(failing_home), str(healthy_home)] # The failing profile stays in rotation: its ledger may still hold jobs. assert set(tick_homes) == {str(failing_home), str(healthy_home)}