381 lines
13 KiB
Python
381 lines
13 KiB
Python
"""Characterization + unit tests for the `run_one_job` shared helper (Phase 4A).
|
|
|
|
`tick`'s per-job body (`_process_job`) is the execute → save → deliver → mark
|
|
sequence that fires ONE due job. Phase 4A extracts it into a module-level
|
|
`run_one_job(job, *, adapters=None, loop=None, verbose=False)` so the external
|
|
Chronos provider's `fire_due` can reuse the IDENTICAL body — no duplicated
|
|
correctness.
|
|
|
|
The first test characterizes the sequence as driven through `tick()` (proving
|
|
the extraction didn't change `tick`'s behavior); the rest unit-test the
|
|
extracted helper directly.
|
|
"""
|
|
import pytest
|
|
|
|
import cron.scheduler as s
|
|
|
|
|
|
def _patch_pipeline(monkeypatch, *, success=True, output="out", final="final response",
|
|
error=None, silent_marker_in=None):
|
|
"""Patch the job pipeline primitives and record the call order."""
|
|
calls = []
|
|
|
|
def fake_run_job(job, *, defer_agent_teardown=None, **kw):
|
|
calls.append(("run_job", job["id"]))
|
|
fr = final if silent_marker_in is None else silent_marker_in
|
|
return (success, output, fr, error)
|
|
|
|
def fake_save(jid, out):
|
|
calls.append(("save", jid))
|
|
return f"/tmp/{jid}.txt"
|
|
|
|
def fake_deliver(job, content, adapters=None, loop=None, **kwargs):
|
|
calls.append(("deliver", job["id"]))
|
|
return None
|
|
|
|
def fake_mark(jid, ok, err=None, delivery_error=None, **_kw):
|
|
calls.append(("mark", jid, ok))
|
|
|
|
monkeypatch.setattr(s, "run_job", fake_run_job)
|
|
monkeypatch.setattr(s, "save_job_output", fake_save)
|
|
monkeypatch.setattr(s, "_deliver_result", fake_deliver)
|
|
monkeypatch.setattr(s, "mark_job_run", fake_mark)
|
|
return calls
|
|
|
|
|
|
def test_tick_process_job_sequence(monkeypatch):
|
|
"""Characterization: a single due job driven through tick() runs the
|
|
sequence run_job → save → deliver → mark, in that order."""
|
|
calls = _patch_pipeline(monkeypatch)
|
|
monkeypatch.setattr(s, "get_due_jobs", lambda: [{"id": "j1", "name": "t"}])
|
|
monkeypatch.setattr(s, "claim_job_for_fire", lambda _job_id, **_kwargs: True)
|
|
|
|
s.tick(verbose=False, sync=True)
|
|
|
|
assert [c[0] for c in calls] == ["run_job", "save", "deliver", "mark"]
|
|
assert calls[-1] == ("mark", "j1", True)
|
|
|
|
|
|
def test_tick_skips_job_when_durable_fire_claim_is_lost(monkeypatch):
|
|
"""A manual/external fire that wins the shared CAS must exclude ticker."""
|
|
calls = _patch_pipeline(monkeypatch)
|
|
monkeypatch.setattr(s, "get_due_jobs", lambda: [{"id": "j1", "name": "t"}])
|
|
monkeypatch.setattr(s, "claim_job_for_fire", lambda _job_id: False)
|
|
|
|
assert s.tick(verbose=False, sync=True) == 0
|
|
assert calls == []
|
|
|
|
|
|
def test_run_one_job_success_sequence(monkeypatch):
|
|
"""The extracted helper runs the same execute→save→deliver→mark sequence
|
|
for a successful job."""
|
|
calls = _patch_pipeline(monkeypatch)
|
|
|
|
ok = s.run_one_job({"id": "j2", "name": "t"})
|
|
|
|
assert ok is True
|
|
assert [c[0] for c in calls] == ["run_job", "save", "deliver", "mark"]
|
|
assert calls[-1] == ("mark", "j2", True)
|
|
|
|
|
|
def test_run_one_job_exception_delivers_failure_alert(monkeypatch):
|
|
"""An exception escaping the run body must not become a silent error row."""
|
|
delivered = []
|
|
marked = []
|
|
finished = []
|
|
|
|
monkeypatch.setattr(
|
|
s, "create_execution", lambda *_a, **_kw: {"id": "exec-j3"}
|
|
)
|
|
monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True)
|
|
monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {})
|
|
monkeypatch.setattr(
|
|
s,
|
|
"run_job",
|
|
lambda *_a, **_kw: (_ for _ in ()).throw(
|
|
RuntimeError("Gemini HTTP 503 (UNAVAILABLE)")
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
s,
|
|
"_deliver_result",
|
|
lambda job, content, **_kw: delivered.append((job["id"], content)) or None,
|
|
)
|
|
monkeypatch.setattr(
|
|
s,
|
|
"mark_job_run",
|
|
lambda *args, **kwargs: marked.append((args, kwargs)),
|
|
)
|
|
monkeypatch.setattr(
|
|
s,
|
|
"finish_execution",
|
|
lambda *args, **kwargs: finished.append((args, kwargs)),
|
|
)
|
|
|
|
ok = s.run_one_job({"id": "j3", "name": "morning", "deliver": "telegram"})
|
|
|
|
assert ok is False
|
|
assert delivered == [
|
|
("j3", "⚠️ Cron 'morning' failed: Gemini HTTP 503 (UNAVAILABLE)")
|
|
]
|
|
assert marked == [
|
|
(("j3", False, "Gemini HTTP 503 (UNAVAILABLE)"), {"delivery_error": None})
|
|
]
|
|
assert finished == [
|
|
(
|
|
("exec-j3",),
|
|
{
|
|
"success": False,
|
|
"error": "Gemini HTTP 503 (UNAVAILABLE)",
|
|
"delivery_outcome": "delivered",
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
def test_run_one_job_exception_records_failure_alert_delivery_error(monkeypatch):
|
|
"""A failed fallback alert must populate last_delivery_error."""
|
|
marked = []
|
|
|
|
monkeypatch.setattr(
|
|
s, "create_execution", lambda *_a, **_kw: {"id": "exec-j4"}
|
|
)
|
|
monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True)
|
|
monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {})
|
|
monkeypatch.setattr(
|
|
s,
|
|
"run_job",
|
|
lambda *_a, **_kw: (_ for _ in ()).throw(RuntimeError("provider failed")),
|
|
)
|
|
monkeypatch.setattr(s, "_deliver_result", lambda *_a, **_kw: "send failed: 502")
|
|
monkeypatch.setattr(
|
|
s,
|
|
"mark_job_run",
|
|
lambda *args, **kwargs: marked.append((args, kwargs)),
|
|
)
|
|
monkeypatch.setattr(s, "finish_execution", lambda *_a, **_kw: None)
|
|
|
|
assert s.run_one_job({"id": "j4", "deliver": "telegram"}) is False
|
|
assert marked == [
|
|
(("j4", False, "provider failed"), {"delivery_error": "send failed: 502"})
|
|
]
|
|
|
|
|
|
def _patch_escaped_failure(monkeypatch, delivered, *, exec_id, err):
|
|
"""Make run_job raise, and capture what the escape handler delivers."""
|
|
monkeypatch.setattr(s, "create_execution", lambda *_a, **_kw: {"id": exec_id})
|
|
monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True)
|
|
monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {})
|
|
monkeypatch.setattr(
|
|
s,
|
|
"run_job",
|
|
lambda *_a, **_kw: (_ for _ in ()).throw(RuntimeError(err)),
|
|
)
|
|
monkeypatch.setattr(
|
|
s,
|
|
"_deliver_result",
|
|
lambda job, content, **_kw: delivered.append(content) or None,
|
|
)
|
|
monkeypatch.setattr(s, "mark_job_run", lambda *_a, **_kw: None)
|
|
monkeypatch.setattr(s, "finish_execution", lambda *_a, **_kw: None)
|
|
# Deterministic threshold: default 3, independent of the host config.
|
|
monkeypatch.setattr(s, "load_config", lambda: {})
|
|
|
|
|
|
def test_escaped_failure_delivery_carries_the_streak_nudge(monkeypatch):
|
|
"""A repeatedly-failing job must be nudged even when it fails at the
|
|
scheduler layer (#88655).
|
|
|
|
``mark_job_run`` increments ``failure_streak`` for an escaped failure just
|
|
as it does for an agent failure, so the counter climbs either way. But the
|
|
nudge that spends it was only composed on the normal delivery path, so a
|
|
job that raises before the run body on every tick - a bad import from a
|
|
half-applied update, a provider client that cannot construct - alerts
|
|
forever and is never told it should be reviewed or paused. Nothing else
|
|
surfaces the streak in chat.
|
|
"""
|
|
delivered = []
|
|
_patch_escaped_failure(
|
|
monkeypatch, delivered, exec_id="exec-j5", err="cannot import name X"
|
|
)
|
|
|
|
ok = s.run_one_job(
|
|
{
|
|
"id": "j5",
|
|
"name": "scout",
|
|
"deliver": "telegram",
|
|
"schedule": {"kind": "interval"},
|
|
"failure_streak": 2, # + this run = 3 = default threshold
|
|
}
|
|
)
|
|
|
|
assert ok is False
|
|
assert len(delivered) == 1
|
|
assert "cannot import name X" in delivered[0]
|
|
assert "failed 3 runs in a row" in delivered[0]
|
|
assert "hermes cron pause scout" in delivered[0]
|
|
|
|
|
|
def test_escaped_failure_delivery_stays_quiet_below_the_threshold(monkeypatch):
|
|
"""The nudge is appended, not always-on: a first failure reads as before."""
|
|
delivered = []
|
|
_patch_escaped_failure(
|
|
monkeypatch, delivered, exec_id="exec-j6", err="provider failed"
|
|
)
|
|
|
|
ok = s.run_one_job(
|
|
{
|
|
"id": "j6",
|
|
"name": "scout",
|
|
"deliver": "telegram",
|
|
"schedule": {"kind": "interval"},
|
|
"failure_streak": 0,
|
|
}
|
|
)
|
|
|
|
assert ok is False
|
|
assert delivered == ["⚠️ Cron 'scout' failed: provider failed"]
|
|
|
|
|
|
def test_run_one_job_exception_after_delivery_does_not_redeliver(monkeypatch):
|
|
"""Once delivery has been attempted, the outer handler must not send again."""
|
|
delivered = []
|
|
mark_calls = []
|
|
|
|
monkeypatch.setattr(
|
|
s, "create_execution", lambda *_a, **_kw: {"id": "exec-j5"}
|
|
)
|
|
monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True)
|
|
monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {})
|
|
monkeypatch.setattr(
|
|
s,
|
|
"run_job",
|
|
lambda *_a, **_kw: (True, "out", "final response", None),
|
|
)
|
|
monkeypatch.setattr(s, "save_job_output", lambda jid, out: f"/tmp/{jid}.txt")
|
|
monkeypatch.setattr(
|
|
s,
|
|
"_deliver_result",
|
|
lambda job, content, **_kw: delivered.append((job["id"], content)) or None,
|
|
)
|
|
|
|
def fake_mark(*args, **kwargs):
|
|
mark_calls.append((args, kwargs))
|
|
if len(mark_calls) == 1:
|
|
raise RuntimeError("bookkeeping boom")
|
|
|
|
monkeypatch.setattr(s, "mark_job_run", fake_mark)
|
|
monkeypatch.setattr(s, "finish_execution", lambda *_a, **_kw: None)
|
|
|
|
ok = s.run_one_job({"id": "j5", "name": "once", "deliver": "telegram"})
|
|
|
|
assert ok is False
|
|
assert delivered == [("j5", "final response")]
|
|
assert mark_calls[0] == (("j5", True, None), {"delivery_error": None})
|
|
assert mark_calls[1] == (
|
|
("j5", False, "bookkeeping boom"),
|
|
{"delivery_error": None},
|
|
)
|
|
|
|
|
|
def test_run_one_job_keyboard_interrupt_skips_delivery_and_reraises(monkeypatch):
|
|
"""Hard interrupts must not attempt failure delivery; they re-raise."""
|
|
delivered = []
|
|
marked = []
|
|
finished = []
|
|
|
|
monkeypatch.setattr(
|
|
s, "create_execution", lambda *_a, **_kw: {"id": "exec-j6"}
|
|
)
|
|
monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True)
|
|
monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {})
|
|
monkeypatch.setattr(
|
|
s,
|
|
"run_job",
|
|
lambda *_a, **_kw: (_ for _ in ()).throw(KeyboardInterrupt()),
|
|
)
|
|
monkeypatch.setattr(
|
|
s,
|
|
"_deliver_result",
|
|
lambda job, content, **_kw: delivered.append((job["id"], content)) or None,
|
|
)
|
|
monkeypatch.setattr(
|
|
s,
|
|
"mark_job_run",
|
|
lambda *args, **kwargs: marked.append((args, kwargs)),
|
|
)
|
|
monkeypatch.setattr(
|
|
s,
|
|
"finish_execution",
|
|
lambda *args, **kwargs: finished.append((args, kwargs)),
|
|
)
|
|
|
|
with pytest.raises(KeyboardInterrupt):
|
|
s.run_one_job({"id": "j6", "name": "interrupt", "deliver": "telegram"})
|
|
|
|
assert delivered == []
|
|
assert marked == [(("j6", False, "KeyboardInterrupt"), {})]
|
|
assert finished == [
|
|
(
|
|
("exec-j6",),
|
|
{
|
|
"success": False,
|
|
"error": "KeyboardInterrupt",
|
|
"delivery_outcome": "suppressed",
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
def test_run_one_job_installs_secret_scope_under_multiplex(monkeypatch, tmp_path):
|
|
"""Regression: under profile isolation (multiplex active), run_one_job must
|
|
keep one profile secret scope active through execution and delivery so
|
|
credential reads do not fail closed or fall through to another profile,
|
|
then tear the scope down after the complete job lifecycle.
|
|
|
|
Behavior contract: the same scope is present during run_job and
|
|
_deliver_result, and no scope remains after run_one_job returns.
|
|
"""
|
|
from agent import secret_scope as ss
|
|
|
|
# Point cron's home resolution at a profile whose .env carries a secret.
|
|
(tmp_path / ".env").write_text("OPENROUTER_BASE_URL=https://openrouter.ai/api/v1\n")
|
|
monkeypatch.setattr(s, "_get_hermes_home", lambda: tmp_path)
|
|
|
|
scope_during_run = {}
|
|
scope_during_delivery = {}
|
|
|
|
def fake_run_job(job, *, defer_agent_teardown=None, **kw):
|
|
# This is where resolve_runtime_provider() would read a secret. Prove a
|
|
# scope is installed and the profile's secret resolves without raising.
|
|
scope_during_run["scope"] = ss.current_secret_scope()
|
|
scope_during_run["base_url"] = ss.get_secret("OPENROUTER_BASE_URL")
|
|
return (True, "out", "final", None)
|
|
|
|
def fake_deliver(*args, **kwargs):
|
|
scope_during_delivery["scope"] = ss.current_secret_scope()
|
|
scope_during_delivery["base_url"] = ss.get_secret("OPENROUTER_BASE_URL")
|
|
return None
|
|
|
|
monkeypatch.setattr(s, "run_job", fake_run_job)
|
|
monkeypatch.setattr(s, "save_job_output", lambda jid, out: f"/tmp/{jid}.txt")
|
|
monkeypatch.setattr(s, "_deliver_result", fake_deliver)
|
|
monkeypatch.setattr(s, "mark_job_run", lambda *a, **k: None)
|
|
|
|
ss.set_multiplex_active(True)
|
|
try:
|
|
ok = s.run_one_job({"id": "j7", "name": "t"})
|
|
finally:
|
|
ss.set_multiplex_active(False)
|
|
|
|
assert ok is True
|
|
# The same profile scope covered both execution and delivery.
|
|
assert scope_during_run["scope"] is not None
|
|
assert scope_during_run["base_url"] == "https://openrouter.ai/api/v1"
|
|
assert scope_during_delivery["scope"] == scope_during_run["scope"]
|
|
assert scope_during_delivery["base_url"] == "https://openrouter.ai/api/v1"
|
|
# And it was torn down after the full lifecycle returned (no leak).
|
|
assert ss.current_secret_scope() is None
|
|
|
|
|