Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,462 @@
|
||||
"""Per-job ``failure_deliver`` routing (NS-788).
|
||||
|
||||
A job's FAILURE notices (run failed, escaped scheduler exception, drift-skip /
|
||||
blocked-config alerts) resolve their delivery targets from ``failure_deliver``
|
||||
when the job sets it, falling back to ``deliver`` when unset — so existing
|
||||
jobs behave byte-identically. ``failure_deliver: local`` is structural silence
|
||||
for failures: nothing is sent, but state (last_status, run history, output
|
||||
file) is still recorded. Success-path delivery never reads ``failure_deliver``.
|
||||
|
||||
The grammar is exactly the ``deliver`` grammar — same normalization, same
|
||||
validation — reused, not duplicated.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
import cron.scheduler as s
|
||||
from cron.scheduler import _resolve_delivery_targets
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cron_env(tmp_path, monkeypatch):
|
||||
"""Isolated cron environment with temp HERMES_HOME."""
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
(hermes_home / "cron").mkdir()
|
||||
(hermes_home / "cron" / "output").mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
|
||||
import cron.jobs as jobs_mod
|
||||
monkeypatch.setattr(jobs_mod, "HERMES_DIR", hermes_home)
|
||||
monkeypatch.setattr(jobs_mod, "CRON_DIR", hermes_home / "cron")
|
||||
monkeypatch.setattr(jobs_mod, "JOBS_FILE", hermes_home / "cron" / "jobs.json")
|
||||
monkeypatch.setattr(jobs_mod, "OUTPUT_DIR", hermes_home / "cron" / "output")
|
||||
|
||||
return hermes_home
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_env(monkeypatch, tmp_path):
|
||||
"""Drive run_one_job with the REAL delivery path down to a fake sender.
|
||||
|
||||
Bookkeeping primitives are stubbed (recorded), but _deliver_result and
|
||||
_resolve_delivery_targets are the genuine articles — the send that would
|
||||
leave the process is captured at the platform-registry sender seam,
|
||||
exactly where a real slack delivery exits.
|
||||
"""
|
||||
home = tmp_path / "hermes-home"
|
||||
home.mkdir()
|
||||
(home / "config.yaml").write_text(
|
||||
"platforms:\n slack:\n enabled: true\n token: xoxb-test\n"
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
|
||||
send_calls = []
|
||||
|
||||
async def fake_sender(pconfig, chat_id, message, *, thread_id=None,
|
||||
media_files=None, force_document=False, caption=None):
|
||||
send_calls.append({"chat_id": chat_id, "message": message})
|
||||
return {"success": True, "chat_id": chat_id, "message_id": "1.2"}
|
||||
|
||||
import gateway.platform_registry as reg
|
||||
import hermes_cli.plugins as hp
|
||||
|
||||
entry = reg.platform_registry.get("slack")
|
||||
if entry is None:
|
||||
hp.discover_plugins()
|
||||
entry = reg.platform_registry.get("slack")
|
||||
if entry is None:
|
||||
pytest.skip("slack platform entry not registered")
|
||||
monkeypatch.setattr(entry, "standalone_sender_fn", fake_sender)
|
||||
monkeypatch.setattr(hp, "discover_plugins", lambda *a, **k: None)
|
||||
|
||||
state = {"send": send_calls, "marked": [], "saved": [], "finished": []}
|
||||
|
||||
monkeypatch.setattr(s, "create_execution", lambda *_a, **_kw: {"id": "exec-t"})
|
||||
monkeypatch.setattr(s, "claim_dispatch", lambda _job_id: True)
|
||||
monkeypatch.setattr(s, "mark_execution_running", lambda _execution_id: {})
|
||||
monkeypatch.setattr(
|
||||
s, "save_job_output",
|
||||
lambda jid, out: state["saved"].append(jid) or f"/tmp/{jid}.txt",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
s, "mark_job_run",
|
||||
lambda *a, **kw: state["marked"].append((a, kw)) or True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
s, "finish_execution",
|
||||
lambda *a, **kw: state["finished"].append((a, kw)),
|
||||
)
|
||||
# No durable incident store in play: never acked, no id.
|
||||
monkeypatch.setattr(
|
||||
s, "_upsert_incident_for_failure", lambda *_a, **_kw: (False, None)
|
||||
)
|
||||
monkeypatch.setattr(s, "load_config", lambda: {})
|
||||
return state
|
||||
|
||||
|
||||
def _failing_run_job(error="provider exploded"):
|
||||
def _fake(job, **_kw):
|
||||
return (False, "raw output", "", error)
|
||||
return _fake
|
||||
|
||||
|
||||
def _succeeding_run_job(final="all good, here is the brief"):
|
||||
def _fake(job, **_kw):
|
||||
return (True, "raw output", final, None)
|
||||
return _fake
|
||||
|
||||
|
||||
class TestFailureDeliverRouting:
|
||||
def test_failure_without_failure_deliver_goes_to_deliver_targets(
|
||||
self, run_env, monkeypatch
|
||||
):
|
||||
"""(a) Unset failure_deliver = today's behavior: failure summary to
|
||||
the job's deliver targets."""
|
||||
monkeypatch.setattr(s, "run_job", _failing_run_job())
|
||||
|
||||
s.run_one_job({"id": "j1", "name": "scout", "deliver": "slack:D0MAIN"})
|
||||
|
||||
assert [c["chat_id"] for c in run_env["send"]] == ["D0MAIN"]
|
||||
assert "failed" in run_env["send"][0]["message"].lower()
|
||||
|
||||
def test_failure_deliver_local_is_silent_but_state_is_recorded(
|
||||
self, run_env, monkeypatch
|
||||
):
|
||||
"""(b) failure_deliver: local — no delivery leaves the process, but
|
||||
the run is still saved and marked failed."""
|
||||
monkeypatch.setattr(s, "run_job", _failing_run_job())
|
||||
|
||||
s.run_one_job({
|
||||
"id": "j2", "name": "scout",
|
||||
"deliver": "slack:D0MAIN", "failure_deliver": "local",
|
||||
})
|
||||
|
||||
assert run_env["send"] == []
|
||||
# State recording is untouched by the silence.
|
||||
assert run_env["saved"] == ["j2"]
|
||||
assert len(run_env["marked"]) == 1
|
||||
args, _kw = run_env["marked"][0]
|
||||
assert args[0] == "j2" and args[1] is False
|
||||
assert "provider exploded" in args[2]
|
||||
|
||||
def test_failure_deliver_explicit_target_wins_over_deliver(
|
||||
self, run_env, monkeypatch
|
||||
):
|
||||
"""(c) failure_deliver set to a different target: the failure notice
|
||||
goes THERE, and nothing goes to the deliver target."""
|
||||
monkeypatch.setattr(s, "run_job", _failing_run_job())
|
||||
|
||||
s.run_one_job({
|
||||
"id": "j3", "name": "scout",
|
||||
"deliver": "slack:D0MAIN", "failure_deliver": "slack:D0ALERTS",
|
||||
})
|
||||
|
||||
assert [c["chat_id"] for c in run_env["send"]] == ["D0ALERTS"]
|
||||
assert "failed" in run_env["send"][0]["message"].lower()
|
||||
|
||||
def test_success_ignores_failure_deliver(self, run_env, monkeypatch):
|
||||
"""(d) Success output still goes to deliver — failure_deliver is
|
||||
never consulted on the success path."""
|
||||
monkeypatch.setattr(s, "run_job", _succeeding_run_job())
|
||||
|
||||
ok = s.run_one_job({
|
||||
"id": "j4", "name": "scout",
|
||||
"deliver": "slack:D0MAIN", "failure_deliver": "slack:D0ALERTS",
|
||||
})
|
||||
|
||||
assert ok is True
|
||||
assert [c["chat_id"] for c in run_env["send"]] == ["D0MAIN"]
|
||||
assert "all good, here is the brief" in run_env["send"][0]["message"]
|
||||
|
||||
|
||||
class TestEscapedExceptionPath:
|
||||
"""The scheduler-layer exception handler is the second failure-delivery
|
||||
site — it must honor failure_deliver identically."""
|
||||
|
||||
def _raise_run_job(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
s, "run_job",
|
||||
lambda *_a, **_kw: (_ for _ in ()).throw(
|
||||
RuntimeError("cannot import name X")
|
||||
),
|
||||
)
|
||||
|
||||
def test_escaped_failure_honors_failure_deliver_target(
|
||||
self, run_env, monkeypatch
|
||||
):
|
||||
self._raise_run_job(monkeypatch)
|
||||
|
||||
ok = s.run_one_job({
|
||||
"id": "j5", "name": "scout",
|
||||
"deliver": "slack:D0MAIN", "failure_deliver": "slack:D0ALERTS",
|
||||
})
|
||||
|
||||
assert ok is False
|
||||
assert [c["chat_id"] for c in run_env["send"]] == ["D0ALERTS"]
|
||||
|
||||
def test_escaped_failure_with_failure_deliver_local_is_silent(
|
||||
self, run_env, monkeypatch
|
||||
):
|
||||
self._raise_run_job(monkeypatch)
|
||||
|
||||
ok = s.run_one_job({
|
||||
"id": "j6", "name": "scout",
|
||||
"deliver": "slack:D0MAIN", "failure_deliver": "local",
|
||||
})
|
||||
|
||||
assert ok is False
|
||||
assert run_env["send"] == []
|
||||
# Failure is still recorded.
|
||||
assert len(run_env["marked"]) == 1
|
||||
args, _kw = run_env["marked"][0]
|
||||
assert args[1] is False and "cannot import name X" in args[2]
|
||||
|
||||
|
||||
class TestResolutionGrammar:
|
||||
"""(e) failure_deliver shares deliver's exact value grammar — the same
|
||||
normalization/expansion path, not a parallel one."""
|
||||
|
||||
def test_for_failure_resolves_failure_deliver_value(self):
|
||||
job = {"deliver": "local", "failure_deliver": "slack:D0ALERTS"}
|
||||
targets = _resolve_delivery_targets(job, for_failure=True)
|
||||
assert [(t["platform"], t["chat_id"]) for t in targets] == [
|
||||
("slack", "D0ALERTS")
|
||||
]
|
||||
|
||||
def test_for_failure_falls_back_to_deliver_when_unset(self):
|
||||
job = {"deliver": "slack:D0MAIN"}
|
||||
targets = _resolve_delivery_targets(job, for_failure=True)
|
||||
assert [(t["platform"], t["chat_id"]) for t in targets] == [
|
||||
("slack", "D0MAIN")
|
||||
]
|
||||
|
||||
def test_success_resolution_never_reads_failure_deliver(self):
|
||||
job = {"deliver": "slack:D0MAIN", "failure_deliver": "slack:D0ALERTS"}
|
||||
targets = _resolve_delivery_targets(job)
|
||||
assert [(t["platform"], t["chat_id"]) for t in targets] == [
|
||||
("slack", "D0MAIN")
|
||||
]
|
||||
|
||||
def test_local_yields_zero_failure_targets(self):
|
||||
job = {"deliver": "slack:D0MAIN", "failure_deliver": "local"}
|
||||
assert _resolve_delivery_targets(job, for_failure=True) == []
|
||||
|
||||
def test_comma_list_and_thread_grammar(self):
|
||||
"""The comma-combine + platform:chat:thread forms deliver's grammar
|
||||
supports work identically for failure_deliver."""
|
||||
job = {
|
||||
"deliver": "local",
|
||||
"failure_deliver": "slack:D0ALERTS,telegram:-1001:17",
|
||||
}
|
||||
targets = _resolve_delivery_targets(job, for_failure=True)
|
||||
assert [(t["platform"], t["chat_id"], t.get("thread_id")) for t in targets] == [
|
||||
("slack", "D0ALERTS", None),
|
||||
("telegram", "-1001", "17"),
|
||||
]
|
||||
|
||||
def test_legacy_list_value_is_flattened_like_deliver(self):
|
||||
"""Same list/tuple tolerance _normalize_deliver_value grants deliver."""
|
||||
job = {"deliver": "local", "failure_deliver": ["slack:D0ALERTS"]}
|
||||
targets = _resolve_delivery_targets(job, for_failure=True)
|
||||
assert [(t["platform"], t["chat_id"]) for t in targets] == [
|
||||
("slack", "D0ALERTS")
|
||||
]
|
||||
|
||||
|
||||
class TestToolSurface:
|
||||
"""cronjob(action=create/update) accepts failure_deliver with deliver's
|
||||
validation — reusing the same normalize/validate helpers."""
|
||||
|
||||
def test_create_stores_failure_deliver(self, cron_env):
|
||||
from tools.cronjob_tools import cronjob
|
||||
from cron.jobs import get_job
|
||||
|
||||
result = json.loads(cronjob(
|
||||
action="create",
|
||||
prompt="scan",
|
||||
schedule="every 1h",
|
||||
deliver="slack:D0MAIN",
|
||||
failure_deliver="local",
|
||||
))
|
||||
assert result["success"] is True
|
||||
assert get_job(result["job_id"])["failure_deliver"] == "local"
|
||||
|
||||
def test_create_without_failure_deliver_does_not_persist_the_key(self, cron_env):
|
||||
"""Existing-job byte-identity: the field only exists when set."""
|
||||
from tools.cronjob_tools import cronjob
|
||||
from cron.jobs import get_job
|
||||
|
||||
result = json.loads(cronjob(
|
||||
action="create", prompt="scan", schedule="every 1h",
|
||||
))
|
||||
assert result["success"] is True
|
||||
assert "failure_deliver" not in get_job(result["job_id"])
|
||||
|
||||
def test_create_flattens_list_value_like_deliver(self, cron_env):
|
||||
from tools.cronjob_tools import cronjob
|
||||
from cron.jobs import get_job
|
||||
|
||||
result = json.loads(cronjob(
|
||||
action="create",
|
||||
prompt="scan",
|
||||
schedule="every 1h",
|
||||
failure_deliver=["slack", "telegram"],
|
||||
))
|
||||
assert result["success"] is True
|
||||
assert get_job(result["job_id"])["failure_deliver"] == "slack,telegram"
|
||||
|
||||
def test_create_rejects_bad_bot_chat_profile_same_as_deliver(self, cron_env):
|
||||
from tools.cronjob_tools import cronjob
|
||||
|
||||
via_failure = json.loads(cronjob(
|
||||
action="create", prompt="scan", schedule="every 1h",
|
||||
failure_deliver="bot-chat:no-such-profile-xyz",
|
||||
))
|
||||
via_deliver = json.loads(cronjob(
|
||||
action="create", prompt="scan", schedule="every 1h",
|
||||
deliver="bot-chat:no-such-profile-xyz",
|
||||
))
|
||||
assert via_failure["success"] is False
|
||||
assert via_deliver["success"] is False
|
||||
# Same validator, same message.
|
||||
assert via_failure["error"] == via_deliver["error"]
|
||||
|
||||
def test_update_sets_and_clears_failure_deliver(self, cron_env):
|
||||
from cron.jobs import create_job, get_job
|
||||
from tools.cronjob_tools import cronjob
|
||||
|
||||
job = create_job(prompt="scan", schedule="every 1h")
|
||||
result = json.loads(cronjob(
|
||||
action="update", job_id=job["id"], failure_deliver="slack:D0ALERTS",
|
||||
))
|
||||
assert result["success"] is True
|
||||
assert get_job(job["id"])["failure_deliver"] == "slack:D0ALERTS"
|
||||
|
||||
# '' clears — job falls back to deliver on failures again.
|
||||
result = json.loads(cronjob(
|
||||
action="update", job_id=job["id"], failure_deliver="",
|
||||
))
|
||||
assert result["success"] is True
|
||||
assert not get_job(job["id"]).get("failure_deliver")
|
||||
|
||||
|
||||
class TestOutcomeBookkeeping:
|
||||
"""Review finding B1 (NS-788): delivery bookkeeping — outcome
|
||||
classification, unresolved-origin, incident 'alerted' marking — must
|
||||
read the SAME lane the notice was actually routed through, or the
|
||||
execution history and incident store record lies (silenced failures
|
||||
logged 'delivered'; delivered failures logged 'not_configured')."""
|
||||
|
||||
@staticmethod
|
||||
def _outcome(state):
|
||||
assert state["finished"], "finish_execution never called"
|
||||
_a, kw = state["finished"][-1]
|
||||
return kw.get("delivery_outcome")
|
||||
|
||||
def test_fd_local_failure_records_suppressed_not_delivered(
|
||||
self, run_env, monkeypatch
|
||||
):
|
||||
alerted = []
|
||||
monkeypatch.setattr(s, "_mark_incident_alerted", alerted.append)
|
||||
monkeypatch.setattr(s, "run_job", _failing_run_job())
|
||||
|
||||
s.run_one_job({
|
||||
"id": "b1a", "name": "scout",
|
||||
"deliver": "slack:D0MAIN", "failure_deliver": "local",
|
||||
})
|
||||
|
||||
assert run_env["send"] == []
|
||||
assert self._outcome(run_env) == "suppressed"
|
||||
assert alerted == [], "silenced failure must NOT mark incident alerted"
|
||||
|
||||
def test_fd_explicit_target_failure_records_delivered(
|
||||
self, run_env, monkeypatch
|
||||
):
|
||||
"""deliver=origin (unresolvable) + failure_deliver=explicit target:
|
||||
the notice IS delivered — outcome must say so, not 'not_configured'."""
|
||||
alerted = []
|
||||
monkeypatch.setattr(s, "_mark_incident_alerted", alerted.append)
|
||||
monkeypatch.setattr(
|
||||
s, "_upsert_incident_for_failure", lambda *_a, **_kw: (False, "inc-b1")
|
||||
)
|
||||
monkeypatch.setattr(s, "run_job", _failing_run_job())
|
||||
|
||||
s.run_one_job({
|
||||
"id": "b1b", "name": "scout",
|
||||
"deliver": "origin", "failure_deliver": "slack:D0OPS",
|
||||
})
|
||||
|
||||
assert [c["chat_id"] for c in run_env["send"]] == ["D0OPS"]
|
||||
assert self._outcome(run_env) == "delivered"
|
||||
assert alerted == ["inc-b1"], "delivered failure ping must mark incident alerted"
|
||||
|
||||
def test_success_outcome_still_reads_deliver_lane(self, run_env, monkeypatch):
|
||||
"""Success bookkeeping is untouched: fd set, success delivers to
|
||||
deliver and records 'delivered'."""
|
||||
monkeypatch.setattr(s, "run_job", _succeeding_run_job())
|
||||
|
||||
s.run_one_job({
|
||||
"id": "b1c", "name": "scout",
|
||||
"deliver": "slack:D0MAIN", "failure_deliver": "local",
|
||||
})
|
||||
|
||||
assert [c["chat_id"] for c in run_env["send"]] == ["D0MAIN"]
|
||||
assert self._outcome(run_env) == "delivered"
|
||||
|
||||
|
||||
class TestPreflightAndDashboardLanes:
|
||||
"""Follow-up (salvage): the failure lane is validated everywhere the
|
||||
deliver lane is — preflight config checks and the dashboard update
|
||||
normalizer — so a typo'd failure target is caught before a failure
|
||||
needs it."""
|
||||
|
||||
def test_preflight_blocks_unknown_failure_platform(self, monkeypatch):
|
||||
"""A bogus failure_deliver platform blocks at preflight, exactly
|
||||
like a bogus deliver platform would."""
|
||||
monkeypatch.setattr(s, "_is_known_delivery_platform", lambda _p: False)
|
||||
err = s._preflight_check_delivery({
|
||||
"id": "p1", "deliver": "local",
|
||||
"failure_deliver": "nonexistent-platform:C1",
|
||||
})
|
||||
assert err is not None and "not a known" in err
|
||||
|
||||
def test_preflight_failure_deliver_local_adds_no_platforms(self):
|
||||
"""failure_deliver: local adds nothing to check — a deliver=local
|
||||
job with suppressed failures stays zero-cost at preflight."""
|
||||
assert s._preflight_check_delivery({
|
||||
"id": "p2", "deliver": "local", "failure_deliver": "local",
|
||||
}) is None
|
||||
|
||||
def test_preflight_duplicate_lane_not_checked_twice(self, monkeypatch):
|
||||
"""failure_deliver equal to deliver must not double-check (or
|
||||
double-report) the same platform."""
|
||||
seen = []
|
||||
|
||||
def _known(p):
|
||||
seen.append(p)
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(s, "_is_known_delivery_platform", _known)
|
||||
s._preflight_check_delivery({
|
||||
"id": "p3", "deliver": "ghost:C1", "failure_deliver": "ghost:C1",
|
||||
})
|
||||
assert seen == ["ghost"]
|
||||
|
||||
def test_dashboard_update_normalizes_failure_deliver(self, tmp_path):
|
||||
"""The dashboard update lane normalizes failure_deliver like
|
||||
deliver: text stripped, empty clears (None) instead of
|
||||
coalescing to a target."""
|
||||
from hermes_cli.web_server import _normalize_dashboard_cron_updates
|
||||
|
||||
out = _normalize_dashboard_cron_updates(
|
||||
{"failure_deliver": " slack:D0ALERTS "}, tmp_path
|
||||
)
|
||||
assert out["failure_deliver"] == "slack:D0ALERTS"
|
||||
|
||||
cleared = _normalize_dashboard_cron_updates(
|
||||
{"failure_deliver": ""}, tmp_path
|
||||
)
|
||||
assert cleared["failure_deliver"] is None
|
||||
Reference in New Issue
Block a user