Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,578 @@
|
||||
"""Tests for the Chronos cron-fire webhook ON THE DASHBOARD APP (web_server).
|
||||
|
||||
Regression guard for the relocation bug: the fire webhook MUST live on the
|
||||
dashboard FastAPI app (`hermes_cli.web_server.app`) — the agent's public HTTP
|
||||
surface on hosted deployments — not only on the aiohttp APIServerAdapter (which
|
||||
hosted agents don't expose). It must:
|
||||
- be a registered route on the dashboard app,
|
||||
- be in PUBLIC_API_PATHS so the dashboard cookie gate doesn't 401 it before
|
||||
the JWT verifier runs,
|
||||
- reject a bad/missing NAS-JWT with 401 (the JWT is the real gate),
|
||||
- 400 on missing job_id,
|
||||
- on a valid token, FORWARD the fire to the gateway api_server (which owns
|
||||
cron execution and the live delivery adapters) and pass its response
|
||||
through — 503 when the gateway is unreachable so NAS retries.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from hermes_cli import web_server
|
||||
from hermes_cli.dashboard_auth.public_paths import PUBLIC_API_PATHS
|
||||
|
||||
|
||||
def _client(auth_required: bool):
|
||||
prev_auth = getattr(web_server.app.state, "auth_required", None)
|
||||
prev_host = getattr(web_server.app.state, "bound_host", None)
|
||||
web_server.app.state.auth_required = auth_required
|
||||
web_server.app.state.bound_host = None
|
||||
client = TestClient(web_server.app)
|
||||
return client, prev_auth, prev_host
|
||||
|
||||
|
||||
def _restore(prev_auth, prev_host):
|
||||
if prev_auth is None:
|
||||
if hasattr(web_server.app.state, "auth_required"):
|
||||
delattr(web_server.app.state, "auth_required")
|
||||
else:
|
||||
web_server.app.state.auth_required = prev_auth
|
||||
if prev_host is None:
|
||||
if hasattr(web_server.app.state, "bound_host"):
|
||||
delattr(web_server.app.state, "bound_host")
|
||||
else:
|
||||
web_server.app.state.bound_host = prev_host
|
||||
|
||||
|
||||
|
||||
|
||||
def test_fire_path_is_public():
|
||||
"""Must bypass the dashboard cookie gate so the NAS bearer-JWT callback
|
||||
reaches the verifier (the JWT is the real auth)."""
|
||||
assert "/api/cron/fire" in PUBLIC_API_PATHS
|
||||
|
||||
|
||||
def test_bad_token_401(monkeypatch):
|
||||
"""Invalid NAS-JWT -> 401, even with the dashboard auth gate ENGAGED
|
||||
(proves the route is reachable past the cookie gate and the verifier is the
|
||||
gate). fire_due must NOT run."""
|
||||
fired = []
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: None), # verification fails
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default")
|
||||
monkeypatch.setattr(web_server, "_fire_cron_job_for_profile",
|
||||
lambda p, j: fired.append((p, j)))
|
||||
|
||||
client, pa, ph = _client(auth_required=True)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer forged"},
|
||||
json={"job_id": "abc"})
|
||||
assert resp.status_code == 401
|
||||
assert fired == []
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
|
||||
|
||||
def test_missing_job_id_400(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: {"purpose": "cron_fire"}),
|
||||
)
|
||||
client, pa, ph = _client(auth_required=False)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer good"},
|
||||
json={})
|
||||
assert resp.status_code == 400
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
|
||||
|
||||
def test_unknown_job_200_gone(monkeypatch):
|
||||
"""Valid token but the job isn't found in any profile -> 200 'gone'
|
||||
(NAS shouldn't retry a fire for a cancelled/completed job)."""
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: {"purpose": "cron_fire"}),
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: None)
|
||||
client, pa, ph = _client(auth_required=False)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer good"},
|
||||
json={"job_id": "ghost"})
|
||||
assert resp.status_code == 200
|
||||
assert resp.json().get("status") == "gone"
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
|
||||
|
||||
def test_valid_fire_forwards_to_gateway(monkeypatch):
|
||||
"""The dashboard is the public door only: a verified fire is FORWARDED to
|
||||
the gateway api_server (which owns the live adapters — relay/E2EE
|
||||
delivery), with the NAS bearer preserved, and the gateway's response is
|
||||
passed through. The deprecated in-dashboard execution must NOT run."""
|
||||
forwarded = []
|
||||
executed = []
|
||||
|
||||
async def fake_forward(profile, job_id, authorization):
|
||||
forwarded.append((profile, job_id, authorization))
|
||||
return 202, {"status": "accepted", "job_id": job_id}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: {"purpose": "cron_fire"}),
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default")
|
||||
monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward)
|
||||
monkeypatch.setattr(web_server, "_fire_cron_job_for_profile",
|
||||
lambda p, j: executed.append((p, j)))
|
||||
|
||||
client, pa, ph = _client(auth_required=False)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer nas-jwt"},
|
||||
json={"job_id": "j1"})
|
||||
assert resp.status_code == 202
|
||||
assert resp.json().get("status") == "accepted"
|
||||
assert forwarded == [("default", "j1", "Bearer nas-jwt")]
|
||||
assert executed == [] # no local execution — gateway owns cron
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
|
||||
|
||||
def test_gateway_unreachable_503_for_nas_retry(monkeypatch):
|
||||
"""Gateway down (scale-to-zero wake window / restart) -> 503 so NAS
|
||||
retries per the Chronos contract. Deliberately NO local-execution
|
||||
fallback — delivering from the dashboard process cannot serve
|
||||
relay-fronted or E2EE targets."""
|
||||
executed = []
|
||||
|
||||
async def fake_forward(profile, job_id, authorization):
|
||||
return None # unreachable
|
||||
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: {"purpose": "cron_fire"}),
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default")
|
||||
monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward)
|
||||
monkeypatch.setattr(web_server, "_fire_cron_job_for_profile",
|
||||
lambda p, j: executed.append((p, j)))
|
||||
|
||||
client, pa, ph = _client(auth_required=False)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer nas-jwt"},
|
||||
json={"job_id": "j2"})
|
||||
assert resp.status_code == 503
|
||||
assert executed == []
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
|
||||
|
||||
def test_gateway_error_status_passes_through(monkeypatch):
|
||||
"""A non-2xx gateway response (e.g. its own 401 on a replayed JWT) passes
|
||||
through unchanged — the dashboard adds no interpretation of its own."""
|
||||
|
||||
async def fake_forward(profile, job_id, authorization):
|
||||
return 401, {"error": "invalid fire token"}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: {"purpose": "cron_fire"}),
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default")
|
||||
monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward)
|
||||
|
||||
client, pa, ph = _client(auth_required=False)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer nas-jwt"},
|
||||
json={"job_id": "j3"})
|
||||
assert resp.status_code == 401
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
|
||||
|
||||
# ── _gateway_fire_endpoint URL resolution ────────────────────────────────
|
||||
|
||||
|
||||
def test_fire_endpoint_default_port(tmp_path, monkeypatch):
|
||||
monkeypatch.delenv("API_SERVER_PORT", raising=False)
|
||||
monkeypatch.delenv("GATEWAY_MULTIPLEX_PROFILES", raising=False)
|
||||
monkeypatch.setattr(web_server, "load_config", lambda: {})
|
||||
url = web_server._gateway_fire_endpoint("default", tmp_path)
|
||||
assert url == "http://127.0.0.1:8642/api/cron/fire"
|
||||
|
||||
|
||||
def test_fire_endpoint_config_yaml_port_wins(tmp_path, monkeypatch):
|
||||
"""The profile config.yaml port (read via the CANONICAL load_config, per
|
||||
the config-read guard) wins over the process env API_SERVER_PORT."""
|
||||
monkeypatch.setenv("API_SERVER_PORT", "9999")
|
||||
monkeypatch.delenv("GATEWAY_MULTIPLEX_PROFILES", raising=False)
|
||||
monkeypatch.setattr(
|
||||
web_server,
|
||||
"load_config",
|
||||
lambda: {"platforms": {"api_server": {"extra": {"port": 8700}}}},
|
||||
)
|
||||
url = web_server._gateway_fire_endpoint("default", tmp_path)
|
||||
assert url == "http://127.0.0.1:8700/api/cron/fire"
|
||||
|
||||
|
||||
def test_fire_endpoint_profile_env_port(tmp_path, monkeypatch):
|
||||
"""A non-default profile reads API_SERVER_PORT from its own .env, not the
|
||||
dashboard process env (per-profile-gateway topology)."""
|
||||
monkeypatch.setenv("API_SERVER_PORT", "9999") # dashboard process env
|
||||
monkeypatch.delenv("GATEWAY_MULTIPLEX_PROFILES", raising=False)
|
||||
monkeypatch.setattr(web_server, "load_config", lambda: {})
|
||||
monkeypatch.setattr(web_server, "_cron_default_profile", lambda: "default")
|
||||
(tmp_path / ".env").write_text("API_SERVER_PORT=8701\n", encoding="utf-8")
|
||||
url = web_server._gateway_fire_endpoint("worker_alpha", tmp_path)
|
||||
assert url == "http://127.0.0.1:8701/api/cron/fire"
|
||||
|
||||
|
||||
def test_fire_endpoint_multiplex_profile_prefix(tmp_path, monkeypatch):
|
||||
"""Multiplex mode: a non-default profile routes through the default
|
||||
gateway's port with the /p/<profile>/ prefix mirror."""
|
||||
monkeypatch.delenv("API_SERVER_PORT", raising=False)
|
||||
monkeypatch.setenv("GATEWAY_MULTIPLEX_PROFILES", "1")
|
||||
monkeypatch.setattr(web_server, "load_config", lambda: {})
|
||||
url = web_server._gateway_fire_endpoint("worker_alpha", tmp_path)
|
||||
assert url == "http://127.0.0.1:8642/p/worker_alpha/api/cron/fire"
|
||||
|
||||
|
||||
def test_fire_endpoint_multiplex_reads_port_from_default_listener(tmp_path, monkeypatch):
|
||||
"""Multiplex mode: only the DEFAULT profile's api_server is bound, so a
|
||||
secondary's fire URL must use the default home's port — not the
|
||||
secondary's own config.yaml/.env port, which nothing listens on
|
||||
(PR #84755). Real config files, real load_config()."""
|
||||
default_home = tmp_path / "root"
|
||||
worker_home = default_home / "profiles" / "worker_alpha"
|
||||
default_home.mkdir()
|
||||
worker_home.mkdir(parents=True)
|
||||
(default_home / "config.yaml").write_text(
|
||||
"gateway:\n multiplex_profiles: true\n"
|
||||
"platforms:\n api_server:\n extra:\n port: 8650\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(worker_home / "config.yaml").write_text(
|
||||
"platforms:\n api_server:\n enabled: false\n extra:\n port: 8702\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(worker_home / ".env").write_text("API_SERVER_PORT=8701\n", encoding="utf-8")
|
||||
monkeypatch.setenv("HERMES_HOME", str(default_home))
|
||||
monkeypatch.delenv("API_SERVER_PORT", raising=False)
|
||||
monkeypatch.delenv("GATEWAY_MULTIPLEX_PROFILES", raising=False)
|
||||
monkeypatch.setattr(web_server, "_cron_default_profile", lambda: "default")
|
||||
|
||||
url = web_server._gateway_fire_endpoint("worker_alpha", worker_home)
|
||||
|
||||
assert url == "http://127.0.0.1:8650/p/worker_alpha/api/cron/fire"
|
||||
# The GATEWAY_MULTIPLEX_PROFILES env override is still honored (parity
|
||||
# with gateway/config.py): forcing it off restores per-profile routing.
|
||||
monkeypatch.setenv("GATEWAY_MULTIPLEX_PROFILES", "0")
|
||||
assert web_server._gateway_fire_endpoint("worker_alpha", worker_home) == (
|
||||
"http://127.0.0.1:8702/api/cron/fire"
|
||||
)
|
||||
|
||||
|
||||
# ── OOF-266: intentional-stop drop + Retry-After on transient 503 ─────────
|
||||
|
||||
|
||||
def test_gateway_unreachable_503_carries_retry_after(monkeypatch):
|
||||
"""Transient unreachable window (wake/restart) -> 503 WITH Retry-After so
|
||||
the scheduler spaces retries past the window instead of exhausting its
|
||||
budget inside it (OOF-266)."""
|
||||
|
||||
async def fake_forward(profile, job_id, authorization):
|
||||
return None # unreachable
|
||||
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: {"purpose": "cron_fire"}),
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default")
|
||||
monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward)
|
||||
monkeypatch.setattr(
|
||||
web_server, "_gateway_intentionally_stopped", lambda p: False
|
||||
)
|
||||
|
||||
client, pa, ph = _client(auth_required=False)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer nas-jwt"},
|
||||
json={"job_id": "j4"})
|
||||
assert resp.status_code == 503
|
||||
assert resp.headers.get("Retry-After") == "60"
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
|
||||
|
||||
def test_gateway_intentionally_stopped_drops_with_200(monkeypatch):
|
||||
"""Operator-stopped gateway (durable desired_state == "stopped") -> the
|
||||
fire is DROPPED with 200: retrying can never succeed until a human starts
|
||||
the gateway, so burning the scheduler retry budget only produces
|
||||
502-retry storms at NAS (OOF-266). Jobs re-arm via the Chronos provider's
|
||||
reconcile on the next gateway start."""
|
||||
executed = []
|
||||
|
||||
async def fake_forward(profile, job_id, authorization):
|
||||
return None # unreachable — gateway process is down
|
||||
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: {"purpose": "cron_fire"}),
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default")
|
||||
monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward)
|
||||
monkeypatch.setattr(
|
||||
web_server, "_gateway_intentionally_stopped", lambda p: True
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_fire_cron_job_for_profile",
|
||||
lambda p, j: executed.append((p, j)))
|
||||
|
||||
client, pa, ph = _client(auth_required=False)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer nas-jwt"},
|
||||
json={"job_id": "j5"})
|
||||
assert resp.status_code == 200
|
||||
assert resp.json().get("status") == "gateway_stopped"
|
||||
assert executed == [] # dropped, never locally executed
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
|
||||
|
||||
def test_stopped_check_only_consulted_when_gateway_unreachable(monkeypatch):
|
||||
"""A reachable gateway's response passes through untouched — the
|
||||
intentional-stop check must not run (a stale desired_state file must
|
||||
never shadow a live gateway)."""
|
||||
consulted = []
|
||||
|
||||
async def fake_forward(profile, job_id, authorization):
|
||||
return 202, {"status": "accepted", "job_id": job_id}
|
||||
|
||||
def fake_stopped(profile):
|
||||
consulted.append(profile)
|
||||
return True # would drop if (wrongly) consulted
|
||||
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: {"purpose": "cron_fire"}),
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default")
|
||||
monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward)
|
||||
monkeypatch.setattr(web_server, "_gateway_intentionally_stopped", fake_stopped)
|
||||
|
||||
client, pa, ph = _client(auth_required=False)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer nas-jwt"},
|
||||
json={"job_id": "j6"})
|
||||
assert resp.status_code == 202
|
||||
assert consulted == []
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
|
||||
|
||||
def test_gateway_own_503_also_carries_retry_after(monkeypatch):
|
||||
"""The gateway's OWN 503 (draining / admission failure) is equally
|
||||
transient — the pass-through stamps the same Retry-After hint. Other
|
||||
statuses stay header-free."""
|
||||
|
||||
async def fake_forward(profile, job_id, authorization):
|
||||
return 503, {"error": "gateway draining"}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: {"purpose": "cron_fire"}),
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default")
|
||||
monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward)
|
||||
|
||||
client, pa, ph = _client(auth_required=False)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer nas-jwt"},
|
||||
json={"job_id": "j7"})
|
||||
assert resp.status_code == 503
|
||||
assert resp.headers.get("Retry-After") == "60"
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
|
||||
|
||||
# ── _gateway_intentionally_stopped (durable operator intent) ──────────────
|
||||
|
||||
|
||||
def _stopped_check_home(monkeypatch, tmp_path):
|
||||
"""Point the profile resolver at tmp_path so the check reads our file."""
|
||||
monkeypatch.setattr(
|
||||
web_server, "_cron_profile_home", lambda p: ("default", tmp_path)
|
||||
)
|
||||
|
||||
|
||||
def test_intentionally_stopped_true_on_desired_state_stopped(tmp_path, monkeypatch):
|
||||
_stopped_check_home(monkeypatch, tmp_path)
|
||||
(tmp_path / "gateway_state.json").write_text(
|
||||
'{"gateway_state":"stopped","desired_state":"stopped"}', encoding="utf-8"
|
||||
)
|
||||
assert web_server._gateway_intentionally_stopped("default") is True
|
||||
|
||||
|
||||
def test_intentionally_stopped_false_when_desired_running(tmp_path, monkeypatch):
|
||||
"""Crash loop / transient outage: runtime says stopped but operator intent
|
||||
is running -> stay on the retryable path."""
|
||||
_stopped_check_home(monkeypatch, tmp_path)
|
||||
(tmp_path / "gateway_state.json").write_text(
|
||||
'{"gateway_state":"startup_failed","desired_state":"running"}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert web_server._gateway_intentionally_stopped("default") is False
|
||||
|
||||
|
||||
def test_intentionally_stopped_false_on_legacy_file_without_desired_state(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
"""Legacy gateway_state.json (no desired_state) must NOT count as
|
||||
intentional — the volatile runtime field can read 'stopped' during
|
||||
transient windows. Fail open to the retryable path."""
|
||||
_stopped_check_home(monkeypatch, tmp_path)
|
||||
(tmp_path / "gateway_state.json").write_text(
|
||||
'{"gateway_state":"stopped"}', encoding="utf-8"
|
||||
)
|
||||
assert web_server._gateway_intentionally_stopped("default") is False
|
||||
|
||||
|
||||
def test_intentionally_stopped_false_on_missing_or_bad_file(tmp_path, monkeypatch):
|
||||
_stopped_check_home(monkeypatch, tmp_path)
|
||||
assert web_server._gateway_intentionally_stopped("default") is False
|
||||
(tmp_path / "gateway_state.json").write_text("{not json", encoding="utf-8")
|
||||
assert web_server._gateway_intentionally_stopped("default") is False
|
||||
(tmp_path / "gateway_state.json").write_text('["list"]', encoding="utf-8")
|
||||
assert web_server._gateway_intentionally_stopped("default") is False
|
||||
|
||||
|
||||
def test_intentionally_stopped_false_when_profile_resolution_fails(monkeypatch):
|
||||
"""Exception-safe: any resolution failure (unknown profile, bad name)
|
||||
returns False — never blocks the retryable path."""
|
||||
def boom(profile):
|
||||
raise RuntimeError("no such profile")
|
||||
|
||||
monkeypatch.setattr(web_server, "_cron_profile_home", boom)
|
||||
assert web_server._gateway_intentionally_stopped("ghost") is False
|
||||
|
||||
|
||||
# ── last_fire_error stamp on forward failure (missed-fire visibility) ─────
|
||||
|
||||
|
||||
def test_forward_failure_stamps_last_fire_error(monkeypatch):
|
||||
"""Gateway unreachable -> the job record gets a durable last_fire_error
|
||||
stamp (via note_fire_forward_failure) so the miss is visible in
|
||||
`cronjob list` / the dashboard instead of only in gui.log."""
|
||||
stamped = []
|
||||
|
||||
async def fake_forward(profile, job_id, authorization):
|
||||
return None # unreachable
|
||||
|
||||
def fake_call_cron(profile, func_name, *args, **kwargs):
|
||||
stamped.append((profile, func_name, args))
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: {"purpose": "cron_fire"}),
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default")
|
||||
monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward)
|
||||
monkeypatch.setattr(web_server, "_gateway_intentionally_stopped", lambda p: False)
|
||||
monkeypatch.setattr(web_server, "_call_cron_for_profile", fake_call_cron)
|
||||
|
||||
client, pa, ph = _client(auth_required=False)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer nas-jwt"},
|
||||
json={"job_id": "j8"})
|
||||
assert resp.status_code == 503 # retry contract unchanged
|
||||
assert len(stamped) == 1
|
||||
profile, func_name, args = stamped[0]
|
||||
assert profile == "default"
|
||||
assert func_name == "note_fire_forward_failure"
|
||||
assert args[0] == "j8"
|
||||
assert "api_server" in args[1] # detail names the dead hop
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
|
||||
|
||||
def test_forward_failure_stamp_error_never_breaks_retry_contract(monkeypatch):
|
||||
"""Stamping is best-effort observability: if the jobs store write blows
|
||||
up, the webhook still returns the retryable 503 + Retry-After."""
|
||||
|
||||
async def fake_forward(profile, job_id, authorization):
|
||||
return None
|
||||
|
||||
def boom(profile, func_name, *args, **kwargs):
|
||||
raise RuntimeError("jobs.json locked")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: {"purpose": "cron_fire"}),
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default")
|
||||
monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward)
|
||||
monkeypatch.setattr(web_server, "_gateway_intentionally_stopped", lambda p: False)
|
||||
monkeypatch.setattr(web_server, "_call_cron_for_profile", boom)
|
||||
|
||||
client, pa, ph = _client(auth_required=False)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer nas-jwt"},
|
||||
json={"job_id": "j9"})
|
||||
assert resp.status_code == 503
|
||||
assert resp.headers.get("Retry-After") == "60"
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
|
||||
|
||||
def test_reachable_gateway_does_not_stamp(monkeypatch):
|
||||
"""A successful forward must not touch the job record."""
|
||||
stamped = []
|
||||
|
||||
async def fake_forward(profile, job_id, authorization):
|
||||
return 202, {"status": "accepted", "job_id": job_id}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"plugins.cron_providers.chronos.verify.get_fire_verifier",
|
||||
lambda: (lambda **kw: {"purpose": "cron_fire"}),
|
||||
)
|
||||
monkeypatch.setattr(web_server, "_find_cron_job_profile", lambda jid: "default")
|
||||
monkeypatch.setattr(web_server, "_forward_cron_fire_to_gateway", fake_forward)
|
||||
monkeypatch.setattr(
|
||||
web_server, "_call_cron_for_profile",
|
||||
lambda *a, **k: stamped.append(a),
|
||||
)
|
||||
|
||||
client, pa, ph = _client(auth_required=False)
|
||||
try:
|
||||
resp = client.post("/api/cron/fire",
|
||||
headers={"Authorization": "Bearer nas-jwt"},
|
||||
json={"job_id": "j10"})
|
||||
assert resp.status_code == 202
|
||||
assert stamped == []
|
||||
finally:
|
||||
_restore(pa, ph)
|
||||
client.close()
|
||||
Reference in New Issue
Block a user