"""Durable at-most-once delivery handoff for restart-safe cron workers.""" from __future__ import annotations import sqlite3 from unittest.mock import Mock import pytest def test_pending_delivery_is_claimed_and_sent_once(tmp_path, monkeypatch): import cron.delivery_queue as queue monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") queue.enqueue("exec-1", {"id": "job-1"}, "brief") send = Mock(return_value=None) assert queue.drain(send) == 1 assert queue.drain(send) == 0 send.assert_called_once_with({"id": "job-1"}, "brief", False) status = queue.get_status("exec-1") assert status["status"] == "delivered" assert status["job_json"] == "{}" assert status["content"] == "" def test_terminal_delivery_retention_is_bounded(tmp_path, monkeypatch): import cron.delivery_queue as queue monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") monkeypatch.setattr(queue, "MAX_TERMINAL_DELIVERIES", 2, raising=False) for index in range(4): execution_id = f"exec-{index}" queue.enqueue(execution_id, {"id": f"job-{index}"}, f"brief-{index}") assert queue.claim_next()["execution_id"] == execution_id assert queue._finish(execution_id, error=None) # Pruning may discard verbose outcome rows, but never the durable # idempotency tombstone for an execution that could be replayed later. pruned = queue.get_status("exec-0") assert pruned is not None assert pruned["status"] == "delivered" assert queue.get_status("exec-1")["status"] == "delivered" assert queue.get_status("exec-2")["status"] == "delivered" assert queue.get_status("exec-3")["status"] == "delivered" # Pruning may discard verbose outcome rows, but never the durable # idempotency tombstone for an execution that could be replayed later. queue.enqueue("exec-0", {"id": "job-replayed"}, "duplicate brief") send = Mock(return_value=None) assert queue.drain(send) == 0 send.assert_not_called() def test_failure_delivery_lane_survives_durable_handoff(tmp_path, monkeypatch): import cron.delivery_queue as queue monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") queue.enqueue( "exec-failure", {"id": "job-failure", "failure_deliver": "local"}, "failed", for_failure=True, ) send = Mock(return_value=None) assert queue.drain(send) == 1 send.assert_called_once_with( {"id": "job-failure", "failure_deliver": "local"}, "failed", True, ) def test_legacy_queue_schema_adds_failure_lane_before_enqueue(tmp_path, monkeypatch): import cron.delivery_queue as queue db = tmp_path / "deliveries.db" with sqlite3.connect(db) as conn: conn.execute( """CREATE TABLE deliveries ( execution_id TEXT PRIMARY KEY, job_json TEXT NOT NULL, content TEXT NOT NULL, status TEXT NOT NULL, owner_process_id TEXT, owner_pid INTEGER, owner_started_at INTEGER, created_at TEXT NOT NULL, finished_at TEXT, error TEXT )""" ) monkeypatch.setattr(queue, "DELIVERY_DB", db) queue.enqueue( "exec-migrated", {"id": "job-migrated"}, "failed", for_failure=True, ) assert queue.get_status("exec-migrated")["for_failure"] == 1 def test_wait_timeout_marks_inflight_delivery_unknown_without_retry( tmp_path, monkeypatch ): import cron.delivery_queue as queue monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") queue.enqueue("exec-inflight", {"id": "job-inflight"}, "result") assert queue.claim_next() is not None error = queue.enqueue_and_wait( "exec-inflight", {"id": "job-inflight"}, "result", timeout=0 ) assert error is not None assert "outcome is unknown" in error status = queue.get_status("exec-inflight") assert status is not None assert status["status"] == "unknown" send = Mock(return_value=None) assert queue.drain(send) == 0 send.assert_not_called() def test_dead_delivery_owner_becomes_unknown_and_is_not_retried( tmp_path, monkeypatch ): import cron.delivery_queue as queue monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") queue.enqueue("exec-1", {"id": "job-1"}, "brief") assert queue.claim_next() is not None monkeypatch.setattr(queue, "_PROCESS_ID", "replacement-gateway") monkeypatch.setattr(queue, "_owner_is_live", lambda _pid, _started: False) assert queue.recover_abandoned() == 1 send = Mock() assert queue.drain(send) == 0 send.assert_not_called() assert queue.get_status("exec-1")["status"] == "unknown" def test_delivery_failure_is_terminal_not_retried_and_redacted( tmp_path, monkeypatch ): import cron.delivery_queue as queue monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") queue.enqueue("exec-1", {"id": "job-1"}, "brief") send = Mock(return_value="request failed: https://example.test/?token=TOKEN123") assert queue.drain(send) == 1 assert queue.drain(send) == 0 assert send.call_count == 1 status = queue.get_status("exec-1") assert status is not None assert status["status"] == "failed" assert "TOKEN123" not in status["error"] assert "token=***" in status["error"] def test_wait_timeout_leaves_unclaimed_delivery_queued_for_next_gateway( tmp_path, monkeypatch ): """A row nobody claimed was never attempted: it is not uncertain, so a gateway outage longer than the worker's wait budget must not lose it.""" import cron.delivery_queue as queue monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") job = {"id": "job-3", "deliver": "origin"} error = queue.enqueue_and_wait("exec-3", job, "result", timeout=0) # Deferred, not failed: the worker must not record delivery_failed. assert error is None status = queue.get_status("exec-3") assert status is not None assert status["status"] == "pending" send = Mock(return_value=None) assert queue.drain(send) == 1 send.assert_called_once_with(job, "result", False) assert queue.get_status("exec-3")["status"] == "delivered" def test_same_gateway_recovers_terminalization_failure_without_resending( tmp_path, monkeypatch ): import cron.delivery_queue as queue monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db") queue.enqueue("exec-4", {"id": "job-4"}, "result") send = Mock(return_value=None) original_finish = queue._finish monkeypatch.setattr( queue, "_finish", Mock(side_effect=OSError("database temporarily unavailable")), ) with pytest.raises(OSError, match="temporarily unavailable"): queue.drain(send) monkeypatch.setattr(queue, "_finish", original_finish) assert queue.drain(send) == 0 send.assert_called_once() status = queue.get_status("exec-4") assert status["status"] == "unknown" assert "not retried" in status["error"]