Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/cron/test_delivery_queue.py

@@ -0,0 +1,212 @@
+"""Durable at-most-once delivery handoff for restart-safe cron workers."""
+
+from __future__ import annotations
+
+import sqlite3
+from unittest.mock import Mock
+
+import pytest
+
+
+def test_pending_delivery_is_claimed_and_sent_once(tmp_path, monkeypatch):
+    import cron.delivery_queue as queue
+
+    monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db")
+    queue.enqueue("exec-1", {"id": "job-1"}, "brief")
+    send = Mock(return_value=None)
+
+    assert queue.drain(send) == 1
+    assert queue.drain(send) == 0
+    send.assert_called_once_with({"id": "job-1"}, "brief", False)
+    status = queue.get_status("exec-1")
+    assert status["status"] == "delivered"
+    assert status["job_json"] == "{}"
+    assert status["content"] == ""
+
+
+def test_terminal_delivery_retention_is_bounded(tmp_path, monkeypatch):
+    import cron.delivery_queue as queue
+
+    monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db")
+    monkeypatch.setattr(queue, "MAX_TERMINAL_DELIVERIES", 2, raising=False)
+    for index in range(4):
+        execution_id = f"exec-{index}"
+        queue.enqueue(execution_id, {"id": f"job-{index}"}, f"brief-{index}")
+        assert queue.claim_next()["execution_id"] == execution_id
+        assert queue._finish(execution_id, error=None)
+
+    # Pruning may discard verbose outcome rows, but never the durable
+    # idempotency tombstone for an execution that could be replayed later.
+    pruned = queue.get_status("exec-0")
+    assert pruned is not None
+    assert pruned["status"] == "delivered"
+    assert queue.get_status("exec-1")["status"] == "delivered"
+    assert queue.get_status("exec-2")["status"] == "delivered"
+    assert queue.get_status("exec-3")["status"] == "delivered"
+
+    # Pruning may discard verbose outcome rows, but never the durable
+    # idempotency tombstone for an execution that could be replayed later.
+    queue.enqueue("exec-0", {"id": "job-replayed"}, "duplicate brief")
+    send = Mock(return_value=None)
+    assert queue.drain(send) == 0
+    send.assert_not_called()
+
+
+def test_failure_delivery_lane_survives_durable_handoff(tmp_path, monkeypatch):
+    import cron.delivery_queue as queue
+
+    monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db")
+    queue.enqueue(
+        "exec-failure",
+        {"id": "job-failure", "failure_deliver": "local"},
+        "failed",
+        for_failure=True,
+    )
+    send = Mock(return_value=None)
+
+    assert queue.drain(send) == 1
+    send.assert_called_once_with(
+        {"id": "job-failure", "failure_deliver": "local"},
+        "failed",
+        True,
+    )
+
+
+def test_legacy_queue_schema_adds_failure_lane_before_enqueue(tmp_path, monkeypatch):
+    import cron.delivery_queue as queue
+
+    db = tmp_path / "deliveries.db"
+    with sqlite3.connect(db) as conn:
+        conn.execute(
+            """CREATE TABLE deliveries (
+                 execution_id TEXT PRIMARY KEY,
+                 job_json TEXT NOT NULL,
+                 content TEXT NOT NULL,
+                 status TEXT NOT NULL,
+                 owner_process_id TEXT,
+                 owner_pid INTEGER,
+                 owner_started_at INTEGER,
+                 created_at TEXT NOT NULL,
+                 finished_at TEXT,
+                 error TEXT
+               )"""
+        )
+    monkeypatch.setattr(queue, "DELIVERY_DB", db)
+
+    queue.enqueue(
+        "exec-migrated",
+        {"id": "job-migrated"},
+        "failed",
+        for_failure=True,
+    )
+
+    assert queue.get_status("exec-migrated")["for_failure"] == 1
+
+
+def test_wait_timeout_marks_inflight_delivery_unknown_without_retry(
+    tmp_path, monkeypatch
+):
+    import cron.delivery_queue as queue
+
+    monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db")
+    queue.enqueue("exec-inflight", {"id": "job-inflight"}, "result")
+    assert queue.claim_next() is not None
+
+    error = queue.enqueue_and_wait(
+        "exec-inflight", {"id": "job-inflight"}, "result", timeout=0
+    )
+
+    assert error is not None
+    assert "outcome is unknown" in error
+    status = queue.get_status("exec-inflight")
+    assert status is not None
+    assert status["status"] == "unknown"
+    send = Mock(return_value=None)
+    assert queue.drain(send) == 0
+    send.assert_not_called()
+
+
+def test_dead_delivery_owner_becomes_unknown_and_is_not_retried(
+    tmp_path, monkeypatch
+):
+    import cron.delivery_queue as queue
+
+    monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db")
+    queue.enqueue("exec-1", {"id": "job-1"}, "brief")
+    assert queue.claim_next() is not None
+    monkeypatch.setattr(queue, "_PROCESS_ID", "replacement-gateway")
+    monkeypatch.setattr(queue, "_owner_is_live", lambda _pid, _started: False)
+
+    assert queue.recover_abandoned() == 1
+    send = Mock()
+    assert queue.drain(send) == 0
+    send.assert_not_called()
+    assert queue.get_status("exec-1")["status"] == "unknown"
+
+
+def test_delivery_failure_is_terminal_not_retried_and_redacted(
+    tmp_path, monkeypatch
+):
+    import cron.delivery_queue as queue
+
+    monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db")
+    queue.enqueue("exec-1", {"id": "job-1"}, "brief")
+    send = Mock(return_value="request failed: https://example.test/?token=TOKEN123")
+
+    assert queue.drain(send) == 1
+    assert queue.drain(send) == 0
+    assert send.call_count == 1
+    status = queue.get_status("exec-1")
+    assert status is not None
+    assert status["status"] == "failed"
+    assert "TOKEN123" not in status["error"]
+    assert "token=***" in status["error"]
+
+
+def test_wait_timeout_leaves_unclaimed_delivery_queued_for_next_gateway(
+    tmp_path, monkeypatch
+):
+    """A row nobody claimed was never attempted: it is not uncertain, so a
+    gateway outage longer than the worker's wait budget must not lose it."""
+    import cron.delivery_queue as queue
+
+    monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db")
+    job = {"id": "job-3", "deliver": "origin"}
+
+    error = queue.enqueue_and_wait("exec-3", job, "result", timeout=0)
+
+    # Deferred, not failed: the worker must not record delivery_failed.
+    assert error is None
+    status = queue.get_status("exec-3")
+    assert status is not None
+    assert status["status"] == "pending"
+    send = Mock(return_value=None)
+    assert queue.drain(send) == 1
+    send.assert_called_once_with(job, "result", False)
+    assert queue.get_status("exec-3")["status"] == "delivered"
+
+
+def test_same_gateway_recovers_terminalization_failure_without_resending(
+    tmp_path, monkeypatch
+):
+    import cron.delivery_queue as queue
+
+    monkeypatch.setattr(queue, "DELIVERY_DB", tmp_path / "deliveries.db")
+    queue.enqueue("exec-4", {"id": "job-4"}, "result")
+    send = Mock(return_value=None)
+    original_finish = queue._finish
+    monkeypatch.setattr(
+        queue,
+        "_finish",
+        Mock(side_effect=OSError("database temporarily unavailable")),
+    )
+
+    with pytest.raises(OSError, match="temporarily unavailable"):
+        queue.drain(send)
+
+    monkeypatch.setattr(queue, "_finish", original_finish)
+    assert queue.drain(send) == 0
+    send.assert_called_once()
+    status = queue.get_status("exec-4")
+    assert status["status"] == "unknown"
+    assert "not retried" in status["error"]