Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/agent/test_credential_pool_unmatched_rotation_bound.py

@@ -0,0 +1,110 @@
+"""#70401: the unmatched-identity rotation branch in
+``mark_exhausted_and_rotate()`` must be bounded and must not write cooldowns
+onto innocent healthy keys.
+
+With OAuth-token auth (provider ``nous``), the upstream 401's ``api_key_hint``
+never matches any pool entry's ``runtime_api_key`` — the wrapper's runtime key
+rotates. The no-match branch deliberately marks nothing exhausted (marking
+would quarantine an innocent healthy key for the full cooldown TTL) and hands
+back a fresh selection. But because nothing is ever marked, the pool can never
+converge to the "no available entries" state: with the old code the caller
+retried the same dead token forever (~6/sec), starving the event loop so chat
+``/stop`` interrupts were never processed; only killing the gateway ended it.
+
+The fix keeps the don't-mark-innocent-keys semantics (see the breaker/cooldown
+design notes in ``mark_exhausted_and_rotate`` — the pool only trips on
+confirmed-empty state, and no cooldown is invented here) but BOUNDS the
+branch: after one full lap of the available entries with no recovery, the
+rotation returns None so the caller surfaces the error / activates fallback.
+Healthy keys carry no cooldown and are immediately available next turn — this
+does not reintroduce hammering, it stops it.
+"""
+import json
+
+import pytest
+
+
+def _seed_pool(tmp_path, monkeypatch, entries, provider="openrouter"):
+    hermes_home = tmp_path / "hermes"
+    hermes_home.mkdir(parents=True, exist_ok=True)
+    (hermes_home / "auth.json").write_text(
+        json.dumps({"version": 1, "credential_pool": {provider: entries}})
+    )
+    monkeypatch.setenv("HERMES_HOME", str(hermes_home))
+    from agent.credential_pool import load_pool
+
+    return load_pool(provider)
+
+
+def _entry(idx, key):
+    return {
+        "id": f"cred-{idx}",
+        "label": f"key-{idx}",
+        "auth_type": "api_key",
+        "priority": idx,
+        "source": "manual",
+        "access_token": key,
+    }
+
+
+class TestUnmatchedHintRotationIsBounded:
+
+    def test_multi_entry_pool_unmatched_hint_loop_terminates(
+        self, tmp_path, monkeypatch
+    ):
+        """Multi-entry pool: consecutive unmatched-hint rotations must reach
+        None within one lap of the pool instead of ping-ponging forever."""
+        pool = _seed_pool(
+            tmp_path, monkeypatch,
+            [_entry(0, "key-a"), _entry(1, "key-b"), _entry(2, "key-c")],
+        )
+        assert pool.select() is not None
+
+        results = []
+        for _ in range(10):  # caller's retry loop
+            nxt = pool.mark_exhausted_and_rotate(
+                status_code=401,
+                error_context={"reason": "unauthorized"},
+                api_key_hint="oauth-runtime-token-that-matches-nothing",
+            )
+            results.append(nxt)
+            if nxt is None:
+                break
+        else:
+            pytest.fail(
+                "unbounded 401 retry loop: 10 unmatched-hint rotations never "
+                "returned None (#70401)"
+            )
+
+        # Bounded within one lap (3 available entries → at most 3 rotations
+        # before the streak trips).
+        assert len(results) <= 4
+        assert results[-1] is None
+        # The escape must NOT have invented cooldowns for healthy keys.
+        statuses = {e.id: e.last_status for e in pool._entries}
+        assert all(status != "exhausted" for status in statuses.values()), (
+            f"innocent keys were quarantined: {statuses}"
+        )
+
+
+
+    def test_matched_hint_path_unaffected(self, tmp_path, monkeypatch):
+        """Regression guard: the normal matched-hint path still marks the
+        failing entry and rotates to the healthy one."""
+        pool = _seed_pool(
+            tmp_path, monkeypatch,
+            [_entry(0, "key-healthy"), _entry(1, "key-failed")],
+        )
+        assert pool.select().access_token == "key-healthy"
+
+        nxt = pool.mark_exhausted_and_rotate(
+            status_code=401,
+            error_context={"reason": "unauthorized"},
+            api_key_hint="key-failed",
+        )
+
+        statuses = {e.id: e.last_status for e in pool._entries}
+        assert statuses["cred-1"] == "exhausted"
+        assert statuses["cred-0"] != "exhausted"
+        assert nxt is not None
+        assert nxt.access_token == "key-healthy"