"""#70401: the unmatched-identity rotation branch in ``mark_exhausted_and_rotate()`` must be bounded and must not write cooldowns onto innocent healthy keys. With OAuth-token auth (provider ``nous``), the upstream 401's ``api_key_hint`` never matches any pool entry's ``runtime_api_key`` — the wrapper's runtime key rotates. The no-match branch deliberately marks nothing exhausted (marking would quarantine an innocent healthy key for the full cooldown TTL) and hands back a fresh selection. But because nothing is ever marked, the pool can never converge to the "no available entries" state: with the old code the caller retried the same dead token forever (~6/sec), starving the event loop so chat ``/stop`` interrupts were never processed; only killing the gateway ended it. The fix keeps the don't-mark-innocent-keys semantics (see the breaker/cooldown design notes in ``mark_exhausted_and_rotate`` — the pool only trips on confirmed-empty state, and no cooldown is invented here) but BOUNDS the branch: after one full lap of the available entries with no recovery, the rotation returns None so the caller surfaces the error / activates fallback. Healthy keys carry no cooldown and are immediately available next turn — this does not reintroduce hammering, it stops it. """ import json import pytest def _seed_pool(tmp_path, monkeypatch, entries, provider="openrouter"): hermes_home = tmp_path / "hermes" hermes_home.mkdir(parents=True, exist_ok=True) (hermes_home / "auth.json").write_text( json.dumps({"version": 1, "credential_pool": {provider: entries}}) ) monkeypatch.setenv("HERMES_HOME", str(hermes_home)) from agent.credential_pool import load_pool return load_pool(provider) def _entry(idx, key): return { "id": f"cred-{idx}", "label": f"key-{idx}", "auth_type": "api_key", "priority": idx, "source": "manual", "access_token": key, } class TestUnmatchedHintRotationIsBounded: def test_multi_entry_pool_unmatched_hint_loop_terminates( self, tmp_path, monkeypatch ): """Multi-entry pool: consecutive unmatched-hint rotations must reach None within one lap of the pool instead of ping-ponging forever.""" pool = _seed_pool( tmp_path, monkeypatch, [_entry(0, "key-a"), _entry(1, "key-b"), _entry(2, "key-c")], ) assert pool.select() is not None results = [] for _ in range(10): # caller's retry loop nxt = pool.mark_exhausted_and_rotate( status_code=401, error_context={"reason": "unauthorized"}, api_key_hint="oauth-runtime-token-that-matches-nothing", ) results.append(nxt) if nxt is None: break else: pytest.fail( "unbounded 401 retry loop: 10 unmatched-hint rotations never " "returned None (#70401)" ) # Bounded within one lap (3 available entries → at most 3 rotations # before the streak trips). assert len(results) <= 4 assert results[-1] is None # The escape must NOT have invented cooldowns for healthy keys. statuses = {e.id: e.last_status for e in pool._entries} assert all(status != "exhausted" for status in statuses.values()), ( f"innocent keys were quarantined: {statuses}" ) def test_matched_hint_path_unaffected(self, tmp_path, monkeypatch): """Regression guard: the normal matched-hint path still marks the failing entry and rotates to the healthy one.""" pool = _seed_pool( tmp_path, monkeypatch, [_entry(0, "key-healthy"), _entry(1, "key-failed")], ) assert pool.select().access_token == "key-healthy" nxt = pool.mark_exhausted_and_rotate( status_code=401, error_context={"reason": "unauthorized"}, api_key_hint="key-failed", ) statuses = {e.id: e.last_status for e in pool._entries} assert statuses["cred-1"] == "exhausted" assert statuses["cred-0"] != "exhausted" assert nxt is not None assert nxt.access_token == "key-healthy"