131 lines
4.8 KiB
Python
131 lines
4.8 KiB
Python
"""Concurrent Nous 401 recovery must not stampede the shared OAuth grant.
|
|
|
|
Sep 2 2026 incident: ~120 subagent processes shared one Nous OAuth pool entry
|
|
whose access token hit its hourly expiry. Every process got a 401, every
|
|
process force-refreshed, and each rotation invalidated the token a sibling had
|
|
just adopted — 81 refreshes and ~540 401s in eight minutes. Processes that
|
|
lost the auth-store flock race had their only entry benched ("matched no nous
|
|
entry ... pool size 0") and surfaced the 401 to the user as "out of funds".
|
|
|
|
Two invariants pinned here:
|
|
|
|
1. ``resolve_nous_runtime_credentials(force_refresh=True, stale_access_token=X)``
|
|
does NOT POST a refresh when the store already holds a usable token that
|
|
is not X — a peer already rotated; adopt it.
|
|
2. A lock-timeout during a pool-level Nous refresh leaves the entry
|
|
untouched instead of marking it exhausted.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
|
|
import hermes_cli.auth as auth_mod
|
|
from agent.credential_pool import CredentialPool, PooledCredential
|
|
|
|
from tests.hermes_cli.test_auth_nous_provider import _invoke_jwt, _setup_nous_auth
|
|
|
|
|
|
def test_forced_refresh_adopts_peer_rotation_instead_of_reposting(tmp_path, monkeypatch):
|
|
hermes_home = tmp_path / "hermes"
|
|
peer_token = _invoke_jwt(seconds=3600)
|
|
failed_token = _invoke_jwt(seconds=3000) # what THIS process still holds
|
|
_setup_nous_auth(
|
|
hermes_home,
|
|
access_token=peer_token,
|
|
refresh_token="rt-after-peer-rotation",
|
|
scope=auth_mod.DEFAULT_NOUS_SCOPE,
|
|
expires_at=auth_mod.datetime.fromtimestamp(
|
|
auth_mod.time.time() + 3600, tz=auth_mod.timezone.utc
|
|
).isoformat(),
|
|
expires_in=3600,
|
|
)
|
|
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
|
|
|
posts = []
|
|
|
|
def _fake_refresh_access_token(*, client, portal_base_url, client_id, refresh_token):
|
|
posts.append(refresh_token)
|
|
return {
|
|
"access_token": _invoke_jwt(seconds=7200),
|
|
"refresh_token": "rt-should-not-happen",
|
|
"expires_in": 7200,
|
|
"token_type": "Bearer",
|
|
"scope": auth_mod.DEFAULT_NOUS_SCOPE,
|
|
}
|
|
|
|
monkeypatch.setattr(auth_mod, "_refresh_access_token", _fake_refresh_access_token)
|
|
|
|
creds = auth_mod.resolve_nous_runtime_credentials(
|
|
force_refresh=True, stale_access_token=failed_token
|
|
)
|
|
|
|
assert posts == [], "peer already rotated — must not consume the refresh token again"
|
|
assert creds["api_key"] == peer_token
|
|
|
|
# Same call WITHOUT the hint keeps the pre-existing force semantics.
|
|
auth_mod.resolve_nous_runtime_credentials(force_refresh=True)
|
|
assert posts == ["rt-after-peer-rotation"]
|
|
|
|
|
|
def test_lock_timeout_during_nous_refresh_does_not_bench_entry(monkeypatch, caplog):
|
|
entry = PooledCredential(
|
|
id="267aed",
|
|
provider="nous",
|
|
auth_type="oauth",
|
|
access_token=_invoke_jwt(seconds=3600),
|
|
refresh_token="rt",
|
|
label="test@nous",
|
|
source="device_code",
|
|
priority=0,
|
|
)
|
|
pool = CredentialPool.__new__(CredentialPool)
|
|
pool._lock = __import__("threading").RLock()
|
|
pool._entries = [entry]
|
|
pool._active_leases = {}
|
|
pool._current_id = None
|
|
pool._max_concurrent = 2
|
|
pool._unmatched_rotation_streak = 0
|
|
pool.provider = "nous"
|
|
|
|
monkeypatch.setattr(pool, "_sync_nous_entry_from_auth_store", lambda e: e)
|
|
monkeypatch.setattr(pool, "_persist", lambda *a, **k: None)
|
|
|
|
def _busy(*a, **k):
|
|
raise TimeoutError("Timed out waiting for auth store lock")
|
|
|
|
monkeypatch.setattr(auth_mod, "resolve_nous_runtime_credentials", _busy)
|
|
|
|
result = pool._refresh_entry_impl(entry, force=True)
|
|
|
|
assert result is entry
|
|
assert pool._entries[0].last_status is None, "lock contention is not a credential failure"
|
|
|
|
|
|
def test_agent_401_refresh_passes_failed_bearer_as_stale_hint(monkeypatch):
|
|
"""Every 401-recovery caller must hand the auth store the bearer that
|
|
failed — without it ``_already_rotated_by_peer`` can never fire and each
|
|
subagent rotates the shared grant again (the "no crash, N refreshes"
|
|
variant of the Sep 2 stampede).
|
|
"""
|
|
from run_agent import AIAgent
|
|
|
|
agent = AIAgent.__new__(AIAgent)
|
|
agent.provider = "nous"
|
|
agent.api_mode = "chat_completions"
|
|
agent.api_key = "jwt-that-just-401d"
|
|
agent.base_url = "https://inference-api.nousresearch.com/v1"
|
|
agent._client_kwargs = {}
|
|
monkeypatch.setattr(agent, "_replace_primary_openai_client", lambda **k: True)
|
|
|
|
seen = {}
|
|
|
|
def _fake_resolve(**kwargs):
|
|
seen.update(kwargs)
|
|
return {"api_key": "fresh", "base_url": agent.base_url}
|
|
|
|
monkeypatch.setattr(auth_mod, "resolve_nous_runtime_credentials", _fake_resolve)
|
|
|
|
assert agent._try_refresh_nous_client_credentials(force=True) is True
|
|
assert seen["force_refresh"] is True
|
|
assert seen["stale_access_token"] == "jwt-that-just-401d"
|