Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/agent/test_credential_pool_nous_refresh_stampede.py

@@ -0,0 +1,130 @@
+"""Concurrent Nous 401 recovery must not stampede the shared OAuth grant.
+
+Sep 2 2026 incident: ~120 subagent processes shared one Nous OAuth pool entry
+whose access token hit its hourly expiry. Every process got a 401, every
+process force-refreshed, and each rotation invalidated the token a sibling had
+just adopted — 81 refreshes and ~540 401s in eight minutes. Processes that
+lost the auth-store flock race had their only entry benched ("matched no nous
+entry ... pool size 0") and surfaced the 401 to the user as "out of funds".
+
+Two invariants pinned here:
+
+1. ``resolve_nous_runtime_credentials(force_refresh=True, stale_access_token=X)``
+   does NOT POST a refresh when the store already holds a usable token that
+   is not X — a peer already rotated; adopt it.
+2. A lock-timeout during a pool-level Nous refresh leaves the entry
+   untouched instead of marking it exhausted.
+"""
+
+import json
+import logging
+
+import hermes_cli.auth as auth_mod
+from agent.credential_pool import CredentialPool, PooledCredential
+
+from tests.hermes_cli.test_auth_nous_provider import _invoke_jwt, _setup_nous_auth
+
+
+def test_forced_refresh_adopts_peer_rotation_instead_of_reposting(tmp_path, monkeypatch):
+    hermes_home = tmp_path / "hermes"
+    peer_token = _invoke_jwt(seconds=3600)
+    failed_token = _invoke_jwt(seconds=3000)  # what THIS process still holds
+    _setup_nous_auth(
+        hermes_home,
+        access_token=peer_token,
+        refresh_token="rt-after-peer-rotation",
+        scope=auth_mod.DEFAULT_NOUS_SCOPE,
+        expires_at=auth_mod.datetime.fromtimestamp(
+            auth_mod.time.time() + 3600, tz=auth_mod.timezone.utc
+        ).isoformat(),
+        expires_in=3600,
+    )
+    monkeypatch.setenv("HERMES_HOME", str(hermes_home))
+
+    posts = []
+
+    def _fake_refresh_access_token(*, client, portal_base_url, client_id, refresh_token):
+        posts.append(refresh_token)
+        return {
+            "access_token": _invoke_jwt(seconds=7200),
+            "refresh_token": "rt-should-not-happen",
+            "expires_in": 7200,
+            "token_type": "Bearer",
+            "scope": auth_mod.DEFAULT_NOUS_SCOPE,
+        }
+
+    monkeypatch.setattr(auth_mod, "_refresh_access_token", _fake_refresh_access_token)
+
+    creds = auth_mod.resolve_nous_runtime_credentials(
+        force_refresh=True, stale_access_token=failed_token
+    )
+
+    assert posts == [], "peer already rotated — must not consume the refresh token again"
+    assert creds["api_key"] == peer_token
+
+    # Same call WITHOUT the hint keeps the pre-existing force semantics.
+    auth_mod.resolve_nous_runtime_credentials(force_refresh=True)
+    assert posts == ["rt-after-peer-rotation"]
+
+
+def test_lock_timeout_during_nous_refresh_does_not_bench_entry(monkeypatch, caplog):
+    entry = PooledCredential(
+        id="267aed",
+        provider="nous",
+        auth_type="oauth",
+        access_token=_invoke_jwt(seconds=3600),
+        refresh_token="rt",
+        label="test@nous",
+        source="device_code",
+        priority=0,
+    )
+    pool = CredentialPool.__new__(CredentialPool)
+    pool._lock = __import__("threading").RLock()
+    pool._entries = [entry]
+    pool._active_leases = {}
+    pool._current_id = None
+    pool._max_concurrent = 2
+    pool._unmatched_rotation_streak = 0
+    pool.provider = "nous"
+
+    monkeypatch.setattr(pool, "_sync_nous_entry_from_auth_store", lambda e: e)
+    monkeypatch.setattr(pool, "_persist", lambda *a, **k: None)
+
+    def _busy(*a, **k):
+        raise TimeoutError("Timed out waiting for auth store lock")
+
+    monkeypatch.setattr(auth_mod, "resolve_nous_runtime_credentials", _busy)
+
+    result = pool._refresh_entry_impl(entry, force=True)
+
+    assert result is entry
+    assert pool._entries[0].last_status is None, "lock contention is not a credential failure"
+
+
+def test_agent_401_refresh_passes_failed_bearer_as_stale_hint(monkeypatch):
+    """Every 401-recovery caller must hand the auth store the bearer that
+    failed — without it ``_already_rotated_by_peer`` can never fire and each
+    subagent rotates the shared grant again (the "no crash, N refreshes"
+    variant of the Sep 2 stampede).
+    """
+    from run_agent import AIAgent
+
+    agent = AIAgent.__new__(AIAgent)
+    agent.provider = "nous"
+    agent.api_mode = "chat_completions"
+    agent.api_key = "jwt-that-just-401d"
+    agent.base_url = "https://inference-api.nousresearch.com/v1"
+    agent._client_kwargs = {}
+    monkeypatch.setattr(agent, "_replace_primary_openai_client", lambda **k: True)
+
+    seen = {}
+
+    def _fake_resolve(**kwargs):
+        seen.update(kwargs)
+        return {"api_key": "fresh", "base_url": agent.base_url}
+
+    monkeypatch.setattr(auth_mod, "resolve_nous_runtime_credentials", _fake_resolve)
+
+    assert agent._try_refresh_nous_client_credentials(force=True) is True
+    assert seen["force_refresh"] is True
+    assert seen["stale_access_token"] == "jwt-that-just-401d"