Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/tools/test_web_providers_xai.py

@@ -0,0 +1,584 @@
+"""Tests for the xAI Web Search provider (plugins/web/xai/).
+
+Covers:
+- XAIWebSearchProvider.is_available() — cheap probe (env var + auth.json)
+- search() — JSON happy path, annotation fallback, citations fallback, empty results
+- search() error paths — HTTP error, request error, missing creds, mutually-exclusive domain filters,
+  200-OK error envelope
+- Request payload shape — model, tools list, allowed_domains/excluded_domains filters
+- OAuth credential resolution end-to-end through tools.xai_http
+- _is_backend_available("xai") integration with tools.web_tools
+- _get_backend() accepts "xai" as a configured backend
+"""
+from __future__ import annotations
+
+import json
+from unittest.mock import MagicMock, patch
+
+
+def _creds(api_key: str = "xai-test-key", base_url: str = "https://api.x.ai/v1") -> dict:
+    return {"provider": "xai", "api_key": api_key, "base_url": base_url}
+
+
+def _mock_resp(json_data, status_code: int = 200):
+    m = MagicMock()
+    m.status_code = status_code
+    m.json.return_value = json_data
+    m.raise_for_status = MagicMock()
+    return m
+
+
+def _responses_payload(text: str, annotations=None, citations=None) -> dict:
+    """Build a minimal Responses-API reply with one message + output_text block."""
+    chunk: dict = {"type": "output_text", "text": text}
+    if annotations is not None:
+        chunk["annotations"] = annotations
+    payload: dict = {
+        "output": [
+            {
+                "type": "message",
+                "content": [chunk],
+            }
+        ],
+    }
+    if citations is not None:
+        payload["citations"] = citations
+    return payload
+
+
+# ---------------------------------------------------------------------------
+# Provider identity / availability
+# ---------------------------------------------------------------------------
+
+
+class TestXAIProviderIdentity:
+    def test_provider_name(self):
+        from plugins.web.xai.provider import XAIWebSearchProvider
+        assert XAIWebSearchProvider().name == "xai"
+
+
+    def test_display_name(self):
+        from plugins.web.xai.provider import XAIWebSearchProvider
+        assert "Grok" in XAIWebSearchProvider().display_name
+
+
+class TestXAIProviderIsAvailable:
+    """``is_available()`` MUST be cheap — no network, no token refresh, no
+    auth-store lock. It runs on every ``hermes tools`` repaint and at
+    tool-registration time, so any I/O regression here would surface as
+    visible CLI latency.
+    """
+
+    def test_available_via_env_var(self, monkeypatch):
+        monkeypatch.setenv("XAI_API_KEY", "sk-xai-test")
+        from plugins.web.xai.provider import XAIWebSearchProvider
+        assert XAIWebSearchProvider().is_available() is True
+
+
+    def test_unavailable_when_auth_store_corrupted(self, monkeypatch, tmp_path):
+        """A malformed auth.json must not crash availability scans."""
+        monkeypatch.delenv("XAI_API_KEY", raising=False)
+        monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+        (tmp_path / "auth.json").write_text("not json at all }{")
+
+        from plugins.web.xai.provider import XAIWebSearchProvider
+        assert XAIWebSearchProvider().is_available() is False
+
+    def test_is_available_does_not_call_resolver(self, monkeypatch):
+        """Regression guard: ``is_available()`` must NEVER touch the resolver,
+        because the OAuth resolver can trigger a network refresh."""
+        monkeypatch.setenv("XAI_API_KEY", "sk-xai-test")
+        from plugins.web.xai import provider as xai_provider
+
+        with patch.object(
+            xai_provider, "resolve_xai_http_credentials",
+            side_effect=AssertionError("is_available must not call the resolver"),
+        ):
+            assert xai_provider.XAIWebSearchProvider().is_available() is True
+
+
+# ---------------------------------------------------------------------------
+# search() happy + parse paths
+# ---------------------------------------------------------------------------
+
+
+class TestXAIProviderSearchJSONPath:
+    _GROK_JSON = json.dumps({
+        "results": [
+            {"title": "xAI", "url": "https://x.ai", "description": "The company."},
+            {"title": "Grok docs", "url": "https://docs.x.ai", "description": "API reference."},
+            {"title": "Grokipedia", "url": "https://grokipedia.com", "description": "Wiki."},
+        ]
+    })
+
+    def test_happy_path_normalizes_results(self):
+        from plugins.web.xai import provider as xai_provider
+
+        with patch.object(xai_provider, "resolve_xai_http_credentials", return_value=_creds()), \
+             patch.object(xai_provider, "_load_xai_web_config", return_value={}), \
+             patch("httpx.post", return_value=_mock_resp(_responses_payload(self._GROK_JSON))):
+            result = xai_provider.XAIWebSearchProvider().search("what is xai", limit=5)
+
+        assert result["success"] is True
+        web = result["data"]["web"]
+        assert len(web) == 3
+        assert web[0] == {
+            "title": "xAI",
+            "url": "https://x.ai",
+            "description": "The company.",
+            "position": 1,
+        }
+        assert web[2]["position"] == 3
+
+
+    def test_parses_json_with_leading_prose(self):
+        """Reasoning models sometimes narrate before the JSON block; we tolerate it."""
+        from plugins.web.xai import provider as xai_provider
+
+        text = "Here are the results:\n" + self._GROK_JSON
+        with patch.object(xai_provider, "resolve_xai_http_credentials", return_value=_creds()), \
+             patch.object(xai_provider, "_load_xai_web_config", return_value={}), \
+             patch("httpx.post", return_value=_mock_resp(_responses_payload(text))):
+            result = xai_provider.XAIWebSearchProvider().search("q", limit=5)
+
+        assert result["success"] is True
+        assert len(result["data"]["web"]) == 3
+
+    def test_drops_rows_without_url(self):
+        from plugins.web.xai import provider as xai_provider
+
+        bad_json = json.dumps({
+            "results": [
+                {"title": "no url", "description": "skip me"},
+                {"title": "good", "url": "https://ok.com", "description": "keep"},
+            ]
+        })
+        with patch.object(xai_provider, "resolve_xai_http_credentials", return_value=_creds()), \
+             patch.object(xai_provider, "_load_xai_web_config", return_value={}), \
+             patch("httpx.post", return_value=_mock_resp(_responses_payload(bad_json))):
+            result = xai_provider.XAIWebSearchProvider().search("q", limit=5)
+
+        assert result["success"] is True
+        web = result["data"]["web"]
+        assert len(web) == 1
+        assert web[0]["url"] == "https://ok.com"
+        assert web[0]["position"] == 1
+
+
+class TestXAIProviderSearchFallbacks:
+    def test_falls_back_to_annotations_when_json_missing(self):
+        """If Grok ignores the JSON instruction, derive results from url_citation annotations."""
+        from plugins.web.xai import provider as xai_provider
+
+        body = "xAI is an AI company founded in 2023. They make Grok."
+        annotations = [
+            {
+                "type": "url_citation",
+                "url": "https://x.ai/about",
+                "title": "1",
+                "start_index": 4,
+                "end_index": 9,
+            },
+            {
+                "type": "url_citation",
+                "url": "https://docs.x.ai",
+                "title": "2",
+                "start_index": 47,
+                "end_index": 52,
+            },
+        ]
+        with patch.object(xai_provider, "resolve_xai_http_credentials", return_value=_creds()), \
+             patch.object(xai_provider, "_load_xai_web_config", return_value={}), \
+             patch("httpx.post", return_value=_mock_resp(_responses_payload(body, annotations=annotations))):
+            result = xai_provider.XAIWebSearchProvider().search("xai", limit=5)
+
+        assert result["success"] is True
+        urls = [r["url"] for r in result["data"]["web"]]
+        assert urls == ["https://x.ai/about", "https://docs.x.ai"]
+        assert result["data"]["web"][0]["position"] == 1
+        assert result["data"]["web"][1]["position"] == 2
+
+
+    def test_empty_response_returns_empty_success(self):
+        from plugins.web.xai import provider as xai_provider
+
+        payload = _responses_payload("", citations=[])
+        with patch.object(xai_provider, "resolve_xai_http_credentials", return_value=_creds()), \
+             patch.object(xai_provider, "_load_xai_web_config", return_value={}), \
+             patch("httpx.post", return_value=_mock_resp(payload)):
+            result = xai_provider.XAIWebSearchProvider().search("q", limit=5)
+
+        assert result["success"] is True
+        assert result["data"]["web"] == []
+
+
+# ---------------------------------------------------------------------------
+# Request payload shape
+# ---------------------------------------------------------------------------
+
+
+class TestXAIProviderRequestShape:
+    def test_posts_to_responses_endpoint_with_bearer_token(self):
+        from plugins.web.xai import provider as xai_provider
+
+        captured: dict = {}
+
+        def fake_post(url, **kwargs):
+            captured["url"] = url
+            captured["headers"] = kwargs.get("headers", {})
+            captured["json"] = kwargs.get("json", {})
+            return _mock_resp(_responses_payload(json.dumps({"results": []})))
+
+        with patch.object(xai_provider, "resolve_xai_http_credentials", return_value=_creds("secret-key")), \
+             patch.object(xai_provider, "_load_xai_web_config", return_value={}), \
+             patch("httpx.post", side_effect=fake_post):
+            xai_provider.XAIWebSearchProvider().search("q", limit=5)
+
+        assert captured["url"] == "https://api.x.ai/v1/responses"
+        assert captured["headers"].get("Authorization") == "Bearer secret-key"
+        body = captured["json"]
+        # Assert against the module constant rather than the literal value,
+        # so renaming DEFAULT_MODEL (when xAI deprecates grok-4.3) doesn't
+        # turn this into a change-detector failure.
+        assert body["model"] == xai_provider.DEFAULT_MODEL
+        assert body["tools"] == [{"type": "web_search"}]
+        assert body["input"][0]["role"] == "user"
+        # No-inline-citations is opt-in via `include` per xAI Responses docs.
+        assert "no_inline_citations" in body.get("include", [])
+
+
+    def test_allowed_domains_capped_at_five(self):
+        """xAI caps domain filters at 5; we trim silently to avoid 400s."""
+        from plugins.web.xai import provider as xai_provider
+
+        captured: dict = {}
+
+        def fake_post(url, **kwargs):
+            captured["json"] = kwargs.get("json", {})
+            return _mock_resp(_responses_payload(json.dumps({"results": []})))
+
+        cfg = {"allowed_domains": [f"d{i}.com" for i in range(10)]}
+        with patch.object(xai_provider, "resolve_xai_http_credentials", return_value=_creds()), \
+             patch.object(xai_provider, "_load_xai_web_config", return_value=cfg), \
+             patch("httpx.post", side_effect=fake_post):
+            xai_provider.XAIWebSearchProvider().search("q", limit=5)
+
+        domains = captured["json"]["tools"][0]["filters"]["allowed_domains"]
+        assert len(domains) == 5
+
+
+# ---------------------------------------------------------------------------
+# Error paths
+# ---------------------------------------------------------------------------
+
+
+class TestXAIProviderSearchErrors:
+    def test_missing_creds_returns_failure(self):
+        from plugins.web.xai import provider as xai_provider
+
+        with patch.object(xai_provider, "resolve_xai_http_credentials", return_value=_creds("")):
+            result = xai_provider.XAIWebSearchProvider().search("q", limit=5)
+
+        assert result["success"] is False
+        assert "xAI" in result["error"]
+
+    def test_mutually_exclusive_domain_filters_rejected_locally(self):
+        from plugins.web.xai import provider as xai_provider
+
+        cfg = {"allowed_domains": ["a.com"], "excluded_domains": ["b.com"]}
+        with patch.object(xai_provider, "resolve_xai_http_credentials", return_value=_creds()), \
+             patch.object(xai_provider, "_load_xai_web_config", return_value=cfg), \
+             patch("httpx.post") as posted:
+            result = xai_provider.XAIWebSearchProvider().search("q", limit=5)
+
+        assert result["success"] is False
+        assert "cannot both be set" in result["error"]
+        posted.assert_not_called()
+
+
+    def test_401_on_oauth_path_triggers_force_refresh_and_retry(self):
+        """OAuth credentials → 401 must force-refresh and retry once.
+
+        Closes the two-gap scenario the resolver's JWT-exp shortcut doesn't
+        cover: opaque (non-JWT) tokens and mid-window revocation. We expect
+        ``httpx.post`` to be called twice with two different Bearer tokens.
+        """
+        import httpx
+        from plugins.web.xai import provider as xai_provider
+
+        bad = MagicMock()
+        bad.status_code = 401
+        bad.text = "Unauthorized"
+        unauthorized = httpx.HTTPStatusError("401", request=MagicMock(), response=bad)
+
+        calls = {"posts": [], "refresh_count": 0}
+
+        def fake_post(url, **kwargs):
+            calls["posts"].append(kwargs.get("headers", {}).get("Authorization"))
+            if len(calls["posts"]) == 1:
+                raise unauthorized
+            return _mock_resp(_responses_payload(json.dumps({"results": []})))
+
+        def fake_resolve(*, force_refresh=False, api_key_hint=None):
+            if force_refresh:
+                calls["refresh_count"] += 1
+                assert api_key_hint == "stale-token"
+                return {
+                    "provider": "xai-oauth",
+                    "api_key": "fresh-after-refresh",
+                    "base_url": "https://api.x.ai/v1",
+                }
+            return {
+                "provider": "xai-oauth",
+                "api_key": "stale-token",
+                "base_url": "https://api.x.ai/v1",
+            }
+
+        with patch.object(xai_provider, "resolve_xai_http_credentials", side_effect=fake_resolve), \
+             patch.object(xai_provider, "_load_xai_web_config", return_value={}), \
+             patch("httpx.post", side_effect=fake_post):
+            result = xai_provider.XAIWebSearchProvider().search("q", limit=5)
+
+        assert result["success"] is True
+        assert calls["refresh_count"] == 1
+        assert calls["posts"] == ["Bearer stale-token", "Bearer fresh-after-refresh"]
+
+    def test_401_on_env_var_path_does_not_retry(self):
+        """Env-var (XAI_API_KEY) creds can't be refreshed — must not retry."""
+        import httpx
+        from plugins.web.xai import provider as xai_provider
+
+        bad = MagicMock()
+        bad.status_code = 401
+        bad.text = "Unauthorized"
+        unauthorized = httpx.HTTPStatusError("401", request=MagicMock(), response=bad)
+
+        calls = {"posts": 0, "refreshed": False}
+
+        def fake_post(url, **kwargs):
+            calls["posts"] += 1
+            raise unauthorized
+
+        def fake_resolve(*, force_refresh=False, api_key_hint=None):
+            if force_refresh:
+                calls["refreshed"] = True
+            # provider=="xai" signals env-var path; retry must be skipped.
+            return {"provider": "xai", "api_key": "«redacted:sk-…»", "base_url": "https://api.x.ai/v1"}
+
+        with patch.object(xai_provider, "resolve_xai_http_credentials", side_effect=fake_resolve), \
+             patch.object(xai_provider, "_load_xai_web_config", return_value={}), \
+             patch("httpx.post", side_effect=fake_post):
+            result = xai_provider.XAIWebSearchProvider().search("q", limit=5)
+
+        assert result["success"] is False
+        assert "401" in result["error"]
+        assert calls["posts"] == 1
+        assert calls["refreshed"] is False
+
+
+    def test_http_200_with_error_envelope_surfaces_failure(self):
+        """xAI sometimes returns 200 with ``{"error": {...}}`` (model
+        overloaded, refusal, etc.). Must be surfaced as a failure rather
+        than silently masked as success-with-empty-results.
+        """
+        from plugins.web.xai import provider as xai_provider
+
+        payload = {"error": {"message": "model overloaded", "type": "server_error"}}
+        with patch.object(xai_provider, "resolve_xai_http_credentials", return_value=_creds()), \
+             patch.object(xai_provider, "_load_xai_web_config", return_value={}), \
+             patch("httpx.post", return_value=_mock_resp(payload)):
+            result = xai_provider.XAIWebSearchProvider().search("q", limit=5)
+
+        assert result["success"] is False
+        assert "model overloaded" in result["error"]
+
+
+# ---------------------------------------------------------------------------
+# Integration with tools/web_tools.py backend wiring
+# ---------------------------------------------------------------------------
+
+
+class TestXAIBackendWiring:
+    def test_is_backend_available_true_via_env_var(self, monkeypatch):
+        from tools import web_tools
+
+        monkeypatch.setenv("XAI_API_KEY", "sk-xai-test")
+        assert web_tools._is_backend_available("xai") is True
+
+
+    def test_is_backend_available_does_not_call_resolver(self, monkeypatch):
+        """Regression guard — `_is_backend_available` runs on every web_search
+        dispatch and every `hermes tools` repaint. It must not invoke the
+        OAuth resolver (which can trigger a network refresh)."""
+        from tools import web_tools
+
+        monkeypatch.setenv("XAI_API_KEY", "sk-xai-test")
+        with patch(
+            "tools.xai_http.resolve_xai_http_credentials",
+            side_effect=AssertionError("must not call resolver"),
+        ):
+            assert web_tools._is_backend_available("xai") is True
+
+    def test_configured_backend_xai_accepted(self, monkeypatch):
+        from tools import web_tools
+        monkeypatch.setattr(web_tools, "_load_web_config", lambda: {"backend": "xai"})
+        assert web_tools._get_backend() == "xai"
+
+    def test_xai_not_in_legacy_backend_candidate_chain(self, monkeypatch):
+        """The hardcoded ``backend_candidates`` tuple in ``_get_backend()``
+        does not include xAI — by design, since the no-config legacy
+        chain is for users who set env vars but never ran ``hermes tools``,
+        and we don't want a stray ``XAI_API_KEY`` (perhaps set for chat
+        inference) to silently re-route web_search through Grok.
+
+        Note: this does NOT prevent the registry's single-provider
+        shortcut (``agent.web_search_registry._resolve``) from selecting
+        xAI when it's the only available web provider. That path is the
+        normal "pick the one provider the user actually configured"
+        behavior shared by every other backend.
+        """
+        from tools import web_tools
+
+        monkeypatch.setattr(web_tools, "_load_web_config", lambda: {})
+        for key in (
+            "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "PARALLEL_API_KEY",
+            "EXA_API_KEY", "SEARXNG_URL", "BRAVE_SEARCH_API_KEY", "KEENABLE_API_KEY",
+        ):
+            monkeypatch.delenv(key, raising=False)
+        monkeypatch.setenv("XAI_API_KEY", "xai-test-key")
+        monkeypatch.setattr(web_tools, "_is_tool_gateway_ready", lambda: False)
+        monkeypatch.setattr(web_tools, "_ddgs_package_importable", lambda: False)
+        assert web_tools._get_backend() != "xai"
+
+
+# ---------------------------------------------------------------------------
+# OAuth credential resolution (end-to-end through tools.xai_http)
+# ---------------------------------------------------------------------------
+
+
+class TestXAIProviderOAuthPath:
+    """Verifies the provider works when credentials come from the OAuth
+    credential pool (``hermes auth`` sign-in) rather than an env-var key.
+    The full ``tools.xai_http.resolve_xai_http_credentials`` chain is exercised
+    against a temporary auth store.
+    """
+
+    def test_search_uses_oauth_bearer_token_and_base_url(self, monkeypatch, tmp_path):
+        from plugins.web.xai import provider as xai_provider
+
+        # Force the env-var fallback to fail so resolution must go via OAuth.
+        monkeypatch.delenv("XAI_API_KEY", raising=False)
+        monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+        monkeypatch.setenv("HERMES_XAI_BASE_URL", "https://proxy.x.ai/v1/")
+        (tmp_path / "auth.json").write_text(json.dumps({
+            "version": 1,
+            "active_provider": "xai-oauth",
+            "providers": {
+                "xai-oauth": {
+                    "tokens": {
+                        "access_token": "ya29.fake-oauth-access-token",
+                        "refresh_token": "fake-oauth-refresh-token",
+                    },
+                },
+            },
+        }))
+
+        captured: dict = {}
+
+        def fake_post(url, **kwargs):
+            captured["url"] = url
+            captured["headers"] = kwargs.get("headers", {})
+            return _mock_resp(_responses_payload(json.dumps({"results": []})))
+
+        with patch.object(xai_provider, "_load_xai_web_config", return_value={}), \
+             patch("httpx.post", side_effect=fake_post):
+            result = xai_provider.XAIWebSearchProvider().search("q", limit=3)
+
+        assert result["success"] is True
+        assert captured["url"] == "https://proxy.x.ai/v1/responses"
+        assert captured["headers"].get("Authorization") == "Bearer ya29.fake-oauth-access-token"
+
+    def test_pool_only_direct_refresh_updates_main_runtime(self, monkeypatch, tmp_path):
+        """A direct 401 refresh must rotate the exact manual pool row.
+
+        xAI refresh tokens are one-time credentials. Persisting the refreshed
+        pair only to ``providers.xai-oauth`` leaves the manual row stale and
+        breaks the next main-runtime load.
+        """
+        from hermes_cli.runtime_provider import resolve_runtime_provider
+        from tools.xai_http import resolve_xai_http_credentials
+
+        monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+        monkeypatch.delenv("XAI_API_KEY", raising=False)
+        monkeypatch.delenv("XAI_OAUTH_ACCESS_TOKEN", raising=False)
+        (tmp_path / "config.yaml").write_text(
+            "credential_pool_strategies:\n  xai-oauth: round_robin\n"
+        )
+        auth_path = tmp_path / "auth.json"
+        auth_path.write_text(json.dumps({
+            "version": 1,
+            "active_provider": "xai-oauth",
+            "providers": {},
+            "credential_pool": {
+                "xai-oauth": [{
+                    "id": "manual-xai",
+                    "label": "pool-only",
+                    "auth_type": "oauth",
+                    "priority": 0,
+                    "source": "manual:device_code",
+                    "access_token": "rejected-access",
+                    "refresh_token": "one-time-refresh",
+                    "base_url": "https://api.x.ai/v1",
+                }, {
+                    "id": "backup-xai",
+                    "label": "backup",
+                    "auth_type": "oauth",
+                    "priority": 1,
+                    "source": "manual:device_code",
+                    "access_token": "backup-access",
+                    "refresh_token": "backup-refresh",
+                    "base_url": "https://api.x.ai/v1",
+                }],
+            },
+        }))
+
+        refresh_calls = []
+
+        def fake_refresh(access_token, refresh_token, **_kwargs):
+            refresh_calls.append((access_token, refresh_token))
+            return {
+                "access_token": "fresh-access",
+                "refresh_token": "rotated-refresh",
+                "last_refresh": "2026-07-12T12:00:00Z",
+            }
+
+        monkeypatch.setattr(
+            "hermes_cli.auth.refresh_xai_oauth_pure",
+            fake_refresh,
+        )
+
+        initial = resolve_xai_http_credentials()
+        assert initial["api_key"] == "rejected-access"
+
+        refreshed = resolve_xai_http_credentials(
+            force_refresh=True,
+            api_key_hint=initial["api_key"],
+        )
+        assert refreshed["api_key"] == "fresh-access"
+        assert refresh_calls == [("rejected-access", "one-time-refresh")]
+
+        stored = json.loads(auth_path.read_text())
+        entries = {
+            item["id"]: item
+            for item in stored["credential_pool"]["xai-oauth"]
+        }
+        entry = entries["manual-xai"]
+        assert entry["access_token"] == "fresh-access"
+        assert entry["refresh_token"] == "rotated-refresh"
+        assert entries["backup-xai"]["access_token"] == "backup-access"
+        assert entries["backup-xai"]["refresh_token"] == "backup-refresh"
+        assert "xai-oauth" not in stored.get("providers", {})
+
+        runtime = resolve_runtime_provider(requested="xai-oauth")
+        assert runtime["api_key"] == "backup-access"
+        assert refresh_calls == [("rejected-access", "one-time-refresh")]