Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

plugins/web/__init__.py

@@ -0,0 +1,7 @@
+# Bundled web search providers — plugins/web/.
+#
+# Each subdirectory follows the image_gen plugin layout:
+#   plugins/web/<name>/{plugin.yaml, __init__.py, provider.py}
+#
+# They auto-load via kind: backend and register via
+# ctx.register_web_search_provider() into agent.web_search_registry.

plugins/web/brave_free/__init__.py

@@ -0,0 +1,14 @@
+"""Brave Search (free tier) plugin — bundled, auto-loaded.
+
+Mirrors the ``plugins/image_gen/openai/`` layout: ``provider.py`` holds the
+provider class, ``__init__.py::register(ctx)`` registers an instance.
+"""
+
+from __future__ import annotations
+
+from plugins.web.brave_free.provider import BraveFreeWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the Brave-free provider with the plugin context."""
+    ctx.register_web_search_provider(BraveFreeWebSearchProvider())

plugins/web/brave_free/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-brave-free
+version: 1.0.0
+description: "Brave Search (free tier) — web search via Brave's Data-for-Search API. Requires BRAVE_SEARCH_API_KEY (free signup at https://brave.com/search/api/, 2k queries/month)."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - brave-free

plugins/web/brave_free/provider.py

@@ -0,0 +1,141 @@
+"""Brave Search (free tier) — plugin form.
+
+Subclasses :class:`agent.web_search_provider.WebSearchProvider` (the
+plugin-facing ABC). The legacy in-tree module
+``tools.web_providers.brave_free`` was removed in the same commit that
+moved this code under ``plugins/``; this file is now the canonical
+implementation.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "brave-free"     # explicit per-capability
+      backend: "brave-free"            # shared fallback
+
+Auth env var::
+
+    BRAVE_SEARCH_API_KEY=...    # https://brave.com/search/api/ (free tier)
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Dict
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+_BRAVE_ENDPOINT = "https://api.search.brave.com/res/v1/web/search"
+
+
+class BraveFreeWebSearchProvider(WebSearchProvider):
+    """Search-only Brave provider using the free-tier Data-for-Search API.
+
+    Free tier is 2,000 queries/month (1 qps). No content-extraction capability —
+    users pair this with Firecrawl/Tavily/Exa for ``web_extract``.
+    """
+
+    @property
+    def name(self) -> str:
+        # Hyphen form preserved for backward compat with the existing
+        # ``web.search_backend: "brave-free"`` config keys users have set.
+        return "brave-free"
+
+    @property
+    def display_name(self) -> str:
+        return "Brave Search (Free)"
+
+    def is_available(self) -> bool:
+        """Return True when ``BRAVE_SEARCH_API_KEY`` is set to a non-empty value."""
+        from agent.web_search_provider import get_provider_env
+
+        return bool(get_provider_env("BRAVE_SEARCH_API_KEY"))
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return False
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a search against the Brave Search API.
+
+        Returns ``{"success": True, "data": {"web": [{"title", "url", "description", "position"}]}}``
+        on success, or ``{"success": False, "error": str}`` on failure.
+        """
+        import httpx
+
+        from agent.web_search_provider import get_provider_env
+
+        api_key = get_provider_env("BRAVE_SEARCH_API_KEY")
+        if not api_key:
+            return {"success": False, "error": "BRAVE_SEARCH_API_KEY is not set"}
+
+        # Brave's `count` is capped at 20.
+        count = max(1, min(int(limit), 20))
+
+        try:
+            resp = httpx.get(
+                _BRAVE_ENDPOINT,
+                params={"q": query, "count": count},
+                headers={
+                    "X-Subscription-Token": api_key,
+                    "Accept": "application/json",
+                },
+                timeout=15,
+            )
+            resp.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            logger.warning("Brave Search HTTP error: %s", exc)
+            return {
+                "success": False,
+                "error": f"Brave Search returned HTTP {exc.response.status_code}",
+            }
+        except httpx.RequestError as exc:
+            logger.warning("Brave Search request error: %s", exc)
+            return {"success": False, "error": f"Could not reach Brave Search: {exc}"}
+
+        try:
+            data = resp.json()
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Brave Search response parse error: %s", exc)
+            return {"success": False, "error": "Could not parse Brave Search response as JSON"}
+
+        raw_results = (data.get("web") or {}).get("results", []) or []
+        truncated = raw_results[:limit]
+
+        web_results = [
+            {
+                "title": str(r.get("title", "")),
+                "url": str(r.get("url", "")),
+                "description": str(r.get("description", "")),
+                "position": i + 1,
+            }
+            for i, r in enumerate(truncated)
+        ]
+
+        logger.info(
+            "Brave Search '%s': %d results (from %d raw, limit %d)",
+            query,
+            len(web_results),
+            len(raw_results),
+            limit,
+        )
+
+        return {"success": True, "data": {"web": web_results}}
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "Brave Search (Free)",
+            "badge": "free",
+            "tag": "Free-tier API key — 2k queries/mo, search only.",
+            "env_vars": [
+                {
+                    "key": "BRAVE_SEARCH_API_KEY",
+                    "prompt": "Brave Search API key (free tier)",
+                    "url": "https://brave.com/search/api/",
+                },
+            ],
+        }

plugins/web/ddgs/__init__.py

@@ -0,0 +1,15 @@
+"""DuckDuckGo search plugin — bundled, auto-loaded.
+
+Backed by the community ``ddgs`` Python package which scrapes DDG's HTML
+results page. No API key required, but the package itself must be installed
+(it's an optional dep — gated via :meth:`is_available`).
+"""
+
+from __future__ import annotations
+
+from plugins.web.ddgs.provider import DDGSWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the DDGS provider with the plugin context."""
+    ctx.register_web_search_provider(DDGSWebSearchProvider())

plugins/web/ddgs/_search_worker.py

@@ -0,0 +1,113 @@
+"""DDGS search child-process entrypoint (#68096).
+
+Invoked as ``python plugins/web/ddgs/_search_worker.py`` (script path from the
+parent provider). Reads one JSON request from stdin, writes one JSON envelope
+to stdout, then exits.
+
+Request::
+    {"query": str, "safe_limit": int}
+
+Envelope::
+    {"ok": true, "results": [...]}
+    {"ok": false, "error": str}
+
+Optional test hooks (only when ``HERMES_DDGS_ALLOW_TEST_HOOKS=1``)::
+    {"query": ..., "safe_limit": ..., "test_hook": "sleep"|"gil"|"success"|"error"|"empty"}
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+
+
+def _hold_gil(secs: int) -> None:
+    """Block in a foreign call that keeps the GIL (ctypes.PyDLL).
+
+    Mirrors native ``primp`` holding the interpreter lock. ``PyDLL`` (unlike
+    ``CDLL``/``WinDLL``) does not release the GIL around the call.
+    """
+    import ctypes
+
+    if sys.platform == "win32":
+        lib = ctypes.PyDLL("kernel32")
+        lib.Sleep.argtypes = [ctypes.c_uint]
+        lib.Sleep(int(secs * 1000))
+        return
+
+    lib = ctypes.PyDLL(None)
+    try:
+        sleep = lib.sleep
+    except AttributeError:  # pragma: no cover — macOS libSystem fallback
+        sleep = ctypes.PyDLL("/usr/lib/libSystem.B.dylib").sleep
+    sleep.argtypes = [ctypes.c_uint]
+    sleep(int(secs))
+
+
+def _run_test_hook(hook: str) -> dict:
+    if hook == "sleep":
+        time.sleep(30)
+        return {"ok": False, "error": "sleep hook returned unexpectedly"}
+    if hook == "gil":
+        _hold_gil(30)
+        return {"ok": False, "error": "gil hook returned unexpectedly"}
+    if hook == "success":
+        return {
+            "ok": True,
+            "results": [
+                {
+                    "title": "Hit",
+                    "url": "https://example.com",
+                    "description": "body",
+                    "position": 1,
+                }
+            ],
+        }
+    if hook == "empty":
+        return {"ok": True, "results": []}
+    if hook == "error":
+        return {"ok": False, "error": "RuntimeError: boom"}
+    return {"ok": False, "error": f"unknown test_hook: {hook!r}"}
+
+
+def _write_envelope(envelope: dict) -> None:
+    json.dump(envelope, sys.stdout)
+    sys.stdout.flush()
+
+
+def main() -> int:
+    try:
+        request = json.load(sys.stdin)
+    except Exception as exc:  # noqa: BLE001
+        _write_envelope({"ok": False, "error": f"invalid request: {exc}"})
+        return 2
+
+    hook = request.get("test_hook")
+    if hook:
+        if os.environ.get("HERMES_DDGS_ALLOW_TEST_HOOKS") != "1":
+            _write_envelope(
+                {"ok": False, "error": "test_hook refused (hooks not enabled)"}
+            )
+            return 3
+        envelope = _run_test_hook(str(hook))
+        _write_envelope(envelope)
+        return 0 if envelope.get("ok") else 1
+
+    query = str(request.get("query") or "")
+    safe_limit = max(1, int(request.get("safe_limit") or 1))
+    try:
+        # Import inside main so script startup stays light / patchable.
+        from plugins.web.ddgs.provider import _run_ddgs_search
+
+        results = _run_ddgs_search(query, safe_limit)
+        _write_envelope({"ok": True, "results": results})
+        return 0
+    except Exception as exc:  # noqa: BLE001
+        _write_envelope({"ok": False, "error": f"{type(exc).__name__}: {exc}"})
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

plugins/web/ddgs/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-ddgs
+version: 1.0.0
+description: "DuckDuckGo web search via the ddgs Python package — no API key required. Install with `pip install ddgs`."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - ddgs

plugins/web/ddgs/provider.py

@@ -0,0 +1,362 @@
+"""DuckDuckGo search — plugin form (via the ``ddgs`` package).
+
+Subclasses the plugin-facing :class:`agent.web_search_provider.WebSearchProvider`.
+The legacy in-tree module ``tools.web_providers.ddgs`` was removed in the
+same commit that moved this code under ``plugins/``; this file is now the
+canonical implementation.
+
+The ``ddgs`` package is an optional dependency. ``is_available()`` reflects
+whether the package is importable; the plugin still registers either way so
+``hermes tools`` can prompt the user to install it.
+
+Isolation note (#68096): ``ddgs``/``primp`` can block inside native code while
+holding the Python GIL. A ``ThreadPoolExecutor`` + ``future.result(timeout=…)``
+cap (see #52118) cannot fire in that state — the waiter never reacquires the
+GIL — so the whole Hermes process freezes through Ctrl+C/SIGTERM. Each search
+therefore runs in a disposable child process the parent can terminate/kill.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures as cf
+import json
+import logging
+import os
+import subprocess
+import sys
+import time
+from typing import Any, Dict, Optional
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+# Overall wall-clock cap for a single ddgs search. The DDGS constructor's
+# ``timeout`` only bounds individual HTTP requests; ddgs's multi-engine retry
+# loop has no overall cap, so a slow/rate-limited DuckDuckGo response can hang
+# the (single, shared) agent loop indefinitely (#36776). Enforce a hard cap
+# here by killing a disposable worker process (#68096).
+_SEARCH_TIMEOUT_SECS = 30
+
+# How often the parent polls stdout / interrupt flag while waiting.
+_POLL_INTERVAL_SECS = 0.1
+
+# After terminate(), wait this long before escalating to kill().
+_TERMINATE_GRACE_SECS = 1.0
+
+
+class _SearchInterrupted(Exception):
+    """Raised when tools.interrupt.is_interrupted() trips during a search wait."""
+
+
+def _run_ddgs_search(query: str, safe_limit: int) -> list[dict[str, Any]]:
+    """Run the blocking ddgs query and return normalized hits.
+
+    Module-level (not a closure) so the child worker can import it and so
+    tests can patch it for in-process unit tests. ``DDGS(timeout=…)`` bounds
+    each individual HTTP request; the overall wall-clock cap is enforced by
+    the parent via process timeout (#68096).
+    """
+    from ddgs import DDGS  # type: ignore
+
+    results: list[dict[str, Any]] = []
+    with DDGS(timeout=10) as client:
+        for i, hit in enumerate(client.text(query, max_results=safe_limit)):
+            if i >= safe_limit:
+                break
+            url = str(hit.get("href") or hit.get("url") or "")
+            results.append(
+                {
+                    "title": str(hit.get("title", "")),
+                    "url": url,
+                    "description": str(hit.get("body", "")),
+                    "position": i + 1,
+                }
+            )
+    return results
+
+
+# Optional test-only hook name forwarded to the child (see _search_worker.py).
+# Production search() never sets this.
+_test_hook: Optional[str] = None
+
+# Last worker Popen started by ``_run_ddgs_search_bounded`` (test reap checks).
+_last_worker_proc: Optional[subprocess.Popen] = None
+
+
+def _plugins_path_entry() -> str:
+    """Return the ``sys.path`` entry that makes ``import plugins`` work.
+
+    Prefer the live ``plugins`` package location over counting ``dirname``s from
+    this file — that stays correct for source checkouts and site-packages.
+    """
+    try:
+        import plugins as plugins_pkg
+
+        pkg_file = getattr(plugins_pkg, "__file__", None)
+        if pkg_file:
+            return os.path.dirname(os.path.dirname(os.path.abspath(pkg_file)))
+    except Exception:  # noqa: BLE001 — fall through to path-walk fallback
+        pass
+    return os.path.dirname(
+        os.path.dirname(
+            os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        )
+    )
+
+
+def _terminate_and_reap(
+    proc: Optional[subprocess.Popen],
+    *,
+    grace: float = _TERMINATE_GRACE_SECS,
+) -> None:
+    """Terminate a worker, escalate to kill, and wait so no orphan remains.
+
+    Does not close the parent's pipe ends — the caller must finish any
+    ``communicate()``/reader first. Closing stdout while another thread is
+    blocked in ``read()`` deadlocks on some platforms.
+    """
+    if proc is None:
+        return
+
+    def _wait_until_dead(seconds: float) -> bool:
+        deadline = time.monotonic() + seconds
+        while time.monotonic() < deadline:
+            if proc.poll() is not None:
+                return True
+            time.sleep(0.05)
+        return proc.poll() is not None
+
+    try:
+        if proc.poll() is None:
+            proc.terminate()
+            _wait_until_dead(grace)
+        if proc.poll() is None:
+            proc.kill()
+            if not _wait_until_dead(grace):
+                logger.warning("DDGS worker pid=%s did not exit after kill", proc.pid)
+    except Exception as exc:  # noqa: BLE001 — best-effort cleanup
+        logger.debug("DDGS worker reap error: %s", exc)
+
+
+def _run_ddgs_search_bounded(query: str, safe_limit: int) -> list[dict[str, Any]]:
+    """Run ``_run_ddgs_search`` in a disposable process with a hard deadline.
+
+    The parent never joins the child while it may be inside native code holding
+    *its* GIL — it only polls a communicator thread and, on timeout/interrupt,
+    terminates the child OS process. Raises ``TimeoutError``,
+    ``_SearchInterrupted``, or ``RuntimeError``.
+    """
+    # Imported lazily so plugin import stays light for ``hermes tools`` probes.
+    from tools.interrupt import is_interrupted
+
+    global _last_worker_proc
+
+    request: dict[str, Any] = {"query": query, "safe_limit": safe_limit}
+    if _test_hook:
+        request["test_hook"] = _test_hook
+
+    from tools.environments.local import _sanitize_subprocess_env
+
+    env = _sanitize_subprocess_env(dict(os.environ))
+    if _test_hook:
+        env["HERMES_DDGS_ALLOW_TEST_HOOKS"] = "1"
+
+    # Running the worker as a script puts ``plugins/web/ddgs/`` on ``sys.path[0]``,
+    # which breaks ``import plugins...``. Prepend the path entry that makes the
+    # live ``plugins`` package importable (source tree or site-packages).
+    child_pythonpath = env.get("PYTHONPATH", "")
+    path_entry = _plugins_path_entry()
+    if path_entry and path_entry not in child_pythonpath.split(os.pathsep):
+        env["PYTHONPATH"] = (
+            path_entry + os.pathsep + child_pythonpath if child_pythonpath else path_entry
+        )
+
+    worker_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_search_worker.py")
+    # Platform-only spawn knobs — stdin/stdout/stderr must stay as explicit
+    # keyword args on the Popen call so scripts/check_subprocess_stdin.py can
+    # see them (TUI gateway inherits stdin; #14036).
+    extra_kwargs: dict[str, Any] = {}
+    if sys.platform == "win32":
+        # New process group so terminate/kill reach the worker cleanly on Windows.
+        extra_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
+    else:
+        # Own session so a hung primp/libcurl grandchild can be reaped with the worker.
+        extra_kwargs["start_new_session"] = True
+
+    proc = subprocess.Popen(
+        [sys.executable, worker_path],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        # DEVNULL avoids the classic deadlock where a chatty child fills the
+        # stderr pipe buffer while the parent only drains stdout.
+        stderr=subprocess.DEVNULL,
+        env=env,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        **extra_kwargs,
+    )
+    _last_worker_proc = proc
+
+    # ``communicate`` runs in a side thread so the parent can poll interrupt /
+    # deadline without blocking. Killing the child unblocks communicate.
+    pool = cf.ThreadPoolExecutor(max_workers=1)
+    fut = pool.submit(proc.communicate, json.dumps(request))
+    timed_out = False
+    interrupted = False
+    raw = ""
+    try:
+        deadline = time.monotonic() + _SEARCH_TIMEOUT_SECS
+        while True:
+            if is_interrupted():
+                interrupted = True
+                break
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                timed_out = True
+                break
+            try:
+                out, _err = fut.result(timeout=min(_POLL_INTERVAL_SECS, remaining))
+                raw = out or ""
+                break
+            except cf.TimeoutError:
+                continue
+    finally:
+        _terminate_and_reap(proc)
+        # After kill, communicate should return promptly; don't block forever.
+        if not fut.done():
+            try:
+                out, _err = fut.result(timeout=_TERMINATE_GRACE_SECS)
+                if not raw:
+                    raw = out or ""
+            except Exception:  # noqa: BLE001
+                pass
+        pool.shutdown(wait=False, cancel_futures=True)
+
+    if interrupted:
+        raise _SearchInterrupted("DuckDuckGo search interrupted")
+    if timed_out:
+        raise TimeoutError(
+            f"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s"
+        )
+
+    raw = raw.strip()
+    if not raw:
+        raise RuntimeError(
+            f"DDGS worker exited without a result (code={proc.poll()})"
+        )
+
+    try:
+        envelope = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise RuntimeError(
+            f"DDGS worker returned invalid JSON: {raw[:200]!r}"
+        ) from exc
+
+    if not isinstance(envelope, dict):
+        raise RuntimeError(f"DDGS worker returned an invalid envelope: {envelope!r}")
+    if envelope.get("ok"):
+        results = envelope.get("results") or []
+        if not isinstance(results, list):
+            raise RuntimeError("DDGS worker returned non-list results")
+        return results
+    raise RuntimeError(str(envelope.get("error") or "DDGS worker failed"))
+
+
+class DDGSWebSearchProvider(WebSearchProvider):
+    """DuckDuckGo HTML-scrape search provider.
+
+    No API key needed. Rate limits are enforced server-side by DuckDuckGo;
+    the provider surfaces ``DuckDuckGoSearchException`` and other ddgs errors
+    as ``{"success": False, "error": ...}`` rather than raising.
+    """
+
+    @property
+    def name(self) -> str:
+        return "ddgs"
+
+    @property
+    def display_name(self) -> str:
+        return "DuckDuckGo (ddgs)"
+
+    def is_available(self) -> bool:
+        """Return True when the ``ddgs`` package is importable.
+
+        Probes the import once; cheap because Python caches the import. Must
+        NOT perform network I/O — runs at tool-registration time and on every
+        ``hermes tools`` paint.
+        """
+        try:
+            import ddgs  # noqa: F401
+
+            return True
+        except ImportError:
+            return False
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return False
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a DuckDuckGo search and return normalized results.
+
+        The synchronous ``ddgs`` call runs in a disposable child process with
+        a hard wall-clock timeout (``_SEARCH_TIMEOUT_SECS``) so a hung native
+        ``primp`` call cannot freeze the Hermes process (#36776, #68096).
+        """
+        try:
+            import ddgs  # type: ignore  # noqa: F401 — availability probe
+        except ImportError:
+            return {
+                "success": False,
+                "error": "ddgs package is not installed — run `pip install ddgs`",
+            }
+
+        # DDGS().text yields at most `max_results` items; we cap defensively
+        # in case the package ignores the hint.
+        safe_limit = max(1, int(limit))
+
+        try:
+            web_results = _run_ddgs_search_bounded(query, safe_limit)
+        except TimeoutError:
+            logger.warning(
+                "DDGS search timed out after %ds for query: %r",
+                _SEARCH_TIMEOUT_SECS,
+                query,
+            )
+            return {
+                "success": False,
+                "error": (
+                    f"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s — "
+                    "DuckDuckGo may be rate-limiting or slow. Try again later "
+                    "or switch to a different search provider."
+                ),
+            }
+        except _SearchInterrupted:
+            logger.info("DDGS search interrupted for query: %r", query)
+            return {
+                "success": False,
+                "error": "DuckDuckGo search interrupted",
+            }
+        except Exception as exc:  # noqa: BLE001 — ddgs raises its own exceptions
+            logger.warning("DDGS search error: %s", exc)
+            return {"success": False, "error": f"DuckDuckGo search failed: {exc}"}
+
+        logger.info(
+            "DDGS search '%s': %d results (limit %d)", query, len(web_results), limit
+        )
+        return {"success": True, "data": {"web": web_results}}
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "DuckDuckGo (ddgs)",
+            "badge": "free · no key · search only",
+            "tag": "Search via the ddgs Python package — no API key (pair with any extract provider)",
+            "env_vars": [],
+            # Trigger `_run_post_setup("ddgs")` after the user picks this row
+            # so the ddgs Python package gets pip-installed on first selection.
+            "post_setup": "ddgs",
+        }

plugins/web/exa/__init__.py

@@ -0,0 +1,15 @@
+"""Exa web search + extract plugin — bundled, auto-loaded.
+
+Backed by the official Exa SDK (``exa-py``). Both search and extract are
+sync; the dispatcher in :mod:`tools.web_tools` handles the wrap when the
+caller is async.
+"""
+
+from __future__ import annotations
+
+from plugins.web.exa.provider import ExaWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the Exa provider with the plugin context."""
+    ctx.register_web_search_provider(ExaWebSearchProvider())

plugins/web/exa/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-exa
+version: 1.0.0
+description: "Exa web search and content extraction. Requires EXA_API_KEY — sign up at https://exa.ai."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - exa

plugins/web/exa/provider.py

@@ -0,0 +1,268 @@
+"""Exa web search + content extraction — plugin form.
+
+Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Uses the
+official Exa SDK (``exa-py``) which is lazy-loaded via
+:func:`tools.lazy_deps.ensure` so that cold-start CLI users don't pay the
+SDK import cost when Exa isn't configured.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "exa"      # explicit per-capability
+      extract_backend: "exa"     # explicit per-capability
+      backend: "exa"             # shared fallback for both
+
+Env var::
+
+    EXA_API_KEY=...    # https://exa.ai (paid tier; free trial available)
+
+The previous in-tree implementation lived at
+``tools.web_tools._exa_search`` / ``_exa_extract``; this file is the
+canonical replacement. Behavior is bit-for-bit identical aside from the
+ABC method-name change.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Dict, List
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+# Module-level note: the canonical ``_exa_client`` cache slot lives on
+# :mod:`tools.web_tools` so tests that do ``tools.web_tools._exa_client =
+# None`` between cases see fresh state. The plugin reads/writes through
+# that public module (see :func:`_get_exa_client`).
+
+
+def _get_exa_client() -> Any:
+    """Lazy-import and cache an Exa SDK client.
+
+    Cache lives on :mod:`tools.web_tools` (as ``_exa_client``) so unit
+    tests that reset that name between cases keep working. Raises
+    ``ValueError`` when ``EXA_API_KEY`` is unset.
+    """
+    import tools.web_tools as _wt
+
+    cached = getattr(_wt, "_exa_client", None)
+    if cached is not None:
+        return cached
+
+    from agent.web_search_provider import get_provider_env
+
+    api_key = get_provider_env("EXA_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "EXA_API_KEY environment variable not set. "
+            "Get your API key at https://exa.ai"
+        )
+
+    try:
+        from tools.lazy_deps import ensure as _lazy_ensure
+
+        _lazy_ensure("search.exa", prompt=False)
+    except ImportError:
+        pass
+    except Exception as exc:  # noqa: BLE001 — lazy_deps surfaces install hints
+        raise ImportError(str(exc))
+
+    from exa_py import Exa  # noqa: WPS433 — deliberately lazy
+
+    client = Exa(api_key=api_key)
+    client.headers["x-exa-integration"] = "hermes-agent"
+    _wt._exa_client = client
+    return client
+
+
+def _reset_client_for_tests() -> None:
+    """Drop the cached Exa client so tests can re-instantiate cleanly."""
+    import tools.web_tools as _wt
+
+    _wt._exa_client = None
+
+
+class ExaWebSearchProvider(WebSearchProvider):
+    """Exa search + extract provider.
+
+    Both methods are sync — Exa's SDK is sync-only. The web_extract_tool
+    dispatcher wraps sync extracts via ``asyncio.to_thread`` when it
+    needs to keep the event loop responsive.
+    """
+
+    @property
+    def name(self) -> str:
+        return "exa"
+
+    @property
+    def display_name(self) -> str:
+        return "Exa"
+
+    def is_available(self) -> bool:
+        """Return True when ``EXA_API_KEY`` is set to a non-empty value.
+
+        Deliberately does NOT consider the keyless free tier — that would
+        let the legacy preference walk route keyed users of lower-priority
+        backends onto Exa's anonymous tier. Keyless availability is a
+        separate, last-resort signal (:meth:`is_keyless_available`).
+        """
+        from agent.web_search_provider import get_provider_env
+
+        return bool(get_provider_env("EXA_API_KEY"))
+
+    def is_keyless_available(self) -> bool:
+        """Exa serves anonymous free-tier calls via its public MCP endpoint.
+
+        False when the user forced ``web.provider_tier.exa: paid`` — an
+        explicit paid selection must never silently resolve keyless.
+        """
+        from plugins.web.keyless_mcp import keyless_enabled, provider_tier
+
+        return keyless_enabled() and provider_tier("exa") != "paid"
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return True
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute an Exa search.
+
+        Returns ``{"success": True, "data": {"web": [{...}, ...]}}`` on
+        success, ``{"success": False, "error": str}`` on failure (incl.
+        missing API key and SDK install errors).
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return {"success": False, "error": "Interrupted"}
+
+            from agent.web_search_provider import get_provider_env
+
+            from plugins.web.keyless_mcp import search_with_failover, use_keyless
+
+            if use_keyless("exa", get_provider_env("EXA_API_KEY")):
+                # Keyless free tier — public MCP endpoint, no SDK needed.
+                logger.info(
+                    "Exa keyless search: '%s' (limit=%d)", query, limit
+                )
+                return search_with_failover("exa", query, limit)
+
+            logger.info("Exa search: '%s' (limit=%d)", query, limit)
+            response = _get_exa_client().search(
+                query,
+                num_results=limit,
+                contents={"highlights": True},
+            )
+
+            web_results = []
+            for i, result in enumerate(response.results or []):
+                highlights = result.highlights or []
+                web_results.append(
+                    {
+                        "url": result.url or "",
+                        "title": result.title or "",
+                        "description": " ".join(highlights) if highlights else "",
+                        "position": i + 1,
+                    }
+                )
+
+            return {"success": True, "data": {"web": web_results}}
+        except ValueError as exc:
+            # Raised by _get_exa_client when EXA_API_KEY missing
+            return {"success": False, "error": str(exc)}
+        except ImportError as exc:
+            return {"success": False, "error": f"Exa SDK not installed: {exc}"}
+        except Exception as exc:  # noqa: BLE001 — surface as failure
+            logger.warning("Exa search error: %s", exc)
+            return {"success": False, "error": f"Exa search failed: {exc}"}
+
+    def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
+        """Extract content from one or more URLs via Exa.
+
+        Returns a list of result dicts shaped for the legacy LLM
+        post-processing pipeline. On per-URL or whole-batch failure,
+        results carry an ``error`` field rather than raising.
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return [
+                    {"url": u, "error": "Interrupted", "title": ""} for u in urls
+                ]
+
+            from agent.web_search_provider import get_provider_env
+
+            from plugins.web.keyless_mcp import extract_with_failover, use_keyless
+
+            if use_keyless("exa", get_provider_env("EXA_API_KEY")):
+                # Keyless free tier — public MCP endpoint, no SDK needed.
+                logger.info("Exa keyless extract: %d URL(s)", len(urls))
+                return extract_with_failover("exa", list(urls))
+
+            logger.info("Exa extract: %d URL(s)", len(urls))
+            response = _get_exa_client().get_contents(urls, text=True)
+
+            results: List[Dict[str, Any]] = []
+            for result in response.results or []:
+                content = result.text or ""
+                url = result.url or ""
+                title = result.title or ""
+                results.append(
+                    {
+                        "url": url,
+                        "title": title,
+                        "content": content,
+                        "raw_content": content,
+                        "metadata": {"sourceURL": url, "title": title},
+                    }
+                )
+            return results
+        except ValueError as exc:
+            return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls]
+        except ImportError as exc:
+            return [
+                {"url": u, "title": "", "content": "", "error": f"Exa SDK not installed: {exc}"}
+                for u in urls
+            ]
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Exa extract error: %s", exc)
+            return [
+                {"url": u, "title": "", "content": "", "error": f"Exa extract failed: {exc}"}
+                for u in urls
+            ]
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "Exa · Free (keyless)",
+            "badge": "free · no key",
+            "tag": (
+                "Semantic + neural web search with content extraction on "
+                "Exa's anonymous free tier. Rate-limited under burst load."
+            ),
+            "env_vars": [],
+            "web_tier": "free",
+            "variants": [
+                {
+                    "name": "Exa · Paid (API key)",
+                    "badge": "paid",
+                    "tag": (
+                        "Semantic + neural web search with content extraction "
+                        "via the Exa SDK. Unthrottled, guaranteed service."
+                    ),
+                    "env_vars": [
+                        {
+                            "key": "EXA_API_KEY",
+                            "prompt": "Exa API key",
+                            "url": "https://exa.ai",
+                        },
+                    ],
+                    "web_tier": "paid",
+                },
+            ],
+        }

plugins/web/firecrawl/__init__.py

@@ -0,0 +1,28 @@
+"""Firecrawl web search + extract plugin — bundled, auto-loaded.
+
+Largest single plugin in this PR. Captures everything the previous
+inline implementation in tools/web_tools.py did:
+
+  - Lazy import of the firecrawl SDK (~200ms cold-start cost) via a
+    callable proxy that defers the actual import to first use.
+  - Dual client paths: direct (FIRECRAWL_API_KEY / FIRECRAWL_API_URL)
+    OR Nous-hosted tool-gateway routing for subscribers, with
+    web.use_gateway as the tie-breaker.
+  - Per-URL scrape loop with 60s timeout, SSRF re-check after redirect,
+    website-policy gating, and format-aware content selection.
+  - Robust response shape normalization across SDK / direct API /
+    gateway variants (search returns differ by transport).
+
+The plugin re-exports ``Firecrawl`` (the lazy proxy) and
+``check_firecrawl_api_key`` for backward-compatibility with tests and
+external code that imports those names from ``tools.web_tools``.
+"""
+
+from __future__ import annotations
+
+from plugins.web.firecrawl.provider import FirecrawlWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the Firecrawl provider with the plugin context."""
+    ctx.register_web_search_provider(FirecrawlWebSearchProvider())

plugins/web/firecrawl/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-firecrawl
+version: 1.0.0
+description: "Firecrawl web search + content extraction. Supports keyless cloud, direct API, and Nous-hosted tool-gateway routing for subscribers."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - firecrawl

plugins/web/firecrawl/provider.py

@@ -0,0 +1,804 @@
+"""Firecrawl web search + extract — plugin form.
+
+Subclasses :class:`agent.web_search_provider.WebSearchProvider`. This is
+the largest provider migrated in this PR; it captures the full inline
+firecrawl implementation that previously lived in tools/web_tools.py:
+
+  - :data:`Firecrawl` lazy proxy that defers the ~200ms SDK import to
+    first use (re-exported by tools.web_tools for backward compat with
+    existing tests that mock that name).
+  - :func:`_get_firecrawl_client` with direct + managed-gateway dual
+    mode, controlled by ``web.use_gateway`` config when both are
+    configured.
+  - :func:`check_firecrawl_api_key` re-exported (tests + tools_config
+    setup hint depend on this name living in tools.web_tools).
+  - :func:`_extract_web_search_results` / :func:`_extract_scrape_payload`
+    response-shape normalizers that handle SDK / direct API / gateway
+    response variants.
+  - Per-URL extract loop with 60s timeout, redirect-aware SSRF re-check,
+    website-policy gating, and format-aware content selection.
+
+Async note: the underlying SDK is sync. ``extract()`` is declared
+``async def`` because it performs per-URL I/O that benefits from
+running in an executor; the implementation wraps each scrape in
+:func:`asyncio.to_thread` with :func:`asyncio.wait_for(timeout=60)` to
+guard against hung fetches.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "firecrawl"     # explicit per-capability
+      extract_backend: "firecrawl"    # explicit per-capability
+      backend: "firecrawl"            # shared fallback (default)
+      use_gateway: false              # prefer managed gateway when both
+                                      # direct + gateway credentials exist
+
+Env vars::
+
+    FIRECRAWL_API_KEY=...            # direct cloud auth
+    FIRECRAWL_API_URL=...            # self-hosted Firecrawl
+    FIRECRAWL_GATEWAY_URL=...        # Nous tool-gateway (subscribers)
+    TOOL_GATEWAY_DOMAIN=...          # alternate gateway env
+    TOOL_GATEWAY_SCHEME=...
+    TOOL_GATEWAY_USER_TOKEN=...
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from typing import Any, Dict, List, NoReturn, Optional, TYPE_CHECKING
+
+import httpx
+
+from agent.web_search_provider import WebSearchProvider
+from tools.url_safety import is_safe_url
+from tools.website_policy import check_website_access
+
+logger = logging.getLogger(__name__)
+
+_FIRECRAWL_CLOUD_API_URL = "https://api.firecrawl.dev"
+
+
+# ---------------------------------------------------------------------------
+# Lazy Firecrawl SDK proxy
+# ---------------------------------------------------------------------------
+# The firecrawl SDK pulls ~200ms of imports (httpcore, firecrawl.v1/v2 type
+# trees) on a cold CLI. We only need it when the backend is actually
+# "firecrawl", so defer the import to first use via a callable proxy.
+#
+# Tests that do ``patch("tools.web_tools.Firecrawl", ...)`` continue to
+# work because tools/web_tools.py re-exports ``Firecrawl`` from this
+# module — so the patched name still references the same proxy instance.
+
+if TYPE_CHECKING:
+    from firecrawl import Firecrawl as FirecrawlSDK  # noqa: F401 — type hints only
+
+_FIRECRAWL_CLS_CACHE: Optional[type] = None
+
+
+def _load_firecrawl_cls() -> type:
+    """Import and cache ``firecrawl.Firecrawl``."""
+    global _FIRECRAWL_CLS_CACHE
+    if _FIRECRAWL_CLS_CACHE is None:
+        try:
+            from tools.lazy_deps import ensure as _lazy_ensure
+
+            _lazy_ensure("search.firecrawl", prompt=False)
+        except ImportError:
+            pass
+        except Exception as exc:  # noqa: BLE001 — surface install hint
+            raise ImportError(str(exc))
+        from firecrawl import Firecrawl as _cls  # noqa: WPS433 — deliberately lazy
+
+        _FIRECRAWL_CLS_CACHE = _cls
+    return _FIRECRAWL_CLS_CACHE
+
+
+class _FirecrawlProxy:
+    """Callable proxy that looks like ``firecrawl.Firecrawl`` but imports lazily."""
+
+    __slots__ = ()
+
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        return _load_firecrawl_cls()(*args, **kwargs)
+
+    def __instancecheck__(self, obj: Any) -> bool:
+        return isinstance(obj, _load_firecrawl_cls())
+
+    def __repr__(self) -> str:
+        return "<lazy firecrawl.Firecrawl proxy>"
+
+
+Firecrawl = _FirecrawlProxy()
+
+
+# ---------------------------------------------------------------------------
+# Client construction (direct vs managed-gateway)
+# ---------------------------------------------------------------------------
+#
+# The canonical cache slots live on :mod:`tools.web_tools` so tests that do
+# ``tools.web_tools._firecrawl_client = None`` between cases see fresh
+# state. The plugin reads/writes through that public module — see
+# :func:`_get_firecrawl_client` below.
+
+
+def _get_direct_firecrawl_config() -> Optional[tuple]:
+    """Return direct Firecrawl (mode, kwargs, cache key), or None when unavailable.
+
+    ``mode`` is ``"sdk"`` (keyed / self-hosted via the Firecrawl SDK) or
+    ``"keyless"`` (explicit Firecrawl selection with no credentials — served
+    by :class:`_KeylessFirecrawlClient` against the public cloud API, which
+    accepts anonymous rate-limited requests). Keyless requires the explicit
+    selection so an unconfigured install never silently routes to it.
+    """
+    from hermes_cli.config import get_env_value
+
+    api_key = (get_env_value("FIRECRAWL_API_KEY") or "").strip()
+    api_url = (get_env_value("FIRECRAWL_API_URL") or "").strip().rstrip("/")
+
+    if not api_key and not api_url:
+        if _is_explicit_firecrawl_selection():
+            return (
+                "keyless",
+                {"api_url": _FIRECRAWL_CLOUD_API_URL},
+                ("direct-keyless", _FIRECRAWL_CLOUD_API_URL, None),
+            )
+        return None
+
+    kwargs: Dict[str, str] = {}
+    if api_key:
+        kwargs["api_key"] = api_key
+    if api_url:
+        kwargs["api_url"] = api_url
+
+    return "sdk", kwargs, ("direct", api_url or None, api_key or None)
+
+
+def _is_explicit_firecrawl_selection() -> bool:
+    """Return True when config explicitly selects Firecrawl for web tools."""
+    import tools.web_tools as _wt
+
+    cfg = _wt._load_web_config()
+    return any(
+        (cfg.get(key) or "").lower().strip() == "firecrawl"
+        for key in ("backend", "search_backend", "extract_backend")
+    )
+
+
+def _use_keyless_ring() -> bool:
+    """True when Firecrawl calls should route via the keyless ring.
+
+    Ring dispatch applies when there are no direct credentials, the
+    managed Nous gateway isn't the selected path, and the keyless tier
+    isn't disabled or pinned paid. Keyed/self-hosted/gateway setups never
+    reach the ring.
+    """
+    from hermes_cli.config import get_env_value
+
+    if (get_env_value("FIRECRAWL_API_KEY") or "").strip():
+        return False
+    if (get_env_value("FIRECRAWL_API_URL") or "").strip():
+        return False
+    import tools.web_tools as _wt
+    from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_selection
+
+    try:
+        if read_selection("web") == NOUS_MANAGED_PROVIDER:
+            return False
+    except Exception:  # noqa: BLE001 — selection helpers optional
+        pass
+    try:
+        if _wt._is_tool_gateway_ready() and not _is_explicit_firecrawl_selection():
+            return False
+    except Exception:  # noqa: BLE001 — probe optional
+        pass
+    from plugins.web.keyless_mcp import use_keyless
+
+    return use_keyless("firecrawl", "")
+
+
+class _KeylessFirecrawlClient:
+    """Minimal REST client for Firecrawl's keyless cloud mode.
+
+    Duck-types the two SDK methods the provider calls (``search`` /
+    ``scrape``) so the rest of the pipeline (result normalizers, caching)
+    is unchanged. No Authorization header is ever sent.
+    """
+
+    def __init__(self, api_url: str = _FIRECRAWL_CLOUD_API_URL):
+        self.api_url = api_url.rstrip("/")
+
+    def _post(self, path: str, payload: Dict[str, Any]) -> Dict[str, Any]:
+        response = httpx.post(
+            f"{self.api_url}{path}",
+            json=payload,
+            headers={"Content-Type": "application/json"},
+            timeout=60.0,
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def search(self, *, query: str, limit: int = 5) -> Dict[str, Any]:
+        return self._post("/v2/search", {"query": query, "limit": limit})
+
+    def scrape(self, *, url: str, formats: List[str]) -> Dict[str, Any]:
+        return self._post("/v2/scrape", {"url": url, "formats": formats})
+
+
+def _get_firecrawl_gateway_url() -> str:
+    """Return the configured Firecrawl gateway URL."""
+    import tools.web_tools as _wt
+
+    return _wt.build_vendor_gateway_url("firecrawl")
+
+
+def _is_tool_gateway_ready() -> bool:
+    """Return True when gateway URL + Nous Subscriber token are available.
+
+    Reads ``peek_nous_access_token`` and ``resolve_managed_tool_gateway``
+    via :mod:`tools.web_tools` rather than direct imports, so unit tests
+    that ``patch("tools.web_tools._peek_nous_access_token", ...)`` see
+    their patches honored. The names are re-exported on
+    :mod:`tools.web_tools` for exactly this reason.
+    """
+    import tools.web_tools as _wt
+
+    return _wt.resolve_managed_tool_gateway(
+        "firecrawl", token_reader=_wt._peek_nous_access_token
+    ) is not None
+
+
+def _has_direct_firecrawl_config() -> bool:
+    """Return True when direct Firecrawl config is explicitly configured."""
+    return _get_direct_firecrawl_config() is not None
+
+
+def check_firecrawl_api_key() -> bool:
+    """Return True when the Firecrawl backend selected via `hermes tools`
+    (or, on a never-configured install, either route) is usable.
+
+    Re-exported by :mod:`tools.web_tools` for backward compatibility with
+    existing tests and the ``hermes tools`` setup flow.
+    """
+    from tools.tool_backend_helpers import (
+        NOUS_MANAGED_PROVIDER,
+        read_selection,
+    )
+
+    selected = read_selection("web")
+    if selected == NOUS_MANAGED_PROVIDER:
+        return _is_tool_gateway_ready()
+    if selected is not None:
+        return _has_direct_firecrawl_config()
+    return _has_direct_firecrawl_config() or _is_tool_gateway_ready()
+
+
+def _firecrawl_backend_help_suffix() -> str:
+    """Return optional managed-gateway guidance for Firecrawl help text."""
+    import tools.web_tools as _wt
+
+    if not _wt.managed_nous_tools_enabled():
+        return ""
+    return (
+        ", or use the Nous Tool Gateway via your subscription "
+        "(FIRECRAWL_GATEWAY_URL or TOOL_GATEWAY_DOMAIN)"
+    )
+
+
+def _raise_web_backend_configuration_error() -> "NoReturn":
+    """Raise a clear error for unsupported web backend configuration."""
+    import tools.web_tools as _wt
+
+    message = (
+        "Web tools are not configured. "
+        "Set FIRECRAWL_API_KEY for cloud Firecrawl or set FIRECRAWL_API_URL "
+        "for a self-hosted Firecrawl instance."
+    )
+    if _wt.managed_nous_tools_enabled():
+        message += (
+            " With your Nous subscription you can also use the Tool Gateway. "
+            "run `hermes tools` and select Nous Subscription as the web provider."
+        )
+    else:
+        message += " " + _wt.nous_tool_gateway_unavailable_message(
+            "managed Firecrawl web tools",
+        )
+    raise ValueError(message)
+
+
+def _get_firecrawl_client() -> Any:
+    """Get or create the cached Firecrawl client.
+
+    Strict selection semantics (switch on the stored ``web`` selection):
+    - ``"nous"`` (or legacy ``use_gateway: true``) → managed Tool Gateway
+      ONLY; unavailable is a selection-naming error (a present
+      FIRECRAWL_API_KEY does not reroute).
+    - any other stored web backend → direct Firecrawl ONLY; missing config
+      is a selection-naming error — never a silent managed fallback billed
+      to Nous.
+    - never-configured web section → legacy behavior: direct config when
+      present, else the managed gateway.
+
+    Raises ValueError when the resolved path is unusable.
+
+    The cached client is stored on :mod:`tools.web_tools` (as
+    ``_firecrawl_client`` and ``_firecrawl_client_config``) rather than on
+    this plugin module so that unit tests that reset the cache via
+    ``tools.web_tools._firecrawl_client = None`` keep working. Helper
+    functions (``resolve_managed_tool_gateway``, ``_read_nous_access_token``,
+    ``Firecrawl``) are also looked up via :mod:`tools.web_tools` for the same
+    reason — see :func:`_is_tool_gateway_ready`.
+    """
+    import tools.web_tools as _wt
+    from tools.tool_backend_helpers import (
+        NOUS_MANAGED_PROVIDER,
+        read_selection,
+        selection_error,
+        selection_exists,
+    )
+
+    selected = read_selection("web")
+
+    direct_config = _get_direct_firecrawl_config()
+
+    def _managed_kwargs():
+        managed_gateway = _wt.resolve_managed_tool_gateway(
+            "firecrawl", token_reader=_wt._read_nous_access_token
+        )
+        if managed_gateway is None:
+            return None
+        kwargs = {
+            "api_key": managed_gateway.nous_user_token,
+            "api_url": managed_gateway.gateway_origin,
+        }
+        return kwargs, (
+            "tool-gateway",
+            kwargs["api_url"],
+            managed_gateway.nous_user_token,
+        )
+
+    if selected == NOUS_MANAGED_PROVIDER:
+        managed = _managed_kwargs()
+        if managed is None:
+            logger.error(
+                "Firecrawl client initialization failed: the Nous "
+                "Subscription web selection is stored but the tool gateway "
+                "is unavailable."
+            )
+            raise ValueError(selection_error(
+                "web",
+                NOUS_MANAGED_PROVIDER,
+                "the Nous Tool Gateway is not available (not entitled or "
+                "unreachable)",
+            ))
+        kwargs, client_config = managed
+        client_mode = "sdk"
+    elif selected is not None or selection_exists("web"):
+        # Stored vendor selection (or per-capability web keys routing to
+        # firecrawl): direct Firecrawl only. With no credentials, the
+        # explicit selection unlocks keyless cloud mode instead of erroring.
+        if direct_config is None:
+            logger.error(
+                "Firecrawl client initialization failed: direct Firecrawl "
+                "selected but FIRECRAWL_API_KEY/FIRECRAWL_API_URL is not set."
+            )
+            raise ValueError(selection_error(
+                "web",
+                selected or "firecrawl",
+                "neither FIRECRAWL_API_KEY nor FIRECRAWL_API_URL is set",
+            ))
+        client_mode, kwargs, client_config = direct_config
+    elif direct_config is not None:
+        client_mode, kwargs, client_config = direct_config
+    else:
+        # Never-configured web section: legacy managed fallback.
+        managed = _managed_kwargs()
+        if managed is None:
+            logger.error(
+                "Firecrawl client initialization failed: "
+                "missing direct config and tool-gateway auth."
+            )
+            _raise_web_backend_configuration_error()
+        kwargs, client_config = managed
+        client_mode = "sdk"
+
+    cached = getattr(_wt, "_firecrawl_client", None)
+    cached_config = getattr(_wt, "_firecrawl_client_config", None)
+    if cached is not None and cached_config == client_config:
+        return cached
+
+    # Construct via the re-exported Firecrawl proxy on tools.web_tools so
+    # unit tests patching ``tools.web_tools.Firecrawl`` see their mock.
+    if client_mode == "keyless":
+        _wt._firecrawl_client = _KeylessFirecrawlClient(api_url=kwargs["api_url"])
+    else:
+        _wt._firecrawl_client = _wt.Firecrawl(**kwargs)
+    _wt._firecrawl_client_config = client_config
+    return _wt._firecrawl_client
+
+
+def _reset_client_for_tests() -> None:
+    """Drop the cached Firecrawl client so tests can re-instantiate cleanly.
+
+    Clears the canonical slots on :mod:`tools.web_tools` (where
+    :func:`_get_firecrawl_client` reads/writes them).
+    """
+    import tools.web_tools as _wt
+
+    _wt._firecrawl_client = None
+    _wt._firecrawl_client_config = None
+
+
+# ---------------------------------------------------------------------------
+# Response shape normalization (SDK / direct / gateway differ)
+# ---------------------------------------------------------------------------
+
+
+def _to_plain_object(value: Any) -> Any:
+    """Convert SDK objects to plain python data structures when possible."""
+    if value is None:
+        return None
+
+    if isinstance(value, (dict, list, str, int, float, bool)):
+        return value
+
+    if hasattr(value, "model_dump"):
+        try:
+            return value.model_dump()
+        except Exception:  # noqa: BLE001
+            pass
+
+    if hasattr(value, "__dict__"):
+        try:
+            return {k: v for k, v in value.__dict__.items() if not k.startswith("_")}
+        except Exception:  # noqa: BLE001
+            pass
+
+    return value
+
+
+def _normalize_result_list(values: Any) -> List[Dict[str, Any]]:
+    """Normalize mixed SDK/list payloads into a list of dicts."""
+    if not isinstance(values, list):
+        return []
+
+    normalized: List[Dict[str, Any]] = []
+    for item in values:
+        plain = _to_plain_object(item)
+        if isinstance(plain, dict):
+            normalized.append(plain)
+    return normalized
+
+
+def _extract_web_search_results(response: Any) -> List[Dict[str, Any]]:
+    """Extract Firecrawl search results across SDK/direct/gateway response shapes."""
+    response_plain = _to_plain_object(response)
+
+    if isinstance(response_plain, dict):
+        data = response_plain.get("data")
+        if isinstance(data, list):
+            return _normalize_result_list(data)
+
+        if isinstance(data, dict):
+            data_web = _normalize_result_list(data.get("web"))
+            if data_web:
+                return data_web
+            data_results = _normalize_result_list(data.get("results"))
+            if data_results:
+                return data_results
+
+        top_web = _normalize_result_list(response_plain.get("web"))
+        if top_web:
+            return top_web
+
+        top_results = _normalize_result_list(response_plain.get("results"))
+        if top_results:
+            return top_results
+
+    if hasattr(response, "web"):
+        return _normalize_result_list(getattr(response, "web", []))
+
+    return []
+
+
+def _extract_scrape_payload(scrape_result: Any) -> Dict[str, Any]:
+    """Normalize Firecrawl scrape payload shape across SDK and gateway variants."""
+    result_plain = _to_plain_object(scrape_result)
+    if not isinstance(result_plain, dict):
+        return {}
+
+    nested = result_plain.get("data")
+    if isinstance(nested, dict):
+        return nested
+
+    return result_plain
+
+
+# ---------------------------------------------------------------------------
+# Provider class
+# ---------------------------------------------------------------------------
+
+
+class FirecrawlWebSearchProvider(WebSearchProvider):
+    """Firecrawl search + extract provider with dual auth paths."""
+
+    @property
+    def name(self) -> str:
+        return "firecrawl"
+
+    @property
+    def display_name(self) -> str:
+        return "Firecrawl"
+
+    def is_available(self) -> bool:
+        """Return True when direct Firecrawl OR managed-gateway path is configured."""
+        return check_firecrawl_api_key()
+
+    def is_keyless_available(self) -> bool:
+        """Firecrawl serves keyless cloud requests (public API, no auth).
+
+        Default-on ring member of the keyless free tier: fresh installs
+        rotate across Exa/Parallel/Firecrawl/Keenable. False when
+        the user pinned ``web.provider_tier.firecrawl: paid``.
+        """
+        from plugins.web.keyless_mcp import keyless_enabled, provider_tier
+
+        return keyless_enabled() and provider_tier("firecrawl") != "paid"
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return True
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a Firecrawl search.
+
+        Sync; matches the legacy ``_get_firecrawl_client().search(...)``
+        call directly. Normalizes the response across SDK/direct/gateway
+        shapes via :func:`_extract_web_search_results`.
+
+        Pre-flight errors (``ValueError`` from configuration check,
+        ``ImportError`` from missing SDK) propagate to the dispatcher's
+        top-level handler, which wraps them as ``tool_error(...)`` —
+        matching the legacy ``{"error": "Error searching web: ..."}``
+        envelope. Only in-flight errors are caught and surfaced as
+        ``{"success": False, "error": ...}``.
+        """
+        from tools.interrupt import is_interrupted
+
+        if is_interrupted():
+            return {"success": False, "error": "Interrupted"}
+
+        if _use_keyless_ring():
+            # No credentials and no managed gateway: ring dispatch with
+            # next-in-line failover on rate limits (default-on free tier).
+            from plugins.web.keyless_mcp import search_with_failover
+
+            logger.info(
+                "Firecrawl keyless search: '%s' (limit=%d)", query, limit
+            )
+            return search_with_failover("firecrawl", query, limit)
+
+        logger.info("Firecrawl search: '%s' (limit=%d)", query, limit)
+        # _get_firecrawl_client() raises ValueError on unconfigured systems —
+        # let it propagate so the dispatcher emits the legacy envelope shape.
+        client = _get_firecrawl_client()
+        try:
+            response = client.search(query=query, limit=limit)
+            web_results = _extract_web_search_results(response)
+            logger.info("Firecrawl: found %d search results", len(web_results))
+            return {"success": True, "data": {"web": web_results}}
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Firecrawl search error: %s", exc)
+            return {"success": False, "error": f"Firecrawl search failed: {exc}"}
+
+    async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
+        """Extract content from one or more URLs via Firecrawl.
+
+        Async; each URL is scraped in a background thread with a 60s
+        timeout. After scraping, the final URL (post-redirect) is
+        re-checked against website-access policy.
+
+        Accepted kwargs (others ignored for forward compat):
+          - ``format``: ``"markdown"`` or ``"html"``; default is both
+            (request both, return markdown when available).
+
+        Returns the legacy per-URL list-of-results shape. Per-URL failures
+        (timeout, SSRF block, scrape error, policy block) become items
+        with an ``error`` field rather than raising.
+        """
+        from tools.interrupt import is_interrupted as _is_interrupted
+
+        if _is_interrupted():
+            return [{"url": u, "error": "Interrupted", "title": ""} for u in urls]
+
+        if _use_keyless_ring():
+            # No credentials and no managed gateway: ring dispatch with
+            # next-in-line failover on rate limits (default-on free tier).
+            import asyncio as _asyncio
+
+            from plugins.web.keyless_mcp import extract_with_failover
+
+            logger.info("Firecrawl keyless extract: %d URL(s)", len(urls))
+            return await _asyncio.to_thread(
+                extract_with_failover, "firecrawl", list(urls)
+            )
+
+        format = kwargs.get("format")
+        formats: List[str] = []
+        if format == "markdown":
+            formats = ["markdown"]
+        elif format == "html":
+            formats = ["html"]
+        else:
+            formats = ["markdown", "html"]
+
+        # check_website_access is the legacy policy gate; imported at
+        # module level (lazy-friendly because the website_policy import is
+        # cheap) so monkeypatching it in tests works as expected.
+
+        results: List[Dict[str, Any]] = []
+
+        for url in urls:
+            if _is_interrupted():
+                results.append({"url": url, "error": "Interrupted", "title": ""})
+                continue
+
+            # Pre-scrape website policy gate
+            blocked = check_website_access(url)
+            if blocked:
+                logger.info(
+                    "Blocked web_extract for %s by rule %s",
+                    blocked["host"],
+                    blocked["rule"],
+                )
+                results.append(
+                    {
+                        "url": url,
+                        "title": "",
+                        "content": "",
+                        "error": blocked["message"],
+                        "blocked_by_policy": {
+                            "host": blocked["host"],
+                            "rule": blocked["rule"],
+                            "source": blocked["source"],
+                        },
+                    }
+                )
+                continue
+
+            try:
+                logger.info("Firecrawl scraping: %s", url)
+                try:
+                    scrape_result = await asyncio.wait_for(
+                        asyncio.to_thread(
+                            _get_firecrawl_client().scrape,
+                            url=url,
+                            formats=formats,
+                        ),
+                        timeout=60,
+                    )
+                except asyncio.TimeoutError:
+                    logger.warning("Firecrawl scrape timed out for %s", url)
+                    results.append(
+                        {
+                            "url": url,
+                            "title": "",
+                            "content": "",
+                            "error": (
+                                "Scrape timed out after 60s — page may be too large "
+                                "or unresponsive. Try browser_navigate instead."
+                            ),
+                        }
+                    )
+                    continue
+
+                scrape_payload = _extract_scrape_payload(scrape_result)
+                metadata = scrape_payload.get("metadata", {})
+                content_markdown = scrape_payload.get("markdown")
+                content_html = scrape_payload.get("html")
+
+                # Ensure metadata is a dict (SDK may return a typed object)
+                if not isinstance(metadata, dict):
+                    if hasattr(metadata, "model_dump"):
+                        metadata = metadata.model_dump()
+                    elif hasattr(metadata, "__dict__"):
+                        metadata = metadata.__dict__
+                    else:
+                        metadata = {}
+
+                title = metadata.get("title", "")
+                final_url = metadata.get("sourceURL", url)
+
+                # Re-check SSRF safety after any redirect reported by Firecrawl.
+                if not is_safe_url(final_url):
+                    logger.info(
+                        "Blocked redirected web_extract for unsafe final URL: %s",
+                        final_url,
+                    )
+                    results.append(
+                        {
+                            "url": final_url,
+                            "title": title,
+                            "content": "",
+                            "raw_content": "",
+                            "error": (
+                                "Blocked: URL targets a private or internal "
+                                "network address"
+                            ),
+                        }
+                    )
+                    continue
+
+                # Re-check website-access policy after any redirect
+                final_blocked = check_website_access(final_url)
+                if final_blocked:
+                    logger.info(
+                        "Blocked redirected web_extract for %s by rule %s",
+                        final_blocked["host"],
+                        final_blocked["rule"],
+                    )
+                    results.append(
+                        {
+                            "url": final_url,
+                            "title": title,
+                            "content": "",
+                            "raw_content": "",
+                            "error": final_blocked["message"],
+                            "blocked_by_policy": {
+                                "host": final_blocked["host"],
+                                "rule": final_blocked["rule"],
+                                "source": final_blocked["source"],
+                            },
+                        }
+                    )
+                    continue
+
+                # Choose markdown vs html according to the requested format
+                if format == "markdown" or (format is None and content_markdown):
+                    chosen_content = content_markdown
+                else:
+                    chosen_content = content_html or content_markdown or ""
+
+                results.append(
+                    {
+                        "url": final_url,
+                        "title": title,
+                        "content": chosen_content,
+                        "raw_content": chosen_content,
+                        "metadata": metadata,
+                    }
+                )
+            except Exception as scrape_err:  # noqa: BLE001
+                logger.debug("Firecrawl scrape failed for %s: %s", url, scrape_err)
+                results.append(
+                    {
+                        "url": url,
+                        "title": "",
+                        "content": "",
+                        "raw_content": "",
+                        "error": str(scrape_err),
+                    }
+                )
+
+        return results
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "Firecrawl",
+            "badge": "keyless/paid · optional gateway",
+            "tag": (
+                "Full search + extract; supports keyless cloud, direct API, "
+                "and Nous tool-gateway routing."
+            ),
+            "env_vars": [
+                {
+                    "key": "FIRECRAWL_API_KEY",
+                    "prompt": "Firecrawl API key (optional; blank = keyless cloud or self-hosted)",
+                    "url": "https://docs.firecrawl.dev/introduction",
+                },
+            ],
+        }

plugins/web/keenable/__init__.py

@@ -0,0 +1,13 @@
+"""Keenable web search + extract plugin — bundled, auto-loaded.
+
+Keyless-ring member (keyed via KEENABLE_API_KEY for higher limits).
+"""
+
+from __future__ import annotations
+
+from plugins.web.keenable.provider import KeenableWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the Keenable provider with the plugin context."""
+    ctx.register_web_search_provider(KeenableWebSearchProvider())

plugins/web/keenable/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-keenable
+version: 1.0.0
+description: "Keenable web search + page fetch (independent web index for AI apps). Works keyless on Keenable's free tier as part of the default rotation; set KEENABLE_API_KEY for higher limits — https://keenable.ai."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - keenable

plugins/web/keenable/provider.py

@@ -0,0 +1,233 @@
+"""Keenable web search + content extraction — bundled plugin.
+
+Keenable (https://keenable.ai) operates an independent web index for AI
+apps with public keyless endpoints (rate-limited free tier; keyed access
+via KEENABLE_API_KEY for higher limits). Integrated as a keyless-ring
+member following the Exa/Parallel/Firecrawl pattern: fresh installs with
+zero web credentials rotate across the ring vendors' free tiers.
+
+Credit: Keenable integration originally proposed by Ilya Gusev (Keenable)
+in PR #49758; the native provider form follows the salvage of that work
+plus the keyless-ring design.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "keenable"      # explicit per-capability
+      extract_backend: "keenable"     # explicit per-capability
+      backend: "keenable"             # shared fallback
+      provider_tier:
+        keenable: free|paid           # pin the tier (unset = auto)
+
+Env var::
+
+    KEENABLE_API_KEY=...   # optional — keyless free tier works without it
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+_KEENABLE_API_URL = "https://api.keenable.ai"
+
+
+def _keenable_headers(api_key: str) -> Dict[str, str]:
+    """Build Keenable request headers for keyed or keyless access.
+
+    Their keyless tier structurally requires an app-identifier header
+    (X-Keenable-Title); no user identifiers are sent.
+    """
+    headers = {"X-Keenable-Title": "hermes-agent"}
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    return headers
+
+
+class KeenableWebSearchProvider(WebSearchProvider):
+    """Keenable search + extract provider (keyed or keyless)."""
+
+    @property
+    def name(self) -> str:
+        return "keenable"
+
+    @property
+    def display_name(self) -> str:
+        return "Keenable"
+
+    def is_available(self) -> bool:
+        """Return True when ``KEENABLE_API_KEY`` is set to a non-empty value."""
+        from agent.web_search_provider import get_provider_env
+
+        return bool(get_provider_env("KEENABLE_API_KEY"))
+
+    def is_keyless_available(self) -> bool:
+        """Keenable serves anonymous free-tier calls via its public endpoints.
+
+        Default-on ring member of the keyless free tier. False when the
+        user pinned ``web.provider_tier.keenable: paid``.
+        """
+        from plugins.web.keyless_mcp import keyless_enabled, provider_tier
+
+        return keyless_enabled() and provider_tier("keenable") != "paid"
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return True
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a Keenable search (keyed path or keyless ring)."""
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return {"success": False, "error": "Interrupted"}
+
+            from agent.web_search_provider import get_provider_env
+
+            from plugins.web.keyless_mcp import search_with_failover, use_keyless
+
+            api_key = get_provider_env("KEENABLE_API_KEY")
+            if use_keyless("keenable", api_key):
+                logger.info(
+                    "Keenable keyless search: '%s' (limit=%d)", query, limit
+                )
+                return search_with_failover("keenable", query, limit)
+
+            import requests
+
+            logger.info("Keenable search: '%s' (limit=%d)", query, limit)
+            response = requests.post(
+                f"{_KEENABLE_API_URL}/v1/search",
+                json={"query": query, "max_results": min(max(1, int(limit)), 20)},
+                headers=_keenable_headers(api_key),
+                timeout=30,
+            )
+            if response.status_code >= 400:
+                detail = (response.text or "").strip() or f"HTTP {response.status_code}"
+                return {"success": False, "error": f"Keenable search failed: {detail}"}
+            data = response.json()
+
+            web_results = []
+            for i, result in enumerate(data.get("results") or []):
+                web_results.append(
+                    {
+                        "url": result.get("url") or "",
+                        "title": result.get("title") or "",
+                        "description": result.get("snippet")
+                        or result.get("description")
+                        or "",
+                        "position": i + 1,
+                    }
+                )
+            return {"success": True, "data": {"web": web_results}}
+        except Exception as exc:  # noqa: BLE001 — surface as failure
+            logger.warning("Keenable search error: %s", exc)
+            return {"success": False, "error": f"Keenable search failed: {exc}"}
+
+    def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
+        """Extract content via Keenable's fetch endpoint (per-URL).
+
+        Sync — the dispatcher wraps in a thread when the caller is async.
+        Returns the legacy list-of-results shape; per-URL failures become
+        items with an ``error`` field.
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return [
+                    {"url": u, "error": "Interrupted", "title": ""} for u in urls
+                ]
+
+            from agent.web_search_provider import get_provider_env
+
+            from plugins.web.keyless_mcp import extract_with_failover, use_keyless
+
+            api_key = get_provider_env("KEENABLE_API_KEY")
+            if use_keyless("keenable", api_key):
+                logger.info("Keenable keyless extract: %d URL(s)", len(urls))
+                return extract_with_failover("keenable", list(urls))
+
+            import requests
+
+            logger.info("Keenable extract: %d URL(s)", len(urls))
+            results: List[Dict[str, Any]] = []
+            for url in urls:
+                try:
+                    response = requests.get(
+                        f"{_KEENABLE_API_URL}/v1/fetch",
+                        params={"url": url},
+                        headers=_keenable_headers(api_key),
+                        timeout=30,
+                    )
+                    if response.status_code >= 400:
+                        raise ValueError(
+                            (response.text or "").strip()
+                            or f"HTTP {response.status_code}"
+                        )
+                    data = response.json()
+                    content = data.get("content") or ""
+                    title = data.get("title") or ""
+                    results.append(
+                        {
+                            "url": data.get("url") or url,
+                            "title": title,
+                            "content": content,
+                            "raw_content": content,
+                            "metadata": {"sourceURL": url, "title": title},
+                        }
+                    )
+                except Exception as exc:  # noqa: BLE001 — per-URL error entry
+                    results.append(
+                        {
+                            "url": url,
+                            "title": "",
+                            "content": "",
+                            "error": f"Keenable extract failed: {exc}",
+                        }
+                    )
+            return results
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Keenable extract error: %s", exc)
+            return [
+                {"url": u, "title": "", "content": "",
+                 "error": f"Keenable extract failed: {exc}"}
+                for u in urls
+            ]
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "Keenable · Free (keyless)",
+            "badge": "free · no key",
+            "tag": (
+                "Independent web index for AI apps — fast search + page "
+                "fetch on Keenable's anonymous free tier."
+            ),
+            "env_vars": [],
+            "web_tier": "free",
+            "variants": [
+                {
+                    "name": "Keenable · Paid (API key)",
+                    "badge": "paid",
+                    "tag": (
+                        "Independent web index for AI apps. Keyed access "
+                        "with higher limits and guaranteed service."
+                    ),
+                    "env_vars": [
+                        {
+                            "key": "KEENABLE_API_KEY",
+                            "prompt": "Keenable API key",
+                            "url": "https://keenable.ai",
+                        },
+                    ],
+                    "web_tier": "paid",
+                },
+            ],
+        }

plugins/web/keyless_mcp.py

@@ -0,0 +1,760 @@
+"""Keyless web search/extract via public MCP endpoints.
+
+Exa and Parallel both operate public, anonymous MCP endpoints with a free
+tier (the same endpoints the opencode CLI ships as its default search
+path):
+
+- Exa:      https://mcp.exa.ai/mcp           (tools: web_search_exa, web_fetch_exa)
+- Parallel: https://search.parallel.ai/mcp   (tools: web_search, web_fetch)
+
+This module implements a minimal JSON-RPC ``tools/call`` client for those
+two endpoints so a fresh Hermes install with **zero web credentials** still
+gets working ``web_search`` / ``web_extract`` tools. The keyless tier is
+resolved strictly LAST — after every keyed backend, the managed tool
+gateway, ddgs, and custom plugin providers — so it never pre-empts a
+deliberate setup (see ``tools.web_tools._get_backend`` and the registry's
+``_KEYLESS_PREFERENCE`` walk).
+
+Privacy: requests carry no user identifiers. Parallel's free tier asks for
+a ``session_id`` used for rate limiting; we send a random per-process UUID
+(rotates every restart, never persisted). Their optional ``model_name``
+analytics field is deliberately omitted.
+
+Disable the whole tier with ``web.keyless_fallback: false`` in config.yaml.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import uuid
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+EXA_MCP_URL = "https://mcp.exa.ai/mcp"
+PARALLEL_MCP_URL = "https://search.parallel.ai/mcp"
+
+# Free-tier rate-limit correlation id for Parallel — random per process,
+# never persisted, not derived from any user/machine identifier.
+_SESSION_ID = uuid.uuid4().hex
+
+_TIMEOUT_SECONDS = 30
+
+
+class KeylessMCPError(RuntimeError):
+    """A keyless MCP call failed (transport, rate limit, or tool error)."""
+
+
+_RATE_LIMIT_MARKERS = (
+    "rate limit",
+    "rate-limit",
+    "ratelimit",
+    "too many requests",
+    "429",
+    "quota exceeded",
+    "slow down",
+)
+
+
+def _is_rate_limitish(message: str) -> bool:
+    """Heuristic: does an error message look like free-tier throttling?"""
+    lowered = (message or "").lower()
+    return any(marker in lowered for marker in _RATE_LIMIT_MARKERS)
+
+
+def keyless_enabled() -> bool:
+    """Return True when the keyless fallback tier is enabled.
+
+    Delegates to :func:`agent.web_search_registry._keyless_tier_enabled` so
+    the config chokepoint (``web.keyless_fallback``, default on) lives in
+    one place alongside the rest of backend resolution.
+    """
+    try:
+        from agent.web_search_registry import _keyless_tier_enabled
+
+        return _keyless_tier_enabled()
+    except Exception as exc:  # noqa: BLE001 — resolver optional in stripped envs
+        logger.debug("keyless_enabled(): registry helper unavailable: %s", exc)
+        return True
+
+
+def provider_tier(name: str) -> str:
+    """Return the user-selected tier for *name*: ``free``, ``paid``, or ``auto``.
+
+    Reads ``web.provider_tier.<name>`` from config.yaml (set by the
+    ``hermes tools`` picker's Free/Paid rows). ``free`` forces the keyless
+    public endpoint even when the vendor API key is present; ``paid``
+    forces the keyed SDK path (missing key surfaces the standard
+    "X_API_KEY not set" error instead of silently downgrading to the free
+    tier). Anything else — including unset — is ``auto``: key present →
+    keyed, otherwise keyless when the tier is enabled.
+    """
+    try:
+        from hermes_cli.config import load_config
+
+        web_cfg = load_config().get("web") or {}
+        tiers = web_cfg.get("provider_tier") or {}
+        value = str(tiers.get(name, "") or "").lower().strip()
+        return value if value in ("free", "paid") else "auto"
+    except Exception as exc:  # noqa: BLE001 — config layer optional
+        logger.debug("provider_tier(%r) config read failed: %s", name, exc)
+        return "auto"
+
+
+def use_keyless(name: str, api_key: str) -> bool:
+    """Decide whether provider *name* should route via the keyless endpoint.
+
+    Single chokepoint shared by the Exa/Parallel search + extract paths so
+    tier semantics can't drift between capabilities:
+
+    - tier ``free``  → keyless, even when *api_key* is set
+    - tier ``paid``  → keyed, even when *api_key* is missing (the keyed
+      path then raises its usual missing-key error)
+    - tier ``auto``  → keyed when *api_key* is set; otherwise keyless when
+      ``web.keyless_fallback`` is enabled
+    """
+    tier = provider_tier(name)
+    if tier == "free":
+        return True
+    if tier == "paid":
+        return False
+    return not api_key and keyless_enabled()
+
+
+def _parse_mcp_body(body: str) -> str:
+    """Extract the first text content item from an MCP tools/call response.
+
+    Handles both plain-JSON bodies and SSE (``data: {...}`` lines) — the
+    Exa endpoint answers as an event stream, Parallel as direct JSON.
+    Raises :class:`KeylessMCPError` for JSON-RPC errors and ``isError``
+    tool results (e.g. Exa's free-tier rate-limit message).
+    """
+
+    def _from_payload(payload: str) -> Optional[str]:
+        payload = payload.strip()
+        if not payload.startswith("{"):
+            return None
+        data = json.loads(payload)
+        err = data.get("error")
+        if err:
+            raise KeylessMCPError(str(err.get("message") or err))
+        result = data.get("result") or {}
+        content = result.get("content") or []
+        if result.get("isError"):
+            texts = [c.get("text", "") for c in content if isinstance(c, dict)]
+            raise KeylessMCPError(
+                " ".join(t for t in texts if t) or "MCP tool call failed"
+            )
+        for item in content:
+            if isinstance(item, dict) and item.get("text"):
+                return str(item["text"])
+        return None
+
+    stripped = body.strip()
+    if stripped.startswith("{"):
+        try:
+            text = _from_payload(stripped)
+            if text is not None:
+                return text
+        except json.JSONDecodeError:
+            pass
+
+    for line in body.splitlines():
+        if not line.startswith("data: "):
+            continue
+        try:
+            text = _from_payload(line[len("data: "):])
+        except json.JSONDecodeError:
+            continue
+        if text is not None:
+            return text
+
+    raise KeylessMCPError("Unrecognized MCP response shape")
+
+
+def mcp_call(
+    url: str,
+    tool: str,
+    arguments: Dict[str, Any],
+    timeout: int = _TIMEOUT_SECONDS,
+) -> str:
+    """POST a JSON-RPC ``tools/call`` to *url* and return the text payload.
+
+    Raises :class:`KeylessMCPError` on transport failures, non-2xx
+    statuses, JSON-RPC errors, and error-shaped tool results.
+    """
+    import requests
+
+    payload = {
+        "jsonrpc": "2.0",
+        "id": 1,
+        "method": "tools/call",
+        "params": {"name": tool, "arguments": arguments},
+    }
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json, text/event-stream",
+        "User-Agent": "hermes-agent",
+    }
+    try:
+        response = requests.post(url, json=payload, headers=headers, timeout=timeout)
+    except requests.RequestException as exc:
+        raise KeylessMCPError(f"request failed: {exc}") from exc
+    if response.status_code >= 400:
+        raise KeylessMCPError(
+            f"HTTP {response.status_code}: {response.text[:300]}"
+        )
+    return _parse_mcp_body(response.text)
+
+
+# ---------------------------------------------------------------------------
+# Parallel (search.parallel.ai) — JSON text payloads
+# ---------------------------------------------------------------------------
+
+
+def parallel_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]:
+    """Keyless Parallel web search → legacy search response shape."""
+    try:
+        text = mcp_call(
+            PARALLEL_MCP_URL,
+            "web_search",
+            {
+                "objective": query,
+                "search_queries": [query],
+                "session_id": _SESSION_ID,
+            },
+        )
+        data = json.loads(text)
+        web_results = []
+        for i, result in enumerate(data.get("results") or []):
+            if limit and i >= limit:
+                break
+            excerpts = result.get("excerpts") or []
+            web_results.append(
+                {
+                    "url": result.get("url") or "",
+                    "title": result.get("title") or "",
+                    "description": " ".join(excerpts) if excerpts else "",
+                    "position": i + 1,
+                }
+            )
+        return {"success": True, "data": {"web": web_results}}
+    except KeylessMCPError as exc:
+        return {
+            "success": False,
+            "error": (
+                f"Keyless Parallel search failed: {exc}. "
+                "Set PARALLEL_API_KEY (https://parallel.ai) or another web "
+                "backend via `hermes tools` for reliable service."
+            ),
+        }
+    except (json.JSONDecodeError, TypeError, KeyError) as exc:
+        return {"success": False, "error": f"Keyless Parallel search returned an unexpected payload: {exc}"}
+
+
+def parallel_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]:
+    """Keyless Parallel web fetch → legacy extract result list."""
+    try:
+        text = mcp_call(
+            PARALLEL_MCP_URL,
+            "web_fetch",
+            {
+                "urls": list(urls),
+                "objective": "Full page content",
+                "session_id": _SESSION_ID,
+            },
+        )
+        data = json.loads(text)
+    except (KeylessMCPError, json.JSONDecodeError, TypeError) as exc:
+        message = (
+            f"Keyless Parallel extract failed: {exc}. "
+            "Set PARALLEL_API_KEY (https://parallel.ai) or another web "
+            "backend via `hermes tools` for reliable service."
+        )
+        return [
+            {"url": u, "title": "", "content": "", "error": message}
+            for u in urls
+        ]
+
+    results: List[Dict[str, Any]] = []
+    seen = set()
+    for result in data.get("results") or []:
+        url = result.get("url") or ""
+        title = result.get("title") or ""
+        content = (
+            result.get("full_content")
+            or result.get("content")
+            or "\n\n".join(result.get("excerpts") or [])
+        )
+        seen.add(url)
+        results.append(
+            {
+                "url": url,
+                "title": title,
+                "content": content,
+                "raw_content": content,
+                "metadata": {"sourceURL": url, "title": title},
+            }
+        )
+    for error in data.get("errors") or []:
+        url = error.get("url") or ""
+        seen.add(url)
+        results.append(
+            {
+                "url": url,
+                "title": "",
+                "content": "",
+                "error": str(
+                    error.get("content") or error.get("error_type") or "extraction failed"
+                ),
+                "metadata": {"sourceURL": url},
+            }
+        )
+    # Any URL the endpoint silently dropped still gets an error entry so the
+    # caller's per-URL contract holds.
+    for u in urls:
+        if u not in seen:
+            results.append(
+                {"url": u, "title": "", "content": "", "error": "no content returned"}
+            )
+    return results
+
+
+# ---------------------------------------------------------------------------
+# Exa (mcp.exa.ai) — formatted plain-text payloads
+# ---------------------------------------------------------------------------
+
+
+def _parse_exa_search_text(text: str, limit: int) -> List[Dict[str, Any]]:
+    """Parse Exa's formatted search text into result dicts.
+
+    The payload is blocks separated by ``---`` lines, each shaped like::
+
+        Title: <title>
+        URL: <url>
+        Published: ...
+        Author: ...
+        Highlights:
+        <free text>
+    """
+    results: List[Dict[str, Any]] = []
+    for block in text.split("\n---\n"):
+        title = ""
+        url = ""
+        highlight_lines: List[str] = []
+        in_highlights = False
+        for line in block.splitlines():
+            stripped = line.strip()
+            if stripped.startswith("Title:"):
+                title = stripped[len("Title:"):].strip()
+                in_highlights = False
+            elif stripped.startswith("URL:"):
+                url = stripped[len("URL:"):].strip()
+                in_highlights = False
+            elif stripped.startswith("Highlights:"):
+                in_highlights = True
+            elif stripped.startswith(("Published:", "Author:")):
+                in_highlights = False
+            elif in_highlights and stripped:
+                highlight_lines.append(stripped)
+        if url:
+            results.append(
+                {
+                    "url": url,
+                    "title": title,
+                    "description": " ".join(highlight_lines),
+                    "position": len(results) + 1,
+                }
+            )
+        if limit and len(results) >= limit:
+            break
+    return results
+
+
+def exa_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]:
+    """Keyless Exa web search → legacy search response shape."""
+    try:
+        text = mcp_call(
+            EXA_MCP_URL,
+            "web_search_exa",
+            {"query": query, "numResults": max(1, int(limit))},
+        )
+    except KeylessMCPError as exc:
+        return {
+            "success": False,
+            "error": (
+                f"Keyless Exa search failed: {exc}. "
+                "Set EXA_API_KEY (https://exa.ai) or another web backend "
+                "via `hermes tools` for reliable service."
+            ),
+        }
+    return {"success": True, "data": {"web": _parse_exa_search_text(text, limit)}}
+
+
+def exa_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]:
+    """Keyless Exa web fetch → legacy extract result list.
+
+    ``web_fetch_exa`` takes a ``urls`` array but returns one combined text
+    payload; we call it per-URL so each result maps cleanly.
+    """
+    results: List[Dict[str, Any]] = []
+    for url in urls:
+        try:
+            text = mcp_call(EXA_MCP_URL, "web_fetch_exa", {"urls": [url]})
+        except KeylessMCPError as exc:
+            results.append(
+                {
+                    "url": url,
+                    "title": "",
+                    "content": "",
+                    "error": (
+                        f"Keyless Exa extract failed: {exc}. "
+                        "Set EXA_API_KEY (https://exa.ai) or another web "
+                        "backend via `hermes tools` for reliable service."
+                    ),
+                }
+            )
+            continue
+        title = ""
+        for line in text.splitlines():
+            stripped = line.strip()
+            if stripped.startswith("# "):
+                title = stripped[2:].strip()
+                break
+            if stripped.startswith("Title:"):
+                title = stripped[len("Title:"):].strip()
+                break
+        results.append(
+            {
+                "url": url,
+                "title": title,
+                "content": text,
+                "raw_content": text,
+                "metadata": {"sourceURL": url, "title": title},
+            }
+        )
+    return results
+
+
+
+# ---------------------------------------------------------------------------
+# Firecrawl keyless (public cloud API, no auth header)
+# ---------------------------------------------------------------------------
+
+
+def firecrawl_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]:
+    """Keyless Firecrawl cloud search → legacy search response shape."""
+    from plugins.web.firecrawl.provider import (
+        _KeylessFirecrawlClient,
+        _extract_web_search_results,
+    )
+
+    try:
+        response = _KeylessFirecrawlClient().search(query=query, limit=limit)
+        return {"success": True, "data": {"web": _extract_web_search_results(response)}}
+    except Exception as exc:  # noqa: BLE001 — normalized below
+        return {
+            "success": False,
+            "error": (
+                f"Keyless Firecrawl search failed: {exc}. "
+                "Set FIRECRAWL_API_KEY (https://firecrawl.dev) or another web "
+                "backend via `hermes tools` for reliable service."
+            ),
+        }
+
+
+def firecrawl_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]:
+    """Keyless Firecrawl cloud scrape → legacy extract result list."""
+    from plugins.web.firecrawl.provider import (
+        _KeylessFirecrawlClient,
+        _extract_scrape_payload,
+    )
+
+    client = _KeylessFirecrawlClient()
+    results: List[Dict[str, Any]] = []
+    for url in urls:
+        try:
+            response = client.scrape(url=url, formats=["markdown"])
+            payload = _extract_scrape_payload(response) or {}
+            metadata = payload.get("metadata") or {}
+            if not isinstance(metadata, dict):
+                metadata = {}
+            content = payload.get("markdown") or payload.get("html") or ""
+            title = metadata.get("title") or ""
+            results.append(
+                {
+                    "url": url,
+                    "title": title,
+                    "content": content,
+                    "raw_content": content,
+                    "metadata": {"sourceURL": url, "title": title},
+                }
+            )
+        except Exception as exc:  # noqa: BLE001 — per-URL error entry
+            results.append(
+                {
+                    "url": url,
+                    "title": "",
+                    "content": "",
+                    "error": (
+                        f"Keyless Firecrawl extract failed: {exc}. "
+                        "Set FIRECRAWL_API_KEY (https://firecrawl.dev) for "
+                        "reliable service."
+                    ),
+                }
+            )
+    return results
+
+
+# ---------------------------------------------------------------------------
+# Keenable keyless (api.keenable.ai public endpoints)
+# ---------------------------------------------------------------------------
+
+
+KEENABLE_API_URL = "https://api.keenable.ai"
+_KEENABLE_TITLE = "hermes-agent"
+
+
+def keenable_search_keyless(query: str, limit: int = 5) -> Dict[str, Any]:
+    """Keyless Keenable search → legacy search response shape.
+
+    POST /v1/search/public with the mandatory X-Keenable-Title app
+    identifier (their keyless tier requires an app name; no user
+    identifiers are sent). Response: {results: [{title, url, snippet}]}.
+    """
+    import requests
+
+    try:
+        response = requests.post(
+            f"{KEENABLE_API_URL}/v1/search/public",
+            json={"query": query, "max_results": max(1, int(limit))},
+            headers={
+                "Content-Type": "application/json",
+                "X-Keenable-Title": _KEENABLE_TITLE,
+            },
+            timeout=_TIMEOUT_SECONDS,
+        )
+        if response.status_code >= 400:
+            raise KeylessMCPError(
+                (response.text or "").strip() or f"HTTP {response.status_code}"
+            )
+        data = response.json()
+    except KeylessMCPError as exc:
+        return {
+            "success": False,
+            "error": (
+                f"Keyless Keenable search failed: {exc}. "
+                "Set KEENABLE_API_KEY (https://keenable.ai) or another web "
+                "backend via `hermes tools` for reliable service."
+            ),
+        }
+    except Exception as exc:  # noqa: BLE001 — transport/JSON errors
+        return {
+            "success": False,
+            "error": f"Keyless Keenable search failed: {exc}.",
+        }
+    web_results = []
+    for i, result in enumerate(data.get("results") or []):
+        web_results.append(
+            {
+                "url": result.get("url") or "",
+                "title": result.get("title") or "",
+                "description": result.get("snippet")
+                or result.get("description")
+                or "",
+                "position": i + 1,
+            }
+        )
+    return {"success": True, "data": {"web": web_results}}
+
+
+def keenable_extract_keyless(urls: List[str]) -> List[Dict[str, Any]]:
+    """Keyless Keenable page fetch → legacy extract result list.
+
+    GET /v1/fetch/public?url=... returns {url, title, content} (markdown).
+    Called per-URL; failures become per-URL error entries.
+    """
+    import requests
+
+    results: List[Dict[str, Any]] = []
+    for url in urls:
+        try:
+            response = requests.get(
+                f"{KEENABLE_API_URL}/v1/fetch/public",
+                params={"url": url},
+                headers={"X-Keenable-Title": _KEENABLE_TITLE},
+                timeout=_TIMEOUT_SECONDS,
+            )
+            if response.status_code >= 400:
+                raise KeylessMCPError(
+                    (response.text or "").strip() or f"HTTP {response.status_code}"
+                )
+            data = response.json()
+            content = data.get("content") or ""
+            title = data.get("title") or ""
+            results.append(
+                {
+                    "url": data.get("url") or url,
+                    "title": title,
+                    "content": content,
+                    "raw_content": content,
+                    "metadata": {"sourceURL": url, "title": title},
+                }
+            )
+        except Exception as exc:  # noqa: BLE001 — per-URL error entry
+            results.append(
+                {
+                    "url": url,
+                    "title": "",
+                    "content": "",
+                    "error": (
+                        f"Keyless Keenable extract failed: {exc}. "
+                        "Set KEENABLE_API_KEY (https://keenable.ai) for "
+                        "reliable service."
+                    ),
+                }
+            )
+    return results
+
+
+# ---------------------------------------------------------------------------
+# Round-robin ring + next-in-line failover (rate-limited free tiers)
+# ---------------------------------------------------------------------------
+
+_KEYLESS_RING = ("exa", "parallel", "firecrawl", "keenable")
+
+_KEYLESS_SEARCHERS = {
+    "exa": lambda query, limit: exa_search_keyless(query, limit),
+    "parallel": lambda query, limit: parallel_search_keyless(query, limit),
+    "firecrawl": lambda query, limit: firecrawl_search_keyless(query, limit),
+    "keenable": lambda query, limit: keenable_search_keyless(query, limit),
+}
+
+_KEYLESS_EXTRACTORS = {
+    "exa": lambda urls: exa_extract_keyless(urls),
+    "parallel": lambda urls: parallel_extract_keyless(urls),
+    "firecrawl": lambda urls: firecrawl_extract_keyless(urls),
+    "keenable": lambda urls: keenable_extract_keyless(urls),
+}
+
+# Per-process round-robin cursor, seeded by the random session id so the
+# fleet spreads evenly across all five free tiers; advances once per
+# unpinned keyless request so a single process also rotates.
+_ring_lock = __import__("threading").Lock()
+_ring_cursor = int(_SESSION_ID, 16) % len(_KEYLESS_RING)
+
+
+def _vendor_pinned(name: str) -> bool:
+    """True when config explicitly routes web traffic to *name*.
+
+    A pinned vendor starts every keyless request (rotation off); the ring
+    is only walked past it on throttle. Pin signals: web.backend /
+    web.search_backend / web.extract_backend naming the vendor, or a
+    free-tier pin in web.provider_tier.
+    """
+    if provider_tier(name) == "free":
+        return True
+    try:
+        import tools.web_tools as _wt
+
+        web_cfg = _wt._load_web_config()
+        return any(
+            (web_cfg.get(key) or "").lower().strip() == name
+            for key in ("backend", "search_backend", "extract_backend")
+        )
+    except Exception as exc:  # noqa: BLE001 — config layer optional
+        logger.debug("_vendor_pinned(%r) config read failed: %s", name, exc)
+        return False
+
+
+def _ring_order(name: str) -> List[str]:
+    """Return the vendor walk order for a request entering via *name*.
+
+    Pinned vendor → start at it (its position in the ring determines the
+    failover succession). Unpinned → true round-robin: start at the next
+    cursor position, advancing the cursor per request. Vendors whose tier
+    is pinned ``paid`` are excluded entirely (an explicit paid selection
+    opts that vendor's free endpoint out).
+    """
+    global _ring_cursor
+    if _vendor_pinned(name):
+        start = _KEYLESS_RING.index(name) if name in _KEYLESS_RING else 0
+    else:
+        with _ring_lock:
+            start = _ring_cursor
+            _ring_cursor = (_ring_cursor + 1) % len(_KEYLESS_RING)
+    ordered = [
+        _KEYLESS_RING[(start + i) % len(_KEYLESS_RING)]
+        for i in range(len(_KEYLESS_RING))
+    ]
+    return [v for v in ordered if provider_tier(v) != "paid"]
+
+
+def search_with_failover(name: str, query: str, limit: int = 5) -> Dict[str, Any]:
+    """Keyless search across the vendor ring with next-in-line failover.
+
+    Starts at *name* when the user pinned it, otherwise at the round-robin
+    cursor. Rate-limit-shaped errors advance to the next ring vendor;
+    non-throttle errors stop the walk (a malformed query fails everywhere).
+    The result notes the serving vendor via ``data.served_by`` whenever it
+    differs from *name*.
+    """
+    order = _ring_order(name)
+    if not order:
+        return {
+            "success": False,
+            "error": "All keyless web providers are pinned to paid tiers.",
+        }
+    last: Dict[str, Any] = {}
+    for i, vendor in enumerate(order):
+        result = _KEYLESS_SEARCHERS[vendor](query, limit)
+        if result.get("success"):
+            if vendor != name:
+                result.setdefault("data", {})["served_by"] = vendor
+            return result
+        last = result
+        if not _is_rate_limitish(result.get("error", "")):
+            return result
+        nxt = order[i + 1] if i + 1 < len(order) else None
+        if nxt:
+            logger.info(
+                "keyless %s search throttled; failing over to %s", vendor, nxt
+            )
+    last["error"] = (
+        f"{last.get('error', '')} (all keyless vendors throttled: "
+        f"{', '.join(order)})"
+    )
+    return last
+
+
+def extract_with_failover(name: str, urls: List[str]) -> List[Dict[str, Any]]:
+    """Keyless extract across the vendor ring, failing over per-batch.
+
+    Advances to the next ring vendor only when EVERY url in a batch comes
+    back with a rate-limit-shaped error — partial failures are page
+    problems, not throttling, and return as-is.
+    """
+    order = _ring_order(name)
+    if not order:
+        return [
+            {"url": u, "title": "", "content": "",
+             "error": "All keyless web providers are pinned to paid tiers."}
+            for u in urls
+        ]
+    last: List[Dict[str, Any]] = []
+    for i, vendor in enumerate(order):
+        results = _KEYLESS_EXTRACTORS[vendor](list(urls))
+        errors = [r.get("error", "") for r in results]
+        all_throttled = bool(results) and all(
+            e and _is_rate_limitish(e) for e in errors
+        )
+        if not all_throttled:
+            return results
+        last = results
+        nxt = order[i + 1] if i + 1 < len(order) else None
+        if nxt:
+            logger.info(
+                "keyless %s extract throttled; failing over to %s", vendor, nxt
+            )
+    return last

plugins/web/parallel/__init__.py

@@ -0,0 +1,16 @@
+"""Parallel.ai web search + extract plugin — bundled, auto-loaded.
+
+First plugin in this repo to expose an async :meth:`extract` — Parallel's
+SDK is async-native (``AsyncParallel.beta.extract``). The web_extract_tool
+dispatcher detects coroutines via :func:`inspect.iscoroutinefunction` and
+awaits.
+"""
+
+from __future__ import annotations
+
+from plugins.web.parallel.provider import ParallelWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the Parallel provider with the plugin context."""
+    ctx.register_web_search_provider(ParallelWebSearchProvider())

plugins/web/parallel/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-parallel
+version: 1.0.0
+description: "Parallel.ai web search + content extraction. Search returns objective-tuned results; extract uses the async SDK for parallel page fetches. Requires PARALLEL_API_KEY — sign up at https://parallel.ai."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - parallel

plugins/web/parallel/provider.py

@@ -0,0 +1,353 @@
+"""Parallel.ai web search + content extraction — plugin form.
+
+Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Uses two
+distinct Parallel SDK clients:
+
+- ``Parallel`` (sync)        — for :meth:`search`
+- ``AsyncParallel`` (async)  — for :meth:`extract`
+
+This is the first plugin to exercise the **async-extract** code path in
+the ABC: :meth:`extract` is declared ``async def``, and the dispatcher
+in :func:`tools.web_tools.web_extract_tool` detects coroutines via
+:func:`inspect.iscoroutinefunction` and awaits.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "parallel"      # explicit per-capability
+      extract_backend: "parallel"     # explicit per-capability
+      backend: "parallel"             # shared fallback
+      # Optional: search mode (default "agentic"; also "fast" or "one-shot")
+      # via the PARALLEL_SEARCH_MODE env var.
+
+Env vars::
+
+    PARALLEL_API_KEY=...             # https://parallel.ai (required)
+    PARALLEL_SEARCH_MODE=agentic     # optional: agentic|fast|one-shot
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Dict, List
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+# Module-level note: the canonical cache slots ``_parallel_client`` and
+# ``_async_parallel_client`` live on :mod:`tools.web_tools` so tests that do
+# ``tools.web_tools._parallel_client = None`` between cases see fresh state.
+# The plugin reads/writes through that public module (see
+# :func:`_get_sync_client` / :func:`_get_async_client`).
+
+
+def _ensure_parallel_sdk_installed() -> None:
+    """Trigger lazy install of the parallel SDK if it isn't present.
+
+    Mirrors the lazy-deps pattern used by the legacy implementation.
+    Swallows benign ImportError from the lazy_deps helper itself; if the
+    SDK is genuinely missing the subsequent ``from parallel import ...``
+    raises ImportError that the caller can handle.
+    """
+    try:
+        from tools.lazy_deps import ensure as _lazy_ensure
+
+        _lazy_ensure("search.parallel", prompt=False)
+    except ImportError:
+        pass
+    except Exception as exc:  # noqa: BLE001 — surface install hint as ImportError
+        raise ImportError(str(exc))
+
+
+def _get_sync_client() -> Any:
+    """Lazy-load + cache the sync Parallel client.
+
+    Cache lives on :mod:`tools.web_tools` (as ``_parallel_client``) so unit
+    tests that reset that name between cases keep working.
+    """
+    import tools.web_tools as _wt
+
+    cached = getattr(_wt, "_parallel_client", None)
+    if cached is not None:
+        return cached
+
+    from agent.web_search_provider import get_provider_env
+
+    api_key = get_provider_env("PARALLEL_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "PARALLEL_API_KEY environment variable not set. "
+            "Get your API key at https://parallel.ai"
+        )
+
+    _ensure_parallel_sdk_installed()
+    from parallel import Parallel  # noqa: WPS433 — deliberately lazy
+
+    client = Parallel(api_key=api_key)
+    _wt._parallel_client = client
+    return client
+
+
+def _get_async_client() -> Any:
+    """Lazy-load + cache the async Parallel client.
+
+    Cache lives on :mod:`tools.web_tools` (as ``_async_parallel_client``).
+    """
+    import tools.web_tools as _wt
+
+    cached = getattr(_wt, "_async_parallel_client", None)
+    if cached is not None:
+        return cached
+
+    from agent.web_search_provider import get_provider_env
+
+    api_key = get_provider_env("PARALLEL_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "PARALLEL_API_KEY environment variable not set. "
+            "Get your API key at https://parallel.ai"
+        )
+
+    _ensure_parallel_sdk_installed()
+    from parallel import AsyncParallel  # noqa: WPS433 — deliberately lazy
+
+    client = AsyncParallel(api_key=api_key)
+    _wt._async_parallel_client = client
+    return client
+
+
+def _reset_clients_for_tests() -> None:
+    """Drop both cached clients so tests can re-instantiate cleanly.
+
+    Clears the canonical slots on :mod:`tools.web_tools` (where
+    :func:`_get_sync_client` / :func:`_get_async_client` read/write them).
+    """
+    import tools.web_tools as _wt
+
+    _wt._parallel_client = None
+    _wt._async_parallel_client = None
+
+
+# Backward-compatible aliases for the names that lived in tools.web_tools
+# before the migration (matches existing tests + external callers).
+_get_parallel_client = _get_sync_client
+_get_async_parallel_client = _get_async_client
+
+
+def _resolve_search_mode() -> str:
+    """Return the validated PARALLEL_SEARCH_MODE value (default "agentic")."""
+    mode = os.getenv("PARALLEL_SEARCH_MODE", "agentic").lower().strip()
+    if mode not in {"fast", "one-shot", "agentic"}:
+        mode = "agentic"
+    return mode
+
+
+class ParallelWebSearchProvider(WebSearchProvider):
+    """Parallel.ai search + async extract provider."""
+
+    @property
+    def name(self) -> str:
+        return "parallel"
+
+    @property
+    def display_name(self) -> str:
+        return "Parallel"
+
+    def is_available(self) -> bool:
+        """Return True when ``PARALLEL_API_KEY`` is set to a non-empty value.
+
+        Deliberately does NOT consider the keyless free tier — that would
+        let the legacy preference walk route keyed users of lower-priority
+        backends onto Parallel's anonymous tier. Keyless availability is a
+        separate, last-resort signal (:meth:`is_keyless_available`).
+        """
+        from agent.web_search_provider import get_provider_env
+
+        return bool(get_provider_env("PARALLEL_API_KEY"))
+
+    def is_keyless_available(self) -> bool:
+        """Parallel serves anonymous free-tier calls via its public MCP endpoint.
+
+        False when the user forced ``web.provider_tier.parallel: paid`` —
+        an explicit paid selection must never silently resolve keyless.
+        """
+        from plugins.web.keyless_mcp import keyless_enabled, provider_tier
+
+        return keyless_enabled() and provider_tier("parallel") != "paid"
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return True
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a Parallel search (sync).
+
+        Uses the ``beta.search`` endpoint with the configured mode
+        (``PARALLEL_SEARCH_MODE`` env var, default "agentic"). Limit is
+        capped at 20 server-side.
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return {"success": False, "error": "Interrupted"}
+
+            from agent.web_search_provider import get_provider_env
+
+            from plugins.web.keyless_mcp import search_with_failover, use_keyless
+
+            if use_keyless("parallel", get_provider_env("PARALLEL_API_KEY")):
+                # Keyless free tier — public MCP endpoint, no SDK needed.
+                logger.info(
+                    "Parallel keyless search: '%s' (limit=%d)", query, limit
+                )
+                return search_with_failover("parallel", query, limit)
+
+            mode = _resolve_search_mode()
+            logger.info(
+                "Parallel search: '%s' (mode=%s, limit=%d)", query, mode, limit
+            )
+            response = _get_sync_client().beta.search(
+                search_queries=[query],
+                objective=query,
+                mode=mode,
+                max_results=min(limit, 20),
+            )
+
+            web_results = []
+            for i, result in enumerate(response.results or []):
+                excerpts = result.excerpts or []
+                web_results.append(
+                    {
+                        "url": result.url or "",
+                        "title": result.title or "",
+                        "description": " ".join(excerpts) if excerpts else "",
+                        "position": i + 1,
+                    }
+                )
+
+            return {"success": True, "data": {"web": web_results}}
+        except ValueError as exc:
+            return {"success": False, "error": str(exc)}
+        except ImportError as exc:
+            return {
+                "success": False,
+                "error": f"Parallel SDK not installed: {exc}",
+            }
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Parallel search error: %s", exc)
+            return {"success": False, "error": f"Parallel search failed: {exc}"}
+
+    async def extract(
+        self, urls: List[str], **kwargs: Any
+    ) -> List[Dict[str, Any]]:
+        """Extract content from one or more URLs via the async SDK.
+
+        Returns the legacy list-of-results shape that
+        :func:`tools.web_tools.web_extract_tool` expects: one entry per
+        successful URL plus one entry per failed URL with an ``error``
+        field. Errors are not raised — they're returned as per-URL items.
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return [
+                    {"url": u, "error": "Interrupted", "title": ""} for u in urls
+                ]
+
+            from agent.web_search_provider import get_provider_env
+
+            from plugins.web.keyless_mcp import extract_with_failover, use_keyless
+
+            if use_keyless("parallel", get_provider_env("PARALLEL_API_KEY")):
+                # Keyless free tier — blocking HTTP, so hop off the loop.
+                import asyncio
+
+                logger.info("Parallel keyless extract: %d URL(s)", len(urls))
+                return await asyncio.to_thread(
+                    extract_with_failover, "parallel", list(urls)
+                )
+
+            logger.info("Parallel extract: %d URL(s)", len(urls))
+            response = await _get_async_client().beta.extract(
+                urls=urls,
+                full_content=True,
+            )
+
+            results: List[Dict[str, Any]] = []
+            for result in response.results or []:
+                content = result.full_content or ""
+                if not content:
+                    content = "\n\n".join(result.excerpts or [])
+                url = result.url or ""
+                title = result.title or ""
+                results.append(
+                    {
+                        "url": url,
+                        "title": title,
+                        "content": content,
+                        "raw_content": content,
+                        "metadata": {"sourceURL": url, "title": title},
+                    }
+                )
+
+            for error in response.errors or []:
+                results.append(
+                    {
+                        "url": error.url or "",
+                        "title": "",
+                        "content": "",
+                        "error": error.content or error.error_type or "extraction failed",
+                        "metadata": {"sourceURL": error.url or ""},
+                    }
+                )
+
+            return results
+        except ValueError as exc:
+            return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls]
+        except ImportError as exc:
+            return [
+                {"url": u, "title": "", "content": "", "error": f"Parallel SDK not installed: {exc}"}
+                for u in urls
+            ]
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Parallel extract error: %s", exc)
+            return [
+                {"url": u, "title": "", "content": "", "error": f"Parallel extract failed: {exc}"}
+                for u in urls
+            ]
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "Parallel · Free (keyless)",
+            "badge": "free · no key",
+            "tag": (
+                "Objective-tuned search + page extraction on Parallel's "
+                "anonymous free tier. Rate-limited under burst load."
+            ),
+            "env_vars": [],
+            "web_tier": "free",
+            "variants": [
+                {
+                    "name": "Parallel · Paid (API key)",
+                    "badge": "paid",
+                    "tag": (
+                        "Objective-tuned search + parallel page extraction "
+                        "via the Parallel SDK. Unthrottled, guaranteed service."
+                    ),
+                    "env_vars": [
+                        {
+                            "key": "PARALLEL_API_KEY",
+                            "prompt": "Parallel API key",
+                            "url": "https://parallel.ai",
+                        },
+                    ],
+                    "web_tier": "paid",
+                },
+            ],
+        }

plugins/web/searxng/__init__.py

@@ -0,0 +1,15 @@
+"""SearXNG search plugin — bundled, auto-loaded.
+
+Backed by a user-hosted SearXNG instance (URL configured via ``SEARXNG_URL``).
+Search-only — pair with an extract provider (firecrawl/tavily/exa) for
+``web_extract`` calls.
+"""
+
+from __future__ import annotations
+
+from plugins.web.searxng.provider import SearXNGWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the SearXNG provider with the plugin context."""
+    ctx.register_web_search_provider(SearXNGWebSearchProvider())

plugins/web/searxng/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-searxng
+version: 1.0.0
+description: "SearXNG web search — free, self-hosted, privacy-respecting metasearch engine. Requires SEARXNG_URL pointing at your instance."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - searxng

plugins/web/searxng/provider.py

@@ -0,0 +1,153 @@
+"""SearXNG search — plugin form.
+
+Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Same JSON
+API call (``/search?format=json``), same result normalization. The legacy
+in-tree module ``tools.web_providers.searxng`` was removed in the same
+commit that moved this code under ``plugins/``; this file is now the
+canonical implementation.
+
+Search-only — SearXNG aggregates results from upstream engines but does not
+fetch/extract arbitrary URLs. ``supports_extract()`` returns False.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "searxng"     # explicit per-capability
+      backend: "searxng"            # shared fallback
+
+Env var::
+
+    SEARXNG_URL=http://localhost:8080
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Dict
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+
+def _searxng_url() -> str:
+    """Return SEARXNG_URL from Hermes config-aware env, falling back to process env."""
+    try:
+        from hermes_cli.config import get_env_value
+
+        val = get_env_value("SEARXNG_URL")
+    except Exception:
+        val = None
+    if val is None:
+        val = os.getenv("SEARXNG_URL", "")
+    return (val or "").strip()
+
+
+class SearXNGWebSearchProvider(WebSearchProvider):
+    """Search via a user-hosted SearXNG instance."""
+
+    @property
+    def name(self) -> str:
+        return "searxng"
+
+    @property
+    def display_name(self) -> str:
+        return "SearXNG"
+
+    def is_available(self) -> bool:
+        """Return True when ``SEARXNG_URL`` is set."""
+        return bool(_searxng_url())
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return False
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a search against the configured SearXNG instance."""
+        import httpx
+
+        base_url = _searxng_url().rstrip("/")
+        if not base_url:
+            return {"success": False, "error": "SEARXNG_URL is not set"}
+
+        params: Dict[str, Any] = {
+            "q": query,
+            "format": "json",
+            "pageno": 1,
+        }
+
+        try:
+            resp = httpx.get(
+                f"{base_url}/search",
+                params=params,
+                timeout=15,
+                headers={"Accept": "application/json"},
+            )
+            resp.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            logger.warning("SearXNG HTTP error: %s", exc)
+            return {
+                "success": False,
+                "error": f"SearXNG returned HTTP {exc.response.status_code}",
+            }
+        except httpx.RequestError as exc:
+            logger.warning("SearXNG request error: %s", exc)
+            return {
+                "success": False,
+                "error": f"Could not reach SearXNG at {base_url}: {exc}",
+            }
+
+        try:
+            data = resp.json()
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("SearXNG response parse error: %s", exc)
+            return {
+                "success": False,
+                "error": "Could not parse SearXNG response as JSON",
+            }
+
+        raw_results = data.get("results", [])
+
+        # SearXNG may return a score field; sort descending and cap to limit.
+        sorted_results = sorted(
+            raw_results,
+            key=lambda r: float(r.get("score", 0)),
+            reverse=True,
+        )[:limit]
+
+        web_results = [
+            {
+                "title": str(r.get("title", "")),
+                "url": str(r.get("url", "")),
+                "description": str(r.get("content", "")),
+                "position": i + 1,
+            }
+            for i, r in enumerate(sorted_results)
+        ]
+
+        logger.info(
+            "SearXNG search '%s': %d results (from %d raw, limit %d)",
+            query,
+            len(web_results),
+            len(raw_results),
+            limit,
+        )
+
+        return {"success": True, "data": {"web": web_results}}
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "SearXNG",
+            "badge": "free · self-hosted",
+            "tag": "Free, privacy-respecting metasearch. Point SEARXNG_URL at your instance.",
+            "env_vars": [
+                {
+                    "key": "SEARXNG_URL",
+                    "prompt": "SearXNG instance URL (e.g. http://localhost:8080)",
+                    "url": "https://searx.space/",
+                },
+            ],
+        }

plugins/web/tavily/__init__.py

@@ -0,0 +1,10 @@
+"""Tavily web search + extract plugin — bundled, auto-loaded."""
+
+from __future__ import annotations
+
+from plugins.web.tavily.provider import TavilyWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the Tavily provider with the plugin context."""
+    ctx.register_web_search_provider(TavilyWebSearchProvider())

plugins/web/tavily/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-tavily
+version: 1.0.0
+description: "Tavily web search + extract. Opt-in keyless via hermes tools; set TAVILY_API_KEY for higher limits — https://app.tavily.com/home."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - tavily

plugins/web/tavily/provider.py

@@ -0,0 +1,313 @@
+"""Tavily web search + content extraction — plugin form.
+
+Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Two
+capabilities advertised:
+
+- ``supports_search()``  -> True (Tavily ``/search``)
+- ``supports_extract()`` -> True (Tavily ``/extract``)
+
+Both are sync — the underlying call is ``httpx.post(...)``.
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "tavily"     # explicit per-capability
+      extract_backend: "tavily"    # explicit per-capability
+      backend: "tavily"            # shared fallback for both
+
+Env vars::
+
+    TAVILY_API_KEY=...           # https://app.tavily.com/home (optional)
+    TAVILY_BASE_URL=...          # optional override of https://api.tavily.com
+
+Auth is header-based. A key uses ``Authorization: Bearer``; without a
+key the request is keyless (``X-Tavily-Access-Mode: keyless``). Both
+paths send ``X-Client-Name: hermes-agent``.
+
+Tavily is **not** a member of the zero-config keyless ring
+(``plugins.web.keyless_mcp._KEYLESS_RING``). Keyless access is opt-in:
+select Tavily in ``hermes tools`` (or set ``web.backend: tavily``).
+Fresh installs with no web credentials rotate across Exa / Parallel /
+Firecrawl / Keenable instead.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+import httpx
+
+from agent.web_search_provider import WebSearchProvider
+
+logger = logging.getLogger(__name__)
+
+_CLIENT_NAME = "hermes-agent"
+
+_SEARCH_PAYLOAD = {
+    "include_raw_content": False,
+    "include_images": False,
+}
+
+
+def _tavily_headers(api_key: str) -> Dict[str, str]:
+    """Build Tavily request headers for keyed or keyless access."""
+    headers = {"X-Client-Name": _CLIENT_NAME}
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    else:
+        headers["X-Tavily-Access-Mode"] = "keyless"
+    return headers
+
+
+def _tavily_request(
+    endpoint: str,
+    payload: Dict[str, Any],
+    *,
+    api_key: Optional[str] = None,
+) -> Dict[str, Any]:
+    """POST to the Tavily API and return the parsed JSON response.
+
+    Keyed when *api_key* (or ``TAVILY_API_KEY``) is set (Bearer auth);
+    otherwise keyless. Pass ``api_key=""`` to force the keyless header even
+    when a key is present (``web.provider_tier.tavily: free``). Non-2xx
+    responses raise ``ValueError`` with the response body so Tavily's
+    keyless rate-limit / upgrade text reaches the model.
+    """
+    from agent.web_search_provider import get_provider_env
+
+    if api_key is None:
+        api_key = get_provider_env("TAVILY_API_KEY")
+    base_url = get_provider_env("TAVILY_BASE_URL") or "https://api.tavily.com"
+    url = f"{base_url}/{endpoint.lstrip('/')}"
+    logger.info("Tavily %s request to %s", endpoint, url)
+
+    response = httpx.post(
+        url,
+        json=payload,
+        timeout=60,
+        headers=_tavily_headers(api_key),
+    )
+    if response.status_code >= 400:
+        body = (response.text or "").strip()
+        detail = body or f"HTTP {response.status_code}"
+        raise ValueError(detail)
+    return response.json()
+
+
+def _normalize_tavily_search_results(response: Dict[str, Any]) -> Dict[str, Any]:
+    """Map Tavily ``/search`` response to ``{success, data: {web: [...]}}``."""
+    web_results = []
+    for i, result in enumerate(response.get("results", [])):
+        web_results.append(
+            {
+                "title": result.get("title", ""),
+                "url": result.get("url", ""),
+                "description": result.get("content", ""),
+                "position": i + 1,
+            }
+        )
+    return {"success": True, "data": {"web": web_results}}
+
+
+def _normalize_tavily_documents(
+    response: Dict[str, Any], fallback_url: str = ""
+) -> List[Dict[str, Any]]:
+    """Map Tavily ``/extract`` response to standard documents.
+
+    Documents follow the legacy LLM post-processing shape::
+
+        {"url", "title", "content", "raw_content", "metadata"}
+
+    Failures (``failed_results``, ``failed_urls``) become result entries
+    with an ``error`` field rather than raising.
+    """
+    documents: List[Dict[str, Any]] = []
+    for result in response.get("results", []):
+        url = result.get("url", fallback_url)
+        raw = result.get("raw_content", "") or result.get("content", "")
+        documents.append(
+            {
+                "url": url,
+                "title": result.get("title", ""),
+                "content": raw,
+                "raw_content": raw,
+                "metadata": {"sourceURL": url, "title": result.get("title", "")},
+            }
+        )
+    for fail in response.get("failed_results", []):
+        documents.append(
+            {
+                "url": fail.get("url", fallback_url),
+                "title": "",
+                "content": "",
+                "raw_content": "",
+                "error": fail.get("error", "extraction failed"),
+                "metadata": {"sourceURL": fail.get("url", fallback_url)},
+            }
+        )
+    for fail_url in response.get("failed_urls", []):
+        url_str = fail_url if isinstance(fail_url, str) else str(fail_url)
+        documents.append(
+            {
+                "url": url_str,
+                "title": "",
+                "content": "",
+                "raw_content": "",
+                "error": "extraction failed",
+                "metadata": {"sourceURL": url_str},
+            }
+        )
+    return documents
+
+
+def _missing_key_error(action: str) -> str:
+    return (
+        f"TAVILY_API_KEY is not set. Get a key at https://app.tavily.com/home "
+        f"or select Tavily in `hermes tools` for opt-in keyless {action}."
+    )
+
+
+class TavilyWebSearchProvider(WebSearchProvider):
+    """Tavily search + extract provider (keyed, or opt-in keyless)."""
+
+    @property
+    def name(self) -> str:
+        return "tavily"
+
+    @property
+    def display_name(self) -> str:
+        return "Tavily"
+
+    def is_available(self) -> bool:
+        """Return True when ``TAVILY_API_KEY`` is set to a non-empty value."""
+        from agent.web_search_provider import get_provider_env
+
+        return bool(get_provider_env("TAVILY_API_KEY"))
+
+    def is_keyless_available(self) -> bool:
+        """Tavily serves anonymous keyless requests (X-Tavily-Access-Mode).
+
+        Opt-in only — Tavily is not a member of the zero-config keyless
+        ring. ``is_keyless_available`` is True so an explicit
+        ``web.backend: tavily`` (or ``hermes tools`` pick) works without a
+        key. False when the user pinned ``web.provider_tier.tavily: paid``.
+        """
+        from plugins.web.keyless_mcp import keyless_enabled, provider_tier
+
+        return keyless_enabled() and provider_tier("tavily") != "paid"
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return True
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a Tavily search (keyed path or opt-in keyless)."""
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return {"success": False, "error": "Interrupted"}
+
+            from agent.web_search_provider import get_provider_env
+
+            from plugins.web.keyless_mcp import use_keyless
+
+            api_key = get_provider_env("TAVILY_API_KEY")
+            force_keyless = use_keyless("tavily", api_key)
+            if not force_keyless and not api_key:
+                return {"success": False, "error": _missing_key_error("search")}
+
+            logger.info(
+                "Tavily %ssearch: '%s' (limit=%d)",
+                "keyless " if force_keyless else "",
+                query,
+                limit,
+            )
+            raw = _tavily_request(
+                "search",
+                {
+                    "query": query,
+                    "max_results": min(limit, 20),
+                    **_SEARCH_PAYLOAD,
+                },
+                api_key="" if force_keyless else api_key,
+            )
+            return _normalize_tavily_search_results(raw)
+        except ValueError as exc:
+            return {"success": False, "error": str(exc)}
+        except Exception as exc:  # noqa: BLE001 — including httpx errors
+            logger.warning("Tavily search error: %s", exc)
+            return {"success": False, "error": f"Tavily search failed: {exc}"}
+
+    def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]:
+        """Extract content from one or more URLs via Tavily.
+
+        Sync — the underlying call is httpx.post(...). Returns the legacy
+        list-of-results shape; per-URL failures become items with ``error``.
+        Keyless uses Tavily's own endpoint, not the keyless ring.
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return [
+                    {"url": u, "error": "Interrupted", "title": ""} for u in urls
+                ]
+
+            from agent.web_search_provider import get_provider_env
+
+            from plugins.web.keyless_mcp import use_keyless
+
+            api_key = get_provider_env("TAVILY_API_KEY")
+            force_keyless = use_keyless("tavily", api_key)
+            if not force_keyless and not api_key:
+                err = _missing_key_error("extract")
+                return [
+                    {"url": u, "title": "", "content": "", "error": err}
+                    for u in urls
+                ]
+
+            logger.info(
+                "Tavily %sextract: %d URL(s)",
+                "keyless " if force_keyless else "",
+                len(urls),
+            )
+            raw = _tavily_request(
+                "extract",
+                {
+                    "urls": urls,
+                    "include_images": False,
+                },
+                api_key="" if force_keyless else api_key,
+            )
+            return _normalize_tavily_documents(
+                raw, fallback_url=urls[0] if urls else ""
+            )
+        except ValueError as exc:
+            return [{"url": u, "title": "", "content": "", "error": str(exc)} for u in urls]
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Tavily extract error: %s", exc)
+            return [
+                {"url": u, "title": "", "content": "", "error": f"Tavily extract failed: {exc}"}
+                for u in urls
+            ]
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        return {
+            "name": "Tavily",
+            "badge": "free · key optional",
+            "tag": (
+                "Search + extract. Opt-in keyless; "
+                "set TAVILY_API_KEY for higher limits."
+            ),
+            "env_vars": [
+                {
+                    "key": "TAVILY_API_KEY",
+                    "prompt": "Tavily API key (optional — keyless works when Tavily is selected)",
+                    "url": "https://app.tavily.com/home",
+                },
+            ],
+        }

plugins/web/xai/__init__.py

@@ -0,0 +1,14 @@
+"""xAI web search plugin — bundled, auto-loaded.
+
+Mirrors the ``plugins/web/brave_free/`` layout: ``provider.py`` holds the
+provider class, ``__init__.py::register(ctx)`` registers an instance.
+"""
+
+from __future__ import annotations
+
+from plugins.web.xai.provider import XAIWebSearchProvider
+
+
+def register(ctx) -> None:
+    """Register the xAI Web Search provider with the plugin context."""
+    ctx.register_web_search_provider(XAIWebSearchProvider())

plugins/web/xai/plugin.yaml

@@ -0,0 +1,7 @@
+name: web-xai
+version: 1.0.0
+description: "xAI Web Search — search the web via Grok's agentic web_search tool (Responses API). Requires xAI Grok OAuth (via `hermes auth`) or XAI_API_KEY (https://x.ai)."
+author: NousResearch
+kind: backend
+provides_web_providers:
+  - xai

plugins/web/xai/provider.py

@@ -0,0 +1,560 @@
+"""xAI Web Search — plugin form.
+
+Routes ``web_search`` tool calls through xAI's agentic Web Search tool
+(server-side ``web_search`` on the Responses API). Grok runs the actual
+searching and page-browsing server-side; we ask it to return the top
+results as structured JSON so we can hand back the same
+``{title, url, description, position}`` rows every other Hermes web
+provider produces.
+
+Reference: https://docs.x.ai/developers/tools/web-search
+
+Config keys this provider responds to::
+
+    web:
+      search_backend: "xai"           # explicit per-capability
+      backend: "xai"                  # shared fallback
+
+Optional knobs (under ``web.xai`` in ``config.yaml``)::
+
+    web:
+      xai:
+        model: "grok-build-0.1"       # reasoning model required by web_search
+        allowed_domains: ["x.ai"]     # max 5 — mutually exclusive with excluded_domains
+        excluded_domains: ["bad.com"] # max 5 — mutually exclusive with allowed_domains
+        timeout: 90                   # seconds (default 90)
+
+Auth: reuses :func:`tools.xai_http.resolve_xai_http_credentials`, which
+prefers Hermes-managed xAI Grok OAuth (via ``hermes auth``) and falls back
+to ``XAI_API_KEY`` (resolved through ``~/.hermes/.env``, then
+``os.environ``).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any, Dict, List, Optional
+
+from agent.web_search_provider import WebSearchProvider
+from tools.xai_http import (
+    has_xai_credentials,
+    hermes_xai_user_agent,
+    resolve_xai_http_credentials,
+)
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_MODEL = "grok-build-0.1"
+DEFAULT_TIMEOUT = 90
+_MAX_DOMAIN_FILTERS = 5  # xAI hard cap on allowed_domains / excluded_domains
+
+# Match the JSON object Grok is asked to emit. Tolerates leading/trailing
+# prose since reasoning models occasionally narrate before the JSON block
+# even when explicitly asked not to.
+_JSON_BLOCK_RE = re.compile(r"\{[\s\S]*\}", re.MULTILINE)
+
+
+# ---------------------------------------------------------------------------
+# Config
+# ---------------------------------------------------------------------------
+
+
+def _load_xai_web_config() -> Dict[str, Any]:
+    """Read ``web.xai`` from config.yaml (returns {} on miss)."""
+    try:
+        from hermes_cli.config import load_config
+
+        cfg = load_config()
+        web_section = cfg.get("web") if isinstance(cfg, dict) else None
+        xai_section = web_section.get("xai") if isinstance(web_section, dict) else None
+        return xai_section if isinstance(xai_section, dict) else {}
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("Could not load web.xai config: %s", exc)
+        return {}
+
+
+def _coerce_domain_list(value: Any) -> List[str]:
+    """Coerce a config value to a clean list of <=5 domain strings."""
+    if not isinstance(value, list):
+        return []
+    cleaned: List[str] = []
+    for item in value:
+        if isinstance(item, str) and item.strip():
+            cleaned.append(item.strip())
+        if len(cleaned) >= _MAX_DOMAIN_FILTERS:
+            break
+    return cleaned
+
+
+# ---------------------------------------------------------------------------
+# Provider
+# ---------------------------------------------------------------------------
+
+
+class XAIWebSearchProvider(WebSearchProvider):
+    """Search-only provider backed by xAI's agentic Web Search tool.
+
+    Sends a structured prompt to Grok with ``tools=[{"type": "web_search"}]``
+    enabled and asks it to return the top *limit* results as JSON. Falls
+    back to the Responses API ``citations`` list if Grok ignores the JSON
+    schema instruction (rare for grok-4.3 but cheap insurance).
+
+    No extract capability — pair with Firecrawl / Tavily / Exa for
+    ``web_extract`` if you need page content.
+
+    Trust model
+    -----------
+    Unlike index-backed providers (Brave / Tavily / Exa) which return
+    verbatim search-engine results, this backend is an LLM in a trench
+    coat: Grok decides which URLs to surface, generates the titles and
+    descriptions itself, and is influenced by the *content of the query*.
+    A maliciously crafted query (e.g. injected via untrusted upstream
+    input the agent picked up) can in principle steer Grok into emitting
+    attacker-chosen URLs. Callers that pipe untrusted text directly into
+    ``web_search`` should treat returned URLs the same way they would
+    treat any model-generated link — validate before fetching.
+    """
+
+    @property
+    def name(self) -> str:
+        return "xai"
+
+    @property
+    def display_name(self) -> str:
+        return "xAI Web Search (Grok)"
+
+    def is_available(self) -> bool:
+        """Cheap availability probe — env var OR auth-store has OAuth tokens.
+
+        Delegates to :func:`tools.xai_http.has_xai_credentials`, which is
+        deliberately *not* the same as :func:`resolve_xai_http_credentials`:
+        it never triggers OAuth token refresh or acquires the auth-store
+        lock. The ABC contract requires this method to be safe to call on
+        every ``hermes tools`` repaint and at tool-registration time.
+        Token freshness / refresh is handled inside :meth:`search`.
+        """
+        return has_xai_credentials()
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return False
+
+    # -- Search -----------------------------------------------------------
+
+    def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Execute a Grok-backed web search.
+
+        Returns ``{"success": True, "data": {"web": [{title, url, description, position}, ...]}}``
+        on success, ``{"success": False, "error": str}`` on failure.
+        """
+        try:
+            from tools.interrupt import is_interrupted
+
+            if is_interrupted():
+                return {"success": False, "error": "Interrupted"}
+        except Exception:  # noqa: BLE001 — interrupt module is best-effort
+            pass
+
+        creds = resolve_xai_http_credentials()
+        api_key = str(creds.get("api_key") or "").strip()
+        base_url = str(creds.get("base_url") or "https://api.x.ai/v1").strip().rstrip("/")
+        if not api_key:
+            return {
+                "success": False,
+                "error": (
+                    "No xAI credentials found. Run `hermes auth` to sign in with "
+                    "xAI Grok OAuth, or set XAI_API_KEY."
+                ),
+            }
+
+        # Clamp limit to the same range the caller (web_search_tool) accepts,
+        # so we don't silently downgrade explicit limits. Grok happily
+        # produces longer lists; cost scales linearly with the requested
+        # count via reasoning tokens, but that's the caller's call to make.
+        try:
+            limit = int(limit)
+        except (TypeError, ValueError):
+            limit = 5
+        limit = max(1, min(limit, 100))
+
+        cfg = _load_xai_web_config()
+        model = cfg.get("model") if isinstance(cfg.get("model"), str) else DEFAULT_MODEL
+        model = model.strip() or DEFAULT_MODEL
+
+        try:
+            timeout = float(cfg.get("timeout", DEFAULT_TIMEOUT))
+        except (TypeError, ValueError):
+            timeout = DEFAULT_TIMEOUT
+
+        allowed = _coerce_domain_list(cfg.get("allowed_domains"))
+        excluded = _coerce_domain_list(cfg.get("excluded_domains"))
+        if allowed and excluded:
+            # xAI explicitly rejects this combo — surface a clear error
+            # rather than a 400 from the API.
+            return {
+                "success": False,
+                "error": (
+                    "web.xai.allowed_domains and web.xai.excluded_domains "
+                    "cannot both be set (xAI restriction)."
+                ),
+            }
+
+        web_search_tool: Dict[str, Any] = {"type": "web_search"}
+        if allowed:
+            web_search_tool["filters"] = {"allowed_domains": allowed}
+        elif excluded:
+            web_search_tool["filters"] = {"excluded_domains": excluded}
+
+        prompt = self._build_prompt(query, limit)
+
+        payload: Dict[str, Any] = {
+            "model": model,
+            "input": [{"role": "user", "content": prompt}],
+            "tools": [web_search_tool],
+            # Drop inline citation markdown — we want the JSON block clean,
+            # and we read URLs from annotations / citations separately.
+            "include": ["no_inline_citations"],
+        }
+
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+            "User-Agent": hermes_xai_user_agent(),
+        }
+
+        try:
+            import httpx
+        except ImportError:
+            return {
+                "success": False,
+                "error": "httpx is not installed (required for xAI web search)",
+            }
+
+        logger.info(
+            "xAI web search via %s: '%s' (limit=%d, model=%s)",
+            base_url, query, limit, model,
+        )
+
+        # Two-attempt loop: if the first call returns 401 and our creds came
+        # from the OAuth path, force-refresh the token once and retry. This
+        # closes two gaps the proactive resolver check doesn't cover:
+        # (1) opaque (non-JWT) access tokens — `_xai_access_token_is_expiring`
+        #     can't decode them and returns False, so refresh never fires
+        #     until the server hands us a 401.
+        # (2) mid-window revocation — admin revoke, refresh-token rotation,
+        #     or clock skew can produce 401s on a token whose JWT `exp` claim
+        #     is still in the future.
+        # Env-var (`XAI_API_KEY`) credentials skip the retry entirely — we
+        # can't refresh those and an immediate retry would just burn quota.
+        is_oauth_path = (creds.get("provider") == "xai-oauth")
+        resp = None
+        for attempt in range(2):
+            try:
+                resp = httpx.post(
+                    f"{base_url}/responses",
+                    headers=headers,
+                    json=payload,
+                    timeout=timeout,
+                )
+                resp.raise_for_status()
+                break
+            except httpx.HTTPStatusError as exc:
+                status = exc.response.status_code if exc.response is not None else 0
+                if status == 401 and attempt == 0 and is_oauth_path:
+                    logger.info(
+                        "xAI web search got 401 on first attempt; forcing OAuth "
+                        "refresh and retrying once.",
+                    )
+                    try:
+                        refreshed = resolve_xai_http_credentials(
+                            force_refresh=True,
+                            api_key_hint=api_key,
+                        )
+                        refreshed_key = str(refreshed.get("api_key") or "").strip()
+                        if refreshed_key and refreshed_key != api_key:
+                            api_key = refreshed_key
+                            headers["Authorization"] = f"Bearer {api_key}"
+                            continue
+                        # Refresh returned the same (or empty) token — no point
+                        # in retrying. Fall through to the error return below.
+                    except Exception as refresh_exc:  # noqa: BLE001
+                        logger.warning(
+                            "xAI web search OAuth refresh after 401 failed: %s",
+                            refresh_exc,
+                        )
+                body = ""
+                try:
+                    body = exc.response.text[:300] if exc.response is not None else ""
+                except Exception:
+                    body = ""
+                logger.warning("xAI web search HTTP %d: %s", status, body)
+                return {
+                    "success": False,
+                    "error": f"xAI web search returned HTTP {status}: {body}".rstrip(),
+                }
+            except httpx.RequestError as exc:
+                logger.warning("xAI web search request error: %s", exc)
+                return {"success": False, "error": f"Could not reach xAI: {exc}"}
+
+        if resp is None:
+            # Defensive — both attempts somehow exited the loop without resp.
+            return {"success": False, "error": "xAI web search produced no response"}
+
+        try:
+            data = resp.json()
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("xAI web search bad JSON: %s", exc)
+            return {
+                "success": False,
+                "error": "Could not parse xAI Responses API reply as JSON",
+            }
+
+        # xAI's Responses surface sometimes returns HTTP 200 with an error
+        # envelope (model overloaded, content-policy refusal, etc.). Without
+        # this check, ``_extract_results`` would silently produce an empty
+        # list and we'd report success-with-no-rows — masking a real failure
+        # the agent should see and decide whether to retry.
+        api_error = data.get("error") if isinstance(data, dict) else None
+        if isinstance(api_error, dict):
+            err_msg = (
+                api_error.get("message")
+                or api_error.get("code")
+                or "unknown error"
+            )
+            logger.warning("xAI web search returned error envelope: %s", err_msg)
+            return {"success": False, "error": f"xAI returned an error: {err_msg}"}
+
+        web_results = self._extract_results(data, limit=limit)
+        if not web_results:
+            # Successful call, just no usable rows — return success with an
+            # empty list so the model can decide whether to retry. Matches
+            # what brave-free / exa do when the upstream API returns 0 hits.
+            return {"success": True, "data": {"web": []}}
+
+        return {"success": True, "data": {"web": web_results}}
+
+    # -- Prompt + parsing -------------------------------------------------
+
+    @staticmethod
+    def _build_prompt(query: str, limit: int) -> str:
+        """Compose the prompt that asks Grok to act as a search engine.
+
+        We deliberately ask for a JSON object (not bare array) so we can
+        match it cheaply with ``_JSON_BLOCK_RE``; we explicitly forbid
+        prose, markdown fences, and inline-citation links to keep the
+        payload parseable.
+        """
+        return (
+            "Use the web_search tool to find current information for the query below, "
+            "then respond with ONLY a single JSON object — no prose, no markdown "
+            "fences, no inline citation links — matching this exact schema:\n\n"
+            '{"results": [{"title": "string", "url": "string", '
+            '"description": "1-2 sentence summary"}]}\n\n'
+            f'Return at most {limit} results, ordered by relevance, with absolute '
+            "https:// URLs. If no usable results exist, return "
+            '{"results": []}.\n\n'
+            f"Query: {query}"
+        )
+
+    @classmethod
+    def _extract_results(
+        cls,
+        response_data: Dict[str, Any],
+        *,
+        limit: int,
+    ) -> List[Dict[str, Any]]:
+        """Pull a ``[{title, url, description, position}, ...]`` list out of a
+        Responses-API reply.
+
+        Strategy:
+
+        1. Walk ``output[*].content[*].text`` for ``output_text`` blocks and
+           try to parse the first JSON object that has a ``results`` list.
+        2. If the JSON path fails, fall back to the message annotations
+           (``url_citation`` entries) — every annotation carries a URL and
+           a ``title`` (citation number); we pair those URLs with surrounding
+           text from the message body as a best-effort description.
+        """
+        text_blocks, annotations = cls._collect_output_text(response_data)
+
+        # Primary path: parse the JSON object Grok was asked for.
+        for block in text_blocks:
+            parsed = cls._try_parse_json_results(block, limit=limit)
+            if parsed:
+                return parsed
+
+        # Secondary path: derive results from message annotations + raw text.
+        # Only short-circuit when annotations actually yielded usable rows;
+        # otherwise fall through to the citations list. (xAI currently only
+        # emits ``url_citation`` annotations, but future annotation types
+        # would silently produce an empty result set if we returned here
+        # unconditionally — masking real data in ``citations``.)
+        if annotations:
+            joined_text = "\n".join(text_blocks)
+            annotation_results = cls._results_from_annotations(
+                annotations, joined_text, limit=limit,
+            )
+            if annotation_results:
+                return annotation_results
+
+        # Last-ditch: raw citations list (no titles or descriptions).
+        citations = response_data.get("citations") or []
+        if isinstance(citations, list):
+            return [
+                {
+                    "title": "",
+                    "url": str(u),
+                    "description": "",
+                    "position": i + 1,
+                }
+                for i, u in enumerate(citations[:limit])
+                if isinstance(u, str) and u.strip()
+            ]
+
+        return []
+
+    @staticmethod
+    def _collect_output_text(
+        response_data: Dict[str, Any],
+    ) -> tuple[List[str], List[Dict[str, Any]]]:
+        """Return (text_blocks, annotations) extracted from ``response.output``."""
+        text_blocks: List[str] = []
+        annotations: List[Dict[str, Any]] = []
+        output = response_data.get("output")
+        if not isinstance(output, list):
+            return text_blocks, annotations
+
+        for item in output:
+            if not isinstance(item, dict) or item.get("type") != "message":
+                continue
+            content = item.get("content")
+            if not isinstance(content, list):
+                continue
+            for chunk in content:
+                if not isinstance(chunk, dict) or chunk.get("type") != "output_text":
+                    continue
+                text = chunk.get("text")
+                if isinstance(text, str) and text.strip():
+                    text_blocks.append(text)
+                chunk_annotations = chunk.get("annotations")
+                if isinstance(chunk_annotations, list):
+                    for ann in chunk_annotations:
+                        if isinstance(ann, dict):
+                            annotations.append(ann)
+        return text_blocks, annotations
+
+    @staticmethod
+    def _try_parse_json_results(
+        text: str,
+        *,
+        limit: int,
+    ) -> Optional[List[Dict[str, Any]]]:
+        """Parse a JSON object with a ``results`` array out of ``text``.
+
+        Returns the normalized result list on success, ``None`` when the
+        block has no valid JSON object or no ``results`` key. Tolerates
+        leading/trailing prose because reasoning models sometimes prefix a
+        short narration even when told not to.
+        """
+        # Try the whole string first — cheapest path when Grok obeys.
+        candidates = [text]
+        match = _JSON_BLOCK_RE.search(text)
+        if match and match.group(0) != text:
+            candidates.append(match.group(0))
+
+        for candidate in candidates:
+            try:
+                parsed = json.loads(candidate)
+            except (json.JSONDecodeError, ValueError):
+                continue
+            if not isinstance(parsed, dict):
+                continue
+            results = parsed.get("results")
+            if not isinstance(results, list):
+                continue
+            normalized: List[Dict[str, Any]] = []
+            for row in results[:limit]:
+                if not isinstance(row, dict):
+                    continue
+                url = str(row.get("url", "")).strip()
+                if not url:
+                    continue
+                normalized.append(
+                    {
+                        "title": str(row.get("title", "")).strip(),
+                        "url": url,
+                        "description": str(row.get("description", "")).strip(),
+                        # Renumber from the kept results, not the raw input
+                        # index, so a dropped malformed row doesn't leave a
+                        # gap in the positions handed back to the agent.
+                        "position": len(normalized) + 1,
+                    }
+                )
+            if normalized:
+                return normalized
+        return None
+
+    @staticmethod
+    def _results_from_annotations(
+        annotations: List[Dict[str, Any]],
+        joined_text: str,
+        *,
+        limit: int,
+    ) -> List[Dict[str, Any]]:
+        """Best-effort fallback when JSON parsing fails.
+
+        Uses each ``url_citation`` annotation's ``url`` (the citation
+        title is just the integer label, so we don't surface it) and
+        slices ~200 characters of surrounding text as the description.
+        """
+        seen: set[str] = set()
+        results: List[Dict[str, Any]] = []
+        for ann in annotations:
+            if ann.get("type") != "url_citation":
+                continue
+            url = str(ann.get("url", "")).strip()
+            if not url or url in seen:
+                continue
+            seen.add(url)
+
+            description = ""
+            start = ann.get("start_index")
+            end = ann.get("end_index")
+            if isinstance(start, int) and isinstance(end, int) and 0 <= start < end <= len(joined_text):
+                window_start = max(0, start - 200)
+                description = joined_text[window_start:start].strip()
+                if len(description) > 200:
+                    description = description[-200:].strip()
+
+            results.append(
+                {
+                    "title": "",
+                    "url": url,
+                    "description": description,
+                    "position": len(results) + 1,
+                }
+            )
+            if len(results) >= limit:
+                break
+        return results
+
+    # -- Setup picker -----------------------------------------------------
+
+    def get_setup_schema(self) -> Dict[str, Any]:
+        # Auth resolution is delegated to the shared ``xai_grok`` post_setup
+        # hook (same one image_gen.xai and tts.xai use) so users see the
+        # familiar OAuth-or-API-key prompt for every xAI service.
+        return {
+            "name": "xAI Web Search (Grok)",
+            "badge": "paid",
+            "tag": (
+                "Agentic web search via Grok's web_search tool — uses xAI "
+                "Grok OAuth or XAI_API_KEY."
+            ),
+            "env_vars": [],
+            "post_setup": "xai_grok",
+        }