Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,410 @@
|
||||
"""
|
||||
Web Search Provider Registry
|
||||
============================
|
||||
|
||||
Central map of registered web providers. Populated by plugins at import-time
|
||||
via :meth:`PluginContext.register_web_search_provider`; consumed by the
|
||||
``web_search`` and ``web_extract`` tool wrappers in :mod:`tools.web_tools` to
|
||||
dispatch each call to the active backend.
|
||||
|
||||
Active selection
|
||||
----------------
|
||||
The active provider is chosen by configuration with this precedence:
|
||||
|
||||
1. ``web.search_backend`` / ``web.extract_backend``
|
||||
(per-capability override).
|
||||
2. ``web.backend`` (shared fallback).
|
||||
3. If exactly one capability-eligible provider is registered AND available,
|
||||
use it.
|
||||
4. Legacy preference order — ``firecrawl`` → ``parallel`` → ``tavily`` →
|
||||
``exa`` → ``searxng`` → ``brave-free`` → ``ddgs`` — filtered by
|
||||
availability. Matches the historic ``tools.web_tools._get_backend()``
|
||||
candidate order so installs that never set a config key keep landing
|
||||
on the same provider they did before the plugin migration.
|
||||
5. Otherwise ``None`` — the tool surfaces a helpful error pointing at
|
||||
``hermes tools``.
|
||||
|
||||
The capability filter (``supports_search`` / ``supports_extract``) is
|
||||
applied at every step so a search-only provider (``brave-free``)
|
||||
configured as ``web.extract_backend`` correctly falls through to an
|
||||
extract-capable backend.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from agent.web_search_provider import WebSearchProvider
|
||||
from hermes_constants import hermes_home_key
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_providers: Dict[str, WebSearchProvider] = {}
|
||||
_scoped_providers: Dict[str, Dict[str, WebSearchProvider]] = {}
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def register_provider(provider: WebSearchProvider, *, scope: Optional[str] = None) -> None:
|
||||
"""Register a web search/extract provider.
|
||||
|
||||
Re-registration (same ``name``) overwrites the previous entry and logs
|
||||
a debug message — makes hot-reload scenarios (tests, dev loops) behave
|
||||
predictably.
|
||||
"""
|
||||
if not isinstance(provider, WebSearchProvider):
|
||||
raise TypeError(
|
||||
f"register_provider() expects a WebSearchProvider instance, "
|
||||
f"got {type(provider).__name__}"
|
||||
)
|
||||
raw_name = provider.name
|
||||
if not isinstance(raw_name, str) or not raw_name.strip():
|
||||
raise ValueError("Web provider .name must be a non-empty string")
|
||||
name = raw_name.strip()
|
||||
with _lock:
|
||||
target = _providers if scope is None else _scoped_providers.setdefault(scope, {})
|
||||
existing = target.get(name)
|
||||
target[name] = provider
|
||||
if existing is not None:
|
||||
logger.debug(
|
||||
"Web provider '%s' re-registered (was %r)",
|
||||
name, type(existing).__name__,
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"Registered web provider '%s' (%s)",
|
||||
name, type(provider).__name__,
|
||||
)
|
||||
|
||||
|
||||
def list_providers(*, scope: Optional[str] = None) -> List[WebSearchProvider]:
|
||||
"""Return all registered providers, sorted by name."""
|
||||
with _lock:
|
||||
merged = dict(_providers)
|
||||
merged.update(_scoped_providers.get(scope or hermes_home_key(), {}))
|
||||
items = list(merged.values())
|
||||
return sorted(items, key=lambda p: p.name)
|
||||
|
||||
|
||||
def get_provider(name: str, *, scope: Optional[str] = None) -> Optional[WebSearchProvider]:
|
||||
"""Return the provider registered under *name*, or None."""
|
||||
if not isinstance(name, str):
|
||||
return None
|
||||
with _lock:
|
||||
key = name.strip()
|
||||
return _scoped_providers.get(scope or hermes_home_key(), {}).get(key) or _providers.get(key)
|
||||
|
||||
|
||||
def snapshot_registration(
|
||||
name: str, *, scope: Optional[str] = None
|
||||
) -> Optional[WebSearchProvider]:
|
||||
with _lock:
|
||||
target = _providers if scope is None else _scoped_providers.get(scope, {})
|
||||
return target.get(name.strip())
|
||||
|
||||
|
||||
def restore_registration(
|
||||
name: str,
|
||||
current: WebSearchProvider,
|
||||
previous: Optional[WebSearchProvider],
|
||||
*,
|
||||
scope: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Restore a plugin registration only when *current* is still installed."""
|
||||
key = name.strip()
|
||||
with _lock:
|
||||
target = _providers if scope is None else _scoped_providers.setdefault(scope, {})
|
||||
if target.get(key) is not current:
|
||||
return False
|
||||
if previous is None:
|
||||
target.pop(key, None)
|
||||
else:
|
||||
target[key] = previous
|
||||
if scope is not None and not target:
|
||||
_scoped_providers.pop(scope, None)
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Active-provider resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _read_config_key(*path: str) -> Optional[str]:
|
||||
"""Resolve a dotted config key from ``config.yaml``. Returns None on miss."""
|
||||
try:
|
||||
from hermes_cli.config import load_config_readonly
|
||||
|
||||
cfg = load_config_readonly()
|
||||
cur = cfg
|
||||
for segment in path:
|
||||
if not isinstance(cur, dict):
|
||||
return None
|
||||
cur = cur.get(segment)
|
||||
if isinstance(cur, str) and cur.strip():
|
||||
return cur.strip()
|
||||
except Exception as exc:
|
||||
logger.debug("Could not read config %s: %s", ".".join(path), exc)
|
||||
return None
|
||||
|
||||
|
||||
# Legacy preference order — preserves behaviour for users who set no
|
||||
# ``web.backend`` / ``web.<capability>_backend`` config key at all. Matches
|
||||
# the historic candidate order in :func:`tools.web_tools._get_backend`
|
||||
# (paid providers first so existing paid setups don't get downgraded to
|
||||
# a free tier on upgrade). Filtered by ``is_available()`` at walk time so
|
||||
# we don't surface a provider the user has no credentials for.
|
||||
_LEGACY_PREFERENCE = (
|
||||
"firecrawl",
|
||||
"parallel",
|
||||
"tavily",
|
||||
"exa",
|
||||
"searxng",
|
||||
"brave-free",
|
||||
"ddgs",
|
||||
)
|
||||
|
||||
# Keyless free-tier walk — strictly LAST-resort, tried only after the
|
||||
# availability-filtered legacy walk finds nothing (i.e. the user has zero
|
||||
# web credentials and no importable ddgs). Ring vendors expose public
|
||||
# anonymous free tiers (see plugins/web/keyless_mcp.py). Unpinned keyless
|
||||
# traffic round-robins across the ring per request (the ring cursor lives
|
||||
# in keyless_mcp; an explicit `hermes tools` pick bypasses this walk
|
||||
# entirely, and rate-limited requests fail over to the next ring vendor).
|
||||
# Disable the tier with ``web.keyless_fallback: false``.
|
||||
_KEYLESS_PREFERENCE = (
|
||||
"exa",
|
||||
"parallel",
|
||||
"firecrawl",
|
||||
"keenable",
|
||||
)
|
||||
|
||||
|
||||
def _keyless_preference() -> tuple:
|
||||
"""Return the keyless walk order for resolution.
|
||||
|
||||
Delegates the entry-vendor choice to the ring cursor in
|
||||
:mod:`plugins.web.keyless_mcp` (round-robin per request, seeded by the
|
||||
per-process random session id) so resolution and dispatch agree on
|
||||
which vendor a fresh install starts at. The remaining vendors follow
|
||||
in ring order as fallbacks for registration gaps.
|
||||
"""
|
||||
try:
|
||||
from plugins.web.keyless_mcp import _KEYLESS_RING, _ring_cursor
|
||||
|
||||
start = _ring_cursor % len(_KEYLESS_RING)
|
||||
return tuple(
|
||||
_KEYLESS_RING[(start + i) % len(_KEYLESS_RING)]
|
||||
for i in range(len(_KEYLESS_RING))
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — ring optional in stripped envs
|
||||
logger.debug("keyless ring order unavailable: %s", exc)
|
||||
return _KEYLESS_PREFERENCE
|
||||
|
||||
|
||||
def _resolve(configured: Optional[str], *, capability: str) -> Optional[WebSearchProvider]:
|
||||
"""Resolve the active provider for a capability ("search" | "extract").
|
||||
|
||||
Resolution rules (in order):
|
||||
|
||||
1. **Explicit config wins, ignoring availability.** If
|
||||
``web.{capability}_backend`` or ``web.backend`` names a registered
|
||||
provider that supports *capability*, return it even if its
|
||||
:meth:`is_available` returns False — the dispatcher will surface a
|
||||
precise "X_API_KEY is not set" error to the user instead of silently
|
||||
routing somewhere else. Matches legacy
|
||||
:func:`tools.web_tools._get_backend` behavior for configured names.
|
||||
|
||||
2. **Single-provider shortcut.** When only one registered provider
|
||||
supports *capability* AND ``is_available()`` reports True, return it.
|
||||
|
||||
3. **Legacy preference walk, filtered by availability.** Walk the
|
||||
:data:`_LEGACY_PREFERENCE` order (firecrawl → parallel → tavily →
|
||||
exa → searxng → brave-free → ddgs) looking for a provider whose
|
||||
``supports_<capability>()`` is True AND whose ``is_available()`` is
|
||||
True. Matches the historic ``tools.web_tools._get_backend()``
|
||||
candidate order so users with credentials but no explicit config
|
||||
key keep landing on the same provider as pre-migration. This is
|
||||
the path that fires when no config key is set — pick the
|
||||
highest-priority backend the user actually has credentials for.
|
||||
|
||||
Returns None when no provider is configured AND no available provider
|
||||
matches the legacy preference; the dispatcher then returns a "set up a
|
||||
provider" error to the user.
|
||||
"""
|
||||
with _lock:
|
||||
snapshot = dict(_providers)
|
||||
snapshot.update(_scoped_providers.get(hermes_home_key(), {}))
|
||||
|
||||
def _capable(p: WebSearchProvider) -> bool:
|
||||
if capability == "search":
|
||||
return bool(p.supports_search())
|
||||
if capability == "extract":
|
||||
return bool(p.supports_extract())
|
||||
return False
|
||||
|
||||
def _is_available_safe(p: WebSearchProvider) -> bool:
|
||||
"""Wrap ``is_available()`` so a buggy provider doesn't kill resolution."""
|
||||
try:
|
||||
return bool(p.is_available())
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("provider %s.is_available() raised %s", p.name, exc)
|
||||
return False
|
||||
|
||||
# 1. Explicit config wins — return regardless of is_available() so the
|
||||
# user gets a precise downstream error message rather than a silent
|
||||
# backend switch. Matches _get_backend() in web_tools.py.
|
||||
if configured:
|
||||
provider = snapshot.get(configured)
|
||||
if provider is not None and _capable(provider):
|
||||
return provider
|
||||
if provider is None:
|
||||
logger.debug(
|
||||
"web backend '%s' configured but not registered; falling back",
|
||||
configured,
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"web backend '%s' configured but does not support '%s'; falling back",
|
||||
configured, capability,
|
||||
)
|
||||
|
||||
# 2. + 3. Fallback path — filter by availability so we don't surface
|
||||
# a provider the user has no credentials for. Without this filter,
|
||||
# a registered-but-unconfigured provider could end up "active" on
|
||||
# a fresh install with no API keys at all.
|
||||
eligible = [
|
||||
p for p in snapshot.values()
|
||||
if _capable(p) and _is_available_safe(p)
|
||||
]
|
||||
if len(eligible) == 1:
|
||||
return eligible[0]
|
||||
|
||||
for legacy in _LEGACY_PREFERENCE:
|
||||
provider = snapshot.get(legacy)
|
||||
if (
|
||||
provider is not None
|
||||
and _capable(provider)
|
||||
and _is_available_safe(provider)
|
||||
):
|
||||
return provider
|
||||
|
||||
# 4. Keyless free-tier walk — the user has NO credentialed/importable
|
||||
# backend at all. Fall back to providers that can serve anonymously
|
||||
# (public MCP free tiers), unless disabled via
|
||||
# ``web.keyless_fallback: false``. This tier never pre-empts a keyed
|
||||
# setup: it is only reachable when the legacy walk found nothing.
|
||||
if _keyless_tier_enabled():
|
||||
for name in _keyless_preference():
|
||||
provider = snapshot.get(name)
|
||||
if provider is None or not _capable(provider):
|
||||
continue
|
||||
try:
|
||||
if provider.is_keyless_available():
|
||||
return provider
|
||||
except Exception as exc: # noqa: BLE001 — buggy provider skipped
|
||||
logger.debug(
|
||||
"provider %s.is_keyless_available() raised %s", name, exc
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _keyless_tier_enabled() -> bool:
|
||||
"""Read ``web.keyless_fallback`` from config.yaml (default: enabled)."""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
web_cfg = load_config().get("web") or {}
|
||||
return bool(web_cfg.get("keyless_fallback", True))
|
||||
except Exception as exc: # noqa: BLE001 — config layer optional
|
||||
logger.debug("keyless_fallback config read failed: %s", exc)
|
||||
return True
|
||||
|
||||
|
||||
def _disabled_web_plugin_for(configured: Optional[str] = None, *, capability: Optional[str] = None) -> Optional[str]:
|
||||
"""Return the plugin key of a *disabled* bundled web plugin that would
|
||||
have provided the configured backend, or None.
|
||||
|
||||
When a user sets ``web.extract_backend: firecrawl`` (or the search
|
||||
equivalent) but also lists ``web-firecrawl`` in ``plugins.disabled``,
|
||||
the provider never registers and the dispatcher would otherwise emit a
|
||||
misleading "No web extract provider configured. Set web.extract_backend
|
||||
to ..." error — even though the backend IS configured correctly. The
|
||||
real fix is to re-enable the plugin. This helper detects that case so
|
||||
the dispatcher can point the user at the actual cause (issue #40190
|
||||
follow-up: pi314's disabled-plugin symptom).
|
||||
|
||||
Pass ``capability`` ("search" | "extract") to resolve the configured
|
||||
name straight from ``config.yaml`` (``web.<capability>_backend`` →
|
||||
``web.backend``). This is more reliable than the resolved backend the
|
||||
dispatcher fell back to, since a disabled provider fails the
|
||||
``_is_backend_available`` gate and the dispatcher silently drops to
|
||||
the shared default. An explicit ``configured`` name still wins when
|
||||
given.
|
||||
|
||||
Matching is by convention: bundled web plugins live under the
|
||||
``web/<vendor>`` key with the provider ``name`` differing only in
|
||||
hyphen/underscore (``brave-free`` provider ⇄ ``web/brave_free`` key,
|
||||
``firecrawl`` ⇄ ``web/firecrawl``). We normalize both sides before
|
||||
comparing so every bundled provider is covered without hardcoding a
|
||||
per-vendor table.
|
||||
"""
|
||||
def _norm(s: str) -> str:
|
||||
return s.strip().lower().replace("-", "_")
|
||||
|
||||
if not configured and capability in ("search", "extract"):
|
||||
configured = (
|
||||
_read_config_key("web", f"{capability}_backend")
|
||||
or _read_config_key("web", "backend")
|
||||
)
|
||||
if not configured:
|
||||
return None
|
||||
|
||||
want = _norm(configured)
|
||||
try:
|
||||
from hermes_cli.plugins import get_plugin_manager
|
||||
|
||||
pm = get_plugin_manager()
|
||||
for key, loaded in pm._plugins.items():
|
||||
if not isinstance(key, str) or not key.startswith("web/"):
|
||||
continue
|
||||
if loaded.enabled:
|
||||
continue
|
||||
if loaded.error != "disabled via config":
|
||||
continue
|
||||
vendor = key.split("/", 1)[1]
|
||||
if _norm(vendor) == want:
|
||||
return key
|
||||
except Exception as exc: # noqa: BLE001 — diagnostics are best-effort
|
||||
logger.debug("disabled-web-plugin lookup failed: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def get_active_search_provider() -> Optional[WebSearchProvider]:
|
||||
"""Resolve the currently-active web search provider.
|
||||
|
||||
Reads ``web.search_backend`` (preferred) or ``web.backend`` (shared
|
||||
fallback) from config.yaml; falls back per the module docstring.
|
||||
"""
|
||||
explicit = _read_config_key("web", "search_backend") or _read_config_key("web", "backend")
|
||||
return _resolve(explicit, capability="search")
|
||||
|
||||
|
||||
def get_active_extract_provider() -> Optional[WebSearchProvider]:
|
||||
"""Resolve the currently-active web extract provider.
|
||||
|
||||
Reads ``web.extract_backend`` (preferred) or ``web.backend`` (shared
|
||||
fallback) from config.yaml; falls back per the module docstring.
|
||||
"""
|
||||
explicit = _read_config_key("web", "extract_backend") or _read_config_key("web", "backend")
|
||||
return _resolve(explicit, capability="extract")
|
||||
|
||||
|
||||
def _reset_for_tests() -> None:
|
||||
"""Clear the registry. **Test-only.**"""
|
||||
with _lock:
|
||||
_providers.clear()
|
||||
_scoped_providers.clear()
|
||||
Reference in New Issue
Block a user