Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
"""Local OpenAI-compatible proxy that forwards to OAuth-authenticated upstreams.
|
||||
|
||||
Lets external apps (OpenViking, Karakeep, Open WebUI, ...) ride the user's
|
||||
already-logged-in provider subscription instead of needing a static API key
|
||||
copy-pasted into each app's config.
|
||||
|
||||
The proxy listens on ``127.0.0.1:<port>``, accepts any bearer (the client's
|
||||
``Authorization`` header is discarded), and attaches the user's real
|
||||
upstream credential to the forwarded request. The credential is refreshed
|
||||
automatically when it approaches expiry.
|
||||
|
||||
First-class adapter:
|
||||
- ``nous`` — Nous Portal (https://inference-api.nousresearch.com/v1)
|
||||
|
||||
Future adapters can plug in by implementing ``UpstreamAdapter``.
|
||||
"""
|
||||
|
||||
from hermes_cli.proxy.adapters.base import UpstreamAdapter
|
||||
|
||||
__all__ = ["UpstreamAdapter"]
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Upstream adapter registry for the local proxy server.
|
||||
|
||||
Each adapter wraps a provider's OAuth state and exposes a uniform interface
|
||||
the proxy server can use to forward requests with a freshly-minted bearer
|
||||
token. See :class:`UpstreamAdapter` for the contract.
|
||||
"""
|
||||
|
||||
from typing import Dict, Type
|
||||
|
||||
from hermes_cli.proxy.adapters.base import UpstreamAdapter
|
||||
from hermes_cli.proxy.adapters.nous_portal import NousPortalAdapter
|
||||
from hermes_cli.proxy.adapters.xai import XAIGrokAdapter
|
||||
|
||||
# Registry of available adapter classes keyed by provider name as used on
|
||||
# the ``hermes proxy start --provider <name>`` CLI flag.
|
||||
ADAPTERS: Dict[str, Type[UpstreamAdapter]] = {
|
||||
"nous": NousPortalAdapter,
|
||||
"xai": XAIGrokAdapter,
|
||||
}
|
||||
|
||||
|
||||
def get_adapter(name: str) -> UpstreamAdapter:
|
||||
"""Instantiate an adapter by provider name.
|
||||
|
||||
Raises:
|
||||
ValueError: if ``name`` is not a registered adapter.
|
||||
"""
|
||||
key = (name or "").strip().lower()
|
||||
if key not in ADAPTERS:
|
||||
available = ", ".join(sorted(ADAPTERS)) or "(none)"
|
||||
raise ValueError(
|
||||
f"Unknown proxy upstream provider: {name!r}. Available: {available}"
|
||||
)
|
||||
return ADAPTERS[key]()
|
||||
|
||||
|
||||
__all__ = ["UpstreamAdapter", "ADAPTERS", "get_adapter"]
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Abstract base for proxy upstream adapters.
|
||||
|
||||
An :class:`UpstreamAdapter` represents one OAuth-authenticated provider the
|
||||
local proxy can forward requests to. The adapter is responsible for:
|
||||
|
||||
- locating the user's auth state for that provider
|
||||
- refreshing/minting credentials when needed
|
||||
- reporting the resolved upstream base URL
|
||||
- declaring which request paths it accepts
|
||||
|
||||
The proxy server is otherwise provider-agnostic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import FrozenSet, Optional
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UpstreamCredential:
|
||||
"""A resolved bearer + base URL ready to forward to."""
|
||||
|
||||
bearer: str
|
||||
"""Authorization header value to send upstream (token only, no ``Bearer`` prefix)."""
|
||||
|
||||
base_url: str
|
||||
"""Upstream base URL, e.g. ``https://inference-api.nousresearch.com/v1``."""
|
||||
|
||||
token_type: str = "Bearer"
|
||||
"""Auth scheme — currently always ``Bearer`` for supported providers."""
|
||||
|
||||
expires_at: Optional[str] = None
|
||||
"""ISO-8601 expiry timestamp for the bearer, when known. Informational."""
|
||||
|
||||
|
||||
class UpstreamAdapter(ABC):
|
||||
"""Contract for an upstream provider the proxy can forward to."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
"""Adapter key used on the CLI (e.g. ``"nous"``)."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def display_name(self) -> str:
|
||||
"""Human-readable provider name for logs and ``proxy status``."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def allowed_paths(self) -> FrozenSet[str]:
|
||||
"""Set of relative request paths the upstream accepts.
|
||||
|
||||
Paths are relative to the proxy's ``/v1`` mount point. For example,
|
||||
``"/chat/completions"`` corresponds to a client request to
|
||||
``http://127.0.0.1:<port>/v1/chat/completions``. Requests to paths
|
||||
not in this set get a 404 with a helpful error body.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def is_authenticated(self) -> bool:
|
||||
"""Return True if the user has usable credentials for this upstream.
|
||||
|
||||
Should be cheap — no network calls. Used by ``proxy start`` for a
|
||||
clear up-front error before binding a port.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_credential(self) -> UpstreamCredential:
|
||||
"""Return a fresh credential, refreshing or rotating if necessary.
|
||||
|
||||
Implementations should:
|
||||
- refresh the access token if it's near expiry
|
||||
- rotate the upstream bearer key if it's near expiry
|
||||
- persist any refreshed state back to disk
|
||||
|
||||
Raises:
|
||||
RuntimeError: if the user isn't authenticated or the upstream
|
||||
refresh fails. The proxy will return 401 to the client.
|
||||
"""
|
||||
|
||||
def get_retry_credential(
|
||||
self,
|
||||
*,
|
||||
failed_credential: UpstreamCredential,
|
||||
status_code: int,
|
||||
) -> Optional[UpstreamCredential]:
|
||||
"""Return an alternate credential after an upstream auth failure.
|
||||
|
||||
The default is no retry. Providers can override this for one-shot
|
||||
fallback paths after the upstream rejects the first request.
|
||||
"""
|
||||
_ = failed_credential, status_code
|
||||
return None
|
||||
|
||||
def describe(self) -> str:
|
||||
"""One-line status summary for ``proxy status``."""
|
||||
try:
|
||||
cred = self.get_credential()
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
return f"{self.display_name}: not ready ({exc})"
|
||||
ttl = f" (expires {cred.expires_at})" if cred.expires_at else ""
|
||||
return f"{self.display_name}: {cred.base_url}{ttl}"
|
||||
|
||||
|
||||
__all__ = ["UpstreamAdapter", "UpstreamCredential"]
|
||||
@@ -0,0 +1,201 @@
|
||||
"""Nous Portal upstream adapter.
|
||||
|
||||
Reads the user's Nous OAuth state from ``~/.hermes/auth.json`` through the
|
||||
shared runtime resolver, validates or refreshes the inference JWT, then exposes
|
||||
the upstream base URL plus bearer for the proxy server to forward to.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from typing import Any, Dict, FrozenSet, Optional
|
||||
|
||||
from hermes_cli.auth import (
|
||||
AuthError,
|
||||
DEFAULT_NOUS_INFERENCE_URL,
|
||||
_load_auth_store,
|
||||
_auth_store_lock,
|
||||
_is_terminal_nous_refresh_error,
|
||||
_nous_inference_env_override,
|
||||
_quarantine_nous_oauth_state,
|
||||
_quarantine_nous_pool_entries,
|
||||
_save_auth_store,
|
||||
_validate_nous_inference_url_from_network,
|
||||
_write_shared_nous_state,
|
||||
resolve_nous_runtime_credentials,
|
||||
)
|
||||
from hermes_cli.proxy.adapters.base import UpstreamAdapter, UpstreamCredential
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Endpoints inference-api.nousresearch.com actually serves. Anything else
|
||||
# the proxy will reject with 404 — keeps stray clients from leaking weird
|
||||
# requests to the upstream.
|
||||
_ALLOWED_PATHS: FrozenSet[str] = frozenset(
|
||||
{
|
||||
"/chat/completions",
|
||||
"/completions",
|
||||
"/embeddings",
|
||||
"/models",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class NousPortalAdapter(UpstreamAdapter):
|
||||
"""Proxy upstream for the Nous Portal inference API."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Serialize proxy requests in this process; cross-process token refresh
|
||||
# and persistence are handled by resolve_nous_runtime_credentials().
|
||||
self._lock = threading.Lock()
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "nous"
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
return "Nous Portal"
|
||||
|
||||
@property
|
||||
def allowed_paths(self) -> FrozenSet[str]:
|
||||
return _ALLOWED_PATHS
|
||||
|
||||
def is_authenticated(self) -> bool:
|
||||
state = self._read_state()
|
||||
if state is None:
|
||||
return False
|
||||
# We need either a usable inference JWT OR (refresh_token + access_token)
|
||||
# to recover. The refresh helper validates and refreshes as needed.
|
||||
return bool(
|
||||
state.get("agent_key")
|
||||
or (state.get("refresh_token") and state.get("access_token"))
|
||||
)
|
||||
|
||||
def get_credential(self) -> UpstreamCredential:
|
||||
return self._get_credential()
|
||||
|
||||
def get_retry_credential(
|
||||
self,
|
||||
*,
|
||||
failed_credential: UpstreamCredential,
|
||||
status_code: int,
|
||||
) -> Optional[UpstreamCredential]:
|
||||
if status_code != 401:
|
||||
return None
|
||||
logger.info("proxy: Nous upstream rejected bearer; force-refreshing invoke JWT")
|
||||
return self._get_credential(
|
||||
force_refresh=True,
|
||||
stale_access_token=failed_credential.bearer,
|
||||
)
|
||||
|
||||
def _get_credential(
|
||||
self,
|
||||
*,
|
||||
force_refresh: bool = False,
|
||||
stale_access_token: Optional[str] = None,
|
||||
) -> UpstreamCredential:
|
||||
with self._lock:
|
||||
state = self._read_state()
|
||||
if state is None:
|
||||
raise RuntimeError(
|
||||
"Not logged into Nous Portal. Run `hermes auth add nous` first."
|
||||
)
|
||||
|
||||
try:
|
||||
refreshed = resolve_nous_runtime_credentials(
|
||||
force_refresh=force_refresh,
|
||||
stale_access_token=stale_access_token or None,
|
||||
)
|
||||
except AuthError as exc:
|
||||
if _is_terminal_nous_refresh_error(exc):
|
||||
_quarantine_nous_oauth_state(
|
||||
state,
|
||||
exc,
|
||||
reason="proxy_refresh_failure",
|
||||
)
|
||||
self._save_state(
|
||||
state,
|
||||
quarantine_error=exc,
|
||||
quarantine_reason="proxy_refresh_failure",
|
||||
)
|
||||
raise RuntimeError(
|
||||
f"Failed to refresh Nous Portal credentials: {exc}"
|
||||
) from exc
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"Failed to refresh Nous Portal credentials: {exc}"
|
||||
) from exc
|
||||
|
||||
runtime_key = refreshed.get("api_key")
|
||||
if not runtime_key:
|
||||
raise RuntimeError(
|
||||
"Nous Portal refresh did not return a usable inference JWT. "
|
||||
"Try `hermes auth add nous` to re-authenticate."
|
||||
)
|
||||
|
||||
# base_url returned by resolve_nous_runtime_credentials() already
|
||||
# honors the NOUS_INFERENCE_BASE_URL env override (the documented
|
||||
# dev/staging escape hatch). Re-validating it here against the prod
|
||||
# host allowlist would wrongly reject a legitimate staging override,
|
||||
# so layer the same env-first overlay on top of the network-validated
|
||||
# value: env override wins, else validate the returned URL, else
|
||||
# fall back to the production default (defense-in-depth for a future
|
||||
# source-layer bypass).
|
||||
base_url = (
|
||||
_nous_inference_env_override()
|
||||
or _validate_nous_inference_url_from_network(refreshed.get("base_url"))
|
||||
or DEFAULT_NOUS_INFERENCE_URL
|
||||
)
|
||||
base_url = base_url.rstrip("/")
|
||||
|
||||
return UpstreamCredential(
|
||||
bearer=runtime_key,
|
||||
base_url=base_url,
|
||||
expires_at=refreshed.get("expires_at"),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers — auth.json access. Kept local rather than added
|
||||
# to hermes_cli.auth to avoid expanding that module's public surface.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _read_state(self) -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
with _auth_store_lock():
|
||||
store = _load_auth_store()
|
||||
except Exception as exc:
|
||||
logger.warning("proxy: failed to load auth store: %s", exc)
|
||||
return None
|
||||
providers = store.get("providers") or {}
|
||||
state = providers.get("nous")
|
||||
if not isinstance(state, dict):
|
||||
return None
|
||||
return dict(state) # copy so the refresh helper can mutate freely
|
||||
|
||||
def _save_state(
|
||||
self,
|
||||
state: Dict[str, Any],
|
||||
*,
|
||||
quarantine_error: Optional[AuthError] = None,
|
||||
quarantine_reason: Optional[str] = None,
|
||||
) -> None:
|
||||
try:
|
||||
with _auth_store_lock():
|
||||
store = _load_auth_store()
|
||||
if quarantine_error is not None and quarantine_reason:
|
||||
_quarantine_nous_pool_entries(
|
||||
store,
|
||||
quarantine_error,
|
||||
reason=quarantine_reason,
|
||||
)
|
||||
providers = store.setdefault("providers", {})
|
||||
providers["nous"] = state
|
||||
_save_auth_store(store)
|
||||
_write_shared_nous_state(state)
|
||||
except Exception as exc:
|
||||
logger.warning("proxy: failed to persist Nous quarantine state: %s", exc)
|
||||
|
||||
|
||||
__all__ = ["NousPortalAdapter"]
|
||||
@@ -0,0 +1,145 @@
|
||||
"""xAI Grok OAuth upstream adapter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from typing import FrozenSet, Optional
|
||||
|
||||
from agent.credential_pool import CredentialPool, PooledCredential, load_pool
|
||||
from hermes_cli.auth import DEFAULT_XAI_OAUTH_BASE_URL
|
||||
from hermes_cli.proxy.adapters.base import UpstreamAdapter, UpstreamCredential
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_POOL_PROVIDER = "xai-oauth"
|
||||
|
||||
# xAI's public API is OpenAI-compatible for the endpoints Hermes commonly
|
||||
# uses. The Responses endpoint is included because Hermes' native xAI runtime
|
||||
# uses codex_responses mode.
|
||||
_ALLOWED_PATHS: FrozenSet[str] = frozenset(
|
||||
{
|
||||
"/responses",
|
||||
"/chat/completions",
|
||||
"/completions",
|
||||
"/embeddings",
|
||||
"/models",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class XAIGrokAdapter(UpstreamAdapter):
|
||||
"""Proxy upstream for xAI Grok via Hermes-managed OAuth credentials."""
|
||||
|
||||
auth_hint = "hermes auth add xai-oauth --type oauth"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._pool: Optional[CredentialPool] = None
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "xai"
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
return "xAI Grok OAuth"
|
||||
|
||||
@property
|
||||
def allowed_paths(self) -> FrozenSet[str]:
|
||||
return _ALLOWED_PATHS
|
||||
|
||||
def is_authenticated(self) -> bool:
|
||||
pool = self._load_pool()
|
||||
return bool(pool and pool.has_available())
|
||||
|
||||
def get_credential(self) -> UpstreamCredential:
|
||||
with self._lock:
|
||||
pool = self._load_pool()
|
||||
if pool is None or not pool.has_credentials():
|
||||
raise RuntimeError(
|
||||
"No xAI OAuth credentials found. Run "
|
||||
"`hermes auth add xai-oauth --type oauth` first."
|
||||
)
|
||||
|
||||
entry = pool.select()
|
||||
if entry is None:
|
||||
raise RuntimeError(
|
||||
"No available xAI OAuth credentials found. Run "
|
||||
"`hermes auth reset xai-oauth` or re-authenticate with "
|
||||
"`hermes auth add xai-oauth --type oauth`."
|
||||
)
|
||||
|
||||
self._pool = pool
|
||||
return self._credential_from_entry(entry)
|
||||
|
||||
def get_retry_credential(
|
||||
self,
|
||||
*,
|
||||
failed_credential: UpstreamCredential,
|
||||
status_code: int,
|
||||
) -> Optional[UpstreamCredential]:
|
||||
if status_code not in {401, 429}:
|
||||
return None
|
||||
|
||||
with self._lock:
|
||||
pool = self._pool or self._load_pool()
|
||||
if pool is None:
|
||||
return None
|
||||
|
||||
if status_code == 429:
|
||||
# Mark the rate-limited key with its 1-hour cooldown and rotate
|
||||
# to the next available credential. Returns None when the pool
|
||||
# has no other key to offer — the 429 will flow back to the client.
|
||||
refreshed = pool.mark_exhausted_and_rotate(status_code=status_code)
|
||||
else:
|
||||
refreshed = pool.try_refresh_current()
|
||||
if refreshed is None:
|
||||
refreshed = pool.mark_exhausted_and_rotate(status_code=status_code)
|
||||
if refreshed is None:
|
||||
return None
|
||||
|
||||
retry_cred = self._credential_from_entry(refreshed)
|
||||
if retry_cred.bearer == failed_credential.bearer:
|
||||
return None
|
||||
logger.info(
|
||||
"proxy: xAI upstream returned %s; retrying with rotated pool credential",
|
||||
status_code,
|
||||
)
|
||||
return retry_cred
|
||||
|
||||
def _load_pool(self) -> Optional[CredentialPool]:
|
||||
try:
|
||||
return load_pool(_POOL_PROVIDER)
|
||||
except Exception as exc:
|
||||
logger.warning("proxy: failed to load xAI OAuth credential pool: %s", exc)
|
||||
return None
|
||||
|
||||
def _credential_from_entry(self, entry: PooledCredential) -> UpstreamCredential:
|
||||
bearer = (
|
||||
getattr(entry, "runtime_api_key", None)
|
||||
or getattr(entry, "access_token", "")
|
||||
or ""
|
||||
)
|
||||
bearer = str(bearer).strip()
|
||||
if not bearer:
|
||||
raise RuntimeError(
|
||||
"xAI OAuth credential pool entry did not contain an access token. "
|
||||
"Re-authenticate with `hermes auth add xai-oauth --type oauth`."
|
||||
)
|
||||
|
||||
base_url = (
|
||||
getattr(entry, "runtime_base_url", None)
|
||||
or getattr(entry, "base_url", None)
|
||||
or DEFAULT_XAI_OAUTH_BASE_URL
|
||||
)
|
||||
base_url = str(base_url or DEFAULT_XAI_OAUTH_BASE_URL).strip().rstrip("/")
|
||||
|
||||
return UpstreamCredential(
|
||||
bearer=bearer,
|
||||
base_url=base_url or DEFAULT_XAI_OAUTH_BASE_URL,
|
||||
expires_at=getattr(entry, "expires_at", None),
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["XAIGrokAdapter"]
|
||||
@@ -0,0 +1,140 @@
|
||||
"""CLI handlers for the ``hermes proxy`` subcommand."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from hermes_cli.proxy.adapters import ADAPTERS, get_adapter
|
||||
from hermes_cli.proxy.server import (
|
||||
AIOHTTP_AVAILABLE,
|
||||
DEFAULT_HOST,
|
||||
DEFAULT_PORT,
|
||||
run_server,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _print_aiohttp_missing() -> None:
|
||||
print(
|
||||
"hermes proxy requires aiohttp. Run `hermes setup` to install it.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
|
||||
def cmd_proxy_start(args: Any) -> int:
|
||||
"""Run the proxy server in the foreground.
|
||||
|
||||
Returns process exit code (0 on clean shutdown).
|
||||
"""
|
||||
if not AIOHTTP_AVAILABLE:
|
||||
_print_aiohttp_missing()
|
||||
return 1
|
||||
|
||||
provider = getattr(args, "provider", None) or "nous"
|
||||
try:
|
||||
adapter = get_adapter(provider)
|
||||
except ValueError as exc:
|
||||
print(f"Error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
if not adapter.is_authenticated():
|
||||
auth_hint = getattr(adapter, "auth_hint", f"hermes auth add {adapter.name}")
|
||||
print(
|
||||
f"Not logged into {adapter.display_name}. "
|
||||
f"Run `{auth_hint}` first.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
host = getattr(args, "host", None) or DEFAULT_HOST
|
||||
port = getattr(args, "port", None) or DEFAULT_PORT
|
||||
|
||||
print(
|
||||
f"Starting Hermes proxy for {adapter.display_name}\n"
|
||||
f" Listening on: http://{host}:{port}/v1\n"
|
||||
f" Forwarding to: (resolved per-request from your subscription)\n"
|
||||
f" Use any bearer token in the client — the proxy attaches your real credential.\n"
|
||||
f"\n"
|
||||
f"Press Ctrl+C to stop.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
try:
|
||||
asyncio.run(run_server(adapter, host=host, port=port))
|
||||
except KeyboardInterrupt:
|
||||
print("\nproxy: stopped", file=sys.stderr)
|
||||
except OSError as exc:
|
||||
print(f"proxy: failed to bind {host}:{port}: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_proxy_status(args: Any) -> int:
|
||||
"""Print the status of each configured upstream adapter."""
|
||||
print("Hermes proxy upstream adapters\n")
|
||||
for name in sorted(ADAPTERS):
|
||||
adapter = get_adapter(name)
|
||||
if not adapter.is_authenticated():
|
||||
print(f" [{name:8s}] {adapter.display_name} — not logged in")
|
||||
continue
|
||||
try:
|
||||
cred = adapter.get_credential()
|
||||
except Exception as exc:
|
||||
print(
|
||||
f" [{name:8s}] {adapter.display_name} — credentials need attention "
|
||||
f"({exc})"
|
||||
)
|
||||
continue
|
||||
expires = f" (bearer expires {cred.expires_at})" if cred.expires_at else ""
|
||||
print(f" [{name:8s}] {adapter.display_name} — ready{expires}")
|
||||
print(
|
||||
"\nStart the proxy with: hermes proxy start [--provider <name>]"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_proxy_list_providers(args: Any) -> int:
|
||||
"""List available proxy upstream providers."""
|
||||
print("Available proxy upstream providers:")
|
||||
for name in sorted(ADAPTERS):
|
||||
adapter = get_adapter(name)
|
||||
print(f" {name} — {adapter.display_name}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_proxy(args: Any) -> int:
|
||||
"""Dispatch ``hermes proxy <subcommand>``."""
|
||||
sub = getattr(args, "proxy_command", None)
|
||||
if sub == "start":
|
||||
return cmd_proxy_start(args)
|
||||
if sub == "status":
|
||||
return cmd_proxy_status(args)
|
||||
if sub in {"providers", "list"}:
|
||||
return cmd_proxy_list_providers(args)
|
||||
# No subcommand → print short help.
|
||||
print(
|
||||
"hermes proxy — local OpenAI-compatible proxy that attaches your\n"
|
||||
"OAuth-authenticated provider credentials to outbound requests.\n"
|
||||
"\n"
|
||||
"Subcommands:\n"
|
||||
" hermes proxy start [--provider nous|xai] [--host 127.0.0.1] [--port 8645]\n"
|
||||
" Run the proxy in the foreground.\n"
|
||||
" hermes proxy status\n"
|
||||
" Show which upstream adapters are ready.\n"
|
||||
" hermes proxy providers\n"
|
||||
" List available upstream providers.\n",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
__all__ = [
|
||||
"cmd_proxy",
|
||||
"cmd_proxy_start",
|
||||
"cmd_proxy_status",
|
||||
"cmd_proxy_list_providers",
|
||||
]
|
||||
@@ -0,0 +1,346 @@
|
||||
"""HTTP server that forwards OpenAI-compatible requests to a configured upstream.
|
||||
|
||||
Listens on ``http://<host>:<port>/v1/<path>`` and forwards each request to
|
||||
``<upstream-base-url>/<path>`` with the client's ``Authorization`` header
|
||||
replaced by a freshly-resolved bearer from the configured adapter. The
|
||||
response body is streamed through unchanged (SSE deltas preserved).
|
||||
|
||||
One narrow SSE compatibility shim applies after a *clean* upstream EOF:
|
||||
when a ``text/event-stream`` response carries a terminal ``finish_reason``
|
||||
or ``lastOne: true`` but omits the OpenAI ``data: [DONE]`` sentinel, the
|
||||
proxy appends a single ``[DONE]`` frame. It never rewrites earlier frames,
|
||||
never duplicates an upstream ``[DONE]``, and never synthesizes ``[DONE]``
|
||||
after an error event or a mid-stream interrupt (see
|
||||
:mod:`hermes_cli.proxy.sse_done`, issue #90848).
|
||||
|
||||
Otherwise the server does not mediate, log, or rewrite request/response
|
||||
bodies — it is a credential-attaching forwarder.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import signal
|
||||
from typing import Optional
|
||||
|
||||
try:
|
||||
import aiohttp
|
||||
from aiohttp import web
|
||||
AIOHTTP_AVAILABLE = True
|
||||
except ImportError:
|
||||
aiohttp = None # type: ignore[assignment]
|
||||
web = None # type: ignore[assignment]
|
||||
AIOHTTP_AVAILABLE = False
|
||||
|
||||
from hermes_cli.proxy.adapters.base import UpstreamAdapter, UpstreamCredential
|
||||
from hermes_cli.proxy.sse_done import (
|
||||
DONE_SSE_FRAME,
|
||||
SseDoneTracker,
|
||||
content_type_is_sse,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Headers we strip when forwarding to the upstream. ``host``/``content-length``
|
||||
# are recomputed by aiohttp; ``authorization`` is replaced with our bearer.
|
||||
# Everything else (content-type, accept, user-agent, x-* headers) passes through.
|
||||
_HOP_BY_HOP_HEADERS = frozenset(
|
||||
{
|
||||
"host",
|
||||
"content-length",
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"te",
|
||||
"trailers",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
"authorization", # we replace this one
|
||||
}
|
||||
)
|
||||
|
||||
DEFAULT_PORT = 8645
|
||||
DEFAULT_HOST = "127.0.0.1"
|
||||
# Body cap for forwarded requests. Chat-completion payloads with long agent
|
||||
# conversations can be large; mirror api_server's MAX_REQUEST_BYTES (10 MB).
|
||||
# client_max_size bounds every read path, including chunked bodies.
|
||||
MAX_REQUEST_BYTES = 10_000_000
|
||||
|
||||
|
||||
def _json_error(status: int, message: str, code: str = "proxy_error") -> "web.Response":
|
||||
"""Return an OpenAI-style error JSON response."""
|
||||
body = {"error": {"message": message, "type": code, "code": code}}
|
||||
return web.json_response(body, status=status)
|
||||
|
||||
|
||||
def _filter_request_headers(headers: "aiohttp.typedefs.LooseHeaders") -> dict:
|
||||
"""Strip hop-by-hop + auth headers from the inbound request."""
|
||||
out = {}
|
||||
for key, value in headers.items():
|
||||
if key.lower() in _HOP_BY_HOP_HEADERS:
|
||||
continue
|
||||
out[key] = value
|
||||
return out
|
||||
|
||||
|
||||
def _filter_response_headers(headers) -> dict:
|
||||
"""Strip hop-by-hop headers from the upstream response."""
|
||||
out = {}
|
||||
for key, value in headers.items():
|
||||
if key.lower() in _HOP_BY_HOP_HEADERS:
|
||||
continue
|
||||
# aiohttp recomputes Content-Encoding/Content-Length on stream — let it.
|
||||
if key.lower() in {"content-encoding", "content-length"}:
|
||||
continue
|
||||
out[key] = value
|
||||
return out
|
||||
|
||||
|
||||
def create_app(adapter: UpstreamAdapter) -> "web.Application":
|
||||
"""Build the aiohttp application bound to a specific upstream adapter."""
|
||||
if not AIOHTTP_AVAILABLE:
|
||||
raise RuntimeError(
|
||||
"aiohttp is required for `hermes proxy`. Run `hermes setup` to install it."
|
||||
)
|
||||
|
||||
app = web.Application(client_max_size=MAX_REQUEST_BYTES)
|
||||
# AppKey ensures forward-compat with future aiohttp versions that strip
|
||||
# bare-string keys.
|
||||
_adapter_key = web.AppKey("adapter", UpstreamAdapter)
|
||||
app[_adapter_key] = adapter
|
||||
|
||||
async def handle_health(request: "web.Request") -> "web.Response":
|
||||
# ``is_authenticated`` is documented as cheap (see UpstreamAdapter),
|
||||
# but both shipped adapters read auth state off disk, and the Nous one
|
||||
# does it under ``_auth_store_lock()`` — 15s cross-process. Offload it
|
||||
# so a healthcheck poll can never freeze the loop behind a lock held by
|
||||
# a concurrent ``hermes auth`` command.
|
||||
authenticated = await asyncio.to_thread(adapter.is_authenticated)
|
||||
return web.json_response(
|
||||
{
|
||||
"status": "ok",
|
||||
"upstream": adapter.display_name,
|
||||
"authenticated": authenticated,
|
||||
}
|
||||
)
|
||||
|
||||
async def handle_proxy(request: "web.Request") -> "web.StreamResponse":
|
||||
# Extract the path *after* /v1
|
||||
rel_path = request.match_info.get("tail", "")
|
||||
rel_path = "/" + rel_path.lstrip("/")
|
||||
|
||||
if rel_path not in adapter.allowed_paths:
|
||||
allowed = ", ".join(sorted(adapter.allowed_paths))
|
||||
return _json_error(
|
||||
404,
|
||||
f"Path /v1{rel_path} is not forwarded by this proxy. "
|
||||
f"Allowed: {allowed}",
|
||||
code="path_not_allowed",
|
||||
)
|
||||
|
||||
# ``UpstreamAdapter.get_credential`` is synchronous and hard-blocking:
|
||||
# the Nous adapter takes ``_auth_store_lock()`` (a cross-process lock
|
||||
# with a 15s timeout), reads auth.json, and may perform a token-refresh
|
||||
# POST, taking the lock a second time to persist a terminal error. Run
|
||||
# it on a worker thread so a refresh or a contended lock cannot freeze
|
||||
# every other in-flight streaming completion on this single loop.
|
||||
try:
|
||||
cred = await asyncio.to_thread(adapter.get_credential)
|
||||
except Exception as exc:
|
||||
logger.warning("proxy: credential resolution failed: %s", exc)
|
||||
return _json_error(401, str(exc), code="upstream_auth_failed")
|
||||
|
||||
# Forward body verbatim. Read into memory once — request bodies for
|
||||
# chat/completions/embeddings are small (<1MB typically). If we ever
|
||||
# need to forward large multipart uploads we'll switch to streaming
|
||||
# the request body too.
|
||||
body = await request.read()
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=None, sock_connect=15, sock_read=300)
|
||||
|
||||
async def _send_upstream(active_cred: UpstreamCredential):
|
||||
upstream_url = f"{active_cred.base_url.rstrip('/')}{rel_path}"
|
||||
# Preserve query string verbatim.
|
||||
if request.query_string:
|
||||
upstream_url = f"{upstream_url}?{request.query_string}"
|
||||
|
||||
fwd_headers = _filter_request_headers(request.headers)
|
||||
fwd_headers["Authorization"] = f"{active_cred.token_type} {active_cred.bearer}"
|
||||
|
||||
logger.debug(
|
||||
"proxy: forwarding %s %s -> %s (body=%d bytes)",
|
||||
request.method, rel_path, upstream_url, len(body),
|
||||
)
|
||||
|
||||
try:
|
||||
session = aiohttp.ClientSession(timeout=timeout)
|
||||
except Exception as exc: # pragma: no cover - aiohttp setup issue
|
||||
raise RuntimeError(f"proxy session init failed: {exc}") from exc
|
||||
|
||||
try:
|
||||
upstream_resp = await session.request(
|
||||
request.method,
|
||||
upstream_url,
|
||||
data=body if body else None,
|
||||
headers=fwd_headers,
|
||||
allow_redirects=False,
|
||||
)
|
||||
except Exception:
|
||||
await session.close()
|
||||
raise
|
||||
return session, upstream_resp
|
||||
|
||||
async def _open_upstream(active_cred: UpstreamCredential):
|
||||
try:
|
||||
return await _send_upstream(active_cred)
|
||||
except RuntimeError as exc:
|
||||
return _json_error(500, str(exc)), None
|
||||
except aiohttp.ClientError as exc:
|
||||
logger.warning("proxy: upstream connection failed: %s", exc)
|
||||
return (
|
||||
_json_error(
|
||||
502,
|
||||
f"upstream connection failed: {exc}",
|
||||
code="upstream_unreachable",
|
||||
),
|
||||
None,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
return (
|
||||
_json_error(
|
||||
504,
|
||||
"upstream request timed out",
|
||||
code="upstream_timeout",
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
session_or_response, upstream_resp = await _open_upstream(cred)
|
||||
if upstream_resp is None:
|
||||
return session_or_response
|
||||
session = session_or_response
|
||||
|
||||
if upstream_resp.status in {401, 429}:
|
||||
# Third and last blocking method on the adapter contract, and the
|
||||
# most expensive: the Nous adapter routes this straight into
|
||||
# ``_get_credential(force_refresh=True)``, so the refresh POST that
|
||||
# ``get_credential`` only performs near expiry is unconditional
|
||||
# here — under the same 15s cross-process ``_auth_store_lock()``.
|
||||
# The xAI adapter loads its key pool off disk and rotates it under
|
||||
# ``self._lock``. Offload it for the same reason as the two above.
|
||||
try:
|
||||
retry_cred = await asyncio.to_thread(
|
||||
adapter.get_retry_credential,
|
||||
failed_credential=cred,
|
||||
status_code=upstream_resp.status,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("proxy: retry credential resolution failed: %s", exc)
|
||||
retry_cred = None
|
||||
|
||||
if retry_cred is not None:
|
||||
upstream_resp.release()
|
||||
await session.close()
|
||||
session_or_response, upstream_resp = await _open_upstream(retry_cred)
|
||||
if upstream_resp is None:
|
||||
return session_or_response
|
||||
session = session_or_response
|
||||
|
||||
# Stream response back. Headers first, then chunked body.
|
||||
resp = web.StreamResponse(
|
||||
status=upstream_resp.status,
|
||||
headers=_filter_response_headers(upstream_resp.headers),
|
||||
)
|
||||
await resp.prepare(request)
|
||||
|
||||
# Track SSE terminal markers so we can append a missing [DONE]
|
||||
# after clean EOF without rewriting any earlier frames.
|
||||
done_tracker: Optional[SseDoneTracker] = None
|
||||
if content_type_is_sse(upstream_resp.headers):
|
||||
done_tracker = SseDoneTracker()
|
||||
|
||||
try:
|
||||
async for chunk in upstream_resp.content.iter_any():
|
||||
if chunk:
|
||||
if done_tracker is not None:
|
||||
done_tracker.feed(chunk)
|
||||
await resp.write(chunk)
|
||||
if done_tracker is not None and done_tracker.should_append_done():
|
||||
try:
|
||||
await resp.write(DONE_SSE_FRAME)
|
||||
except Exception as exc: # client hung up at EOF — harmless
|
||||
logger.debug("proxy: DONE append skipped: %s", exc)
|
||||
except (aiohttp.ClientError, asyncio.CancelledError, OSError) as exc:
|
||||
if done_tracker is not None:
|
||||
done_tracker.mark_interrupted()
|
||||
logger.warning("proxy: streaming interrupted: %s", exc)
|
||||
finally:
|
||||
upstream_resp.release()
|
||||
await session.close()
|
||||
|
||||
await resp.write_eof()
|
||||
return resp
|
||||
|
||||
# /health doesn't go through the upstream
|
||||
app.router.add_get("/health", handle_health)
|
||||
# Catch-all under /v1 — forwards if the path is allowed.
|
||||
app.router.add_route("*", "/v1/{tail:.*}", handle_proxy)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
async def run_server(
|
||||
adapter: UpstreamAdapter,
|
||||
host: str = DEFAULT_HOST,
|
||||
port: int = DEFAULT_PORT,
|
||||
shutdown_event: Optional[asyncio.Event] = None,
|
||||
) -> None:
|
||||
"""Run the proxy in the current event loop until shutdown_event is set.
|
||||
|
||||
If shutdown_event is None, runs until cancelled (Ctrl+C or SIGTERM).
|
||||
"""
|
||||
if not AIOHTTP_AVAILABLE:
|
||||
raise RuntimeError(
|
||||
"aiohttp is required for `hermes proxy`. Run `hermes setup` to install it."
|
||||
)
|
||||
|
||||
app = create_app(adapter)
|
||||
runner = web.AppRunner(app, access_log=None)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, host=host, port=port)
|
||||
await site.start()
|
||||
|
||||
logger.info(
|
||||
"proxy: listening on http://%s:%d/v1 -> %s",
|
||||
host, port, adapter.display_name,
|
||||
)
|
||||
|
||||
stop_event = shutdown_event or asyncio.Event()
|
||||
|
||||
# Wire signal handlers when we own the loop's lifetime.
|
||||
if shutdown_event is None:
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
try:
|
||||
loop.add_signal_handler(sig, stop_event.set) # windows-footgun: ok
|
||||
except NotImplementedError:
|
||||
# Windows / restricted environments — Ctrl+C will still
|
||||
# raise KeyboardInterrupt and unwind us.
|
||||
pass
|
||||
|
||||
try:
|
||||
await stop_event.wait()
|
||||
finally:
|
||||
logger.info("proxy: shutting down")
|
||||
await runner.cleanup()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"create_app",
|
||||
"run_server",
|
||||
"DEFAULT_HOST",
|
||||
"DEFAULT_PORT",
|
||||
"AIOHTTP_AVAILABLE",
|
||||
]
|
||||
@@ -0,0 +1,152 @@
|
||||
"""SSE ``[DONE]`` sentinel normalization for OpenAI-compatible proxies.
|
||||
|
||||
Some upstreams (notably Nous Portal for certain free models) deliver a
|
||||
complete chat-completions stream — content deltas, a non-null
|
||||
``finish_reason``, and often a ``lastOne: true`` usage frame — then close
|
||||
the connection without the conventional OpenAI terminal event::
|
||||
|
||||
data: [DONE]
|
||||
|
||||
Strict OpenAI-compatible clients treat that shape as a truncated stream.
|
||||
This module watches the forwarded SSE byte stream and reports whether the
|
||||
proxy should append a single ``data: [DONE]`` frame after a *clean*
|
||||
upstream EOF.
|
||||
|
||||
Rules (issue #90848):
|
||||
- Retain every original delta unchanged (this helper never rewrites bytes).
|
||||
- Append ``[DONE]`` only after a complete terminal choice
|
||||
(``finish_reason`` non-null) **or** an upstream ``lastOne: true`` marker.
|
||||
- Never synthesize ``[DONE]`` after an error event, or when the stream was
|
||||
interrupted before clean EOF.
|
||||
- Never emit a second ``[DONE]`` when the upstream already sent one.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
DONE_SSE_FRAME = b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SseDoneTracker:
|
||||
"""Incremental scanner over forwarded SSE chunks."""
|
||||
|
||||
saw_done: bool = False
|
||||
saw_terminal_finish: bool = False
|
||||
saw_last_one: bool = False
|
||||
saw_error_event: bool = False
|
||||
saw_malformed_event: bool = False
|
||||
interrupted: bool = False
|
||||
_buf: bytearray = field(default_factory=bytearray, repr=False)
|
||||
_data_lines: list = field(default_factory=list, repr=False)
|
||||
|
||||
def feed(self, chunk: bytes) -> None:
|
||||
"""Observe a forwarded chunk (bytes are not modified)."""
|
||||
if not chunk:
|
||||
return
|
||||
self._buf.extend(chunk)
|
||||
while True:
|
||||
nl = self._buf.find(b"\n")
|
||||
if nl < 0:
|
||||
break
|
||||
line = bytes(self._buf[:nl])
|
||||
del self._buf[: nl + 1]
|
||||
self._consume_line(line)
|
||||
|
||||
def mark_interrupted(self) -> None:
|
||||
"""Upstream stream ended via error/cancel — do not synthesize DONE."""
|
||||
self.interrupted = True
|
||||
|
||||
def should_append_done(self) -> bool:
|
||||
"""True when a single terminal ``[DONE]`` should be appended."""
|
||||
if (
|
||||
self.interrupted
|
||||
or self.saw_done
|
||||
or self.saw_error_event
|
||||
or self.saw_malformed_event
|
||||
):
|
||||
return False
|
||||
# Flush any trailing line without a final newline (rare but valid),
|
||||
# then dispatch a final event that never saw its blank-line boundary.
|
||||
if self._buf:
|
||||
self._consume_line(bytes(self._buf))
|
||||
self._buf.clear()
|
||||
self._dispatch_event()
|
||||
if self.saw_done or self.saw_error_event or self.saw_malformed_event:
|
||||
return False
|
||||
return self.saw_terminal_finish or self.saw_last_one
|
||||
|
||||
def _consume_line(self, line: bytes) -> None:
|
||||
# Strip CR from CRLF-delimited SSE.
|
||||
if line.endswith(b"\r"):
|
||||
line = line[:-1]
|
||||
if not line:
|
||||
# Blank line = SSE event boundary: dispatch accumulated data.
|
||||
self._dispatch_event()
|
||||
return
|
||||
if not line.startswith(b"data:"):
|
||||
return
|
||||
# Per the SSE spec one event may span several consecutive ``data:``
|
||||
# lines whose payloads are joined with "\n" at dispatch time.
|
||||
# Parsing each line independently would misread a split JSON event
|
||||
# as two malformed fragments.
|
||||
self._data_lines.append(line[5:].strip())
|
||||
|
||||
def _dispatch_event(self) -> None:
|
||||
if not self._data_lines:
|
||||
return
|
||||
payload = b"\n".join(self._data_lines)
|
||||
self._data_lines = []
|
||||
payload = payload.strip()
|
||||
if payload == b"[DONE]":
|
||||
self.saw_done = True
|
||||
return
|
||||
if not payload:
|
||||
return
|
||||
try:
|
||||
text = payload.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
self.saw_malformed_event = True
|
||||
return
|
||||
try:
|
||||
event = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
self.saw_malformed_event = True
|
||||
return
|
||||
if not isinstance(event, dict):
|
||||
return
|
||||
if event.get("error") is not None:
|
||||
self.saw_error_event = True
|
||||
return
|
||||
# Accept integer-truthy sentinels too — relabelled upstreams have
|
||||
# been observed sending ``"lastOne": 1`` / ``"true"``.
|
||||
if event.get("lastOne") in (True, 1, "true"):
|
||||
self.saw_last_one = True
|
||||
for choice in event.get("choices") or []:
|
||||
if not isinstance(choice, dict):
|
||||
continue
|
||||
if choice.get("finish_reason") is not None:
|
||||
self.saw_terminal_finish = True
|
||||
# OpenAI error-shaped finish reasons should not unlock DONE.
|
||||
fr = choice.get("finish_reason")
|
||||
if isinstance(fr, str) and fr.lower() in {"error", "provider_error"}:
|
||||
self.saw_error_event = True
|
||||
|
||||
|
||||
def content_type_is_sse(headers) -> bool:
|
||||
"""Return True when response headers advertise an SSE body."""
|
||||
try:
|
||||
value = headers.get("Content-Type") or headers.get("content-type") or ""
|
||||
except Exception:
|
||||
value = ""
|
||||
return "text/event-stream" in str(value).lower()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DONE_SSE_FRAME",
|
||||
"SseDoneTracker",
|
||||
"content_type_is_sse",
|
||||
]
|
||||
Reference in New Issue
Block a user