Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
"""Hermes gateway monitoring.
|
||||
|
||||
Service health monitoring plus redacted operational diagnostics for the
|
||||
gateway daemon, exported over OTLP to an operator-configured endpoint.
|
||||
|
||||
``emitter`` is the in-process event bus: producers (gateway status hooks,
|
||||
the diagnostic log handler) hand typed events to a fire-and-forget queue,
|
||||
and subscribers (the OTLP streamers) consume them off the hot path. The
|
||||
emitter never blocks or raises into gateway code (the hot-path invariant),
|
||||
and nothing is persisted locally — monitoring is an egress path, not a store.
|
||||
|
||||
Deliberately out of scope here: run/model/tool trajectory capture, usage
|
||||
analytics, and any content-bearing signal. Those planes are served by the
|
||||
NeMo Relay integration and its Hermes-owned subscribers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from . import emitter, events
|
||||
|
||||
emit = emitter.emit
|
||||
get_emitter = emitter.get_emitter
|
||||
|
||||
__all__ = [
|
||||
"emitter",
|
||||
"events",
|
||||
"emit",
|
||||
"get_emitter",
|
||||
]
|
||||
@@ -0,0 +1,203 @@
|
||||
"""Content-free cron service-health and execution telemetry projection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
from agent.monitoring.events import CronExecutionEvent
|
||||
from agent.monitoring.gateway_health import GatewayHealthSnapshot, GatewayMetric
|
||||
from cron.jobs import (
|
||||
_compute_grace_seconds,
|
||||
get_catch_up_occurrence_count,
|
||||
get_ticker_heartbeat_age,
|
||||
get_ticker_success_age,
|
||||
load_jobs,
|
||||
)
|
||||
from cron.scheduler import get_running_job_ids
|
||||
from hermes_time import now as _hermes_now
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_KNOWN_STATUSES = {"claimed", "running", "completed", "failed", "unknown"}
|
||||
_KNOWN_SOURCES = {"builtin", "direct", "external"}
|
||||
_KNOWN_DELIVERY_OUTCOMES = {
|
||||
"delivered", "failed", "suppressed", "suppressed_acked", "not_configured",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CronHealthSnapshot:
|
||||
metrics: list[GatewayMetric]
|
||||
events: list[CronExecutionEvent]
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return _hermes_now()
|
||||
|
||||
|
||||
def _job_key(raw: Any) -> str:
|
||||
value = str(raw or "unknown").encode("utf-8", errors="replace")
|
||||
return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}"
|
||||
|
||||
|
||||
def classify_cron_error(raw: Any) -> str:
|
||||
text = str(raw or "").lower()
|
||||
if (
|
||||
re.search(r"\b(?:authentication|authenticated|authenticate|authorization|authorized|authorize|unauthorized|forbidden)\b", text)
|
||||
or re.search(r"\bbearer\b", text)
|
||||
or re.search(r"\b(?:access|api|refresh) token\b", text)
|
||||
or re.search(r"\b(?:401|403)\b", text)
|
||||
):
|
||||
return "auth_failed"
|
||||
if "rate limit" in text or "429" in text or "quota" in text:
|
||||
return "rate_limited"
|
||||
if "timeout" in text or "timed out" in text:
|
||||
return "timeout"
|
||||
if any(value in text for value in ("network", "connection", "dns", "socket", "unreachable")):
|
||||
return "network_error"
|
||||
if "dispatch" in text or "executor" in text:
|
||||
return "dispatch_failed"
|
||||
if "interrupt" in text or "owner exited" in text or "restarted" in text:
|
||||
return "interrupted"
|
||||
if "empty response" in text:
|
||||
return "empty_response"
|
||||
if any(value in text for value in ("config", "missing", "invalid")):
|
||||
return "invalid_config"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _parse_time(raw: Any) -> Optional[datetime]:
|
||||
try:
|
||||
return datetime.fromisoformat(str(raw)) if raw else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _duration_ms(record: dict[str, Any]) -> Optional[int]:
|
||||
start = _parse_time(record.get("started_at")) or _parse_time(record.get("claimed_at"))
|
||||
finish = _parse_time(record.get("finished_at"))
|
||||
if start is None or finish is None:
|
||||
return None
|
||||
try:
|
||||
duration = int((finish - start).total_seconds() * 1000)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return max(0, duration)
|
||||
|
||||
|
||||
def project_execution_event(
|
||||
record: dict[str, Any], *, delivery_outcome: Optional[str] = None
|
||||
) -> CronExecutionEvent:
|
||||
status = str(record.get("status") or "unknown").lower()
|
||||
source = str(record.get("source") or "unknown").lower()
|
||||
if source not in _KNOWN_SOURCES and source != "unknown":
|
||||
source = "external"
|
||||
outcome = str(delivery_outcome).lower() if delivery_outcome is not None else None
|
||||
return CronExecutionEvent(
|
||||
status=status if status in _KNOWN_STATUSES else "unknown",
|
||||
job_key=_job_key(record.get("job_id")),
|
||||
source=source if source in _KNOWN_SOURCES else "unknown",
|
||||
duration_ms=_duration_ms(record),
|
||||
delivery_outcome=(
|
||||
outcome if outcome in _KNOWN_DELIVERY_OUTCOMES else None
|
||||
),
|
||||
error_class=(
|
||||
classify_cron_error(record.get("error"))
|
||||
if status in {"failed", "unknown"}
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def emit_execution_state(
|
||||
record: Optional[dict[str, Any]], *, delivery_outcome: Optional[str] = None
|
||||
) -> None:
|
||||
"""Best-effort lifecycle emit; terminal states synchronously cross the queue barrier."""
|
||||
if not record:
|
||||
return
|
||||
try:
|
||||
from agent.monitoring import emitter
|
||||
|
||||
event = project_execution_event(record, delivery_outcome=delivery_outcome)
|
||||
target = emitter.get_emitter()
|
||||
target.emit(event)
|
||||
if event.status in {"completed", "failed", "unknown"}:
|
||||
target.flush(timeout=1.0)
|
||||
except Exception:
|
||||
logger.debug("cron execution telemetry emit failed", exc_info=True)
|
||||
|
||||
|
||||
def _is_overdue(job: dict[str, Any], now: datetime) -> bool:
|
||||
if not job.get("enabled", True):
|
||||
return False
|
||||
next_run = _parse_time(job.get("next_run_at"))
|
||||
schedule = job.get("schedule")
|
||||
if next_run is None or not isinstance(schedule, dict):
|
||||
return False
|
||||
try:
|
||||
if next_run.tzinfo is None and now.tzinfo is not None:
|
||||
next_run = next_run.replace(tzinfo=now.tzinfo)
|
||||
lateness = (now - next_run).total_seconds()
|
||||
return lateness > _compute_grace_seconds(schedule)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def build_cron_health_snapshot() -> CronHealthSnapshot:
|
||||
metrics: list[GatewayMetric] = []
|
||||
for name, reader in (
|
||||
("hermes.cron.scheduler.heartbeat_age_seconds", get_ticker_heartbeat_age),
|
||||
("hermes.cron.scheduler.last_success_age_seconds", get_ticker_success_age),
|
||||
):
|
||||
try:
|
||||
value = reader()
|
||||
if value is not None:
|
||||
metrics.append(GatewayMetric(name, max(0.0, float(value)), {}))
|
||||
except Exception:
|
||||
logger.debug("cron freshness metric unavailable", exc_info=True)
|
||||
|
||||
try:
|
||||
metrics.append(
|
||||
GatewayMetric(
|
||||
"hermes.cron.scheduler.catch_up_occurrences",
|
||||
get_catch_up_occurrence_count(),
|
||||
{},
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("cron catch-up metric unavailable", exc_info=True)
|
||||
|
||||
try:
|
||||
jobs = load_jobs()
|
||||
enabled = [job for job in jobs if job.get("enabled", True)]
|
||||
metrics.append(GatewayMetric("hermes.cron.jobs.enabled", len(enabled), {}))
|
||||
metrics.append(
|
||||
GatewayMetric(
|
||||
"hermes.cron.jobs.overdue",
|
||||
sum(1 for job in enabled if _is_overdue(job, _now())),
|
||||
{},
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("cron job metrics unavailable", exc_info=True)
|
||||
|
||||
try:
|
||||
metrics.append(
|
||||
GatewayMetric("hermes.cron.jobs.running", len(get_running_job_ids()), {})
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("cron running-job metric unavailable", exc_info=True)
|
||||
return CronHealthSnapshot(metrics=metrics, events=[])
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CronHealthSnapshot",
|
||||
"build_cron_health_snapshot",
|
||||
"classify_cron_error",
|
||||
"emit_execution_state",
|
||||
"project_execution_event",
|
||||
]
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Monitoring emitter: fire-and-forget queue + background dispatcher.
|
||||
|
||||
The emitter is the single seam between producers (gateway status hooks, the
|
||||
diagnostic log handler) and consumers (the OTLP streamers). Its contract is
|
||||
the hot-path invariant:
|
||||
|
||||
``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network,
|
||||
and MUST NEVER raise into the caller. A monitoring failure is logged
|
||||
locally and dropped — it can never affect the gateway or a session.
|
||||
|
||||
Mechanism:
|
||||
* ``emit(event)`` does a non-blocking ``queue.put_nowait`` wrapped in a bare
|
||||
except. On a full queue it drops the *oldest* event and counts the drop.
|
||||
* A daemon thread drains the queue and fans each batch out to subscribers
|
||||
(the OTLP metric/span/log streamers). Each subscriber is fail-isolated —
|
||||
a slow or raising subscriber never affects the hot path or its peers.
|
||||
|
||||
Nothing is persisted here. Monitoring is an egress path, not a local store;
|
||||
if no subscriber is attached, events simply age out of the ring buffer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MAX_QUEUE = 10_000 # ring-buffer depth; oldest dropped when full
|
||||
_DRAIN_BATCH = 256
|
||||
|
||||
|
||||
class MonitoringEmitter:
|
||||
"""Owns the queue, the dispatcher thread, and the subscriber list."""
|
||||
|
||||
def __init__(self, *, enabled: bool = True) -> None:
|
||||
self._enabled = enabled
|
||||
self._q: "queue.Queue[Dict[str, Any]]" = queue.Queue(maxsize=_MAX_QUEUE)
|
||||
self._dropped = 0
|
||||
self._dispatched = 0
|
||||
self._stop = threading.Event()
|
||||
self._started = False
|
||||
self._lock = threading.Lock()
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
# Live subscribers (the OTLP streamers). Called from the dispatcher
|
||||
# thread, fully fail-isolated. Each subscriber is callable(batch: list[dict]).
|
||||
self._subscribers: list = []
|
||||
|
||||
# ── public API (hot path) ───────────────────────────────────────────────
|
||||
def emit(self, event: Any) -> None:
|
||||
"""Enqueue an event. Never blocks, never raises.
|
||||
|
||||
``event`` may be a dataclass with ``to_dict()`` or a plain dict.
|
||||
"""
|
||||
if not self._enabled:
|
||||
return
|
||||
try:
|
||||
payload = event.to_dict() if hasattr(event, "to_dict") else dict(event)
|
||||
payload.setdefault("ts_ns", time.time_ns())
|
||||
self._ensure_started()
|
||||
try:
|
||||
self._q.put_nowait(payload)
|
||||
except queue.Full:
|
||||
# Drop oldest to make room — bounded memory, newest-wins.
|
||||
try:
|
||||
self._q.get_nowait()
|
||||
self._q.task_done()
|
||||
self._dropped += 1
|
||||
self._q.put_nowait(payload)
|
||||
except Exception:
|
||||
self._dropped += 1
|
||||
except Exception: # the hot-path invariant: never propagate
|
||||
logger.debug("monitoring emit failed", exc_info=True)
|
||||
|
||||
# ── lifecycle ───────────────────────────────────────────────────────────
|
||||
def _ensure_started(self) -> None:
|
||||
if self._started:
|
||||
return
|
||||
with self._lock:
|
||||
if self._started:
|
||||
return
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="hermes-monitoring-dispatch", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
self._started = True
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
first = self._q.get(timeout=0.5)
|
||||
except queue.Empty:
|
||||
continue
|
||||
batch = [first]
|
||||
while len(batch) < _DRAIN_BATCH:
|
||||
try:
|
||||
batch.append(self._q.get_nowait())
|
||||
except queue.Empty:
|
||||
break
|
||||
try:
|
||||
self._dispatch(batch)
|
||||
finally:
|
||||
for _ in batch:
|
||||
self._q.task_done()
|
||||
|
||||
def _dispatch(self, batch) -> None:
|
||||
# Fan-out to subscribers (OTLP streamers) — fully fail-isolated.
|
||||
for sub in list(self._subscribers):
|
||||
try:
|
||||
sub(batch)
|
||||
except Exception:
|
||||
logger.debug("monitoring subscriber failed", exc_info=True)
|
||||
self._dispatched += len(batch)
|
||||
|
||||
def subscribe(self, callback) -> None:
|
||||
"""Register a live batch subscriber (callable(batch: list[dict]))."""
|
||||
if callback not in self._subscribers:
|
||||
self._subscribers.append(callback)
|
||||
self._enabled = True
|
||||
|
||||
def unsubscribe(self, callback) -> None:
|
||||
try:
|
||||
self._subscribers.remove(callback)
|
||||
except ValueError:
|
||||
pass
|
||||
if not self._subscribers:
|
||||
self._enabled = False
|
||||
|
||||
# ── introspection / shutdown (tests, CLI) ───────────────────────────────
|
||||
def flush(self, timeout: float = 2.0) -> None:
|
||||
"""Wait boundedly for queued and in-flight batches to finish dispatch."""
|
||||
if timeout <= 0:
|
||||
return
|
||||
|
||||
finished = threading.Event()
|
||||
|
||||
def _wait_for_completion() -> None:
|
||||
self._q.join()
|
||||
finished.set()
|
||||
|
||||
waiter = threading.Thread(
|
||||
target=_wait_for_completion,
|
||||
name="hermes-monitoring-flush",
|
||||
daemon=True,
|
||||
)
|
||||
waiter.start()
|
||||
finished.wait(timeout=timeout)
|
||||
|
||||
def stats(self) -> Dict[str, int]:
|
||||
return {
|
||||
"queued": self._q.qsize(),
|
||||
"dispatched": self._dispatched,
|
||||
"dropped": self._dropped,
|
||||
"subscribers": len(self._subscribers),
|
||||
}
|
||||
|
||||
def close(self) -> None:
|
||||
self._stop.set()
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=2.0)
|
||||
self._started = False
|
||||
|
||||
|
||||
# ── process-wide singleton ──────────────────────────────────────────────────
|
||||
_EMITTER: Optional[MonitoringEmitter] = None
|
||||
_EMITTER_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def get_emitter() -> MonitoringEmitter:
|
||||
"""Return the process-wide monitoring emitter."""
|
||||
global _EMITTER
|
||||
if _EMITTER is not None:
|
||||
return _EMITTER
|
||||
with _EMITTER_LOCK:
|
||||
if _EMITTER is None:
|
||||
# Collection is opt-in. A plane exporter enables the singleton by
|
||||
# attaching its first subscriber; until then producers are no-ops.
|
||||
_EMITTER = MonitoringEmitter(enabled=False)
|
||||
return _EMITTER
|
||||
|
||||
|
||||
def emit(event: Any) -> None:
|
||||
"""Module-level convenience: emit via the singleton."""
|
||||
get_emitter().emit(event)
|
||||
|
||||
|
||||
def reset_emitter_for_tests(emitter: Optional[MonitoringEmitter] = None) -> None:
|
||||
"""Swap the singleton (tests only)."""
|
||||
global _EMITTER
|
||||
with _EMITTER_LOCK:
|
||||
if _EMITTER is not None and emitter is not _EMITTER:
|
||||
try:
|
||||
_EMITTER.close()
|
||||
except Exception:
|
||||
pass
|
||||
_EMITTER = emitter
|
||||
|
||||
|
||||
# Back-compat alias for the salvaged class name used in emozilla's tests.
|
||||
TelemetryEmitter = MonitoringEmitter
|
||||
|
||||
__all__ = [
|
||||
"MonitoringEmitter",
|
||||
"TelemetryEmitter",
|
||||
"get_emitter",
|
||||
"emit",
|
||||
"reset_emitter_for_tests",
|
||||
]
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Typed gateway monitoring events.
|
||||
|
||||
Content-free service-health and redacted diagnostic events for the gateway
|
||||
daemon. These are the only event shapes the monitoring plane emits: no
|
||||
prompts, messages, tool args/results, session history, or usage analytics.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
def _now_ns() -> int:
|
||||
return time.time_ns()
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class GatewayHealthEvent:
|
||||
"""Content-free gateway health snapshot or lifecycle event."""
|
||||
|
||||
name: str
|
||||
gateway_state: Optional[str] = None
|
||||
old_state: Optional[str] = None
|
||||
new_state: Optional[str] = None
|
||||
exit_reason: Optional[str] = None
|
||||
restart_requested: Optional[bool] = None
|
||||
active_agents: int = 0
|
||||
gateway_busy: bool = False
|
||||
gateway_drainable: bool = False
|
||||
platform_count: int = 0
|
||||
fatal_platform_count: int = 0
|
||||
profile: Optional[str] = None
|
||||
install_id: Optional[str] = None
|
||||
version: Optional[str] = None
|
||||
supervision_mode: Optional[str] = None
|
||||
pid: Optional[int] = None
|
||||
ts_ns: int = field(default_factory=_now_ns)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {"event": "gateway_health", **asdict(self)}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class GatewayDiagnosticEvent:
|
||||
"""Redacted gateway diagnostic event for operator-owned observability."""
|
||||
|
||||
name: str
|
||||
subsystem: str
|
||||
error_class: str = "unknown"
|
||||
error_code: Optional[str] = None
|
||||
platform: Optional[str] = None
|
||||
old_state: Optional[str] = None
|
||||
new_state: Optional[str] = None
|
||||
profile: Optional[str] = None
|
||||
version: Optional[str] = None
|
||||
severity: str = "warning"
|
||||
ts_ns: int = field(default_factory=_now_ns)
|
||||
source_logger: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {"event": "gateway_diagnostic", **asdict(self)}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CronExecutionEvent:
|
||||
"""Content-free durable cron execution lifecycle projection."""
|
||||
|
||||
status: str
|
||||
job_key: str
|
||||
source: str = "unknown"
|
||||
duration_ms: Optional[int] = None
|
||||
delivery_outcome: Optional[str] = None
|
||||
error_class: Optional[str] = None
|
||||
ts_ns: int = field(default_factory=_now_ns)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {"event": "cron_execution", **asdict(self)}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GatewayHealthEvent",
|
||||
"GatewayDiagnosticEvent",
|
||||
"CronExecutionEvent",
|
||||
]
|
||||
@@ -0,0 +1,469 @@
|
||||
"""Gateway health and diagnostics signal producer.
|
||||
|
||||
This module keeps the plane narrow: service health monitoring plus
|
||||
redacted operational diagnostics. It reuses the existing gateway runtime-status
|
||||
contract and emits content-free metrics/events. No prompts, messages, tool args,
|
||||
session history, audit records, or product analytics belong here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from agent.monitoring.events import GatewayDiagnosticEvent, GatewayHealthEvent
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GatewayMetric:
|
||||
name: str
|
||||
value: int | float
|
||||
attributes: Dict[str, str]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GatewayHealthSnapshot:
|
||||
metrics: List[GatewayMetric]
|
||||
events: List[GatewayHealthEvent | GatewayDiagnosticEvent]
|
||||
|
||||
|
||||
_RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"}
|
||||
_FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"}
|
||||
_KNOWN_GATEWAY_STATES = {
|
||||
"starting", "draining", "stopping", "stopped", "startup_failed", "unknown"
|
||||
} | _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES
|
||||
_KNOWN_PLATFORM_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | {
|
||||
"connecting", "disconnected", "disabled", "paused", "retrying", "unknown"
|
||||
}
|
||||
_SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"}
|
||||
_SOURCE_LOGGER_RE = re.compile(r"^gateway(?:\.[A-Za-z_][A-Za-z0-9_]*)*$")
|
||||
|
||||
|
||||
def _allowed_logger(name: str) -> bool:
|
||||
return name == "gateway" or name.startswith("gateway.")
|
||||
|
||||
|
||||
def source_logger_for_export(name: Any) -> Optional[str]:
|
||||
"""Return a bounded source-controlled gateway logger name for OTLP scope."""
|
||||
value = str(name or "")
|
||||
return value if len(value) <= 128 and _SOURCE_LOGGER_RE.fullmatch(value) else None
|
||||
|
||||
|
||||
def redact_gateway_message(message: Any) -> str:
|
||||
"""Redact gateway diagnostic free text for operator-owned export.
|
||||
|
||||
Single scrub path: everything goes through
|
||||
``agent.monitoring.redaction.redact_for_export`` (unconditional
|
||||
secrets + PII), then is length-bounded.
|
||||
"""
|
||||
try:
|
||||
from agent.monitoring.redaction import redact_for_export
|
||||
redacted = redact_for_export(str(message or "")) or ""
|
||||
except Exception:
|
||||
redacted = "[redaction-unavailable]"
|
||||
return redacted[:500]
|
||||
|
||||
|
||||
def classify_gateway_error(raw: Any) -> str:
|
||||
s = str(raw or "").lower()
|
||||
if any(k in s for k in ("auth", "token", "unauthorized", "forbidden", "401", "403")):
|
||||
return "auth_failed"
|
||||
if "rate" in s and "limit" in s:
|
||||
return "rate_limited"
|
||||
if "timeout" in s or "timed out" in s:
|
||||
return "timeout"
|
||||
if any(
|
||||
k in s
|
||||
for k in (
|
||||
"network",
|
||||
"connection",
|
||||
"dns",
|
||||
"socket",
|
||||
"connect call failed",
|
||||
"failed to connect",
|
||||
"cannot connect",
|
||||
"unreachable",
|
||||
"name resolution",
|
||||
)
|
||||
):
|
||||
return "network_error"
|
||||
if any(k in s for k in ("config", "missing", "invalid")):
|
||||
return "invalid_config"
|
||||
if "startup" in s:
|
||||
return "startup_failed"
|
||||
if "fatal" in s:
|
||||
return "platform_fatal"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def classify_exit_reason(
|
||||
raw: Any, *, state: Any, restart_requested: bool
|
||||
) -> Optional[str]:
|
||||
"""Reduce free-form shutdown text to a bounded operational class."""
|
||||
if restart_requested:
|
||||
return "restart_requested"
|
||||
state_name = str(state or "").lower()
|
||||
if raw is None and state_name != "startup_failed":
|
||||
return None
|
||||
classified = classify_gateway_error(raw)
|
||||
if state_name == "startup_failed":
|
||||
return classified if classified != "unknown" else "startup_failed"
|
||||
text = str(raw or "").lower()
|
||||
if "signal" in text or "sigterm" in text or "sigint" in text:
|
||||
return "signal"
|
||||
if state_name == "stopped" and any(word in text for word in ("shutdown", "stop")):
|
||||
return "planned_stop"
|
||||
return classified
|
||||
|
||||
|
||||
def _bounded_state(raw: Any, *, allowed: set[str]) -> str:
|
||||
state = str(raw or "unknown").lower()
|
||||
return state if state in allowed else "unknown"
|
||||
|
||||
|
||||
def _safe_metric_value(raw: Any, *, limit: int = 128) -> str:
|
||||
try:
|
||||
from agent.monitoring.redaction import redact_for_export
|
||||
value = redact_for_export(str(raw or "")) or "unknown"
|
||||
except Exception:
|
||||
return "unknown"
|
||||
return value[:limit]
|
||||
|
||||
|
||||
def _safe_instance_id(raw: Any) -> str:
|
||||
"""Return a stable opaque instance key without exporting the source ID."""
|
||||
value = str(raw or "unknown").encode("utf-8", errors="replace")
|
||||
return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}"
|
||||
|
||||
|
||||
def subsystem_for_logger(logger_name: str) -> str:
|
||||
if logger_name == "gateway.relay" or logger_name.startswith("gateway.relay."):
|
||||
return "platform.relay"
|
||||
if logger_name.startswith("gateway.platforms."):
|
||||
parts = logger_name.split(".")
|
||||
if len(parts) >= 3 and parts[2]:
|
||||
return f"platform.{parts[2]}"
|
||||
if logger_name.startswith("gateway.platforms"):
|
||||
return "platform"
|
||||
if logger_name.startswith("gateway"):
|
||||
return "gateway"
|
||||
return "gateway"
|
||||
|
||||
|
||||
def platform_for_subsystem(subsystem: str) -> Optional[str]:
|
||||
if subsystem.startswith("platform."):
|
||||
return subsystem.split(".", 1)[1] or None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_active_agents(raw: Any) -> int:
|
||||
try:
|
||||
from gateway.status import parse_active_agents
|
||||
return parse_active_agents(raw)
|
||||
except Exception:
|
||||
try:
|
||||
return max(0, int(raw))
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _derive_busy(gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool:
|
||||
try:
|
||||
from gateway.status import derive_gateway_busy
|
||||
return derive_gateway_busy(
|
||||
gateway_running=gateway_running,
|
||||
gateway_state=gateway_state,
|
||||
active_agents=active_agents,
|
||||
)
|
||||
except Exception:
|
||||
return bool(gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0)
|
||||
|
||||
|
||||
def _derive_drainable(gateway_running: bool, gateway_state: Any) -> bool:
|
||||
try:
|
||||
from gateway.status import derive_gateway_drainable
|
||||
return derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state)
|
||||
except Exception:
|
||||
return bool(gateway_running and gateway_state == "running")
|
||||
|
||||
|
||||
def _base_attrs(*, profile: str, install_id: str, version: str, supervision_mode: str) -> Dict[str, str]:
|
||||
mode = str(supervision_mode or "unknown").lower()
|
||||
return {
|
||||
"service.instance.id": _safe_instance_id(install_id),
|
||||
"service.version": _safe_metric_value(version, limit=64),
|
||||
"hermes.supervision_mode": mode if mode in _SUPERVISION_MODES else "unknown",
|
||||
}
|
||||
|
||||
|
||||
def _metric(name: str, value: int | float, attrs: Dict[str, str], **extra: str) -> GatewayMetric:
|
||||
out = dict(attrs)
|
||||
for key, val in extra.items():
|
||||
if val is not None:
|
||||
out[key] = _safe_metric_value(val)
|
||||
return GatewayMetric(name=name, value=value, attributes=out)
|
||||
|
||||
|
||||
def build_gateway_health_snapshot(
|
||||
runtime: Optional[dict[str, Any]],
|
||||
*,
|
||||
gateway_running: bool,
|
||||
profile: str,
|
||||
install_id: str,
|
||||
version: str,
|
||||
supervision_mode: str = "unknown",
|
||||
) -> GatewayHealthSnapshot:
|
||||
"""Convert gateway_state.json-compatible runtime state into P0 signals."""
|
||||
runtime = runtime or {}
|
||||
gateway_state = _bounded_state(
|
||||
runtime.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES
|
||||
)
|
||||
active_agents = _parse_active_agents(runtime.get("active_agents", 0))
|
||||
busy = _derive_busy(gateway_running, gateway_state, active_agents)
|
||||
drainable = _derive_drainable(gateway_running, gateway_state)
|
||||
platforms = runtime.get("platforms") if isinstance(runtime.get("platforms"), dict) else {}
|
||||
base = _base_attrs(profile=profile, install_id=install_id, version=version, supervision_mode=supervision_mode)
|
||||
|
||||
metrics: list[GatewayMetric] = [
|
||||
_metric("hermes.gateway.up", 1 if gateway_running else 0, base),
|
||||
_metric("hermes.gateway.active_agents", active_agents, base),
|
||||
_metric("hermes.gateway.busy", 1 if busy else 0, base),
|
||||
_metric("hermes.gateway.drainable", 1 if drainable else 0, base),
|
||||
_metric("hermes.gateway.restart_requested", 1 if runtime.get("restart_requested") else 0, base),
|
||||
]
|
||||
if gateway_state:
|
||||
metrics.append(_metric("hermes.gateway.state", 1, base, **{"hermes.gateway.state": str(gateway_state)}))
|
||||
|
||||
fatal_count = 0
|
||||
events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
|
||||
for platform, pdata in platforms.items():
|
||||
pdata = pdata if isinstance(pdata, dict) else {}
|
||||
state = _bounded_state(
|
||||
pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES
|
||||
)
|
||||
raw_error = pdata.get("error_code") or pdata.get("error_message")
|
||||
error_code = classify_gateway_error(raw_error)
|
||||
is_up = state in _RUNNING_PLATFORM_STATES
|
||||
is_degraded = state in _FATAL_PLATFORM_STATES
|
||||
if is_degraded:
|
||||
fatal_count += 1
|
||||
metrics.append(_metric(
|
||||
"hermes.platform.up",
|
||||
1 if is_up else 0,
|
||||
base,
|
||||
**{"hermes.platform": str(platform), "hermes.platform.state": state},
|
||||
))
|
||||
metrics.append(_metric(
|
||||
"hermes.platform.degraded",
|
||||
1 if is_degraded else 0,
|
||||
base,
|
||||
**{"hermes.platform": str(platform), "hermes.platform.state": state, "hermes.error_code": error_code},
|
||||
))
|
||||
if is_degraded:
|
||||
events.append(GatewayDiagnosticEvent(
|
||||
name="platform.fatal",
|
||||
subsystem=f"platform.{platform}",
|
||||
platform=str(platform),
|
||||
error_code=error_code,
|
||||
error_class=classify_gateway_error(error_code or pdata.get("error_message")),
|
||||
profile=profile,
|
||||
version=version,
|
||||
severity="error" if state == "fatal" else "warning",
|
||||
))
|
||||
|
||||
events.insert(0, GatewayHealthEvent(
|
||||
name="gateway.health_snapshot",
|
||||
gateway_state=str(gateway_state) if gateway_state is not None else None,
|
||||
active_agents=active_agents,
|
||||
gateway_busy=busy,
|
||||
gateway_drainable=drainable,
|
||||
platform_count=len(platforms),
|
||||
fatal_platform_count=fatal_count,
|
||||
profile=profile,
|
||||
install_id=install_id,
|
||||
version=version,
|
||||
supervision_mode=supervision_mode,
|
||||
pid=_coerce_pid(runtime.get("pid")),
|
||||
))
|
||||
return GatewayHealthSnapshot(metrics=metrics, events=events)
|
||||
|
||||
|
||||
def _safe_profile() -> str:
|
||||
try:
|
||||
from hermes_cli.profiles import get_active_profile_name
|
||||
return str(get_active_profile_name() or "default")
|
||||
except Exception:
|
||||
return "default"
|
||||
|
||||
|
||||
def _safe_version() -> str:
|
||||
try:
|
||||
from hermes_cli import __version__
|
||||
return str(__version__)
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: dict[str, Any]) -> None:
|
||||
"""Emit immediate content-free gateway events for runtime status changes.
|
||||
|
||||
Called by gateway.status.write_runtime_status after persisting the new status.
|
||||
Fully fail-open: failures never affect gateway status writes.
|
||||
"""
|
||||
try:
|
||||
from agent.monitoring import emitter
|
||||
out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
|
||||
profile = _safe_profile()
|
||||
version = _safe_version()
|
||||
old_gateway_state = _bounded_state(
|
||||
(previous or {}).get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES
|
||||
) if (previous or {}).get("gateway_state") is not None else None
|
||||
new_gateway_state = _bounded_state(
|
||||
current.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES
|
||||
) if current.get("gateway_state") is not None else None
|
||||
if old_gateway_state != new_gateway_state and new_gateway_state:
|
||||
out.append(GatewayHealthEvent(
|
||||
name="gateway.lifecycle",
|
||||
gateway_state=new_gateway_state,
|
||||
old_state=old_gateway_state,
|
||||
new_state=new_gateway_state,
|
||||
exit_reason=classify_exit_reason(
|
||||
current.get("exit_reason"),
|
||||
state=new_gateway_state,
|
||||
restart_requested=bool(current.get("restart_requested")),
|
||||
),
|
||||
restart_requested=bool(current.get("restart_requested")),
|
||||
active_agents=_parse_active_agents(current.get("active_agents", 0)),
|
||||
profile=profile,
|
||||
version=version,
|
||||
pid=_coerce_pid(current.get("pid")),
|
||||
))
|
||||
if new_gateway_state == "startup_failed":
|
||||
out.append(GatewayDiagnosticEvent(
|
||||
name="gateway.startup_failed",
|
||||
subsystem="gateway",
|
||||
error_class=classify_gateway_error(current.get("exit_reason") or "startup_failed"),
|
||||
error_code=classify_gateway_error(current.get("exit_reason") or "startup_failed"),
|
||||
profile=profile,
|
||||
version=version,
|
||||
severity="error",
|
||||
))
|
||||
if new_gateway_state == "stopped":
|
||||
out.append(GatewayHealthEvent(
|
||||
name="gateway.exit",
|
||||
gateway_state=new_gateway_state,
|
||||
old_state=old_gateway_state,
|
||||
new_state=new_gateway_state,
|
||||
exit_reason=classify_exit_reason(
|
||||
current.get("exit_reason"),
|
||||
state=new_gateway_state,
|
||||
restart_requested=bool(current.get("restart_requested")),
|
||||
),
|
||||
restart_requested=bool(current.get("restart_requested")),
|
||||
active_agents=_parse_active_agents(current.get("active_agents", 0)),
|
||||
profile=profile,
|
||||
version=version,
|
||||
pid=_coerce_pid(current.get("pid")),
|
||||
))
|
||||
|
||||
old_platforms_raw = (previous or {}).get("platforms")
|
||||
new_platforms_raw = current.get("platforms")
|
||||
old_platforms = old_platforms_raw if isinstance(old_platforms_raw, dict) else {}
|
||||
new_platforms = new_platforms_raw if isinstance(new_platforms_raw, dict) else {}
|
||||
for platform, pdata in new_platforms.items():
|
||||
pdata = pdata if isinstance(pdata, dict) else {}
|
||||
prev_raw = old_platforms.get(platform, {})
|
||||
prev = prev_raw if isinstance(prev_raw, dict) else {}
|
||||
old_state = _bounded_state(
|
||||
prev.get("state"), allowed=_KNOWN_PLATFORM_STATES
|
||||
) if prev.get("state") is not None else None
|
||||
new_state = _bounded_state(
|
||||
pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES
|
||||
) if pdata.get("state") is not None else None
|
||||
if old_state == new_state or not new_state:
|
||||
continue
|
||||
error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message"))
|
||||
severity = "error" if new_state.lower() in {"fatal", "failed", "error"} else "warning"
|
||||
out.append(GatewayDiagnosticEvent(
|
||||
name="platform.state_change",
|
||||
subsystem=f"platform.{platform}",
|
||||
platform=str(platform),
|
||||
old_state=old_state,
|
||||
new_state=new_state,
|
||||
error_code=error_code,
|
||||
error_class=error_code,
|
||||
profile=profile,
|
||||
version=version,
|
||||
severity=severity,
|
||||
))
|
||||
if new_state.lower() in _FATAL_PLATFORM_STATES:
|
||||
out.append(GatewayDiagnosticEvent(
|
||||
name="platform.fatal",
|
||||
subsystem=f"platform.{platform}",
|
||||
platform=str(platform),
|
||||
error_code=error_code,
|
||||
error_class=error_code,
|
||||
profile=profile,
|
||||
version=version,
|
||||
severity=severity,
|
||||
))
|
||||
for ev in out:
|
||||
emitter.emit(ev)
|
||||
except Exception:
|
||||
logging.getLogger(__name__).debug("gateway runtime status transition emit failed", exc_info=True)
|
||||
|
||||
|
||||
def _coerce_pid(raw: Any) -> Optional[int]:
|
||||
try:
|
||||
pid = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return pid if pid > 0 else None
|
||||
|
||||
|
||||
class GatewayDiagnosticLogHandler(logging.Handler):
|
||||
"""Allowlisted warning/error bridge for gateway-owned diagnostics."""
|
||||
|
||||
def __init__(self, *, profile: str = "default", version: str = "unknown") -> None:
|
||||
super().__init__(level=logging.WARNING)
|
||||
self.profile = profile
|
||||
self.version = version
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
try:
|
||||
if record.levelno < logging.WARNING:
|
||||
return
|
||||
if not _allowed_logger(record.name):
|
||||
return
|
||||
subsystem = subsystem_for_logger(record.name)
|
||||
message = record.getMessage()
|
||||
error_class = classify_gateway_error(message)
|
||||
event = GatewayDiagnosticEvent(
|
||||
name=f"gateway.log.{record.levelname.lower()}",
|
||||
subsystem=subsystem,
|
||||
source_logger=source_logger_for_export(record.name),
|
||||
platform=platform_for_subsystem(subsystem),
|
||||
error_class=error_class,
|
||||
error_code=error_class,
|
||||
profile=self.profile,
|
||||
version=self.version,
|
||||
severity=record.levelname.lower(),
|
||||
)
|
||||
from agent.monitoring import emitter
|
||||
emitter.get_emitter().emit(event)
|
||||
except Exception:
|
||||
logging.getLogger(__name__).debug("gateway diagnostic emit failed", exc_info=True)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GatewayMetric",
|
||||
"GatewayHealthSnapshot",
|
||||
"GatewayDiagnosticLogHandler",
|
||||
"build_gateway_health_snapshot",
|
||||
"classify_gateway_error",
|
||||
"source_logger_for_export",
|
||||
"redact_gateway_message",
|
||||
]
|
||||
@@ -0,0 +1,643 @@
|
||||
"""Gateway Health & Diagnostics OTLP export runtime.
|
||||
|
||||
This exporter emits operator-owned gateway service-health metrics plus
|
||||
narrow redacted diagnostic events. It is deliberately in-process and fail-open so
|
||||
it works under systemd, launchd, s6, containers, tmux, nohup, or a simple shell
|
||||
without a sidecar/watchdog dependency.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_DIAGNOSTIC_SCOPE = "hermes.gateway.diagnostics"
|
||||
|
||||
_RESOURCE_ATTRIBUTE_KEYS = frozenset({
|
||||
"service.name",
|
||||
"service.namespace",
|
||||
"service.version",
|
||||
"service.instance.id",
|
||||
"deployment.environment.name",
|
||||
"cloud.provider",
|
||||
"cloud.platform",
|
||||
"cloud.region",
|
||||
"telemetry.scope",
|
||||
})
|
||||
_DIAGNOSTIC_ATTRIBUTE_KEYS = frozenset({
|
||||
"name",
|
||||
"subsystem",
|
||||
"error_class",
|
||||
"error_code",
|
||||
"platform",
|
||||
"old_state",
|
||||
"new_state",
|
||||
"version",
|
||||
"severity",
|
||||
})
|
||||
_SAFE_RESOURCE_VALUE = re.compile(r"^[A-Za-z0-9._:/-]{1,128}$")
|
||||
|
||||
|
||||
def _redact_string(raw: Any, *, limit: int = 500) -> str:
|
||||
try:
|
||||
from agent.monitoring.redaction import redact_for_export
|
||||
return (redact_for_export(str(raw or "")) or "[redacted]")[:limit]
|
||||
except Exception:
|
||||
return "[redaction-unavailable]"
|
||||
|
||||
|
||||
def _safe_resource_attributes(raw: Any) -> Dict[str, str]:
|
||||
"""Allowlist bounded resource labels and reject values changed by redaction."""
|
||||
attrs: Dict[str, str] = {}
|
||||
if not isinstance(raw, dict):
|
||||
return attrs
|
||||
for key, value in raw.items():
|
||||
key = str(key)
|
||||
if key not in _RESOURCE_ATTRIBUTE_KEYS or value is None:
|
||||
continue
|
||||
if key == "service.instance.id":
|
||||
from agent.monitoring.gateway_health import _safe_instance_id
|
||||
attrs[key] = _safe_instance_id(value)
|
||||
continue
|
||||
text = str(value)
|
||||
if not _SAFE_RESOURCE_VALUE.fullmatch(text):
|
||||
continue
|
||||
if _redact_string(text, limit=128) != text:
|
||||
continue
|
||||
attrs[key] = text
|
||||
return attrs
|
||||
|
||||
|
||||
def _runtime_resource_attributes(
|
||||
config: Dict[str, Any], *, telemetry_scope: str
|
||||
) -> Dict[str, str]:
|
||||
"""Build the safe OTLP resource shared by metrics and diagnostic logs."""
|
||||
gh = _gateway_health_config(config)
|
||||
attrs = _safe_resource_attributes(gh.get("resource_attributes"))
|
||||
from agent.monitoring.gateway_health import _safe_instance_id
|
||||
|
||||
attrs["service.name"] = "hermes-gateway"
|
||||
attrs["service.instance.id"] = _safe_instance_id(_install_id(config))
|
||||
attrs["telemetry.scope"] = telemetry_scope
|
||||
return attrs
|
||||
|
||||
|
||||
def _diagnostic_log_attributes(event: Dict[str, Any]) -> Dict[str, Any]:
|
||||
attrs: Dict[str, Any] = {}
|
||||
for key in _DIAGNOSTIC_ATTRIBUTE_KEYS:
|
||||
value = event.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
attrs[f"hermes.{key}"] = _redact_string(value) if isinstance(value, str) else value
|
||||
return attrs
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class GatewayHealthExportRuntime:
|
||||
enabled: bool
|
||||
reason: str = "disabled"
|
||||
streamer: Any = None
|
||||
metric_provider: Any = None
|
||||
log_handler: Any = None
|
||||
log_streamer: Any = None
|
||||
thread: Optional[threading.Thread] = None
|
||||
stop_event: Optional[threading.Event] = None
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.stop_event is not None:
|
||||
self.stop_event.set()
|
||||
if self.thread is not None:
|
||||
self.thread.join(timeout=0.25)
|
||||
if self.log_handler is not None:
|
||||
try:
|
||||
logging.getLogger().removeHandler(self.log_handler)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# All producers above are now stopped. Drain queued and in-flight
|
||||
# events before detaching subscribers so the terminal lifecycle event
|
||||
# cannot race exporter shutdown. The barrier is bounded and fail-open.
|
||||
try:
|
||||
from agent.monitoring.emitter import get_emitter
|
||||
emitter = get_emitter()
|
||||
emitter.flush(timeout=1.0)
|
||||
if self.streamer is not None:
|
||||
emitter.unsubscribe(self.streamer)
|
||||
if self.log_streamer is not None:
|
||||
emitter.unsubscribe(self.log_streamer)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Network flush/close runs under one bounded daemon-thread deadline and
|
||||
# can never delay gateway teardown indefinitely.
|
||||
closeables = [
|
||||
item for item in (self.streamer, self.log_streamer, self.metric_provider)
|
||||
if item is not None
|
||||
]
|
||||
|
||||
def _close() -> None:
|
||||
for item in closeables:
|
||||
try:
|
||||
item.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if closeables:
|
||||
worker = threading.Thread(
|
||||
target=_close,
|
||||
name="hermes-gateway-health-export-shutdown",
|
||||
daemon=True,
|
||||
)
|
||||
worker.start()
|
||||
worker.join(timeout=2.0)
|
||||
|
||||
self.streamer = None
|
||||
self.log_streamer = None
|
||||
self.metric_provider = None
|
||||
self.thread = None
|
||||
self.stop_event = None
|
||||
|
||||
|
||||
def _gateway_health_config(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
mon = (config or {}).get("monitoring") or {}
|
||||
return mon.get("gateway_health_export") or {}
|
||||
|
||||
|
||||
def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
mon = (config or {}).get("monitoring") or {}
|
||||
export = mon.get("export") or {}
|
||||
return export.get("otlp") or {}
|
||||
|
||||
|
||||
def _enabled(config: Dict[str, Any]) -> bool:
|
||||
gh = _gateway_health_config(config)
|
||||
otlp = _otlp_config(config)
|
||||
return bool(gh.get("enabled") and otlp.get("enabled") and otlp.get("endpoint"))
|
||||
|
||||
|
||||
def _require_metrics_sdk(*, auto_install: bool = True, prompt: bool = False) -> Dict[str, Any]:
|
||||
if auto_install:
|
||||
try:
|
||||
from tools.lazy_deps import ensure as _lazy_ensure
|
||||
_lazy_ensure("export.otlp", prompt=prompt)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.metrics import Observation
|
||||
from opentelemetry.trace import INVALID_SPAN_ID, INVALID_TRACE_ID, TraceFlags
|
||||
from opentelemetry._logs import LogRecord
|
||||
from opentelemetry._logs.severity import SeverityNumber
|
||||
from opentelemetry.sdk._logs import LoggerProvider
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
return {
|
||||
"OTLPLogExporter": OTLPLogExporter,
|
||||
"OTLPMetricExporter": OTLPMetricExporter,
|
||||
"Observation": Observation,
|
||||
"LogRecord": LogRecord,
|
||||
"LoggerProvider": LoggerProvider,
|
||||
"INVALID_SPAN_ID": INVALID_SPAN_ID,
|
||||
"INVALID_TRACE_ID": INVALID_TRACE_ID,
|
||||
"TraceFlags": TraceFlags,
|
||||
"SeverityNumber": SeverityNumber,
|
||||
"BatchLogRecordProcessor": BatchLogRecordProcessor,
|
||||
"MeterProvider": MeterProvider,
|
||||
"PeriodicExportingMetricReader": PeriodicExportingMetricReader,
|
||||
"Resource": Resource,
|
||||
}
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"OTLP metrics SDK unavailable: {exc}") from exc
|
||||
|
||||
|
||||
def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]:
|
||||
resolved: Dict[str, str] = {}
|
||||
for header_name, env_name in (headers_env or {}).items():
|
||||
val = os.environ.get(str(env_name))
|
||||
if val:
|
||||
resolved[str(header_name)] = val
|
||||
return resolved
|
||||
|
||||
|
||||
def _metric_endpoint(endpoint: str) -> str:
|
||||
if endpoint.endswith("/v1/traces"):
|
||||
return endpoint[: -len("/v1/traces")] + "/v1/metrics"
|
||||
return endpoint
|
||||
|
||||
|
||||
def _logs_endpoint(endpoint: str) -> str:
|
||||
if endpoint.endswith("/v1/traces"):
|
||||
return endpoint[: -len("/v1/traces")] + "/v1/logs"
|
||||
if endpoint.endswith("/v1/metrics"):
|
||||
return endpoint[: -len("/v1/metrics")] + "/v1/logs"
|
||||
return endpoint
|
||||
|
||||
|
||||
def _version() -> str:
|
||||
try:
|
||||
from hermes_cli import __version__
|
||||
return str(__version__)
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _profile() -> str:
|
||||
try:
|
||||
from hermes_cli.profiles import get_active_profile_name
|
||||
return str(get_active_profile_name() or "default")
|
||||
except Exception:
|
||||
return "default"
|
||||
|
||||
|
||||
def _install_id(config: Dict[str, Any]) -> str:
|
||||
try:
|
||||
from agent.monitoring.policy import ensure_install_id
|
||||
return str(ensure_install_id(config))
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _supervision_mode() -> str:
|
||||
if os.environ.get("INVOCATION_ID"):
|
||||
return "systemd"
|
||||
if os.environ.get("S6_CMD_ARG0") or os.environ.get("S6_VERSION"):
|
||||
return "s6"
|
||||
if os.environ.get("container") or os.path.exists("/.dockerenv"):
|
||||
return "container"
|
||||
if os.environ.get("LAUNCHD_SOCKET"):
|
||||
return "launchd"
|
||||
return "manual"
|
||||
|
||||
|
||||
def _read_gateway_snapshot(config: Dict[str, Any]):
|
||||
from agent.monitoring.gateway_health import build_gateway_health_snapshot
|
||||
try:
|
||||
from gateway.status import read_runtime_status
|
||||
runtime = read_runtime_status() or {}
|
||||
except Exception:
|
||||
runtime = {}
|
||||
return build_gateway_health_snapshot(
|
||||
runtime,
|
||||
gateway_running=True,
|
||||
profile=_profile(),
|
||||
install_id=_install_id(config),
|
||||
version=_version(),
|
||||
supervision_mode=_supervision_mode(),
|
||||
)
|
||||
|
||||
|
||||
def _read_cron_snapshot():
|
||||
from agent.monitoring.cron_health import build_cron_health_snapshot
|
||||
|
||||
return build_cron_health_snapshot()
|
||||
|
||||
|
||||
def _read_background_work_count() -> int:
|
||||
"""Count live background/subagent work that ``active_agents`` does NOT include.
|
||||
|
||||
``hermes.gateway.active_agents`` counts foreground turns + in-flight cron
|
||||
jobs + API runs, but deliberately excludes backgrounded ``delegate_task``
|
||||
subagents, ``terminal(background=true)`` processes, kanban workers, and the
|
||||
runner's own background tasks (they are tracked only for the scale-to-zero
|
||||
suspend guard, ``_scale_to_zero_has_live_background_work``). Without this
|
||||
metric a peer churning through delegated subagents shows ``active_agents=0``
|
||||
on the fleet dashboard. Best-effort and content-free: a single integer,
|
||||
no job/task identity. Returns 0 if a source can't be imported.
|
||||
|
||||
Delegation is counted TASK-granular (``active_task_count``): a fan-out batch
|
||||
of N subagents contributes N, not 1, so the metric reflects real concurrent
|
||||
subagent load rather than dispatch-unit/pool-slot count. This intentionally
|
||||
differs from the async pool's capacity accounting (one batch = one slot).
|
||||
"""
|
||||
total = 0
|
||||
try:
|
||||
from tools.async_delegation import active_task_count
|
||||
|
||||
total += max(0, int(active_task_count()))
|
||||
except Exception:
|
||||
logger.debug("background-work async-delegation count failed", exc_info=True)
|
||||
try:
|
||||
from tools.process_registry import process_registry
|
||||
|
||||
total += max(0, int(process_registry.count_running()))
|
||||
except Exception:
|
||||
logger.debug("background-work process-registry count failed", exc_info=True)
|
||||
return total
|
||||
|
||||
|
||||
def _read_background_delegations_count() -> int:
|
||||
"""Count live async delegation UNITS (dispatch/pool slots).
|
||||
|
||||
Complements ``_read_background_work_count`` (which is task-granular): this
|
||||
counts each ``delegate_task`` dispatch as ONE regardless of fan-out width,
|
||||
matching the async pool's capacity accounting (a batch = one slot). Together
|
||||
the two metrics let an operator see both slot pressure
|
||||
(``background_delegations``, alert vs ``max_concurrent_children``) and real
|
||||
concurrent subagent load (``background_work``). Delegations only — it does
|
||||
not include ``terminal(background)`` / kanban work, which are already folded
|
||||
into ``background_work``. Best-effort; 0 if the source can't be imported.
|
||||
"""
|
||||
try:
|
||||
from tools.async_delegation import active_count
|
||||
|
||||
return max(0, int(active_count()))
|
||||
except Exception:
|
||||
logger.debug("background-delegations count failed", exc_info=True)
|
||||
return 0
|
||||
|
||||
|
||||
def _read_runtime_snapshot(config: Dict[str, Any]):
|
||||
gateway_snapshot = _read_gateway_snapshot(config)
|
||||
# Background/subagent work — a distinct metric from active_agents (which
|
||||
# never counts it). Appended to the gateway snapshot so it rides the same
|
||||
# base resource attributes (service.instance.id etc.).
|
||||
try:
|
||||
from agent.monitoring.gateway_health import GatewayMetric
|
||||
|
||||
base = dict(gateway_snapshot.metrics[0].attributes) if gateway_snapshot.metrics else {}
|
||||
gateway_snapshot.metrics.append(
|
||||
GatewayMetric(
|
||||
name="hermes.gateway.background_work",
|
||||
value=_read_background_work_count(),
|
||||
attributes=base,
|
||||
)
|
||||
)
|
||||
gateway_snapshot.metrics.append(
|
||||
GatewayMetric(
|
||||
name="hermes.gateway.background_delegations",
|
||||
value=_read_background_delegations_count(),
|
||||
attributes=base,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"background-work snapshot unavailable; metric not exported (error_type=%s)",
|
||||
type(exc).__name__,
|
||||
)
|
||||
logger.debug("background-work snapshot traceback", exc_info=True)
|
||||
try:
|
||||
cron_snapshot = _read_cron_snapshot()
|
||||
except Exception as exc:
|
||||
# Content-free visibility: cron telemetry silently dropping out is a
|
||||
# release-relevant regression, so surface it at WARNING with only the
|
||||
# exception *type* name (never the message, which could carry paths or
|
||||
# other environment detail). exc_info stays on the DEBUG record.
|
||||
logger.warning(
|
||||
"cron health snapshot unavailable; cron telemetry not exported (error_type=%s)",
|
||||
type(exc).__name__,
|
||||
)
|
||||
logger.debug("cron health snapshot traceback", exc_info=True)
|
||||
return gateway_snapshot
|
||||
gateway_snapshot.metrics.extend(cron_snapshot.metrics)
|
||||
return gateway_snapshot
|
||||
|
||||
|
||||
def _emit_snapshot_events(config: Dict[str, Any]) -> None:
|
||||
gh = _gateway_health_config(config)
|
||||
if not gh.get("diagnostic_events_enabled", True):
|
||||
return
|
||||
try:
|
||||
from agent.monitoring import emitter
|
||||
snapshot = _read_runtime_snapshot(config)
|
||||
for event in snapshot.events:
|
||||
emitter.emit(event)
|
||||
except Exception:
|
||||
logger.debug("gateway health snapshot emit failed", exc_info=True)
|
||||
|
||||
|
||||
def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any:
|
||||
gh = _gateway_health_config(config)
|
||||
if not gh.get("metrics_enabled", True):
|
||||
return None
|
||||
otlp = _otlp_config(config)
|
||||
endpoint = _metric_endpoint(str(otlp.get("endpoint")))
|
||||
headers = _resolve_headers(otlp.get("headers_env"))
|
||||
exporter = sdk["OTLPMetricExporter"](endpoint=endpoint, headers=headers or None)
|
||||
interval_ms = max(5, int(gh.get("export_interval_seconds", 60))) * 1000
|
||||
reader = sdk["PeriodicExportingMetricReader"](exporter, export_interval_millis=interval_ms)
|
||||
resource_attrs = _runtime_resource_attributes(
|
||||
config, telemetry_scope="gateway_health"
|
||||
)
|
||||
provider = sdk["MeterProvider"](
|
||||
metric_readers=[reader],
|
||||
resource=sdk["Resource"].create(resource_attrs),
|
||||
)
|
||||
meter = provider.get_meter("hermes.gateway.health")
|
||||
Observation = sdk["Observation"]
|
||||
|
||||
metric_names = [
|
||||
"hermes.gateway.up",
|
||||
"hermes.gateway.state",
|
||||
"hermes.gateway.active_agents",
|
||||
"hermes.gateway.busy",
|
||||
"hermes.gateway.drainable",
|
||||
"hermes.gateway.restart_requested",
|
||||
"hermes.gateway.background_work",
|
||||
"hermes.gateway.background_delegations",
|
||||
"hermes.platform.up",
|
||||
"hermes.platform.degraded",
|
||||
"hermes.cron.scheduler.heartbeat_age_seconds",
|
||||
"hermes.cron.scheduler.last_success_age_seconds",
|
||||
"hermes.cron.scheduler.catch_up_occurrences",
|
||||
"hermes.cron.jobs.enabled",
|
||||
"hermes.cron.jobs.running",
|
||||
"hermes.cron.jobs.overdue",
|
||||
]
|
||||
|
||||
def callback(name: str):
|
||||
def _cb(_options=None):
|
||||
try:
|
||||
snapshot = _read_runtime_snapshot(config)
|
||||
return [Observation(m.value, m.attributes) for m in snapshot.metrics if m.name == name]
|
||||
except Exception:
|
||||
logger.debug("gateway metric callback failed", exc_info=True)
|
||||
return []
|
||||
return _cb
|
||||
|
||||
for metric_name in metric_names:
|
||||
meter.create_observable_gauge(metric_name, callbacks=[callback(metric_name)])
|
||||
return provider
|
||||
|
||||
|
||||
def _severity_number(sdk: Dict[str, Any], severity: Any) -> Any:
|
||||
SeverityNumber = sdk["SeverityNumber"]
|
||||
sev = str(severity or "warning").lower()
|
||||
if sev in {"critical", "fatal"}:
|
||||
return SeverityNumber.FATAL
|
||||
if sev == "error":
|
||||
return SeverityNumber.ERROR
|
||||
if sev in {"info", "information"}:
|
||||
return SeverityNumber.INFO
|
||||
if sev == "debug":
|
||||
return SeverityNumber.DEBUG
|
||||
return SeverityNumber.WARN
|
||||
|
||||
|
||||
class GatewayDiagnosticLogStreamer:
|
||||
"""Emitter subscriber that sends gateway diagnostic events as OTLP logs."""
|
||||
|
||||
def __init__(self, config: Dict[str, Any], sdk: Dict[str, Any]):
|
||||
otlp = _otlp_config(config)
|
||||
headers = _resolve_headers(otlp.get("headers_env"))
|
||||
endpoint = _logs_endpoint(str(otlp.get("endpoint")))
|
||||
resource_attrs = _runtime_resource_attributes(
|
||||
config, telemetry_scope="gateway_diagnostics"
|
||||
)
|
||||
self._provider = sdk["LoggerProvider"](resource=sdk["Resource"].create(resource_attrs))
|
||||
self._processor = sdk["BatchLogRecordProcessor"](
|
||||
sdk["OTLPLogExporter"](endpoint=endpoint, headers=headers or None)
|
||||
)
|
||||
self._provider.add_log_record_processor(self._processor)
|
||||
self._logger = self._provider.get_logger(_DEFAULT_DIAGNOSTIC_SCOPE)
|
||||
self._LogRecord = sdk["LogRecord"]
|
||||
self._sdk = sdk
|
||||
self.exported = 0
|
||||
|
||||
def __call__(self, batch: list[Dict[str, Any]]) -> None:
|
||||
from agent.monitoring.gateway_health import source_logger_for_export
|
||||
|
||||
for ev in batch:
|
||||
if ev.get("event") != "gateway_diagnostic":
|
||||
continue
|
||||
attrs = _diagnostic_log_attributes(ev)
|
||||
# Preserve the source-controlled Python logger as the OTel
|
||||
# instrumentation scope. This adds precise code attribution without
|
||||
# turning a fluid module layout into a maintained subsystem enum.
|
||||
# Rendered messages stay out because they may contain arbitrary IDs,
|
||||
# names, paths, or configured strings. A future, separately gated
|
||||
# ``diagnostic_detail: redacted_message`` mode may add best-effort
|
||||
# free text when an observability plane defines that privacy policy.
|
||||
source_logger = source_logger_for_export(ev.get("source_logger"))
|
||||
otel_logger = (
|
||||
self._provider.get_logger(source_logger)
|
||||
if source_logger is not None
|
||||
else self._logger
|
||||
)
|
||||
body = "gateway diagnostic"
|
||||
record = self._LogRecord(
|
||||
timestamp=ev.get("ts_ns"),
|
||||
trace_id=self._sdk["INVALID_TRACE_ID"],
|
||||
span_id=self._sdk["INVALID_SPAN_ID"],
|
||||
trace_flags=self._sdk["TraceFlags"].DEFAULT,
|
||||
severity_text=str(ev.get("severity") or "warning").upper(),
|
||||
severity_number=_severity_number(self._sdk, ev.get("severity")),
|
||||
body=_redact_string(body),
|
||||
attributes=attrs,
|
||||
)
|
||||
otel_logger.emit(record)
|
||||
self.exported += 1
|
||||
|
||||
def shutdown(self) -> None:
|
||||
try:
|
||||
from agent.monitoring.emitter import get_emitter
|
||||
get_emitter().unsubscribe(self)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self._processor.force_flush()
|
||||
self._provider.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _start_diagnostic_log_streamer(config: Dict[str, Any], sdk: Dict[str, Any]) -> GatewayDiagnosticLogStreamer:
|
||||
from agent.monitoring.emitter import get_emitter
|
||||
streamer = GatewayDiagnosticLogStreamer(config, sdk)
|
||||
get_emitter().subscribe(streamer)
|
||||
return streamer
|
||||
|
||||
|
||||
def _start_snapshot_thread(config: Dict[str, Any], stop_event: threading.Event) -> threading.Thread:
|
||||
interval = max(5, int(_gateway_health_config(config).get("logs_export_interval_seconds", 5)))
|
||||
|
||||
def _run() -> None:
|
||||
while not stop_event.wait(interval):
|
||||
_emit_snapshot_events(config)
|
||||
|
||||
thread = threading.Thread(target=_run, name="hermes-gateway-health-export", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
|
||||
def _attach_log_handler(config: Dict[str, Any]) -> Any:
|
||||
gh = _gateway_health_config(config)
|
||||
if not gh.get("diagnostic_events_enabled", True) or not gh.get("warning_error_events_enabled", True):
|
||||
return None
|
||||
from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler
|
||||
handler = GatewayDiagnosticLogHandler(profile=_profile(), version=_version())
|
||||
root = logging.getLogger()
|
||||
if handler not in root.handlers:
|
||||
root.addHandler(handler)
|
||||
return handler
|
||||
|
||||
|
||||
def _gateway_health_event(ev: Dict[str, Any]) -> bool:
|
||||
return ev.get("event") in {"gateway_health", "cron_execution"}
|
||||
|
||||
|
||||
def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRuntime:
|
||||
"""Start P0 gateway health export if configured. Never raises."""
|
||||
if not _enabled(config):
|
||||
return GatewayHealthExportRuntime(enabled=False, reason="disabled")
|
||||
gh = _gateway_health_config(config)
|
||||
runtime = GatewayHealthExportRuntime(enabled=True, reason="enabled")
|
||||
sdk: Optional[Dict[str, Any]] = None
|
||||
|
||||
if gh.get("metrics_enabled", True) or gh.get("diagnostic_events_enabled", True):
|
||||
try:
|
||||
sdk = _require_metrics_sdk(prompt=False)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"monitoring.gateway_health_export.enabled but OTLP SDK is unavailable; "
|
||||
"install 'hermes-agent[otlp]'",
|
||||
exc_info=True,
|
||||
)
|
||||
return GatewayHealthExportRuntime(enabled=False, reason="otlp_unavailable")
|
||||
|
||||
if gh.get("metrics_enabled", True) and sdk is not None:
|
||||
try:
|
||||
runtime.metric_provider = _start_metric_provider(config, sdk)
|
||||
except Exception:
|
||||
logger.warning("gateway health OTLP metrics failed to start", exc_info=True)
|
||||
runtime.shutdown()
|
||||
return GatewayHealthExportRuntime(enabled=False, reason="metrics_start_failed")
|
||||
|
||||
if gh.get("diagnostic_events_enabled", True) and sdk is not None:
|
||||
try:
|
||||
from agent.monitoring import otlp_exporter
|
||||
runtime.streamer = otlp_exporter.start_streaming(config, event_filter=_gateway_health_event)
|
||||
if runtime.streamer is None:
|
||||
raise RuntimeError("gateway health span streamer did not start")
|
||||
runtime.log_streamer = _start_diagnostic_log_streamer(config, sdk)
|
||||
except Exception:
|
||||
logger.debug("gateway diagnostic OTLP export failed to start", exc_info=True)
|
||||
runtime.shutdown()
|
||||
return GatewayHealthExportRuntime(enabled=False, reason="diagnostics_start_failed")
|
||||
|
||||
try:
|
||||
runtime.log_handler = _attach_log_handler(config)
|
||||
except Exception:
|
||||
logger.debug("gateway diagnostic log handler failed to attach", exc_info=True)
|
||||
if gh.get("diagnostic_events_enabled", True):
|
||||
try:
|
||||
_emit_snapshot_events(config)
|
||||
runtime.stop_event = threading.Event()
|
||||
runtime.thread = _start_snapshot_thread(config, runtime.stop_event)
|
||||
except Exception:
|
||||
logger.debug("gateway health snapshot thread failed to start", exc_info=True)
|
||||
return runtime
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GatewayHealthExportRuntime",
|
||||
"start_gateway_health_export",
|
||||
]
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP.
|
||||
|
||||
Maps gateway monitoring events to OTel spans and sends them to the endpoint
|
||||
configured under ``monitoring.export.otlp``. Lets an operator stream Hermes
|
||||
gateway health into their own observability stack (OTEL Collector, DataDog,
|
||||
and similar).
|
||||
|
||||
Notes:
|
||||
* The destination is operator-configured; this module only sends to that
|
||||
endpoint. No default destination ships.
|
||||
* ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an
|
||||
optional extra (``pip install hermes-agent[otlp]``), imported lazily so the
|
||||
dependency is only required when OTLP export is actually used.
|
||||
* ``headers_env`` maps a header name to an environment variable name; values
|
||||
are read from the environment at export time and never logged or stored.
|
||||
* The continuous subscriber runs in the emitter's dispatcher thread and is
|
||||
fail-isolated, so an export error cannot affect the gateway.
|
||||
|
||||
Only monitoring events (gateway_health / gateway_diagnostic) exist on this
|
||||
plane; the ``event_filter`` seam is kept so future planes sharing the emitter
|
||||
cannot silently ride along on this exporter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OTLPUnavailable(RuntimeError):
|
||||
"""Raised when the optional OpenTelemetry SDK isn't installed."""
|
||||
|
||||
|
||||
def _require_sdk(*, auto_install: bool = True, prompt: bool = True):
|
||||
"""Import the OTel SDK, lazily installing it on first use if needed.
|
||||
|
||||
Routes through tools.lazy_deps (feature 'export.otlp') so a missing SDK
|
||||
triggers the standard venv install flow — same as every other optional
|
||||
backend — gated by security.allow_lazy_installs and TTY-prompted. Falls back
|
||||
to OTLPUnavailable (with a manual install hint) when the SDK can't be made
|
||||
importable (lazy installs disabled, install failed, or auto_install=False).
|
||||
|
||||
``auto_install``: attempt the lazy install when missing (default True).
|
||||
``prompt``: ask before installing when interactive (default True); pass
|
||||
False from non-interactive contexts like the continuous streamer.
|
||||
"""
|
||||
if auto_install:
|
||||
try:
|
||||
from tools.lazy_deps import ensure as _lazy_ensure
|
||||
_lazy_ensure("export.otlp", prompt=prompt)
|
||||
except ImportError:
|
||||
pass # lazy_deps unavailable — fall through to the import attempt
|
||||
except Exception:
|
||||
# FeatureUnavailable (lazy installs disabled / declined / failed) —
|
||||
# fall through; the import below raises OTLPUnavailable with the hint.
|
||||
pass
|
||||
try:
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
|
||||
OTLPSpanExporter,
|
||||
)
|
||||
from opentelemetry.trace import SpanKind
|
||||
return {
|
||||
"TracerProvider": TracerProvider,
|
||||
"BatchSpanProcessor": BatchSpanProcessor,
|
||||
"Resource": Resource,
|
||||
"OTLPSpanExporter": OTLPSpanExporter,
|
||||
"SpanKind": SpanKind,
|
||||
}
|
||||
except Exception as e: # ImportError or partial install
|
||||
raise OTLPUnavailable(
|
||||
"OTLP export requires the optional dependency. Install with:\n"
|
||||
" pip install 'hermes-agent[otlp]'\n"
|
||||
f"(import error: {e})"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]:
|
||||
"""Resolve {header_name: ENV_VAR_NAME} -> {header_name: value} from env.
|
||||
|
||||
The config stores environment variable names, not secret values; values are
|
||||
read from the environment here. Missing variables are skipped (and noted at
|
||||
debug level without the value).
|
||||
"""
|
||||
resolved: Dict[str, str] = {}
|
||||
for header_name, env_name in (headers_env or {}).items():
|
||||
val = os.environ.get(str(env_name))
|
||||
if val:
|
||||
resolved[str(header_name)] = val
|
||||
else:
|
||||
logger.debug("OTLP header %s: env var %s not set; skipping",
|
||||
header_name, env_name)
|
||||
return resolved
|
||||
|
||||
|
||||
def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
mon = (config or {}).get("monitoring") or {}
|
||||
export = mon.get("export") or {}
|
||||
return export.get("otlp") or {}
|
||||
|
||||
|
||||
def build_exporter(config: Dict[str, Any]):
|
||||
"""Construct an OTLP span exporter from config. Raises OTLPUnavailable if no SDK."""
|
||||
sdk = _require_sdk()
|
||||
otlp = _otlp_config(config)
|
||||
endpoint = otlp.get("endpoint")
|
||||
if not endpoint:
|
||||
raise ValueError("monitoring.export.otlp.endpoint is not set")
|
||||
headers = _resolve_headers(otlp.get("headers_env"))
|
||||
return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None)
|
||||
|
||||
|
||||
def _resource_attributes(config: Dict[str, Any]) -> Dict[str, str]:
|
||||
# Lazy import: gateway_health_export imports this module back (for
|
||||
# start_streaming), so the dependency must resolve at call time, not
|
||||
# at module load, to avoid a circular import.
|
||||
from agent.monitoring.gateway_health_export import _runtime_resource_attributes
|
||||
|
||||
return _runtime_resource_attributes(config, telemetry_scope="gateway_monitoring")
|
||||
|
||||
|
||||
def _make_provider(config: Dict[str, Any]):
|
||||
sdk = _require_sdk()
|
||||
resource = sdk["Resource"].create(_resource_attributes(config))
|
||||
provider = sdk["TracerProvider"](resource=resource)
|
||||
processor = sdk["BatchSpanProcessor"](build_exporter(config))
|
||||
provider.add_span_processor(processor)
|
||||
return provider, processor
|
||||
|
||||
|
||||
# ── event -> span attribute mapping ──────────────────────────────────────────
|
||||
def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Span attributes for a monitoring event (content-free by construction)."""
|
||||
kind = ev.get("event")
|
||||
attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"}
|
||||
keep_by_kind = {
|
||||
"gateway_health": ("name", "gateway_state", "old_state", "new_state",
|
||||
"exit_reason", "restart_requested", "active_agents",
|
||||
"gateway_busy", "gateway_drainable", "platform_count",
|
||||
"fatal_platform_count", "version",
|
||||
"supervision_mode", "pid"),
|
||||
"gateway_diagnostic": ("name", "subsystem", "error_class", "error_code",
|
||||
"platform", "old_state", "new_state",
|
||||
"version", "severity"),
|
||||
"cron_execution": ("status", "job_key", "source", "duration_ms",
|
||||
"delivery_outcome", "error_class"),
|
||||
}
|
||||
for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type]
|
||||
v = ev.get(col)
|
||||
if v is not None:
|
||||
if isinstance(v, str):
|
||||
try:
|
||||
from agent.monitoring.redaction import redact_for_export
|
||||
v = (redact_for_export(v) or "[redacted]")[:500]
|
||||
except Exception:
|
||||
v = "[redaction-unavailable]"
|
||||
attrs[f"hermes.{col}"] = v
|
||||
return attrs
|
||||
|
||||
|
||||
def export_batch(provider, batch: List[Dict[str, Any]]) -> int:
|
||||
"""Map a batch of events to OTel spans. Returns spans created."""
|
||||
tracer = provider.get_tracer("hermes.monitoring")
|
||||
n = 0
|
||||
for ev in batch:
|
||||
try:
|
||||
name = f"hermes.{ev.get('event', 'event')}"
|
||||
span = tracer.start_span(name, attributes=_span_attrs(ev))
|
||||
span.end()
|
||||
n += 1
|
||||
except Exception:
|
||||
logger.debug("OTLP span map failed", exc_info=True)
|
||||
return n
|
||||
|
||||
|
||||
# ── continuous streaming subscriber ─────────────────────────────────────────
|
||||
class OTLPStreamer:
|
||||
"""A live subscriber that pushes each emitter batch to OTLP as it lands.
|
||||
|
||||
Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Dict[str, Any],
|
||||
*,
|
||||
event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None,
|
||||
):
|
||||
self._provider, self._processor = _make_provider(config)
|
||||
self._event_filter = event_filter
|
||||
self.exported = 0
|
||||
|
||||
def __call__(self, batch: List[Dict[str, Any]]) -> None:
|
||||
if self._event_filter is not None:
|
||||
batch = [ev for ev in batch if self._event_filter(ev)]
|
||||
if not batch:
|
||||
return
|
||||
self.exported += export_batch(self._provider, batch)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
try:
|
||||
from agent.monitoring.emitter import get_emitter
|
||||
get_emitter().unsubscribe(self)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self._processor.force_flush()
|
||||
self._provider.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
"""True when the OTel SDK is already importable. Does NOT auto-install —
|
||||
this is a pure check (e.g. for status display)."""
|
||||
try:
|
||||
_require_sdk(auto_install=False)
|
||||
return True
|
||||
except OTLPUnavailable:
|
||||
return False
|
||||
|
||||
|
||||
def is_enabled(config: Dict[str, Any]) -> bool:
|
||||
otlp = _otlp_config(config)
|
||||
return bool(otlp.get("enabled") and otlp.get("endpoint"))
|
||||
|
||||
|
||||
def start_streaming(
|
||||
config: Dict[str, Any],
|
||||
*,
|
||||
event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None,
|
||||
) -> Optional[OTLPStreamer]:
|
||||
"""If OTLP is enabled, attach a streamer to the singleton emitter.
|
||||
|
||||
``event_filter`` scopes the exporter to its plane, e.g. gateway-health
|
||||
export, so enabling one plane cannot silently export unrelated events.
|
||||
|
||||
Non-interactive context (startup): attempts a lazy install with prompt=False
|
||||
so a configured-but-missing SDK is installed once (gated by
|
||||
security.allow_lazy_installs), then streams. If it still can't load, logs and
|
||||
no-ops — never blocks or raises into startup.
|
||||
"""
|
||||
if not is_enabled(config):
|
||||
return None
|
||||
try:
|
||||
_require_sdk(prompt=False)
|
||||
except OTLPUnavailable:
|
||||
logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not "
|
||||
"be installed/imported; install 'hermes-agent[otlp]'")
|
||||
return None
|
||||
from agent.monitoring.emitter import get_emitter
|
||||
streamer = OTLPStreamer(config, event_filter=event_filter)
|
||||
get_emitter().subscribe(streamer)
|
||||
return streamer
|
||||
|
||||
|
||||
__all__ = [
|
||||
"OTLPUnavailable",
|
||||
"OTLPStreamer",
|
||||
"build_exporter",
|
||||
"export_batch",
|
||||
"is_available",
|
||||
"is_enabled",
|
||||
"start_streaming",
|
||||
]
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Install identity for gateway monitoring.
|
||||
|
||||
The install id is a stable, resettable pseudonymous identifier attached to
|
||||
exported health signals so an operator can tell instances apart in their
|
||||
collector. It carries no account identity and can be rotated by clearing
|
||||
``monitoring.install_id`` in config.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any, Dict
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ensure_install_id(config: Dict[str, Any]) -> str:
|
||||
"""Return a stable install id, minting and persisting one when empty.
|
||||
|
||||
The id must survive gateway restarts (it becomes ``service.instance.id``
|
||||
on exported signals), so a freshly minted UUID is written back to
|
||||
config.yaml immediately. The write is fail-open: if persisting fails
|
||||
(read-only home, managed scope), the ephemeral id is still returned and
|
||||
a new one is minted next start.
|
||||
|
||||
Clearing ``monitoring.install_id`` (e.g. ``hermes config set
|
||||
monitoring.install_id ""``) rotates the id on the next gateway start.
|
||||
"""
|
||||
mon = config.get("monitoring") if isinstance(config, dict) else None
|
||||
existing = (mon or {}).get("install_id") if isinstance(mon, dict) else None
|
||||
if isinstance(existing, str) and existing.strip():
|
||||
return existing
|
||||
|
||||
minted = str(uuid.uuid4())
|
||||
try:
|
||||
from hermes_cli.config import load_config, save_config
|
||||
|
||||
fresh = load_config()
|
||||
if isinstance(fresh, dict):
|
||||
slot = fresh.setdefault("monitoring", {})
|
||||
if isinstance(slot, dict) and not str(slot.get("install_id") or "").strip():
|
||||
slot["install_id"] = minted
|
||||
save_config(fresh)
|
||||
except Exception:
|
||||
logger.debug("install_id persist failed; using ephemeral id", exc_info=True)
|
||||
# Keep the in-memory config consistent for this process either way.
|
||||
if isinstance(config, dict):
|
||||
config.setdefault("monitoring", {})
|
||||
if isinstance(config["monitoring"], dict):
|
||||
config["monitoring"]["install_id"] = minted
|
||||
return minted
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ensure_install_id",
|
||||
]
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Redaction applied to monitoring data before egress.
|
||||
|
||||
One unconditional scrub, no modes, no knobs. Every string that leaves the
|
||||
process passes through ``redact_for_export``:
|
||||
|
||||
* Secrets first — wraps ``agent/redact.py::redact_sensitive_text(force=True)``
|
||||
plus bearer/token-shape patterns, and fails CLOSED: if the redactor cannot
|
||||
run, the raw string is never emitted.
|
||||
* PII second — e-mail addresses, phone numbers, and UUID-shaped identifiers
|
||||
are rewritten to ``[email]`` / ``[phone]`` / ``[id]``.
|
||||
|
||||
There is deliberately no setting to weaken this. The monitoring plane is
|
||||
content-free by design: rendered log messages are not exported, and bounded
|
||||
structured strings are still scrubbed as defense-in-depth. This redactor also
|
||||
remains available for a future, explicitly gated redacted-message detail mode.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
# ── secret shapes (belt-and-suspenders on top of agent/redact.py) ───────────
|
||||
_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+\-/]+=*", re.IGNORECASE)
|
||||
_TOKEN_RE = re.compile(
|
||||
r"\b(xox[baprs]-[A-Za-z0-9-]+|sk-[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,})\b"
|
||||
)
|
||||
_SECRET_LITERAL_RE = re.compile(r"\*{3,}")
|
||||
_BEARER_RESIDUE_RE = re.compile(r"\bBearer\s+\[[^\]]+\]", re.IGNORECASE)
|
||||
|
||||
# ── PII shapes ───────────────────────────────────────────────────────────────
|
||||
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}")
|
||||
# E.164-ish and common separators; conservative to avoid nuking code/IDs.
|
||||
_PHONE_RE = re.compile(
|
||||
r"(?<!\w)(?:\+?\d{1,3}[\s.\-]?)?(?:\(\d{2,4}\)[\s.\-]?)?\d{3}[\s.\-]?\d{3,4}(?:[\s.\-]?\d{2,4})?(?!\w)"
|
||||
)
|
||||
# Long opaque hex/uuid-ish user identifiers.
|
||||
_UUID_RE = re.compile(
|
||||
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
|
||||
)
|
||||
|
||||
|
||||
def _secret_redact(text: str) -> str:
|
||||
"""Always-on secret redaction. force=True so user config can't disable it."""
|
||||
try:
|
||||
from agent.redact import redact_sensitive_text
|
||||
out = redact_sensitive_text(text, force=True)
|
||||
except Exception:
|
||||
# Fail CLOSED: if the redactor can't run, do not emit the raw string.
|
||||
return "[redaction-unavailable]"
|
||||
out = _BEARER_RE.sub("[redacted]", out)
|
||||
out = _TOKEN_RE.sub("[redacted]", out)
|
||||
out = _SECRET_LITERAL_RE.sub("[redacted]", out)
|
||||
out = _BEARER_RESIDUE_RE.sub("[redacted]", out)
|
||||
return out
|
||||
|
||||
|
||||
def redact_for_export(text: Optional[str]) -> Optional[str]:
|
||||
"""Scrub a string for egress: secrets, then PII. Unconditional."""
|
||||
if text is None:
|
||||
return None
|
||||
out = _secret_redact(str(text))
|
||||
out = _EMAIL_RE.sub("[email]", out)
|
||||
out = _UUID_RE.sub("[id]", out)
|
||||
out = _PHONE_RE.sub("[phone]", out)
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"redact_for_export",
|
||||
]
|
||||
Reference in New Issue
Block a user