yilsem/aiturk-hermes-ide
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 3 Wiki Activity

Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

Browse Source
This commit is contained in:
yilsem
2026-09-05 13:26:46 +03:00
commit 03634b1ca3
11340 changed files with 3442369 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hermes_cli/observability/__init__.py
+31
View File
@@ -0,0 +1,31 @@
"""First-party Hermes observability integrations."""
from __future__ import annotations
import logging
from typing import Any
logger = logging.getLogger(__name__)
def observe_lifecycle(hook_name: str, **kwargs: Any) -> None:
"""Dispatch a Hermes lifecycle event to built-in observability features."""
from . import relay_shared_metrics
_safe_observe(relay_shared_metrics.observe_lifecycle, hook_name, kwargs)
def handles_hook(hook_name: str) -> bool:
"""Return whether any built-in observability feature handles a hook."""
from . import relay_shared_metrics
return relay_shared_metrics.handles_hook(hook_name)
def _safe_observe(callback: Any, hook_name: str, kwargs: dict[str, Any]) -> None:
try:
callback(hook_name, **kwargs)
except Exception:
logger.warning(
"Built-in observability hook failed: %s", hook_name, exc_info=True
)
hermes_cli/observability/relay_runtime.py
+14
View File
@@ -0,0 +1,14 @@
"""Compatibility alias for the core Hermes Relay runtime.
New code should import :mod:`agent.relay_runtime`. This module remains an
alias, rather than a copy, so existing plugins and tests share the same
profile registry and test-reset state during the migration.
"""
from __future__ import annotations
import sys
from agent import relay_runtime as _core_relay_runtime
sys.modules[__name__] = _core_relay_runtime
hermes_cli/observability/relay_shared_metrics.py
+1473
View File
File diff suppressed because it is too large Load Diff
hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json
+338
View File
@@ -0,0 +1,338 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "urn:hermes-agent:schema:shared-metrics:v1",
"title": "Hermes Shared Metrics Package v1",
"type": "object",
"additionalProperties": false,
"required": [
"schema_version",
"package_id",
"install_id",
"period_start",
"period_end",
"generated_at",
"resource",
"metrics"
],
"properties": {
"schema_version": {
"const": "hermes.shared_metrics.v1"
},
"package_id": {
"$ref": "#/$defs/uuid"
},
"install_id": {
"description": "Random persistent identifier scoped to one HERMES_HOME; local-only in schema v1.",
"$ref": "#/$defs/uuid"
},
"period_start": {
"type": "string",
"format": "date-time"
},
"period_end": {
"type": "string",
"format": "date-time"
},
"generated_at": {
"type": "string",
"format": "date-time"
},
"resource": {
"type": "object",
"additionalProperties": false,
"required": [
"hermes_version"
],
"properties": {
"hermes_version": {
"type": "string",
"minLength": 1,
"maxLength": 64
}
}
},
"metrics": {
"type": "array",
"minItems": 1,
"items": {
"oneOf": [
{
"$ref": "#/$defs/model_call_counter"
},
{
"$ref": "#/$defs/task_started_counter"
},
{
"$ref": "#/$defs/task_finished_counter"
}
]
}
}
},
"$defs": {
"uuid": {
"type": "string",
"pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
},
"model_call_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.model_call.count"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"call_role",
"locality",
"model_family",
"outcome",
"provider_family"
],
"properties": {
"call_role": {
"const": "primary"
},
"locality": {
"enum": [
"local",
"remote",
"unknown"
]
},
"model_family": {
"enum": [
"claude",
"deepseek",
"gemini",
"gemma",
"glm",
"gpt",
"grok",
"kimi",
"llama",
"minimax",
"mimo",
"mistral",
"nemotron",
"nova",
"o1",
"o3",
"o4",
"qwen",
"step",
"trinity",
"unknown"
]
},
"outcome": {
"enum": [
"cancelled",
"failed",
"success"
]
},
"provider_family": {
"enum": [
"aggregator",
"custom",
"direct",
"local",
"unknown"
]
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"task_started_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.task_run.started"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"entrypoint",
"execution_surface"
],
"properties": {
"entrypoint": {
"$ref": "#/$defs/task_entrypoint"
},
"execution_surface": {
"$ref": "#/$defs/execution_surface"
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"task_finished_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.task_run.finished"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"duration_bucket",
"end_reason",
"entrypoint",
"execution_surface",
"model_call_count_bucket",
"outcome",
"retry_count_bucket",
"termination",
"tool_call_count_bucket"
],
"properties": {
"duration_bucket": {
"$ref": "#/$defs/duration_bucket"
},
"end_reason": {
"enum": [
"approval_denied",
"completed",
"failed",
"guardrail_blocked",
"iteration_limit",
"system_aborted",
"timed_out",
"unknown",
"user_cancelled"
]
},
"entrypoint": {
"$ref": "#/$defs/task_entrypoint"
},
"execution_surface": {
"$ref": "#/$defs/execution_surface"
},
"model_call_count_bucket": {
"$ref": "#/$defs/count_bucket"
},
"outcome": {
"enum": [
"cancelled",
"failed",
"success",
"timed_out",
"unknown"
]
},
"retry_count_bucket": {
"$ref": "#/$defs/count_bucket"
},
"termination": {
"enum": [
"none",
"system_aborted",
"timed_out",
"unknown",
"user_cancelled"
]
},
"tool_call_count_bucket": {
"$ref": "#/$defs/count_bucket"
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"execution_surface": {
"enum": [
"api",
"batch",
"cli",
"desktop",
"gateway",
"other",
"python",
"scheduled_task",
"tui",
"unknown"
]
},
"task_entrypoint": {
"enum": [
"api",
"background",
"batch",
"delegated",
"gateway_message",
"interactive",
"other",
"python",
"scheduled_task",
"unknown"
]
},
"duration_bucket": {
"enum": [
"1s_to_5s",
"2m_to_10m",
"30s_to_2m",
"5s_to_30s",
"gte_10m",
"lt_1s"
]
},
"count_bucket": {
"enum": [
"0",
"1",
"2",
"3_to_5",
"6_to_10",
"gte_11"
]
}
}
}
hermes_cli/observability/schemas/hermes.shared_metrics.v2.schema.json
+702
View File
@@ -0,0 +1,702 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "urn:hermes-agent:schema:shared-metrics:v2",
"title": "Hermes Shared Metrics Package v2",
"type": "object",
"additionalProperties": false,
"required": [
"schema_version",
"package_id",
"install_id",
"period_start",
"period_end",
"generated_at",
"resource",
"metrics"
],
"properties": {
"schema_version": {
"const": "hermes.shared_metrics.v2"
},
"package_id": {
"$ref": "#/$defs/uuid"
},
"install_id": {
"description": "Random persistent identifier scoped to one HERMES_HOME; local-only in schema v2.",
"$ref": "#/$defs/uuid"
},
"period_start": {
"type": "string",
"format": "date-time"
},
"period_end": {
"type": "string",
"format": "date-time"
},
"generated_at": {
"type": "string",
"format": "date-time"
},
"resource": {
"type": "object",
"additionalProperties": false,
"required": [
"architecture",
"hermes_version",
"install_method",
"os_family"
],
"properties": {
"architecture": {
"type": "string",
"enum": ["arm", "arm64", "unknown", "x86", "x86_64"]
},
"hermes_version": {
"type": "string",
"minLength": 1,
"maxLength": 64
},
"install_method": {
"type": "string",
"enum": ["apt", "docker", "git", "home-manager", "homebrew", "nixos", "pip", "unknown"]
},
"os_family": {
"type": "string",
"enum": ["linux", "macos", "unknown", "windows"]
}
}
},
"metrics": {
"type": "array",
"minItems": 1,
"items": {
"oneOf": [
{
"$ref": "#/$defs/client_active_counter"
},
{
"$ref": "#/$defs/model_call_counter"
},
{
"$ref": "#/$defs/model_route_counter"
},
{
"$ref": "#/$defs/task_started_counter"
},
{
"$ref": "#/$defs/task_finished_counter"
},
{
"$ref": "#/$defs/tool_call_counter"
},
{
"$ref": "#/$defs/tool_approval_counter"
},
{
"$ref": "#/$defs/skill_lifecycle_counter"
},
{
"$ref": "#/$defs/skill_load_counter"
}
]
}
}
},
"$defs": {
"client_active_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.client.active"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"maxProperties": 0
},
"value": {
"const": 1
}
}
},
"uuid": {
"type": "string",
"pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
},
"model_call_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.model_call.count"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"call_role",
"locality",
"model_family",
"outcome",
"provider_family"
],
"properties": {
"call_role": {
"const": "primary"
},
"locality": {
"enum": [
"local",
"remote",
"unknown"
]
},
"model_family": {
"enum": [
"claude",
"deepseek",
"gemini",
"gemma",
"glm",
"gpt",
"grok",
"kimi",
"llama",
"minimax",
"mimo",
"mistral",
"nemotron",
"nova",
"o1",
"o3",
"o4",
"qwen",
"step",
"trinity",
"unknown"
]
},
"outcome": {
"enum": [
"cancelled",
"failed",
"success"
]
},
"provider_family": {
"enum": [
"aggregator",
"custom",
"direct",
"local",
"unknown"
]
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"model_route_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.model_route.count"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"model",
"provider"
],
"properties": {
"model": {
"type": "string",
"minLength": 1,
"maxLength": 256,
"pattern": "^[a-z0-9][a-z0-9._:/@+\\-]*$"
},
"provider": {
"type": "string",
"minLength": 1,
"maxLength": 64,
"pattern": "^[a-z0-9][a-z0-9._:/@+\\-]*$"
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"task_started_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.task_run.started"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"entrypoint",
"execution_surface"
],
"properties": {
"entrypoint": {
"$ref": "#/$defs/task_entrypoint"
},
"execution_surface": {
"$ref": "#/$defs/execution_surface"
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"task_finished_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.task_run.finished"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"duration_bucket",
"end_reason",
"entrypoint",
"execution_surface",
"model_call_count_bucket",
"outcome",
"retry_count_bucket",
"termination",
"tool_call_count_bucket"
],
"properties": {
"duration_bucket": {
"$ref": "#/$defs/duration_bucket"
},
"end_reason": {
"enum": [
"approval_denied",
"completed",
"failed",
"guardrail_blocked",
"iteration_limit",
"system_aborted",
"timed_out",
"unknown",
"user_cancelled"
]
},
"entrypoint": {
"$ref": "#/$defs/task_entrypoint"
},
"execution_surface": {
"$ref": "#/$defs/execution_surface"
},
"model_call_count_bucket": {
"$ref": "#/$defs/count_bucket"
},
"outcome": {
"enum": [
"cancelled",
"failed",
"success",
"timed_out",
"unknown"
]
},
"retry_count_bucket": {
"$ref": "#/$defs/count_bucket"
},
"termination": {
"enum": [
"none",
"system_aborted",
"timed_out",
"unknown",
"user_cancelled"
]
},
"tool_call_count_bucket": {
"$ref": "#/$defs/count_bucket"
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"execution_surface": {
"enum": [
"api",
"batch",
"cli",
"desktop",
"gateway",
"other",
"python",
"scheduled_task",
"tui",
"unknown"
]
},
"task_entrypoint": {
"enum": [
"api",
"background",
"batch",
"delegated",
"gateway_message",
"interactive",
"other",
"python",
"scheduled_task",
"unknown"
]
},
"duration_bucket": {
"enum": [
"1s_to_5s",
"2m_to_10m",
"30s_to_2m",
"5s_to_30s",
"gte_10m",
"lt_1s"
]
},
"count_bucket": {
"enum": [
"0",
"1",
"2",
"3_to_5",
"6_to_10",
"gte_11"
]
},
"tool_latency_bucket": {
"enum": [
"100ms_to_250ms",
"10s_to_30s",
"1s_to_2s",
"250ms_to_500ms",
"2s_to_5s",
"500ms_to_1s",
"5s_to_10s",
"gte_30s",
"lt_100ms",
"unknown"
]
},
"tool_retry_bucket": {
"enum": [
"0",
"1",
"2",
"3_to_5",
"6_to_10",
"gte_11",
"unknown"
]
},
"tool_call_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.tool_call.count"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"approval_outcome",
"latency_bucket",
"outcome",
"retry_count_bucket",
"tool_category"
],
"properties": {
"approval_outcome": {
"enum": [
"approved",
"denied",
"not_required",
"timed_out",
"unknown"
]
},
"latency_bucket": {
"$ref": "#/$defs/tool_latency_bucket"
},
"outcome": {
"enum": [
"blocked",
"cancelled",
"failed",
"success",
"timed_out",
"unknown"
]
},
"retry_count_bucket": {
"$ref": "#/$defs/tool_retry_bucket"
},
"tool_category": {
"enum": [
"browser",
"code_execution",
"communication",
"computer_use",
"delegation",
"file",
"home_automation",
"mcp",
"media",
"memory",
"other",
"planning",
"project",
"scheduler",
"skill",
"terminal",
"unknown",
"web"
]
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"tool_approval_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.tool_approval.count"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"attribution",
"outcome"
],
"properties": {
"attribution": {
"enum": [
"tool_call",
"unattributed"
]
},
"outcome": {
"enum": [
"approved",
"denied",
"timed_out",
"unknown"
]
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"skill_lifecycle_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.skill.lifecycle.count"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"action",
"provenance"
],
"properties": {
"action": {
"enum": [
"archived",
"created",
"edited",
"installed",
"patched",
"restored",
"stale"
]
},
"provenance": {
"$ref": "#/$defs/skill_provenance"
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"skill_load_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.skill.load.count"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"post_patch_state",
"provenance",
"reuse_state",
"use_count_bucket"
],
"properties": {
"post_patch_state": {
"enum": [
"no_new_patch",
"not_applicable",
"reused_after_patch"
]
},
"provenance": {
"$ref": "#/$defs/skill_provenance"
},
"reuse_state": {
"enum": [
"first_use",
"reused"
]
},
"use_count_bucket": {
"$ref": "#/$defs/count_bucket"
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"skill_provenance": {
"enum": [
"agent_created",
"external",
"installed",
"local",
"unknown"
]
}
}
}
hermes_cli/observability/shared_metrics.py
+822
View File
@@ -0,0 +1,822 @@
"""Durable aggregation and local export for Hermes shared metrics."""
from __future__ import annotations
import json
import logging
import sqlite3
import uuid
from collections.abc import Iterator
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
from hermes_cli.sqlite_util import write_txn
from hermes_constants import get_hermes_home
from utils import atomic_json_write
from .shared_metrics_contract import (
CLIENT_ACTIVE_METRIC,
COUNTER_METRICS,
MODEL_ROUTE_METRIC,
client_resource_is_valid,
counter_dimensions_are_valid,
)
_PACKAGE_SCHEMA_VERSION = "hermes.shared_metrics.v2"
_STORE_SCHEMA_VERSION = "2"
_BUSY_TIMEOUT_MS = 250
_SCHEMA_BUSY_TIMEOUT_MS = 5_000
_LOCAL_HISTORY_RETENTION_DAYS = 30
_ACTIVE_INSTALL_STATE_KEY = "client_active_recorded_at"
_ACTIVE_INSTALL_INTERVAL = timedelta(hours=24)
logger = logging.getLogger(__name__)
def _utc_now() -> datetime:
return datetime.now(timezone.utc)
def _isoformat(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
class SharedMetricsStore:
"""Persist allowlisted counters and export immutable delta packages."""
def __init__(
self,
database_path: Path | None = None,
outbox_directory: Path | None = None,
) -> None:
root = get_hermes_home() / "telemetry" / "shared_metrics"
self.database_path = database_path or root / "metrics.sqlite3"
self.outbox_directory = outbox_directory or root / "outbox"
self._ensure_private_directory(self.database_path.parent)
self._ensure_private_directory(self.outbox_directory)
self._ensure_private_file(self.database_path)
self._ensure_schema()
def record_model_call(
self,
dimensions: dict[str, str],
resource: dict[str, str],
) -> None:
"""Increment the terminal model-call counter for the current UTC day."""
self.record_counter(MODEL_ROUTE_METRIC, dimensions, resource)
def record_client_active(self, resource: dict[str, str]) -> bool:
"""Record this install at most once in any rolling 24-hour window."""
dimensions: dict[str, str] = {}
self._validate_counter(CLIENT_ACTIVE_METRIC, dimensions, resource)
now = _utc_now()
with self._connection() as connection:
with write_txn(connection):
row = connection.execute(
"SELECT value FROM telemetry_state WHERE key = ?",
(_ACTIVE_INSTALL_STATE_KEY,),
).fetchone()
if row is not None:
last_recorded = self._parse_state_timestamp(row["value"])
if last_recorded is not None and last_recorded > now:
# A wall-clock correction must not suppress activity until
# the stale future timestamp plus another full interval.
connection.execute(
"""
UPDATE telemetry_state
SET value = ?
WHERE key = ?
""",
(_isoformat(now), _ACTIVE_INSTALL_STATE_KEY),
)
return False
if (
last_recorded is not None
and now < last_recorded + _ACTIVE_INSTALL_INTERVAL
):
return False
self._install_id(connection)
self._record_counter_in_transaction(
connection,
CLIENT_ACTIVE_METRIC,
dimensions,
resource,
period_start=now.date().isoformat(),
)
connection.execute(
"""
INSERT INTO telemetry_state(key, value)
VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
""",
(_ACTIVE_INSTALL_STATE_KEY, _isoformat(now)),
)
return True
def record_counter(
self,
metric_name: str,
dimensions: dict[str, str],
resource: dict[str, str],
) -> None:
"""Increment one allowlisted counter for the current UTC day."""
self._validate_counter(metric_name, dimensions, resource)
with self._connection() as connection:
self._record_counter_in_transaction(
connection,
metric_name,
dimensions,
resource,
period_start=_utc_now().date().isoformat(),
)
@staticmethod
def _validate_counter(
metric_name: str,
dimensions: dict[str, str],
resource: dict[str, str],
) -> None:
if metric_name not in COUNTER_METRICS:
raise ValueError(f"Unsupported shared metric: {metric_name}")
if not counter_dimensions_are_valid(metric_name, dimensions):
raise ValueError(f"Unsupported dimensions for shared metric: {metric_name}")
if not client_resource_is_valid(resource):
raise ValueError("Unsupported shared-metrics client resource")
@staticmethod
def _record_counter_in_transaction(
connection: sqlite3.Connection,
metric_name: str,
dimensions: dict[str, str],
resource: dict[str, str],
*,
period_start: str,
) -> None:
dimensions_json = json.dumps(
dimensions,
sort_keys=True,
separators=(",", ":"),
)
connection.execute(
"""
INSERT INTO counter_aggregates(
period_start,
metric_name,
hermes_version,
os_family,
architecture,
install_method,
dimensions_json,
value,
packaged_value
) VALUES (?, ?, ?, ?, ?, ?, ?, 1, 0)
ON CONFLICT(
period_start,
metric_name,
hermes_version,
os_family,
architecture,
install_method,
dimensions_json
)
DO UPDATE SET value = value + 1
""",
(
period_start,
metric_name,
resource["hermes_version"],
resource["os_family"],
resource["architecture"],
resource["install_method"],
dimensions_json,
),
)
def create_and_export_package(self) -> list[Path]:
"""Commit one pending delta package, then atomically export the outbox."""
pending_periods = self._pending_period_count()
for _ in range(pending_periods):
if self._create_package() is None:
break
return self._export_and_prune()
def create_and_export_package_if_due(self) -> list[Path]:
"""Create pending packages at most once per UTC day, then export them."""
self._create_pending_packages_if_due()
return self._export_and_prune()
def _export_and_prune(self) -> list[Path]:
exported = self._export_pending_packages()
try:
self._prune_expired_history()
except Exception:
logger.warning(
"Unable to prune expired shared-metrics history",
exc_info=True,
)
return exported
def counter_snapshot(self) -> list[dict[str, Any]]:
"""Return cumulative counters for focused tests and local inspection."""
with self._connection() as connection:
rows = connection.execute(
"""
SELECT
period_start,
metric_name,
hermes_version,
os_family,
architecture,
install_method,
dimensions_json,
value,
packaged_value
FROM counter_aggregates
ORDER BY
period_start,
hermes_version,
os_family,
architecture,
install_method,
metric_name,
dimensions_json
"""
).fetchall()
return [
{
"period_start": row["period_start"],
"metric_name": row["metric_name"],
"resource": {
"hermes_version": row["hermes_version"],
"os_family": row["os_family"],
"architecture": row["architecture"],
"install_method": row["install_method"],
},
"dimensions": json.loads(row["dimensions_json"]),
"value": row["value"],
"packaged_value": row["packaged_value"],
}
for row in rows
]
@contextmanager
def _connection(
self,
*,
busy_timeout_ms: int = _BUSY_TIMEOUT_MS,
) -> Iterator[sqlite3.Connection]:
connection = sqlite3.connect(
self.database_path,
timeout=busy_timeout_ms / 1000,
)
try:
connection.row_factory = sqlite3.Row
connection.execute(f"PRAGMA busy_timeout={busy_timeout_ms}")
with connection:
yield connection
finally:
connection.close()
@staticmethod
def _ensure_private_directory(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True, mode=0o700)
try:
path.chmod(0o700)
except OSError:
pass
@staticmethod
def _ensure_private_file(path: Path) -> None:
path.touch(mode=0o600, exist_ok=True)
try:
path.chmod(0o600)
except OSError:
pass
def _ensure_schema(self) -> None:
with self._connection(busy_timeout_ms=_SCHEMA_BUSY_TIMEOUT_MS) as connection:
# Serialize first-run creation and upgrades across Hermes processes.
with write_txn(connection):
self._ensure_schema_in_transaction(connection)
@staticmethod
def _ensure_schema_in_transaction(connection: sqlite3.Connection) -> None:
connection.execute(
"""
CREATE TABLE IF NOT EXISTS telemetry_state (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)
"""
)
schema_row = connection.execute(
"SELECT value FROM telemetry_state WHERE key = 'schema_version'"
).fetchone()
schema_version = str(schema_row["value"]) if schema_row is not None else None
if schema_version == "1":
SharedMetricsStore._migrate_v1_counter_aggregates(connection)
schema_version = _STORE_SCHEMA_VERSION
if schema_version is not None and schema_version != _STORE_SCHEMA_VERSION:
raise RuntimeError(
f"Unsupported shared-metrics store schema version: {schema_version}"
)
SharedMetricsStore._create_counter_aggregates_table(connection)
connection.execute(
"""
CREATE TABLE IF NOT EXISTS package_outbox (
package_id TEXT PRIMARY KEY,
period_start TEXT NOT NULL,
period_end TEXT NOT NULL,
payload_json TEXT NOT NULL,
created_at TEXT NOT NULL,
exported_at TEXT
)
"""
)
SharedMetricsStore._add_send_columns(connection)
SharedMetricsStore._add_consent_tables(connection)
connection.execute(
"""
INSERT INTO telemetry_state(key, value)
VALUES ('schema_version', ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
""",
(_STORE_SCHEMA_VERSION,),
)
@staticmethod
def _add_send_columns(connection: sqlite3.Connection) -> None:
"""Add transmission bookkeeping to ``package_outbox``, idempotently.
These columns are ADDITIVE and nullable, and the store schema version
is deliberately NOT bumped. ``_ensure_schema_in_transaction`` raises on
any version it does not recognise and has no forward-compatibility
branch, so bumping would make an older Hermes — a second profile on an
older build, or a rollback — hard-fail against the same database file.
Old readers select named columns and never ``SELECT *``, so extra
columns are invisible to them.
"""
existing = {
str(row["name"])
for row in connection.execute("PRAGMA table_info(package_outbox)")
}
for column, declaration in (
# When the 202 was received. NULL = never acknowledged.
("sent_at", "TEXT"),
# NULL/'pending' = eligible, 'sent' = done, 'rejected' = permanent 400.
("send_state", "TEXT"),
("send_attempts", "INTEGER NOT NULL DEFAULT 0"),
# Earliest next attempt; enforces backoff across process restarts.
("next_attempt_at", "TEXT"),
("last_error", "TEXT"),
# The identifier actually transmitted, frozen on the first
# attempt so retries stay byte-identical. Since the 2026-08-27
# product decision this is the stable install_id itself.
# Only the ~36-byte id is stored: the body is recomputed from
# payload_json, whose serialisation is deterministic.
("sent_install_id", "TEXT"),
# NULL until first claimed; rewritten on every claim. Settlement
# and the pre-POST revalidation are compare-and-set on this, so a
# claimant whose lease lapsed loses authority the moment another
# process reclaims (PR-review finding: without it, a suspended
# sender resuming after a reclaim double-POSTs the package).
("claim_token", "TEXT"),
):
if column not in existing:
connection.execute(
f"ALTER TABLE package_outbox ADD COLUMN {column} {declaration}"
)
@staticmethod
def _add_consent_tables(connection: sqlite3.Connection) -> None:
"""Create the consent-window tables, idempotently.
Additive like ``_add_send_columns`` — the schema version is
deliberately NOT bumped, and old readers never touch these tables.
``send_consent_windows`` records consent as explicit intervals rather
than a moving day-stamp: a window is opened when send consent is
observed, heartbeat-confirmed on every later observation, and closed
at the LAST CONFIRMED moment (never "now") when consent is observed
withdrawn. Consent is asserted only for time that was actually
observed, so unobserved gaps — a hand-edited config with no process
running — fail closed by construction.
``consent_marks`` holds two monotonic high-water marks with strictly
separated roles:
- ``obs``: the latest observation stamp ever seen. Advanced only by
the reconciler. Confirms consent and clamps window closes.
- ``data``: the latest package ``period_end`` ever stored. Advanced
only by the package writer. Clamps window OPENS, so a rolled-back
clock can never open a window underneath packages that already
exist on disk.
The separation is load-bearing: letting data stamps confirm consent
re-created a refused-window leak (packages stored during an off
window would vouch for it), and letting observation stamps clamp
opens is not enough on its own to stop a rollback sliding a window
under existing refused data.
"""
connection.execute(
"""
CREATE TABLE IF NOT EXISTS send_consent_windows (
opened_at TEXT NOT NULL,
last_confirmed_at TEXT NOT NULL,
closed_at TEXT
)
"""
)
connection.execute(
"""
CREATE TABLE IF NOT EXISTS consent_marks (
name TEXT PRIMARY KEY CHECK (name IN ('obs', 'data')),
stamp TEXT NOT NULL
)
"""
)
@staticmethod
def _create_counter_aggregates_table(connection: sqlite3.Connection) -> None:
connection.execute(
"""
CREATE TABLE IF NOT EXISTS counter_aggregates (
period_start TEXT NOT NULL,
metric_name TEXT NOT NULL,
hermes_version TEXT NOT NULL,
os_family TEXT NOT NULL,
architecture TEXT NOT NULL,
install_method TEXT NOT NULL,
dimensions_json TEXT NOT NULL,
value INTEGER NOT NULL,
packaged_value INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (
period_start,
metric_name,
hermes_version,
os_family,
architecture,
install_method,
dimensions_json
)
)
"""
)
@staticmethod
def _migrate_v1_counter_aggregates(connection: sqlite3.Connection) -> None:
connection.execute(
"ALTER TABLE counter_aggregates RENAME TO counter_aggregates_v1"
)
SharedMetricsStore._create_counter_aggregates_table(connection)
connection.execute(
"""
INSERT INTO counter_aggregates(
period_start,
metric_name,
hermes_version,
os_family,
architecture,
install_method,
dimensions_json,
value,
packaged_value
)
SELECT
period_start,
metric_name,
hermes_version,
'unknown',
'unknown',
'unknown',
dimensions_json,
value,
packaged_value
FROM counter_aggregates_v1
"""
)
connection.execute("DROP TABLE counter_aggregates_v1")
def _install_id(self, connection: sqlite3.Connection) -> str:
row = connection.execute(
"SELECT value FROM telemetry_state WHERE key = 'install_id'"
).fetchone()
if row is not None:
return str(row["value"])
candidate = str(uuid.uuid4())
connection.execute(
"INSERT OR IGNORE INTO telemetry_state(key, value) VALUES ('install_id', ?)",
(candidate,),
)
row = connection.execute(
"SELECT value FROM telemetry_state WHERE key = 'install_id'"
).fetchone()
if row is None:
raise RuntimeError("Unable to create the shared-metrics install identity")
return str(row["value"])
@staticmethod
def _parse_state_timestamp(value: Any) -> datetime | None:
try:
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
except (TypeError, ValueError):
return None
if parsed.tzinfo is None:
return None
return parsed.astimezone(timezone.utc)
def _pending_period_count(self) -> int:
with self._connection() as connection:
row = connection.execute(
"""
SELECT COUNT(*) AS period_count
FROM (
SELECT
period_start,
hermes_version,
os_family,
architecture,
install_method
FROM counter_aggregates
WHERE value > packaged_value
GROUP BY
period_start,
hermes_version,
os_family,
architecture,
install_method
)
"""
).fetchone()
return int(row["period_count"]) if row is not None else 0
def _create_pending_packages_if_due(self) -> None:
now = _utc_now()
with self._connection() as connection:
with write_txn(connection):
# Gate on the committed package, not its file write, so a failed
# outbox export can be retried without packaging deltas twice.
package_created_today = connection.execute(
"""
SELECT 1
FROM package_outbox
WHERE substr(created_at, 1, 10) >= ?
LIMIT 1
""",
(now.date().isoformat(),),
).fetchone()
if package_created_today is not None:
return
while self._create_package_in_transaction(connection, now) is not None:
pass
def _create_package(self) -> dict[str, Any] | None:
now = _utc_now()
with self._connection() as connection:
with write_txn(connection):
return self._create_package_in_transaction(connection, now)
def _create_package_in_transaction(
self,
connection: sqlite3.Connection,
now: datetime,
) -> dict[str, Any] | None:
period_row = connection.execute(
"""
SELECT
period_start,
hermes_version,
os_family,
architecture,
install_method
FROM counter_aggregates
WHERE value > packaged_value
ORDER BY
period_start,
hermes_version,
os_family,
architecture,
install_method
LIMIT 1
"""
).fetchone()
period_value = period_row["period_start"] if period_row is not None else None
if not period_value:
return None
rows = connection.execute(
"""
SELECT metric_name, dimensions_json, value, packaged_value
FROM counter_aggregates
WHERE period_start = ?
AND hermes_version = ?
AND os_family = ?
AND architecture = ?
AND install_method = ?
AND value > packaged_value
ORDER BY metric_name, dimensions_json
""",
(
period_value,
period_row["hermes_version"],
period_row["os_family"],
period_row["architecture"],
period_row["install_method"],
),
).fetchall()
period_start = datetime.fromisoformat(str(period_value)).replace(
tzinfo=timezone.utc
)
period_end = period_start + timedelta(days=1)
package_id = str(uuid.uuid4())
resource = {
"hermes_version": period_row["hermes_version"],
"os_family": period_row["os_family"],
"architecture": period_row["architecture"],
"install_method": period_row["install_method"],
}
if not client_resource_is_valid(resource):
raise ValueError("Unsupported shared-metrics client resource")
payload = {
"schema_version": _PACKAGE_SCHEMA_VERSION,
"package_id": package_id,
"install_id": self._install_id(connection),
"period_start": _isoformat(period_start),
"period_end": _isoformat(period_end),
"generated_at": _isoformat(now),
"resource": resource,
"metrics": [self._package_metric(row) for row in rows],
}
payload_json = json.dumps(
payload,
sort_keys=True,
separators=(",", ":"),
)
connection.execute(
"""
INSERT INTO package_outbox(
package_id,
period_start,
period_end,
payload_json,
created_at
) VALUES (?, ?, ?, ?, ?)
""",
(
package_id,
payload["period_start"],
payload["period_end"],
payload_json,
payload["generated_at"],
),
)
# Advance the data high-water mark. This is the ONLY writer of the
# 'data' mark: it clamps consent-window opens so a rolled-back clock
# can never open a window underneath packages that already exist.
connection.execute(
"""
INSERT INTO consent_marks(name, stamp) VALUES ('data', ?)
ON CONFLICT(name) DO UPDATE SET stamp = MAX(stamp, excluded.stamp)
""",
(payload["period_end"],),
)
for row in rows:
connection.execute(
"""
UPDATE counter_aggregates
SET packaged_value = value
WHERE period_start = ?
AND metric_name = ?
AND hermes_version = ?
AND os_family = ?
AND architecture = ?
AND install_method = ?
AND dimensions_json = ?
""",
(
period_value,
row["metric_name"],
period_row["hermes_version"],
period_row["os_family"],
period_row["architecture"],
period_row["install_method"],
row["dimensions_json"],
),
)
return payload
@staticmethod
def _package_metric(row: sqlite3.Row) -> dict[str, Any]:
metric_name = str(row["metric_name"])
dimensions = json.loads(row["dimensions_json"])
if not isinstance(dimensions, dict) or not counter_dimensions_are_valid(
metric_name, dimensions
):
raise ValueError(f"Unsupported dimensions for shared metric: {metric_name}")
return {
"name": metric_name,
"type": "counter",
"dimensions": dimensions,
"value": row["value"] - row["packaged_value"],
}
def _export_pending_packages(self) -> list[Path]:
with self._connection() as connection:
rows = connection.execute(
"""
SELECT package_id, payload_json
FROM package_outbox
WHERE exported_at IS NULL
ORDER BY created_at, package_id
"""
).fetchall()
exported: list[Path] = []
for row in rows:
package_id = str(row["package_id"])
path = self.outbox_directory / f"{package_id}.json"
atomic_json_write(
path,
json.loads(row["payload_json"]),
indent=2,
sort_keys=True,
mode=0o600,
)
with self._connection() as connection:
connection.execute(
"""
UPDATE package_outbox
SET exported_at = ?
WHERE package_id = ? AND exported_at IS NULL
""",
(_isoformat(_utc_now()), package_id),
)
exported.append(path)
return exported
def _prune_expired_history(self, *, now: datetime | None = None) -> None:
"""Remove exported local history after the bounded retention window."""
cutoff = (now or _utc_now()) - timedelta(
days=_LOCAL_HISTORY_RETENTION_DAYS
)
cutoff_timestamp = _isoformat(cutoff)
cutoff_period = cutoff.date().isoformat()
with self._connection() as connection:
rows = connection.execute(
"""
SELECT package_id
FROM package_outbox
WHERE exported_at IS NOT NULL
AND exported_at < ?
ORDER BY exported_at, package_id
""",
(cutoff_timestamp,),
).fetchall()
removable_package_ids: list[str] = []
for row in rows:
package_id = str(row["package_id"])
try:
(self.outbox_directory / f"{package_id}.json").unlink(
missing_ok=True
)
except OSError:
logger.warning(
"Unable to prune expired shared-metrics package %s",
package_id,
exc_info=True,
)
continue
removable_package_ids.append(package_id)
with self._connection() as connection:
with write_txn(connection):
for package_id in removable_package_ids:
connection.execute(
"""
DELETE FROM package_outbox
WHERE package_id = ?
AND exported_at IS NOT NULL
AND exported_at < ?
""",
(package_id, cutoff_timestamp),
)
connection.execute(
"""
DELETE FROM counter_aggregates
WHERE period_start < ?
AND value = packaged_value
AND NOT EXISTS (
SELECT 1
FROM package_outbox
WHERE exported_at IS NULL
AND substr(package_outbox.period_start, 1, 10)
= counter_aggregates.period_start
)
""",
(cutoff_period,),
)
hermes_cli/observability/shared_metrics_contract.py
+978
View File
@@ -0,0 +1,978 @@
"""Bounded product contract for the first Hermes shared-metrics slice."""
from __future__ import annotations
from math import isfinite
from typing import Any
from agent.relay_runtime import (
LOGICAL_LLM_SCOPE,
RUNTIME_INSTANCE_KEY,
RUNTIME_SCHEMA_KEY,
RUNTIME_SCHEMA_VERSION,
)
SCHEMA_KEY = "hermes.metrics.schema_version"
SCHEMA_VERSION = "hermes.metrics.event.v2"
MODEL_CALL_SCOPE = "hermes.model_call"
MODEL_CALL_PROFILE_MODEL = "unknown"
TASK_SCOPE = "hermes.task_run"
TOOL_CALL_SCOPE = "hermes.tool_call"
CLIENT_ACTIVE_MARK = "hermes.client.active"
TOOL_APPROVAL_MARK = "hermes.tool_approval"
SKILL_LIFECYCLE_MARK = "hermes.skill.lifecycle"
SKILL_LOAD_MARK = "hermes.skill.load"
SUBSCRIBER_NAME = "hermes.nemo_relay.shared_metrics"
CLIENT_ACTIVE_METRIC = "hermes.client.active"
LEGACY_MODEL_CALL_METRIC = "hermes.model_call.count"
MODEL_ROUTE_METRIC = "hermes.model_route.count"
TASK_STARTED_METRIC = "hermes.task_run.started"
TASK_FINISHED_METRIC = "hermes.task_run.finished"
TOOL_CALL_METRIC = "hermes.tool_call.count"
TOOL_APPROVAL_METRIC = "hermes.tool_approval.count"
SKILL_LIFECYCLE_METRIC = "hermes.skill.lifecycle.count"
SKILL_LOAD_METRIC = "hermes.skill.load.count"
MODEL_IDENTIFIER_MAX_LENGTH = 256
PROVIDER_IDENTIFIER_MAX_LENGTH = 64
_METRIC_IDENTIFIER_CHARACTERS = frozenset(
"abcdefghijklmnopqrstuvwxyz0123456789._:/@+-"
)
_METRIC_IDENTIFIER_START_CHARACTERS = frozenset(
"abcdefghijklmnopqrstuvwxyz0123456789"
)
EXECUTION_SURFACES: frozenset[str] = frozenset({
"api",
"batch",
"cli",
"desktop",
"gateway",
"python",
"scheduled_task",
"tui",
"other",
"unknown",
})
TASK_OUTCOMES: frozenset[str] = frozenset({
"cancelled",
"failed",
"success",
"timed_out",
"unknown",
})
TASK_END_REASONS: frozenset[str] = frozenset({
"approval_denied",
"completed",
"failed",
"guardrail_blocked",
"iteration_limit",
"system_aborted",
"timed_out",
"unknown",
"user_cancelled",
})
TASK_TERMINATIONS: frozenset[str] = frozenset({
"none",
"system_aborted",
"timed_out",
"unknown",
"user_cancelled",
})
TASK_ENTRYPOINTS: frozenset[str] = frozenset({
"api",
"background",
"batch",
"delegated",
"gateway_message",
"interactive",
"other",
"python",
"scheduled_task",
"unknown",
})
DURATION_BUCKETS: frozenset[str] = frozenset({
"1s_to_5s",
"2m_to_10m",
"30s_to_2m",
"5s_to_30s",
"gte_10m",
"lt_1s",
})
COUNT_BUCKETS: frozenset[str] = frozenset({
"0",
"1",
"2",
"3_to_5",
"6_to_10",
"gte_11",
})
TOOL_CATEGORIES: frozenset[str] = frozenset({
"browser",
"code_execution",
"communication",
"computer_use",
"delegation",
"file",
"home_automation",
"mcp",
"media",
"memory",
"other",
"planning",
"project",
"scheduler",
"skill",
"terminal",
"unknown",
"web",
})
TOOL_OUTCOMES: frozenset[str] = frozenset({
"blocked",
"cancelled",
"failed",
"success",
"timed_out",
"unknown",
})
TOOL_APPROVAL_OUTCOMES: frozenset[str] = frozenset({
"approved",
"denied",
"not_required",
"timed_out",
"unknown",
})
TOOL_APPROVAL_ATTRIBUTIONS: frozenset[str] = frozenset({
"tool_call",
"unattributed",
})
TOOL_LATENCY_BUCKETS: frozenset[str] = frozenset({
"100ms_to_250ms",
"10s_to_30s",
"1s_to_2s",
"250ms_to_500ms",
"2s_to_5s",
"500ms_to_1s",
"5s_to_10s",
"gte_30s",
"lt_100ms",
"unknown",
})
TOOL_RETRY_BUCKETS: frozenset[str] = COUNT_BUCKETS | frozenset({"unknown"})
SKILL_LIFECYCLE_ACTIONS: frozenset[str] = frozenset({
"archived",
"created",
"edited",
"installed",
"patched",
"restored",
"stale",
})
SKILL_PROVENANCES: frozenset[str] = frozenset({
"agent_created",
"external",
"installed",
"local",
"unknown",
})
SKILL_REUSE_STATES: frozenset[str] = frozenset({"first_use", "reused"})
SKILL_POST_PATCH_STATES: frozenset[str] = frozenset({
"no_new_patch",
"not_applicable",
"reused_after_patch",
})
CLIENT_OS_FAMILIES: frozenset[str] = frozenset({
"linux",
"macos",
"unknown",
"windows",
})
CLIENT_ARCHITECTURES: frozenset[str] = frozenset({
"arm",
"arm64",
"unknown",
"x86",
"x86_64",
})
CLIENT_INSTALL_METHODS: frozenset[str] = frozenset({
"apt",
"docker",
"git",
"home-manager",
"homebrew",
"nixos",
"pip",
"unknown",
})
CLIENT_RESOURCE_KEYS: frozenset[str] = frozenset({
"architecture",
"hermes_version",
"install_method",
"os_family",
})
def client_os_family(value: Any) -> str:
"""Map a platform system name to the shared-metrics OS taxonomy."""
normalized = str(value or "").strip().lower()
return {
"darwin": "macos",
"linux": "linux",
"macos": "macos",
"windows": "windows",
}.get(normalized, "unknown")
def client_architecture(value: Any) -> str:
"""Map a machine architecture to the shared-metrics taxonomy."""
normalized = str(value or "").strip().lower().replace("-", "_")
if normalized in {"amd64", "x64", "x86_64"}:
return "x86_64"
if normalized in {"aarch64", "arm64"}:
return "arm64"
if normalized in {"i386", "i486", "i586", "i686", "x86"}:
return "x86"
if normalized.startswith("armv"):
return "arm"
return "unknown"
def client_install_method(value: Any) -> str:
"""Return an allowlisted Hermes installation method."""
normalized = str(value or "").strip().lower()
if normalized == "nix":
return "nixos"
return normalized if normalized in CLIENT_INSTALL_METHODS else "unknown"
def client_resource(
hermes_version: Any,
*,
os_name: Any,
architecture: Any,
install_method: Any,
) -> dict[str, str]:
"""Build the bounded client resource attached to aggregate packages."""
normalized_version = str(hermes_version or "").strip()
if not normalized_version or len(normalized_version) > 64:
normalized_version = "unknown"
return {
"architecture": client_architecture(architecture),
"hermes_version": normalized_version,
"install_method": client_install_method(install_method),
"os_family": client_os_family(os_name),
}
def client_resource_is_valid(resource: Any) -> bool:
"""Return whether a package resource exactly matches the bounded contract."""
if not isinstance(resource, dict) or set(resource) != CLIENT_RESOURCE_KEYS:
return False
version = resource.get("hermes_version")
return (
isinstance(version, str)
and 0 < len(version) <= 64
and resource.get("os_family") in CLIENT_OS_FAMILIES
and resource.get("architecture") in CLIENT_ARCHITECTURES
and resource.get("install_method") in CLIENT_INSTALL_METHODS
)
_LEGACY_PROVIDER_FAMILIES = frozenset({
"aggregator",
"custom",
"direct",
"local",
"unknown",
})
_LEGACY_MODEL_LOCALITIES = frozenset({"local", "remote", "unknown"})
_LEGACY_MODEL_OUTCOMES = frozenset({"cancelled", "failed", "success"})
_LEGACY_MODEL_FAMILIES = frozenset({
"claude",
"deepseek",
"gemini",
"gemma",
"glm",
"gpt",
"grok",
"kimi",
"llama",
"minimax",
"mimo",
"mistral",
"nemotron",
"nova",
"o1",
"o3",
"o4",
"qwen",
"step",
"trinity",
"unknown",
})
_COUNTER_DIMENSION_VALUES: dict[str, dict[str, frozenset[str]]] = {
CLIENT_ACTIVE_METRIC: {},
# Retained only so pre-v2 pending rows remain packageable.
LEGACY_MODEL_CALL_METRIC: {
"call_role": frozenset({"primary"}),
"locality": _LEGACY_MODEL_LOCALITIES,
"model_family": _LEGACY_MODEL_FAMILIES,
"outcome": _LEGACY_MODEL_OUTCOMES,
"provider_family": _LEGACY_PROVIDER_FAMILIES,
},
TASK_STARTED_METRIC: {
"entrypoint": TASK_ENTRYPOINTS,
"execution_surface": EXECUTION_SURFACES,
},
TASK_FINISHED_METRIC: {
"duration_bucket": DURATION_BUCKETS,
"end_reason": TASK_END_REASONS,
"entrypoint": TASK_ENTRYPOINTS,
"execution_surface": EXECUTION_SURFACES,
"model_call_count_bucket": COUNT_BUCKETS,
"outcome": TASK_OUTCOMES,
"retry_count_bucket": COUNT_BUCKETS,
"termination": TASK_TERMINATIONS,
"tool_call_count_bucket": COUNT_BUCKETS,
},
TOOL_CALL_METRIC: {
"approval_outcome": TOOL_APPROVAL_OUTCOMES,
"latency_bucket": TOOL_LATENCY_BUCKETS,
"outcome": TOOL_OUTCOMES,
"retry_count_bucket": TOOL_RETRY_BUCKETS,
"tool_category": TOOL_CATEGORIES,
},
TOOL_APPROVAL_METRIC: {
"attribution": TOOL_APPROVAL_ATTRIBUTIONS,
"outcome": TOOL_APPROVAL_OUTCOMES - {"not_required"},
},
SKILL_LIFECYCLE_METRIC: {
"action": SKILL_LIFECYCLE_ACTIONS,
"provenance": SKILL_PROVENANCES,
},
SKILL_LOAD_METRIC: {
"post_patch_state": SKILL_POST_PATCH_STATES,
"provenance": SKILL_PROVENANCES,
"reuse_state": SKILL_REUSE_STATES,
"use_count_bucket": COUNT_BUCKETS,
},
}
COUNTER_METRICS: frozenset[str] = frozenset({
CLIENT_ACTIVE_METRIC,
MODEL_ROUTE_METRIC,
SKILL_LIFECYCLE_METRIC,
SKILL_LOAD_METRIC,
TASK_FINISHED_METRIC,
TASK_STARTED_METRIC,
TOOL_APPROVAL_METRIC,
TOOL_CALL_METRIC,
})
def counter_dimensions_are_valid(
metric_name: str,
dimensions: dict[str, Any],
) -> bool:
"""Return whether dimensions match one closed shared-metric contract."""
if metric_name == MODEL_ROUTE_METRIC:
return (
set(dimensions) == {"model", "provider"}
and dimensions["model"]
== _metric_identifier(
dimensions["model"],
max_length=MODEL_IDENTIFIER_MAX_LENGTH,
)
and dimensions["provider"]
== _metric_identifier(
dimensions["provider"],
max_length=PROVIDER_IDENTIFIER_MAX_LENGTH,
)
)
contract = _COUNTER_DIMENSION_VALUES.get(metric_name)
if contract is None or set(dimensions) != set(contract):
return False
return all(
isinstance(dimensions[field], str) and dimensions[field] in allowed_values
for field, allowed_values in contract.items()
)
def _event_metadata_is_valid(event: Any) -> bool:
metadata = getattr(event, "metadata", None)
if not isinstance(metadata, dict) or metadata.get(SCHEMA_KEY) != SCHEMA_VERSION:
return False
relay_metadata = set(metadata) - {SCHEMA_KEY, RUNTIME_INSTANCE_KEY}
return not relay_metadata - {"otel.status_code"} and metadata.get(
"otel.status_code", "OK"
) in {"OK", "ERROR"}
def client_active_counter(event: Any) -> tuple[str, dict[str, str]] | None:
"""Return the active-install counter for one empty allowlisted mark."""
if not _event_metadata_is_valid(event):
return None
if (
str(getattr(event, "kind", "") or "") != "mark"
or str(getattr(event, "name", "") or "") != CLIENT_ACTIVE_MARK
or getattr(event, "category", None) is not None
or getattr(event, "scope_category", None) is not None
or getattr(event, "category_profile", None) is not None
or getattr(event, "data", None) != {}
):
return None
return CLIENT_ACTIVE_METRIC, {}
def model_call_dimensions(event: Any) -> dict[str, str] | None:
"""Return package dimensions for one valid logical model-call end event."""
auxiliary = _auxiliary_model_call_dimensions(event)
if auxiliary is not None:
return auxiliary
if not _event_metadata_is_valid(event):
return None
if (
str(getattr(event, "kind", "") or "") != "scope"
or str(getattr(event, "category", "") or "") != "llm"
or str(getattr(event, "name", "") or "") != MODEL_CALL_SCOPE
or str(getattr(event, "scope_category", "") or "") != "end"
):
return None
category_profile = getattr(event, "category_profile", None)
if not isinstance(category_profile, dict) or set(category_profile) != {
"model_name"
}:
return None
# The synthetic scope can span provider fallback. The accepted terminal
# route is carried in the validated payload rather than this start profile.
if category_profile.get("model_name") != MODEL_CALL_PROFILE_MODEL:
return None
data = getattr(event, "data", None)
expected_fields = {"model", "provider"}
if not isinstance(data, dict) or set(data) != expected_fields:
return None
dimensions = {field: data.get(field) for field in sorted(expected_fields)}
if not counter_dimensions_are_valid(MODEL_ROUTE_METRIC, dimensions):
return None
return dimensions
def _auxiliary_model_call_dimensions(event: Any) -> dict[str, str] | None:
"""Project a terminal auxiliary route from its Hermes logical scope."""
metadata = getattr(event, "metadata", None)
if (
not isinstance(metadata, dict)
or metadata.get(RUNTIME_SCHEMA_KEY) != RUNTIME_SCHEMA_VERSION
):
return None
relay_metadata = set(metadata) - {
RUNTIME_INSTANCE_KEY,
RUNTIME_SCHEMA_KEY,
"hermes.call_role",
}
if relay_metadata - {"otel.status_code"} or metadata.get(
"otel.status_code", "OK"
) not in {"OK", "ERROR"}:
return None
call_role = metadata.get("hermes.call_role")
if not isinstance(call_role, str) or not call_role.startswith("auxiliary:"):
return None
if (
str(getattr(event, "kind", "") or "") != "scope"
or str(getattr(event, "category", "") or "") != "function"
or str(getattr(event, "name", "") or "") != LOGICAL_LLM_SCOPE
or str(getattr(event, "scope_category", "") or "") != "end"
or getattr(event, "category_profile", None) is not None
):
return None
data = getattr(event, "data", None)
if (
not isinstance(data, dict)
or set(data)
not in (
{"model", "outcome", "provider"},
{"model", "outcome", "provider", "response_model"},
)
or data.get("outcome") not in {"cancelled", "failed", "success"}
):
return None
dimensions = model_call_fields(data)
if not counter_dimensions_are_valid(MODEL_ROUTE_METRIC, dimensions):
return None
return dimensions
def task_counter(event: Any) -> tuple[str, dict[str, str]] | None:
"""Return one validated task counter from a task scope event."""
if not _event_metadata_is_valid(event):
return None
if (
str(getattr(event, "kind", "") or "") != "scope"
or str(getattr(event, "category", "") or "") != "function"
or str(getattr(event, "name", "") or "") != TASK_SCOPE
):
return None
if getattr(event, "category_profile", None) is not None:
return None
scope_category = str(getattr(event, "scope_category", "") or "")
data = getattr(event, "data", None)
if scope_category == "start":
expected_fields = {"entrypoint", "execution_surface"}
if not isinstance(data, dict) or set(data) != expected_fields:
return None
dimensions = {
"entrypoint": data.get("entrypoint"),
"execution_surface": data.get("execution_surface"),
}
if not counter_dimensions_are_valid(TASK_STARTED_METRIC, dimensions):
return None
return TASK_STARTED_METRIC, dimensions
expected_fields = {
"duration_bucket",
"end_reason",
"entrypoint",
"execution_surface",
"model_call_count_bucket",
"outcome",
"retry_count_bucket",
"termination",
"tool_call_count_bucket",
}
if (
scope_category != "end"
or not isinstance(data, dict)
or set(data) != expected_fields
):
return None
dimensions = {field: data.get(field) for field in sorted(expected_fields)}
if not counter_dimensions_are_valid(TASK_FINISHED_METRIC, dimensions):
return None
return TASK_FINISHED_METRIC, dimensions
def tool_call_dimensions(event: Any) -> dict[str, str] | None:
"""Return package dimensions for one allowlisted tool lifecycle end event."""
if not _event_metadata_is_valid(event):
return None
if (
str(getattr(event, "kind", "") or "") != "scope"
or str(getattr(event, "category", "") or "") != "tool"
or str(getattr(event, "name", "") or "") != TOOL_CALL_SCOPE
or str(getattr(event, "scope_category", "") or "") != "end"
or getattr(event, "category_profile", None) != {}
):
return None
data = getattr(event, "data", None)
expected_fields = {
"approval_outcome",
"latency_bucket",
"outcome",
"retry_count_bucket",
"tool_category",
}
if not isinstance(data, dict) or set(data) != expected_fields:
return None
dimensions = {field: data.get(field) for field in sorted(expected_fields)}
if not counter_dimensions_are_valid(TOOL_CALL_METRIC, dimensions):
return None
return dimensions
def tool_approval_counter(event: Any) -> tuple[str, dict[str, str]] | None:
"""Return one validated approval counter from a safe Relay mark event."""
if not _event_metadata_is_valid(event):
return None
if (
str(getattr(event, "kind", "") or "") != "mark"
or str(getattr(event, "name", "") or "") != TOOL_APPROVAL_MARK
or getattr(event, "category", None) is not None
or getattr(event, "scope_category", None) is not None
or getattr(event, "category_profile", None) is not None
):
return None
data = getattr(event, "data", None)
expected_fields = {"attribution", "outcome"}
if not isinstance(data, dict) or set(data) != expected_fields:
return None
dimensions = {field: data.get(field) for field in sorted(expected_fields)}
if not counter_dimensions_are_valid(TOOL_APPROVAL_METRIC, dimensions):
return None
return TOOL_APPROVAL_METRIC, dimensions
def skill_counter(event: Any) -> tuple[str, dict[str, str]] | None:
"""Return one validated skill lifecycle or load counter from a safe mark."""
if not _event_metadata_is_valid(event):
return None
if (
str(getattr(event, "kind", "") or "") != "mark"
or getattr(event, "category", None) is not None
or getattr(event, "scope_category", None) is not None
or getattr(event, "category_profile", None) is not None
):
return None
name = str(getattr(event, "name", "") or "")
data = getattr(event, "data", None)
if name == SKILL_LIFECYCLE_MARK:
metric_name = SKILL_LIFECYCLE_METRIC
expected_fields = {"action", "provenance"}
elif name == SKILL_LOAD_MARK:
metric_name = SKILL_LOAD_METRIC
expected_fields = {
"post_patch_state",
"provenance",
"reuse_state",
"use_count_bucket",
}
else:
return None
if not isinstance(data, dict) or set(data) != expected_fields:
return None
dimensions = {field: data.get(field) for field in sorted(expected_fields)}
if not counter_dimensions_are_valid(metric_name, dimensions):
return None
return metric_name, dimensions
def skill_lifecycle_fields(kwargs: dict[str, Any]) -> dict[str, str] | None:
"""Build bounded fields for one successful non-load skill transition."""
action = str(kwargs.get("action") or "").strip().lower()
if action not in SKILL_LIFECYCLE_ACTIONS:
return None
return {
"action": action,
"provenance": skill_provenance(kwargs.get("provenance")),
}
def skill_load_fields(kwargs: dict[str, Any]) -> dict[str, str] | None:
"""Build bounded skill-use fields without exporting local skill identity."""
use_count = kwargs.get("use_count")
reused = kwargs.get("reused")
reuse_after_patch = kwargs.get("reuse_after_patch")
if (
isinstance(use_count, bool)
or not isinstance(use_count, int)
or use_count < 1
or not isinstance(reused, bool)
or not isinstance(reuse_after_patch, bool)
or (reuse_after_patch and not reused)
):
return None
return {
"post_patch_state": (
"not_applicable"
if not reused
else "reused_after_patch"
if reuse_after_patch
else "no_new_patch"
),
"provenance": skill_provenance(kwargs.get("provenance")),
"reuse_state": "reused" if reused else "first_use",
"use_count_bucket": count_bucket(use_count),
}
def skill_provenance(value: Any) -> str:
"""Normalize producer provenance to the closed shared-metrics taxonomy."""
normalized = str(value or "").strip().lower()
return normalized if normalized in SKILL_PROVENANCES else "unknown"
def execution_surface(kwargs: dict[str, Any]) -> str:
"""Normalize the safe session surface carried by the parent Relay scope."""
value = (
str(kwargs.get("execution_surface") or kwargs.get("platform") or "unknown")
.strip()
.lower()
)
if value in EXECUTION_SURFACES:
return value
if value == "api_server":
return "api"
if value in {"cron", "scheduler", "scheduled"}:
return "scheduled_task"
try:
from hermes_cli.platforms import get_all_platforms
if value in get_all_platforms():
return "gateway"
except Exception:
pass
if value in {"discord", "email", "slack", "telegram", "teams", "whatsapp"}:
return "gateway"
return "unknown" if value == "unknown" else "other"
def task_start_fields(kwargs: dict[str, Any]) -> dict[str, str]:
"""Build the bounded fields recorded on a task scope start event."""
surface = execution_surface(kwargs)
return {
"entrypoint": task_entrypoint(kwargs, surface),
"execution_surface": surface,
}
def task_entrypoint(kwargs: dict[str, Any], surface: str | None = None) -> str:
"""Normalize the task dispatch owner without exporting source strings."""
declared = str(kwargs.get("entrypoint") or "").strip().lower()
if declared in TASK_ENTRYPOINTS:
return declared
resolved_surface = surface or execution_surface(kwargs)
if kwargs.get("parent_task_id") or kwargs.get("parent_session_id"):
return "delegated"
return {
"api": "api",
"batch": "batch",
"cli": "interactive",
"desktop": "interactive",
"gateway": "gateway_message",
"python": "python",
"scheduled_task": "scheduled_task",
"tui": "interactive",
"unknown": "unknown",
}.get(resolved_surface, "other")
def task_terminal_fields(
kwargs: dict[str, Any],
*,
duration_ms: int,
model_call_count: int,
tool_call_count: int,
retry_count: int,
) -> dict[str, str]:
"""Build the bounded terminal payload for one task scope."""
start_fields = task_start_fields(kwargs)
outcome, end_reason, termination = task_terminal_state(kwargs)
return {
**start_fields,
"duration_bucket": duration_bucket(duration_ms),
"end_reason": end_reason,
"model_call_count_bucket": count_bucket(model_call_count),
"outcome": outcome,
"retry_count_bucket": count_bucket(retry_count),
"termination": termination,
"tool_call_count_bucket": count_bucket(tool_call_count),
}
def task_terminal_state(kwargs: dict[str, Any]) -> tuple[str, str, str]:
"""Map Hermes terminal state to bounded task outcome dimensions."""
reason = str(kwargs.get("turn_exit_reason") or "").strip().lower()
if kwargs.get("interrupted") or "interrupt" in reason or "cancel" in reason:
return "cancelled", "user_cancelled", "user_cancelled"
if "timeout" in reason or "timed_out" in reason:
return "timed_out", "timed_out", "timed_out"
if "max_iterations" in reason or "budget_exhausted" in reason:
return "failed", "iteration_limit", "system_aborted"
if "approval" in reason and ("denied" in reason or "rejected" in reason):
return "failed", "approval_denied", "none"
if "guardrail" in reason:
return "failed", "guardrail_blocked", "system_aborted"
if reason == "system_aborted":
return "failed", "system_aborted", "system_aborted"
if kwargs.get("completed") is True:
return "success", "completed", "none"
if kwargs.get("failed") is True or (reason and reason != "unknown"):
return "failed", "failed", "none"
return "unknown", "unknown", "unknown"
def duration_bucket(duration_ms: int) -> str:
"""Bucket a non-negative task duration into a fixed low-cardinality range."""
value = max(0, int(duration_ms))
if value < 1_000:
return "lt_1s"
if value < 5_000:
return "1s_to_5s"
if value < 30_000:
return "5s_to_30s"
if value < 120_000:
return "30s_to_2m"
if value < 600_000:
return "2m_to_10m"
return "gte_10m"
def count_bucket(count: int) -> str:
"""Bucket a non-negative per-task count into a fixed range."""
value = max(0, int(count))
if value <= 2:
return str(value)
if value <= 5:
return "3_to_5"
if value <= 10:
return "6_to_10"
return "gte_11"
def tool_category(kwargs: dict[str, Any]) -> str:
"""Map Hermes registry toolset metadata to a low-cardinality category."""
toolset = str(kwargs.get("toolset") or "").strip().lower()
if not toolset:
return "unknown"
if toolset in TOOL_CATEGORIES:
return toolset
if toolset.startswith("mcp"):
return "mcp"
if toolset.startswith("browser"):
return "browser"
if toolset.startswith(("image", "tts", "video", "vision")):
return "media"
if toolset.startswith("homeassistant"):
return "home_automation"
if toolset in {"clarify", "kanban", "todo"}:
return "planning"
if toolset == "session_search":
return "memory"
if toolset == "cronjob":
return "scheduler"
if toolset == "skills":
return "skill"
if toolset == "x_search":
return "web"
if toolset.startswith(
("discord", "email", "feishu", "hermes-yuanbao", "slack", "sms")
):
return "communication"
return "other"
def tool_outcome(kwargs: dict[str, Any]) -> str:
"""Normalize the terminal Hermes tool status without inspecting its result."""
status = str(kwargs.get("status") or "").strip().lower()
return {
"blocked": "blocked",
"cancelled": "cancelled",
"error": "failed",
"failed": "failed",
"ok": "success",
"success": "success",
"timed_out": "timed_out",
"timeout": "timed_out",
}.get(status, "unknown")
def tool_approval_outcome(kwargs: dict[str, Any]) -> str:
"""Normalize a terminal approval choice to a bounded outcome."""
choice = str(kwargs.get("choice") or "").strip().lower()
if choice in {"always", "approve", "approved", "once", "session", "smart_approve"}:
return "approved"
if choice in {"deny", "denied", "smart_deny"}:
return "denied"
if choice in {"timed_out", "timeout"}:
return "timed_out"
return "unknown"
def tool_terminal_fields(
kwargs: dict[str, Any],
*,
category: str | None = None,
approval_outcome: str = "not_required",
fallback_duration_ms: int | None = None,
) -> dict[str, str]:
"""Build one bounded tool-call terminal payload."""
return {
"approval_outcome": (
approval_outcome
if approval_outcome in TOOL_APPROVAL_OUTCOMES
else "unknown"
),
"latency_bucket": tool_latency_bucket(
kwargs.get("duration_ms"),
fallback_duration_ms=fallback_duration_ms,
),
"outcome": tool_outcome(kwargs),
"retry_count_bucket": tool_retry_bucket(kwargs.get("retry_count")),
"tool_category": (
category if category in TOOL_CATEGORIES else tool_category(kwargs)
),
}
def tool_latency_bucket(
value: Any,
*,
fallback_duration_ms: int | None = None,
) -> str:
"""Bucket a tool duration reported in milliseconds."""
duration_ms = _non_negative_number(value)
if duration_ms is None:
duration_ms = _non_negative_number(fallback_duration_ms)
if duration_ms is None:
return "unknown"
if duration_ms < 100:
return "lt_100ms"
if duration_ms < 250:
return "100ms_to_250ms"
if duration_ms < 500:
return "250ms_to_500ms"
if duration_ms < 1_000:
return "500ms_to_1s"
if duration_ms < 2_000:
return "1s_to_2s"
if duration_ms < 5_000:
return "2s_to_5s"
if duration_ms < 10_000:
return "5s_to_10s"
if duration_ms < 30_000:
return "10s_to_30s"
return "gte_30s"
def tool_retry_bucket(value: Any) -> str:
"""Bucket only explicit tool retries; missing relationships stay unknown."""
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
return "unknown"
return count_bucket(value)
def _non_negative_number(value: Any) -> float | None:
if isinstance(value, bool) or not isinstance(value, (int, float)):
return None
try:
number = float(value)
except (OverflowError, TypeError, ValueError):
return None
return number if isfinite(number) and number >= 0 else None
def model_call_fields(kwargs: dict[str, Any]) -> dict[str, str]:
"""Return the terminal model identity and provider route known to Hermes."""
model = _metric_identifier(
kwargs.get("response_model"),
max_length=MODEL_IDENTIFIER_MAX_LENGTH,
)
if model == "unknown":
model = _metric_identifier(
kwargs.get("model"),
max_length=MODEL_IDENTIFIER_MAX_LENGTH,
)
return {
"model": model,
"provider": _metric_identifier(
kwargs.get("provider"),
max_length=PROVIDER_IDENTIFIER_MAX_LENGTH,
),
}
def _metric_identifier(value: Any, *, max_length: int) -> str:
"""Normalize one structurally safe identifier without a product catalog."""
if not isinstance(value, str):
return "unknown"
identifier = value.strip().lower()
if (
not identifier
or len(identifier) > max_length
or identifier[0] not in _METRIC_IDENTIFIER_START_CHARACTERS
or any(
character not in _METRIC_IDENTIFIER_CHARACTERS
for character in identifier
)
):
return "unknown"
return identifier
hermes_cli/observability/shared_metrics_send_config.py
+114
View File
@@ -0,0 +1,114 @@
"""Configuration for shared-metrics transmission.
Collection (``telemetry.shared_metrics.enabled``) and transmission
(``telemetry.shared_metrics.send``) are separate opt-ins. See
``docs/observability/relay-shared-metrics.md`` Appendix A for the consent,
identity, rotation, retention, and deletion decisions behind this module.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass
from urllib.parse import urlparse
logger = logging.getLogger(__name__)
#: Production ingest endpoint. Overridable through config only.
#:
#: Deliberately NOT overridable by an environment variable: AGENTS.md reserves
#: HERMES_* env vars for secrets, and a behavioural override here would be a
#: consent hazard — a user who agreed to send metrics to Nous could have them
#: silently redirected to any host by an inherited variable, with nothing
#: visible in their config to show it. Tests and the staging E2E write this
#: key into a throwaway profile instead.
DEFAULT_ENDPOINT = "https://telemetry.nousresearch.com/v1/telemetry"
_LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "[::1]"})
# Module-level latch: the enabled/send mismatch is a static misconfiguration,
# so it is reported once per process instead of on every hook fire.
_warned_send_without_collection = False
@dataclass(frozen=True)
class SendConfig:
"""Resolved transmission settings."""
#: Collection is on. Nothing is packaged or sent without it.
enabled: bool
#: Transmission is on AND permitted (that is, collection is also on).
send: bool
#: Where packages are POSTed.
endpoint: str
def _endpoint_is_safe(endpoint: str) -> bool:
"""Reject plaintext destinations unless they are loopback.
Telemetry must not leave a machine in clear text because of a typo in a
config file. Loopback stays allowed so tests can use a local HTTP server.
"""
try:
parsed = urlparse(endpoint)
except ValueError:
return False
if parsed.scheme == "https":
return True
if parsed.scheme == "http":
return (parsed.hostname or "") in _LOCAL_HOSTS
return False
def resolve_send_config(config: dict | None) -> SendConfig:
"""Resolve transmission settings from config plus the environment.
Endpoint precedence: config > production default.
``send`` is returned as False whenever transmission cannot legitimately
happen, so callers never have to re-check the combination.
"""
global _warned_send_without_collection
raw = config if isinstance(config, dict) else {}
telemetry = raw.get("telemetry")
telemetry = telemetry if isinstance(telemetry, dict) else {}
shared = telemetry.get("shared_metrics")
shared = shared if isinstance(shared, dict) else {}
enabled = shared.get("enabled") is True
send_requested = shared.get("send") is True
if send_requested and not enabled:
# Loud, not silent: the user believes telemetry is being sent, and it
# never will be. Error level, once per process.
if not _warned_send_without_collection:
_warned_send_without_collection = True
logger.error(
"telemetry.shared_metrics.send is true but "
"telemetry.shared_metrics.enabled is false — nothing is "
"collected, so nothing can be sent. Enable collection or "
"turn sending off."
)
return SendConfig(enabled=False, send=False, endpoint=DEFAULT_ENDPOINT)
endpoint = shared.get("endpoint")
if not isinstance(endpoint, str) or not endpoint.strip():
endpoint = DEFAULT_ENDPOINT
endpoint = endpoint.strip()
if send_requested and not _endpoint_is_safe(endpoint):
logger.error(
"Refusing to send shared metrics to %r: telemetry must use https "
"(or a localhost http endpoint for testing).",
endpoint,
)
return SendConfig(enabled=enabled, send=False, endpoint=endpoint)
return SendConfig(enabled=enabled, send=send_requested, endpoint=endpoint)
def reset_warning_latch_for_tests() -> None:
"""Clear the once-per-process error latch (test support only)."""
global _warned_send_without_collection
_warned_send_without_collection = False
hermes_cli/observability/shared_metrics_sender.py
+792
View File
@@ -0,0 +1,792 @@
"""Transmit exported shared-metrics packages to the Nous telemetry service.
Implements the sender side of the ingest contract (see the telemetry repo's
``CONTRACT.md``):
* ``202`` — durably stored. Mark sent.
* ``400`` — permanently malformed. Never retry.
* ``429`` — keep, retry after ``Retry-After``.
* ``5xx`` / timeout / connection error — keep, retry with backoff.
Two properties are load-bearing and easy to get wrong:
**The outbox directory is the user's local history, not a queue.** Packages
are pruned by age; a ``202`` marks send state in SQLite and never deletes a
file. See Appendix A.7 of ``docs/observability/relay-shared-metrics.md``.
**Consent is gated on the package's PERIOD, not its creation time.** One
period is split across packages created on different days, so a created-at
gate would send a period's tail while dropping its head and silently
undercount the first consented day. The gate itself is interval containment:
the period must fall entirely inside a recorded consent window
(``send_consent_windows``), maintained by the single ``reconcile_send_consent``
writer below.
"""
from __future__ import annotations
import gzip
import json
import logging
import random
import sqlite3
import time
import urllib.error
import urllib.request
import uuid
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from hermes_cli.sqlite_util import write_txn
logger = logging.getLogger(__name__)
#: Contract recommends timing out at 30s and treating a timeout as retryable.
REQUEST_TIMEOUT_SECONDS = 30
#: In-process attempts per package per pass, then the package waits for a
#: later pass. Backoff is 1s/5s/25s with full jitter.
MAX_ATTEMPTS = 3
_BACKOFF_BASE_SECONDS = 1
_BACKOFF_FACTOR = 5
#: Contract recommends gzip above roughly this size.
GZIP_THRESHOLD_BYTES = 4096
#: Packages per pass. Bounds work on an interactive hook even after an outage.
MAX_PACKAGES_PER_PASS = 20
#: How long a claimed row is held. The claim writes a LEASE INTO THE FUTURE:
#: selection requires `next_attempt_at <= now`, so for the length of the lease
#: no other process can take the package.
#:
#: This must exceed the worst case for ONE package — three 30s request
#: timeouts plus 1s+5s of backoff, about 96s — which is why packages are
#: claimed one at a time, immediately before being sent. An earlier revision
#: claimed up to 20 rows under a single shared lease; a full batch can legally
#: run ~1900s, so the later rows' leases expired while the pass still held
#: them in memory and another process re-sent them.
_CLAIM_LEASE_SECONDS = 300
#: Floor applied after a pass fails to deliver, so a hard-down service is not
#: retried on every task completion.
_FAILURE_BACKOFF_SECONDS = 15 * 60
#: Statuses that are permanent per the ingest contract. Deliberately narrow:
#: 400 means the envelope is malformed and will never validate. 413 is added
#: because a package over the service's 1 MiB cap cannot shrink on retry.
#: Everything else — including 403 from the origin guard and 404 from a bad
#: path — is retried, because those are usually deployment or edge
#: misconfiguration that resolves without the package changing.
_PERMANENT_STATUSES = frozenset({400, 413})
#: Attempts after which a package is abandoned. Without a ceiling a
#: permanently-poisoned row is retried until 30-day retention deletes it —
#: measured at ~160 requests — which wastes the user's bandwidth and keeps a
#: doomed package at the head of the queue.
MAX_SEND_ATTEMPTS = 25
def _utc_now() -> datetime:
return datetime.now(timezone.utc)
def _isoformat(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
def _parse_stamp(value: str) -> datetime:
"""Parse a stamp this module itself wrote (Z-suffixed ISO-8601, UTC)."""
return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(
timezone.utc
)
@dataclass
class SendOutcome:
"""What one pass did. Returned for tests and diagnostics."""
sent: int = 0
rejected: int = 0
deferred: int = 0
class _Response:
__slots__ = ("status", "retry_after", "body")
def __init__(self, status: int, retry_after: str | None, body: str) -> None:
self.status = status
self.retry_after = retry_after
self.body = body
def _post(endpoint: str, payload: bytes, *, timeout: int) -> _Response:
"""POST one package. Raises on transport failure; never on HTTP status."""
headers = {
"Content-Type": "application/json",
"User-Agent": "hermes-agent-shared-metrics/1",
}
body = payload
if len(payload) > GZIP_THRESHOLD_BYTES:
# mtime=0: gzip embeds a timestamp by default, which would make two
# sends of one package differ on the wire. The service decompresses
# before storing so it would not change what lands in S3, but a
# deterministic body keeps "a resend is byte-identical" true at the
# transport layer too, and makes the property testable.
body = gzip.compress(payload, mtime=0)
headers["Content-Encoding"] = "gzip"
request = urllib.request.Request(
endpoint, data=body, headers=headers, method="POST"
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
return _Response(
response.status,
response.headers.get("Retry-After"),
response.read(2048).decode("utf-8", "replace"),
)
except urllib.error.HTTPError as exc:
# An HTTP error status is a normal contract outcome, not a failure.
return _Response(
exc.code,
exc.headers.get("Retry-After") if exc.headers else None,
exc.read(2048).decode("utf-8", "replace") if exc.fp else "",
)
def _retry_after_seconds(value: str | None, default: int) -> int:
if not value:
return default
try:
# Contract sends seconds. Clamp so a hostile or bogus value cannot
# park a package for years, and never go below one second.
return max(1, min(int(float(value)), 86_400))
except (TypeError, ValueError):
return default
#: Maximum distance one reconcile call can advance the 'obs' mark. Honest
#: heartbeats arrive hours apart at most, so the cap never binds in normal
#: operation; a machine legitimately off for months catches up in a few
#: hook fires (fail-closed latency only). What it bounds is FORWARD clock
#: poison: without it, a single glitched sample (NTP flap reading 2099)
#: permanently drags the mark — and with it every window open and every
#: confirmation horizon — decades ahead, which round 6 reproduced as a
#: refused-data leak. Capped, one insane sample moves the mark at most
#: this far, and real time overtakes it again.
MAX_OBS_ADVANCE_SECONDS = 30 * 24 * 3600
def reconcile_send_consent(
connection: sqlite3.Connection,
send_enabled: bool,
*,
now: datetime | None = None,
) -> None:
"""Reconcile the consent-window table with the observed config state.
THE ONLY writer of consent state. Must run inside a write transaction.
A pure function of (config, now, store): call it from anywhere, any
number of times, in any order — the resulting windows are the same. This
replaces the previous edge-detection design, whose three partial
observers (wizard, relay, mid-pass) each covered a different subset of
transitions and repeatedly leaked the transitions between the subsets.
Timestamp discipline (each rule is load-bearing; see the validation
harness in tests/hermes_cli/test_shared_metrics_consent_windows.py):
- The 'obs' mark advances to every observation stamp, monotonically —
but by at most ``MAX_OBS_ADVANCE_SECONDS`` per call. Unbounded, the
mark is monotonic in the LEAK direction: one glitched-forward sample
would drag ``last_confirmed_at`` decades ahead, a later close would
stamp that horizon, and the closed window would contain every future
refused period (reproduced in round 6). Bounded, a poisoned sample
costs at most one cap's width, and real time overtakes it.
An open window's ``last_confirmed_at`` follows the mark: consent is
asserted only for time that was actually observed.
- A close is stamped at ``last_confirmed_at`` — never "now" — so an
unobserved gap (hand-edited config, machine off for 90 days) is never
inside a window and fails closed.
- An open clamps to ``max(now, obs, data)``: a rolled-back clock cannot
open a window underneath refused packages already on disk, and cannot
make the new window adjacent to the previous close.
"""
stamp = _isoformat(now or _utc_now())
raw_stamp = stamp # pre-cap observation time, used to clamp closes
previous_obs = connection.execute(
"SELECT stamp FROM consent_marks WHERE name = 'obs'"
).fetchone()
if previous_obs is not None:
ceiling = _isoformat(
_parse_stamp(str(previous_obs[0]))
+ timedelta(seconds=MAX_OBS_ADVANCE_SECONDS)
)
stamp = min(stamp, ceiling)
connection.execute(
"""
INSERT INTO consent_marks(name, stamp) VALUES ('obs', ?)
ON CONFLICT(name) DO UPDATE SET stamp = MAX(stamp, excluded.stamp)
""",
(stamp,),
)
marks = dict(
connection.execute("SELECT name, stamp FROM consent_marks").fetchall()
)
obs = marks["obs"] # >= stamp; immune to clock rollback
data = marks.get("data")
open_row = connection.execute(
"SELECT rowid FROM send_consent_windows WHERE closed_at IS NULL"
).fetchone()
if send_enabled:
if open_row is None:
opened = max(x for x in (obs, data) if x is not None)
connection.execute(
"INSERT INTO send_consent_windows(opened_at, last_confirmed_at)"
" VALUES (?, ?)",
(opened, opened),
)
else:
connection.execute(
"UPDATE send_consent_windows"
" SET last_confirmed_at = MAX(last_confirmed_at, ?)"
" WHERE rowid = ?",
(obs, open_row[0]),
)
elif open_row is not None:
# Close at the last CONFIRMED moment, but never after the closing
# observation's own raw stamp. The two clamps serve different
# adversaries and both are load-bearing:
# - min with last_confirmed_at: an unobserved gap (machine off,
# hand-edited config) is never asserted as consented (v1's leak).
# - min with the RAW stamp (pre-cap, pre-MAX): if last_confirmed_at
# was poisoned by a glitched-forward sample, an honest clock at
# revoke time pulls the close back to the true revoke moment, so
# the refused era that follows falls OUTSIDE the closed window
# (round 6's D1 leak). A rolled-back clock at close time only
# closes EARLIER — fail-closed.
connection.execute(
"UPDATE send_consent_windows"
" SET closed_at = MIN(last_confirmed_at, ?)"
" WHERE rowid = ?",
(raw_stamp, open_row[0]),
)
#: Claim-time consent predicate: the package's period must fall entirely
#: inside SOME recorded consent window. An open window vouches only up to its
#: last confirmed moment, so a package whose period runs past it waits for
#: the next reconcile heartbeat (fail-closed; released within one hook fire).
CONSENT_GATE_SQL = """EXISTS (
SELECT 1 FROM send_consent_windows w
WHERE package_outbox.period_start >= w.opened_at
AND package_outbox.period_end <=
CASE WHEN w.closed_at IS NULL THEN w.last_confirmed_at
ELSE w.closed_at END
)"""
def _state_get(connection: sqlite3.Connection, key: str) -> str | None:
row = connection.execute(
"SELECT value FROM telemetry_state WHERE key = ?", (key,)
).fetchone()
return str(row[0]) if row is not None else None
def _state_set(connection: sqlite3.Connection, key: str, value: str) -> None:
connection.execute(
"""
INSERT INTO telemetry_state(key, value) VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
""",
(key, value),
)
class SharedMetricsSender:
"""Sends exported packages, one bounded pass at a time."""
def __init__(
self,
store,
endpoint: str,
*,
post=_post,
sleep=time.sleep,
now=_utc_now,
max_attempts: int = MAX_ATTEMPTS,
consent_check=None,
) -> None:
self._store = store
self._endpoint = endpoint
self._post = post
self._sleep = sleep
self._now = now
self._max_attempts = max_attempts
# Called before every package. None disables the check for callers
# that have already established consent out of band (tests, E2E).
self._consent_check = consent_check
# -- selection ---------------------------------------------------------
def _claim_next(self, now: datetime, seen: set[str]) -> dict | None:
"""Claim exactly ONE package, immediately before it is sent.
Claiming a whole batch up front does not work: a single shared lease
has to cover the entire pass, and 20 retrying packages can legally run
far longer than any sane lease (three 30s timeouts plus backoff each).
The later rows' leases then expire while this pass still holds them in
memory, and another process re-sends them. Taking one row at a time
keeps the lease covering only the package actually in flight.
``seen`` holds packages this pass has already finished with. They are
excluded IN SQL rather than by rejecting the fetched row: with
``LIMIT 1``, returning None for an already-seen row would make the
caller believe the queue was empty and abandon every healthy package
behind it. A row can legitimately become eligible again mid-pass (a
short Retry-After, or a pass that outlives the 15-minute failure
backoff), so this is reachable in normal operation, not just in tests.
"""
with self._store._connection() as connection:
with write_txn(connection):
stamp = _isoformat(now)
lease_until = now + timedelta(seconds=_CLAIM_LEASE_SECONDS)
placeholders = ",".join("?" for _ in seen)
exclusion = (
f" AND package_id NOT IN ({placeholders})" if seen else ""
)
# Consent is a READ here — the claim must never mutate the
# window table. The old design's opt_in_period() call at this
# exact spot meant selecting a row could rewrite what was
# permitted to be sent (and did, under a rolled-back clock).
row = connection.execute(
f"""
SELECT package_id, payload_json, sent_install_id
FROM package_outbox
WHERE exported_at IS NOT NULL
AND (send_state IS NULL OR send_state = 'pending')
AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
AND {CONSENT_GATE_SQL}
AND send_attempts < ?
{exclusion}
ORDER BY created_at, package_id
LIMIT 1
""",
(stamp, MAX_SEND_ATTEMPTS, *sorted(seen)),
).fetchone()
if row is None:
return None
package_id = str(row[0])
derived = row[2]
if not derived:
derived = self._freeze_identity(
connection, package_id, row[1], now
)
if derived is None:
# Unusable row, already marked rejected. Signal the
# caller to continue rather than stop.
return {"package_id": package_id, "skip": True}
token = str(uuid.uuid4())
connection.execute(
"""
UPDATE package_outbox
SET send_state = 'pending',
send_attempts = send_attempts + 1,
next_attempt_at = ?,
claim_token = ?
WHERE package_id = ?
""",
# Lease INTO THE FUTURE: selection requires
# next_attempt_at <= now, so no other process can take
# this row while it is in flight. Success or a real
# backoff overwrites it; if this process dies, it expires.
# The token is this claim's identity: a reclaim after
# expiry mints a new one, and every later write by THIS
# claimant is compare-and-set against it, so a lapsed
# claimant that resumes cannot settle or transmit.
(_isoformat(lease_until), token, package_id),
)
return {
"package_id": package_id,
"payload_json": str(row[1]),
"derived": str(derived),
"claim_token": token,
"skip": False,
}
def _freeze_identity(
self,
connection: sqlite3.Connection,
package_id: str,
payload_json,
now: datetime,
) -> str | None:
"""Record the transmitted id on the row, or reject an unusable one.
The stable install_id is transmitted as-is (product decision,
2026-08-27 — see the doc's A.2). What remains of "freezing" is the
validation and the audit column: ``sent_install_id`` records exactly
what the wire will carry, and rejecting unusable rows here rather
than raising matters because an exception rolls back the claim
transaction and blocks every healthy package behind this one.
"""
reason = None
install_id = None
try:
payload = json.loads(payload_json)
except (TypeError, ValueError):
reason = "unreadable payload"
else:
# Valid JSON is not enough: a top-level array, string, number or
# null parses cleanly and then has no .get().
if not isinstance(payload, dict):
reason = f"payload is {type(payload).__name__}, expected object"
else:
install_id = payload.get("install_id")
if not isinstance(install_id, str) or not install_id.strip():
reason = "payload has no usable install_id"
if reason is not None:
logger.warning(
"Shared-metrics package %s cannot be sent (%s)", package_id, reason
)
connection.execute(
"""
UPDATE package_outbox
SET send_state = 'rejected', last_error = ?
WHERE package_id = ?
""",
(reason, package_id),
)
return None
connection.execute(
"UPDATE package_outbox SET sent_install_id = ? WHERE package_id = ?",
(install_id, package_id),
)
return str(install_id)
# -- transmission ------------------------------------------------------
def _body(self, payload_json: str, transmitted_id: str) -> bytes:
"""Rebuild the exact bytes to send.
The payload is recomputed from the stored package rather than kept as
a second copy: json.dumps with these options is deterministic. The
install_id is written from the frozen ``sent_install_id`` column
rather than trusted implicitly, keeping "a resend is byte-identical"
anchored to one recorded value.
"""
payload = json.loads(payload_json)
payload = dict(payload)
payload["install_id"] = transmitted_id
return json.dumps(payload, indent=2, sort_keys=True).encode("utf-8")
def _mark(
self,
package_id: str,
*,
only_if_pending: bool = True,
token: str | None = None,
**columns,
) -> None:
"""Write send state for one package.
Guarded on send_state so a pass whose lease lapsed cannot resurrect a
row another process has already finished: without this, a slow sender
could overwrite 'sent' back to 'pending' and cause a re-send.
When ``token`` is given, the write is additionally compare-and-set on
claim_token: it lands only if THIS claim is still the current one. A
claimant that lapsed and was superseded writes zero rows — its
settlement, backoff, and error strings all silently lose to the
newer claim's, which is the correct outcome.
"""
assignments = ", ".join(f"{name} = ?" for name in columns)
predicate = (
" AND (send_state IS NULL OR send_state = 'pending')"
if only_if_pending
else ""
)
params: list = [*columns.values(), package_id]
if token is not None:
predicate += " AND claim_token = ?"
params.append(token)
with self._store._connection() as connection:
with write_txn(connection):
connection.execute(
f"UPDATE package_outbox SET {assignments} "
f"WHERE package_id = ?{predicate}",
params,
)
def _renew_claim(self, package_id: str, token: str | None) -> bool:
"""Atomically re-assert ownership and extend the lease. CAS, one row.
A read-only ownership check is not enough: a claimant whose lease
expired while suspended can pass the check (its token is still in
the row if no one reclaimed yet) and then POST while another process
legitimately reclaims — the check-to-POST expiry race a seventh
review reproduced. Renewal closes it by requiring, in ONE statement:
- the token still matches (nobody reclaimed), AND
- the current lease is UNEXPIRED (this claimant is not stale), AND
- the row is still pending,
and only then pushing next_attempt_at a fresh lease into the future,
so the upcoming POST (30s timeout, well under the 300s lease) runs
entirely inside renewed authority. rowcount == 1 is the only grant.
A claimant that wakes past its own lease fails the unexpired
condition and yields even though its token was never replaced.
"""
if token is None:
return False
try:
now = self._now()
lease_until = now + timedelta(seconds=_CLAIM_LEASE_SECONDS)
with self._store._connection() as connection:
with write_txn(connection):
cursor = connection.execute(
"""
UPDATE package_outbox
SET next_attempt_at = ?
WHERE package_id = ?
AND claim_token = ?
AND (send_state IS NULL OR send_state = 'pending')
AND next_attempt_at > ?
""",
(
_isoformat(lease_until),
package_id,
token,
_isoformat(now),
),
)
return cursor.rowcount == 1
except Exception:
# If renewal itself fails, do not transmit on unproven authority.
logger.warning(
"Unable to renew shared-metrics claim", exc_info=True
)
return False
def _defer(
self,
package_id: str,
delay_seconds: int,
reason: str,
*,
token: str | None = None,
) -> None:
# Defence in depth: no current caller can pass a non-positive delay
# (Retry-After is already clamped to [1, 86400] when parsed, and every
# other call site passes a positive constant), so this clamp is
# deliberately unreachable today and no test can distinguish it. It
# stays because a past deadline would make the row instantly
# re-eligible and let a pass spin on it — a cheap guard against a
# future caller that forgets.
delay = max(1, int(delay_seconds))
retry_at = self._now().timestamp() + delay
self._mark(
package_id,
token=token,
send_state="pending",
next_attempt_at=_isoformat(
datetime.fromtimestamp(retry_at, tz=timezone.utc)
),
last_error=reason[:500],
)
def _send_one(self, package: dict) -> str:
"""Try one package. Returns 'sent', 'rejected', or 'deferred'.
Delivery is at-least-once. The pre-POST ownership check plus the
token-fenced writes close the claim->POST and settle-after-reclaim
gaps, but a suspension landing MID-POST (bytes already on the wire
when the machine sleeps) can still duplicate: no client-side check
can revoke a request in flight. The body is byte-identical across
retries by construction, so the residual duplicate is exactly one
redundant copy of identical content; collapsing it fully would need
package_id-keyed dedupe at the ingest service.
"""
package_id = package["package_id"]
token = package.get("claim_token")
body = self._body(package["payload_json"], package["derived"])
for attempt in range(1, self._max_attempts + 1):
# Atomically renew the claim before EVERY external POST. The
# renewal is compare-and-set on (token, pending, lease unexpired)
# and extends the lease past the request, so a suspended-then-
# resumed claimant whose lease lapsed yields here even if nobody
# has reclaimed yet — a read-only ownership check passed in that
# state and still double-sent (check-to-POST expiry race). The
# ingest key is minute-prefixed, so duplicates become distinct
# stored objects, not overwrites.
if not self._renew_claim(package_id, token):
logger.info(
"Shared-metrics claim on %s superseded or expired; yielding",
package_id,
)
return "deferred"
try:
response = self._post(
self._endpoint, body, timeout=REQUEST_TIMEOUT_SECONDS
)
except Exception as exc: # transport failure: offline, DNS, TLS
reason = f"{type(exc).__name__}: {exc}"
if attempt >= self._max_attempts:
self._defer(
package_id, _FAILURE_BACKOFF_SECONDS, reason, token=token
)
return "deferred"
self._sleep(self._backoff(attempt))
continue
if response.status == 202:
self._mark(
package_id,
token=token,
send_state="sent",
sent_at=_isoformat(self._now()),
last_error=None,
)
return "sent"
if response.status in _PERMANENT_STATUSES:
# Only statuses the contract (or the envelope schema) makes
# terminal. Everything else retries: 403 in particular is the
# ingest service's origin guard, which returns 403 during an
# edge/Transform-Rule misconfiguration — treating that as
# permanent would discard every package sent during the
# incident instead of retrying after recovery.
logger.warning(
"Telemetry package %s rejected with HTTP %s; not retrying",
package_id,
response.status,
)
self._mark(
package_id,
token=token,
send_state="rejected",
last_error=f"HTTP {response.status}: {response.body[:400]}",
)
return "rejected"
if response.status == 429:
self._defer(
package_id,
_retry_after_seconds(response.retry_after, _FAILURE_BACKOFF_SECONDS),
"rate limited",
token=token,
)
return "deferred"
# 5xx and anything unexpected: retryable.
reason = f"HTTP {response.status}"
if attempt >= self._max_attempts:
self._defer(
package_id, _FAILURE_BACKOFF_SECONDS, reason, token=token
)
return "deferred"
self._sleep(self._backoff(attempt))
self._defer(
package_id, _FAILURE_BACKOFF_SECONDS, "attempts exhausted", token=token
)
return "deferred"
@staticmethod
def _backoff(attempt: int) -> float:
"""1s, 5s, 25s with full jitter."""
ceiling = _BACKOFF_BASE_SECONDS * (_BACKOFF_FACTOR ** (attempt - 1))
return random.uniform(0, ceiling)
# -- entry point -------------------------------------------------------
def send_pending(self) -> SendOutcome:
"""Run one bounded pass. Never raises.
Claims and sends ONE package at a time so each row's lease only has to
cover its own transmission, and re-checks consent before every send so
revoking `send` mid-pass stops the remaining packages.
"""
outcome = SendOutcome()
seen: set[str] = set()
for _ in range(MAX_PACKAGES_PER_PASS):
if not self._still_consented():
# The user turned sending off while this pass was running.
# Stop without transmitting anything further, and reconcile
# so the window closes at its last confirmed moment. This is
# the same single writer every other observation point uses —
# not a separate recording mechanism.
logger.info("Shared-metrics sending disabled mid-pass; stopping")
self._reconcile(send_enabled=False)
break
try:
package = self._claim_next(self._now(), seen)
except Exception:
logger.warning(
"Unable to select shared-metrics packages", exc_info=True
)
break
if package is None:
break
seen.add(package["package_id"])
if package.get("skip"):
# Unusable row already marked rejected during the claim.
outcome.rejected += 1
continue
try:
result = self._send_one(package)
except Exception:
logger.warning("Unable to send shared-metrics package", exc_info=True)
outcome.deferred += 1
continue
if result == "sent":
outcome.sent += 1
elif result == "rejected":
outcome.rejected += 1
else:
outcome.deferred += 1
return outcome
def _reconcile(self, *, send_enabled: bool) -> None:
"""Run the single consent writer from within a pass."""
try:
with self._store._connection() as connection:
with write_txn(connection):
reconcile_send_consent(
connection, send_enabled, now=self._now()
)
except Exception:
logger.warning(
"Unable to reconcile shared-metrics consent", exc_info=True
)
def _still_consented(self) -> bool:
"""Re-read profile-owned send consent.
Consent is a boundary, not cached configuration: the documentation
promises that setting `send: false` stops transmission immediately,
and a pass can run for minutes. Injected senders (tests, the staging
E2E) opt out by passing consent_check=None.
"""
if self._consent_check is None:
return True
try:
return bool(self._consent_check())
except Exception:
# Fail CLOSED: if consent cannot be established, do not transmit.
logger.warning(
"Unable to confirm shared-metrics send consent; stopping",
exc_info=True,
)
return False
hermes_cli/observability/shared_metrics_subscriber.py
+101
View File
@@ -0,0 +1,101 @@
"""Relay subscriber for the persisted Hermes shared-metrics slice."""
from __future__ import annotations
import logging
import platform
import threading
from typing import Any
from agent.relay_runtime import RUNTIME_INSTANCE_KEY
from hermes_cli.config import detect_install_method
from .shared_metrics import SharedMetricsStore
from .shared_metrics_contract import (
CLIENT_ACTIVE_METRIC,
MODEL_ROUTE_METRIC,
TOOL_CALL_METRIC,
client_active_counter,
client_resource,
model_call_dimensions,
skill_counter,
task_counter,
tool_approval_counter,
tool_call_dimensions,
)
logger = logging.getLogger(__name__)
class SharedMetricsSubscriber:
"""Persist validated Hermes counters from Relay lifecycle events."""
def __init__(
self,
store: SharedMetricsStore,
hermes_version: str,
*,
runtime_id: str | None = None,
) -> None:
self.store = store
self._client_resource = client_resource(
hermes_version,
os_name=platform.system(),
architecture=platform.machine(),
install_method=detect_install_method(),
)
self._runtime_id = runtime_id
self._active = True
self._lock = threading.RLock()
def deactivate(self) -> None:
"""Stop accepting events before telemetry is disabled or torn down."""
with self._lock:
self._active = False
def __call__(self, event: Any) -> None:
if self._runtime_id is not None:
metadata = getattr(event, "metadata", None)
if (
not isinstance(metadata, dict)
or metadata.get(RUNTIME_INSTANCE_KEY) != self._runtime_id
):
return
metric = client_active_counter(event)
dimensions = None
metric_name = CLIENT_ACTIVE_METRIC
if metric is not None:
metric_name, dimensions = metric
if dimensions is None:
dimensions = model_call_dimensions(event)
metric_name = MODEL_ROUTE_METRIC
if dimensions is None:
dimensions = tool_call_dimensions(event)
metric_name = TOOL_CALL_METRIC
if dimensions is None:
metric = (
task_counter(event)
or tool_approval_counter(event)
or skill_counter(event)
)
if metric is None:
return
metric_name, dimensions = metric
with self._lock:
if not self._active:
return
try:
if metric_name == CLIENT_ACTIVE_METRIC:
self.store.record_client_active(self._client_resource)
else:
self.store.record_counter(
metric_name,
dimensions,
self._client_resource,
)
except Exception:
logger.warning(
"Unable to persist the Hermes shared metric: %s",
metric_name,
exc_info=True,
)