"""Content-free cron service-health and execution telemetry projection.""" from __future__ import annotations import hashlib import logging import re from dataclasses import dataclass from datetime import datetime from typing import Any, Optional from agent.monitoring.events import CronExecutionEvent from agent.monitoring.gateway_health import GatewayHealthSnapshot, GatewayMetric from cron.jobs import ( _compute_grace_seconds, get_catch_up_occurrence_count, get_ticker_heartbeat_age, get_ticker_success_age, load_jobs, ) from cron.scheduler import get_running_job_ids from hermes_time import now as _hermes_now logger = logging.getLogger(__name__) _KNOWN_STATUSES = {"claimed", "running", "completed", "failed", "unknown"} _KNOWN_SOURCES = {"builtin", "direct", "external"} _KNOWN_DELIVERY_OUTCOMES = { "delivered", "failed", "suppressed", "suppressed_acked", "not_configured", } @dataclass(frozen=True, slots=True) class CronHealthSnapshot: metrics: list[GatewayMetric] events: list[CronExecutionEvent] def _now() -> datetime: return _hermes_now() def _job_key(raw: Any) -> str: value = str(raw or "unknown").encode("utf-8", errors="replace") return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}" def classify_cron_error(raw: Any) -> str: text = str(raw or "").lower() if ( re.search(r"\b(?:authentication|authenticated|authenticate|authorization|authorized|authorize|unauthorized|forbidden)\b", text) or re.search(r"\bbearer\b", text) or re.search(r"\b(?:access|api|refresh) token\b", text) or re.search(r"\b(?:401|403)\b", text) ): return "auth_failed" if "rate limit" in text or "429" in text or "quota" in text: return "rate_limited" if "timeout" in text or "timed out" in text: return "timeout" if any(value in text for value in ("network", "connection", "dns", "socket", "unreachable")): return "network_error" if "dispatch" in text or "executor" in text: return "dispatch_failed" if "interrupt" in text or "owner exited" in text or "restarted" in text: return "interrupted" if "empty response" in text: return "empty_response" if any(value in text for value in ("config", "missing", "invalid")): return "invalid_config" return "unknown" def _parse_time(raw: Any) -> Optional[datetime]: try: return datetime.fromisoformat(str(raw)) if raw else None except (TypeError, ValueError): return None def _duration_ms(record: dict[str, Any]) -> Optional[int]: start = _parse_time(record.get("started_at")) or _parse_time(record.get("claimed_at")) finish = _parse_time(record.get("finished_at")) if start is None or finish is None: return None try: duration = int((finish - start).total_seconds() * 1000) except (TypeError, ValueError): return None return max(0, duration) def project_execution_event( record: dict[str, Any], *, delivery_outcome: Optional[str] = None ) -> CronExecutionEvent: status = str(record.get("status") or "unknown").lower() source = str(record.get("source") or "unknown").lower() if source not in _KNOWN_SOURCES and source != "unknown": source = "external" outcome = str(delivery_outcome).lower() if delivery_outcome is not None else None return CronExecutionEvent( status=status if status in _KNOWN_STATUSES else "unknown", job_key=_job_key(record.get("job_id")), source=source if source in _KNOWN_SOURCES else "unknown", duration_ms=_duration_ms(record), delivery_outcome=( outcome if outcome in _KNOWN_DELIVERY_OUTCOMES else None ), error_class=( classify_cron_error(record.get("error")) if status in {"failed", "unknown"} else None ), ) def emit_execution_state( record: Optional[dict[str, Any]], *, delivery_outcome: Optional[str] = None ) -> None: """Best-effort lifecycle emit; terminal states synchronously cross the queue barrier.""" if not record: return try: from agent.monitoring import emitter event = project_execution_event(record, delivery_outcome=delivery_outcome) target = emitter.get_emitter() target.emit(event) if event.status in {"completed", "failed", "unknown"}: target.flush(timeout=1.0) except Exception: logger.debug("cron execution telemetry emit failed", exc_info=True) def _is_overdue(job: dict[str, Any], now: datetime) -> bool: if not job.get("enabled", True): return False next_run = _parse_time(job.get("next_run_at")) schedule = job.get("schedule") if next_run is None or not isinstance(schedule, dict): return False try: if next_run.tzinfo is None and now.tzinfo is not None: next_run = next_run.replace(tzinfo=now.tzinfo) lateness = (now - next_run).total_seconds() return lateness > _compute_grace_seconds(schedule) except (TypeError, ValueError): return False def build_cron_health_snapshot() -> CronHealthSnapshot: metrics: list[GatewayMetric] = [] for name, reader in ( ("hermes.cron.scheduler.heartbeat_age_seconds", get_ticker_heartbeat_age), ("hermes.cron.scheduler.last_success_age_seconds", get_ticker_success_age), ): try: value = reader() if value is not None: metrics.append(GatewayMetric(name, max(0.0, float(value)), {})) except Exception: logger.debug("cron freshness metric unavailable", exc_info=True) try: metrics.append( GatewayMetric( "hermes.cron.scheduler.catch_up_occurrences", get_catch_up_occurrence_count(), {}, ) ) except Exception: logger.debug("cron catch-up metric unavailable", exc_info=True) try: jobs = load_jobs() enabled = [job for job in jobs if job.get("enabled", True)] metrics.append(GatewayMetric("hermes.cron.jobs.enabled", len(enabled), {})) metrics.append( GatewayMetric( "hermes.cron.jobs.overdue", sum(1 for job in enabled if _is_overdue(job, _now())), {}, ) ) except Exception: logger.debug("cron job metrics unavailable", exc_info=True) try: metrics.append( GatewayMetric("hermes.cron.jobs.running", len(get_running_job_ids()), {}) ) except Exception: logger.debug("cron running-job metric unavailable", exc_info=True) return CronHealthSnapshot(metrics=metrics, events=[]) __all__ = [ "CronHealthSnapshot", "build_cron_health_snapshot", "classify_cron_error", "emit_execution_state", "project_execution_event", ]