Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,302 @@
|
||||
"""Durable cron failure incidents with signature dedup and ack.
|
||||
|
||||
The executions ledger (``cron.executions``) records every attempt; this module
|
||||
groups the *failures* into durable incidents keyed by ``(job_id, error
|
||||
signature)`` so the same job failing with the same error does not re-ping the
|
||||
operator every run once they have acknowledged it.
|
||||
|
||||
Lifecycle: ``detected`` → ``alerted`` → ``closed``. Closing
|
||||
(acking) an incident is per-signature: the same job + same normalized error
|
||||
keeps resolving to the SAME incident id, so a closed incident stays closed (no
|
||||
re-alert) until the error text changes, which mints a brand-new incident.
|
||||
``detected`` means the failure was recorded; ``alerted`` means at least one
|
||||
failure ping for the signature actually reached the operator. Richer states
|
||||
(e.g. a dv9.6 ``reviewed``) are deliberately NOT reserved here — state
|
||||
validity lives in ``INCIDENT_STATES`` (Python), not a SQLite CHECK, exactly
|
||||
so a future slice can add states without a table rebuild.
|
||||
|
||||
Incidents live in the SAME ``cron/executions.db`` as ``cron.executions`` so
|
||||
there is one durable cron store per profile. The schema is lazily created on
|
||||
connect and a missing database never raises (directories are created).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import sqlite3
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterator, List, Optional
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
from hermes_time import now as _hermes_now
|
||||
|
||||
# Optional test override (mirrors ``cron.executions.EXECUTIONS_FILE``).
|
||||
EXECUTIONS_FILE: Optional[Path] = None
|
||||
|
||||
INCIDENT_STATES = ("detected", "alerted", "closed")
|
||||
_FAILURE_TYPE_ORDER = (
|
||||
("rate_limit", (r"\b429\b", "rate limit", "usage limit", "quota")),
|
||||
("timeout", ("timeout", "timed out")),
|
||||
("auth", (r"\b401\b", "unauthorized", "authentication", "auth")),
|
||||
("delivery", ("delivery", "deliver", "delivering")),
|
||||
("config", ("config", "configuration", "validation")),
|
||||
("script", ("script", "no_agent")),
|
||||
("agent", ("agent", "model", "provider", "inference")),
|
||||
)
|
||||
MAX_ERROR_CHARS = 500
|
||||
_MAX_SIGNATURE_ERROR_CHARS = 200
|
||||
|
||||
_lock = threading.RLock()
|
||||
|
||||
|
||||
def _connect() -> sqlite3.Connection:
|
||||
from cron.jobs import _ensure_cron_dir
|
||||
|
||||
path = _db_path()
|
||||
_ensure_cron_dir(path.parent)
|
||||
return sqlite3.connect(path, timeout=5)
|
||||
|
||||
|
||||
def _db_path() -> Path:
|
||||
"""Resolve the shared cron DB path.
|
||||
|
||||
Prefer the ``cron.executions`` override when one is installed so an
|
||||
operator/test that redirects the executions ledger also redirects the
|
||||
incident table — they must stay in the SAME database. Falls back to this
|
||||
module's own override, then the canonical profile home.
|
||||
"""
|
||||
try:
|
||||
from cron.executions import EXECUTIONS_FILE as _EXEC_OVERRIDE
|
||||
|
||||
if _EXEC_OVERRIDE is not None:
|
||||
return Path(_EXEC_OVERRIDE)
|
||||
except Exception:
|
||||
pass
|
||||
if EXECUTIONS_FILE is not None:
|
||||
return Path(EXECUTIONS_FILE)
|
||||
return get_hermes_home().resolve() / "cron" / "executions.db"
|
||||
|
||||
|
||||
def _initialize_schema(conn: sqlite3.Connection) -> None:
|
||||
from hermes_state import apply_wal_with_fallback
|
||||
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
apply_wal_with_fallback(conn, db_label="cron/executions.db")
|
||||
conn.execute("PRAGMA synchronous=FULL")
|
||||
conn.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS cron_incidents (
|
||||
id TEXT PRIMARY KEY,
|
||||
job_id TEXT NOT NULL,
|
||||
error_sig TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
failure_type TEXT NOT NULL DEFAULT 'unknown',
|
||||
first_seen_at TEXT NOT NULL,
|
||||
last_seen_at TEXT NOT NULL,
|
||||
acked_at TEXT,
|
||||
closed_at TEXT,
|
||||
error TEXT NOT NULL,
|
||||
output_file TEXT
|
||||
)"""
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_cron_incidents_job "
|
||||
"ON cron_incidents(job_id)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_cron_incidents_state "
|
||||
"ON cron_incidents(state)"
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _transaction() -> Iterator[sqlite3.Connection]:
|
||||
"""Open a connection, commit/rollback on exit, always close.
|
||||
|
||||
Mirrors ``cron.executions._transaction``: schema init runs inside the
|
||||
``try`` so a PRAGMA/DDL failure after a successful ``connect()`` still
|
||||
closes the connection instead of leaking it.
|
||||
"""
|
||||
with _lock:
|
||||
conn = _connect()
|
||||
try:
|
||||
_initialize_schema(conn)
|
||||
with conn:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _normalize_error(error: str) -> str:
|
||||
"""Strip whitespace and lowercase before signing (dedup normalization)."""
|
||||
return re.sub(r"\s+", " ", str(error or "")).strip().lower()
|
||||
|
||||
|
||||
def _redact_error(error: str) -> str:
|
||||
"""Redact secrets then bound the stored error length."""
|
||||
text = str(error or "")
|
||||
try:
|
||||
from agent.redact import redact_sensitive_text
|
||||
|
||||
text = redact_sensitive_text(text)
|
||||
except Exception:
|
||||
# Redaction is best-effort; the scheduler path never fails on it.
|
||||
pass
|
||||
return text[:MAX_ERROR_CHARS]
|
||||
|
||||
|
||||
def _error_signature(job_id: str, error: str) -> str:
|
||||
"""Dedup key: stable for same job + same normalized error prefix."""
|
||||
normalized = _normalize_error(error)[:_MAX_SIGNATURE_ERROR_CHARS]
|
||||
digest = hashlib.sha256(job_id.encode() + normalized.encode()).hexdigest()
|
||||
return digest[:12]
|
||||
|
||||
|
||||
def _incident_id(job_id: str, error_sig: str) -> str:
|
||||
return f"{job_id[:6]}_{error_sig}"
|
||||
|
||||
|
||||
def _classify_failure_type(error: str) -> str:
|
||||
"""Classify a failure from error-text keywords; ``unknown`` is the default."""
|
||||
text = _normalize_error(error)
|
||||
if not text:
|
||||
return "unknown"
|
||||
for kind, patterns in _FAILURE_TYPE_ORDER:
|
||||
for pattern in patterns:
|
||||
if pattern.startswith("\\b") and pattern.endswith("\\b"):
|
||||
if re.search(pattern, text):
|
||||
return kind
|
||||
elif pattern in text:
|
||||
return kind
|
||||
return "unknown"
|
||||
|
||||
|
||||
def upsert_incident(
|
||||
job_id: str,
|
||||
error: str,
|
||||
*,
|
||||
job_name: Optional[str] = None,
|
||||
failure_type: Optional[str] = None,
|
||||
output_file: Optional[str] = None,
|
||||
) -> tuple[str, bool]:
|
||||
"""Record (or refresh) the incident for ``job_id`` + ``error``.
|
||||
|
||||
Returns ``(incident_id, is_new)``. A row for the same signature already
|
||||
existing refreshes ``last_seen_at``/``error``/``output_file`` and keeps its
|
||||
current state — a ``closed`` (acked) incident stays closed for the same
|
||||
signature. A changed error text mints a new incident automatically.
|
||||
"""
|
||||
job_id = str(job_id or "")
|
||||
sig = _error_signature(job_id, error)
|
||||
stored_error = _redact_error(error)
|
||||
incident_id = _incident_id(job_id, sig)
|
||||
now = _hermes_now().isoformat()
|
||||
failure_type = failure_type or _classify_failure_type(error)
|
||||
output_file = str(output_file) if output_file is not None else None
|
||||
|
||||
with _transaction() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id FROM cron_incidents WHERE id=?", (incident_id,)
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
conn.execute(
|
||||
"""UPDATE cron_incidents
|
||||
SET last_seen_at=?, error=?, output_file=?
|
||||
WHERE id=?""",
|
||||
(now, stored_error, output_file, incident_id),
|
||||
)
|
||||
return incident_id, False
|
||||
conn.execute(
|
||||
"""INSERT INTO cron_incidents
|
||||
(id, job_id, error_sig, state, failure_type,
|
||||
first_seen_at, last_seen_at, error, output_file)
|
||||
VALUES (?, ?, ?, 'detected', ?, ?, ?, ?, ?)""",
|
||||
(incident_id, job_id, sig, failure_type, now, now,
|
||||
stored_error, output_file),
|
||||
)
|
||||
return incident_id, True
|
||||
|
||||
|
||||
def set_incident_state(incident_id: str, state: str) -> bool:
|
||||
"""Transition an incident's lifecycle state; return whether it changed.
|
||||
|
||||
``closed`` is terminal for that signature: no transition (including back
|
||||
to ``alerted``) leaves it — re-open happens by the error changing and
|
||||
minting a NEW incident. Unknown states are rejected (no-op, ``False``).
|
||||
"""
|
||||
if state not in INCIDENT_STATES:
|
||||
return False
|
||||
now = _hermes_now().isoformat()
|
||||
with _transaction() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT state FROM cron_incidents WHERE id=?", (incident_id,)
|
||||
).fetchone()
|
||||
if row is None or row["state"] == state:
|
||||
return False
|
||||
if row["state"] == "closed":
|
||||
return False
|
||||
if state == "closed":
|
||||
conn.execute(
|
||||
"""UPDATE cron_incidents
|
||||
SET state='closed', closed_at=?, acked_at=?
|
||||
WHERE id=? AND state != 'closed'""",
|
||||
(now, now, incident_id),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"UPDATE cron_incidents SET state=? WHERE id=?",
|
||||
(state, incident_id),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def ack_incident(incident_id: str) -> bool:
|
||||
"""Acknowledge (close) an incident; return whether the state changed.
|
||||
|
||||
A no-op (``False``) when the incident does not exist or is already closed.
|
||||
"""
|
||||
return set_incident_state(incident_id, "closed")
|
||||
|
||||
|
||||
def list_incidents(state: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Return incidents, newest-activity first, optionally filtered by state."""
|
||||
if state is not None and state not in INCIDENT_STATES:
|
||||
return []
|
||||
with _transaction() as conn:
|
||||
if state is None:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM cron_incidents "
|
||||
"ORDER BY last_seen_at DESC, id DESC"
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM cron_incidents WHERE state=? "
|
||||
"ORDER BY last_seen_at DESC, id DESC",
|
||||
(state,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def get_incident(incident_id: str) -> Optional[Dict[str, Any]]:
|
||||
with _transaction() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM cron_incidents WHERE id=?", (incident_id,)
|
||||
).fetchone()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
def count_incidents(state: Optional[str] = None) -> int:
|
||||
if state is not None and state not in INCIDENT_STATES:
|
||||
return 0
|
||||
with _transaction() as conn:
|
||||
if state is None:
|
||||
row = conn.execute("SELECT COUNT(*) AS n FROM cron_incidents").fetchone()
|
||||
else:
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) AS n FROM cron_incidents WHERE state=?",
|
||||
(state,),
|
||||
).fetchone()
|
||||
return int(row["n"]) if row is not None else 0
|
||||
Reference in New Issue
Block a user