yilsem/aiturk-hermes-ide
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 3 Wiki Activity

Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

Browse Source
This commit is contained in:
yilsem
2026-09-05 13:26:46 +03:00
commit 03634b1ca3
11340 changed files with 3442369 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cron/__init__.py
+44
View File
@@ -0,0 +1,44 @@
"""
Cron job scheduling system for Hermes Agent.
This module provides scheduled task execution, allowing the agent to:
- Run automated tasks on schedules (cron expressions, intervals, one-shot)
- Self-schedule reminders and follow-up tasks
- Execute tasks in isolated sessions (no prior context)
Cron jobs are executed automatically by the gateway daemon:
hermes gateway install # Install as a user service
sudo hermes gateway install --system # Linux servers: boot-time system service
hermes gateway # Or run in foreground
The gateway ticks the scheduler every 60 seconds. A file lock prevents
duplicate execution if multiple processes overlap.
"""
from cron.jobs import (
create_job,
get_job,
list_jobs,
remove_job,
update_job,
pause_job,
resume_job,
trigger_job,
rearm_oneshot,
JOBS_FILE,
)
from cron.scheduler import tick
__all__ = [
"create_job",
"get_job",
"list_jobs",
"remove_job",
"update_job",
"pause_job",
"resume_job",
"trigger_job",
"rearm_oneshot",
"tick",
"JOBS_FILE",
]
cron/blueprint_catalog.py
+799
View File
@@ -0,0 +1,799 @@
"""Automation Blueprints — parameterized automation blueprints with typed slots.
A *blueprint* is a one-place definition of an automation that every surface
renders natively:
* Dashboard / GUI app -> a form (one field per slot)
* CLI / TUI / messenger -> a pre-filled ``/blueprint`` slash command
* Agent -> a seed prompt; it asks for any blank/ambiguous slot
* Docs catalog -> a copy-paste command + a ``hermes://`` deep-link
The single source of truth is the slot schema below. ``blueprint_form_schema``
emits what a form renderer needs; ``blueprint_slash_command`` emits the flattened
one-line command; ``fill_blueprint`` validates user-supplied values and turns a
blueprint into a ``cron.jobs.create_job`` kwargs dict (so there is no second job
engine). The form-where-there's-a-screen / agent-fills-where-there's-a-chat
split both consume this same module.
Design choice: users never type raw cron. A blueprint carries a fixed recurrence
in ``schedule_template`` and parameterizes only the human-friendly parts
(time-of-day, weekday set). Blueprints needing full flexibility expose a ``text``
slot named ``schedule`` that passes through verbatim.
"""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
__all__ = [
"BlueprintSlot",
"AutomationBlueprint",
"CATALOG",
"get_blueprint",
"blueprint_form_schema",
"blueprint_slash_command",
"blueprint_deeplink",
"blueprint_catalog_entry",
"fill_blueprint",
"BlueprintFillError",
"WEEKDAY_PRESETS",
]
class BlueprintFillError(ValueError):
"""Raised when supplied slot values fail validation."""
# Slot types the renderers understand.
_SLOT_TYPES = frozenset({"time", "enum", "text", "weekdays"})
# Named weekday recurrences -> cron day-of-week field.
WEEKDAY_PRESETS: Dict[str, str] = {
"everyday": "*",
"weekdays": "1-5",
"weekends": "0,6",
}
@dataclass(frozen=True)
class BlueprintSlot:
"""A single fillable field on a blueprint."""
name: str
type: str
label: str
default: Any = None
options: tuple = () # for type="enum": allowed values
optional: bool = False
help: str = ""
# When False, ``options`` are suggestions rather than a closed set —
# any value is accepted (e.g. the deliver slot, where the real set of
# valid platforms depends on the user's configured gateways and is
# validated downstream by the cron scheduler).
strict: bool = True
def __post_init__(self) -> None:
if self.type not in _SLOT_TYPES:
raise ValueError(f"unknown slot type {self.type!r} (slot {self.name})")
@dataclass(frozen=True)
class AutomationBlueprint:
"""A parameterized automation blueprint."""
key: str
title: str
description: str
category: str
# Cron expression with ``{slot}`` placeholders, e.g. "{minute} {hour} * * {dow}".
# Placeholders are filled from resolved slot values (time -> minute/hour,
# weekdays -> dow). A literal cron string with no placeholders = fixed schedule.
schedule_template: str
# Seed instruction for the agent / the cron job prompt; may contain {slot}s.
prompt_template: str
slots: List[BlueprintSlot] = field(default_factory=list)
deliver_default: str = "origin"
skills: tuple = () # skills the job loads before running
tags: tuple = ()
# ---------------------------------------------------------------------------
# Curated in-repo catalog
# ---------------------------------------------------------------------------
_TIME = lambda default="08:00": BlueprintSlot( # noqa: E731 - concise factory
name="time", type="time", label="What time?", default=default,
help="24h local time, e.g. 08:00",
)
_DELIVER = BlueprintSlot(
name="deliver", type="enum", label="Where to deliver?",
default="origin", options=("origin", "local", "telegram", "discord", "email"),
optional=False, strict=False,
help="origin = the chat you set this up from (or your configured home "
"channel when created from the dashboard); local = save only, no message; "
"or any connected platform name",
)
CATALOG: List[AutomationBlueprint] = [
AutomationBlueprint(
key="morning-brief",
title="Morning briefing",
description="A short daily briefing: today's calendar, weather, and "
"anything urgent waiting on you.",
category="daily",
schedule_template="{minute} {hour} * * *",
prompt_template=(
"Produce a concise morning briefing for the user: today's calendar "
"events, the local weather, and any urgent items. When Gmail/Google "
"Calendar are connected, follow the google-workspace skill's "
"references/daily-brief.md procedure (exact day window, conflict "
"detection, meeting prep, mail-to-meeting links). Keep it short and "
"scannable. If no data sources are connected, give a brief "
"good-morning with the date and offer to connect calendar/email."
),
slots=[_TIME("08:00"), _DELIVER],
skills=("google-workspace",),
tags=("daily", "briefing"),
),
AutomationBlueprint(
key="important-mail",
title="Important-mail monitor",
description="Check your inbox periodically and ping you ONLY about mail "
"that actually needs attention.",
category="email",
schedule_template="*/{interval_min} * * * *",
prompt_template=(
"Check the user's inbox for new messages since the last run. Surface "
"ONLY mail matching: {criteria}. Score candidates with the urgency "
"classifier and deliver only what clears the bar; if nothing does, "
"respond with [SILENT]. Requires a connected mail source; if none is "
"configured, explain how to connect one and stop."
),
slots=[
BlueprintSlot(
name="interval_min", type="enum", label="How often?",
default="30", options=("15", "30", "60"),
help="minutes between checks",
),
BlueprintSlot(
name="criteria", type="text",
label="Only notify me if the mail…",
default="needs a reply today, is from my manager or family, "
"or mentions a deadline",
),
_DELIVER,
],
skills=("email-inbox-triage",),
tags=("email", "monitor"),
),
AutomationBlueprint(
key="weekly-review",
title="Weekly review",
description="A weekly recap: what got done, what's still open, and "
"what's coming up.",
category="weekly",
schedule_template="{minute} {hour} * * {dow}",
prompt_template=(
"Run the weekly-review-planning skill's procedure for the user: "
"review the completed week and coming 1-2 weeks across connected "
"calendar, tasks, notes, and email; surface commitments, stalled "
"projects, and waiting items; build a capacity-aware plan for next "
"week. Recommendations and drafts only — no mutations without "
"approval. Keep the output in the skill's seven-section shape."
),
slots=[
_TIME("18:00"),
BlueprintSlot(
name="day", type="enum", label="Which day?",
default="sunday",
options=("sunday", "monday", "friday", "saturday"),
),
_DELIVER,
],
skills=("weekly-review-planning",),
tags=("weekly", "review"),
),
AutomationBlueprint(
key="workday-start",
title="Workday start reminder",
description="A weekday nudge with your agenda and top priorities.",
category="daily",
schedule_template="{minute} {hour} * * 1-5",
prompt_template=(
"Give the user a brief weekday start-of-day nudge: today's calendar "
"and the 1-3 highest-priority things to focus on, inferred from "
"recent context and any task tools. Encouraging, short, one message."
),
slots=[_TIME("09:00"), _DELIVER],
tags=("daily", "focus"),
),
AutomationBlueprint(
key="custom-reminder",
title="Custom reminder",
description="A recurring reminder in your own words, on your schedule.",
category="general",
schedule_template="{minute} {hour} * * {dow}",
prompt_template="Remind the user: {what}",
slots=[
BlueprintSlot(name="what", type="text", label="Remind me to…",
default="take a break and stretch"),
_TIME("14:00"),
BlueprintSlot(
name="recurrence", type="weekdays", label="Repeat on",
default="everyday",
options=tuple(WEEKDAY_PRESETS.keys()),
),
_DELIVER,
],
tags=("reminder",),
),
AutomationBlueprint(
key="evening-winddown",
title="Evening wind-down",
description="An end-of-day check-in: tomorrow's calendar at a glance "
"and anything you should prep tonight.",
category="daily",
schedule_template="{minute} {hour} * * *",
prompt_template=(
"Give the user a short evening wind-down: tomorrow's calendar, any "
"early commitments to prep for, and one gentle nudge to wrap up "
"loose ends from today. Keep it calm and brief — one message. If no "
"calendar is connected, just offer a friendly sign-off and the "
"weather for tomorrow."
),
slots=[_TIME("21:00"), _DELIVER],
tags=("daily", "evening"),
),
AutomationBlueprint(
key="news-digest",
title="Topic news digest",
description="A recurring digest on a topic you care about — deduped "
"against what was already sent, so only genuinely new items land.",
category="general",
schedule_template="{minute} {hour} * * {dow}",
prompt_template=(
"Search the web for new and noteworthy items about: {topic}. "
"Dedupe against what you sent in previous runs — only include "
"genuinely new developments. Deliver a tight digest of at most "
"{count} bullets, each one line with a link. If nothing new since "
"last run, respond with [SILENT]."
),
slots=[
BlueprintSlot(
name="topic", type="text", label="What topic?",
default="AI and technology",
help="a subject, product, person, or search phrase",
),
_TIME("18:00"),
BlueprintSlot(
name="recurrence", type="weekdays", label="Repeat on",
default="weekdays",
options=tuple(WEEKDAY_PRESETS.keys()),
),
BlueprintSlot(
name="count", type="enum", label="How many bullets?",
default="5", options=("3", "5", "8"),
),
_DELIVER,
],
tags=("digest", "research"),
),
AutomationBlueprint(
key="bill-renewal-watch",
title="Bills & renewals reminder",
description="A heads-up before a recurring payment, subscription "
"renewal, or due date — so nothing auto-charges by surprise.",
category="general",
schedule_template="{minute} {hour} * * {dow}",
prompt_template=(
"Remind the user about an upcoming payment or renewal: {what}. "
"Phrase it as an actionable heads-up (e.g. 'review or cancel before "
"it renews'), not just a notification. One short message."
),
slots=[
BlueprintSlot(
name="what", type="text", label="What's due?",
default="my streaming subscription renews soon",
),
_TIME("10:00"),
BlueprintSlot(
name="recurrence", type="weekdays", label="Repeat on",
default="everyday",
options=tuple(WEEKDAY_PRESETS.keys()),
),
_DELIVER,
],
tags=("reminder", "finance"),
),
AutomationBlueprint(
key="price-watch",
title="Price & availability watch",
description="Watch an exact product, flight, hotel, or listing and "
"alert when your price or availability condition is met.",
category="general",
schedule_template="0 */{interval_h} * * *",
prompt_template=(
"Load the product-price-monitor skill and run the tick for this "
"watch: {item}. Alert condition: {condition}. Compare the "
"normalized all-in price/availability against stored state, "
"suppress duplicate alerts, and never overwrite last-known-good "
"state with a failed fetch. If no condition is met, respond with "
"[SILENT]. On the first run, execute the skill's setup phase "
"first: pin the exact item, verify one live fetch, and write the "
"watch contract state file."
),
slots=[
BlueprintSlot(
name="item", type="text", label="What exactly to watch?",
default="a product URL or exact flight/hotel/listing description",
help="URL or precise description — variant, dates, seller",
),
BlueprintSlot(
name="condition", type="text", label="Alert me when…",
default="the all-in price drops below my target",
help="threshold price (state the currency), availability, or terms change",
),
BlueprintSlot(
name="interval_h", type="enum", label="How often?",
default="6", options=("1", "3", "6", "12", "24"),
help="hours between checks — be gentle with rate limits",
),
_DELIVER,
],
skills=("product-price-monitor",),
tags=("prices", "shopping", "travel", "monitor"),
),
AutomationBlueprint(
key="competitor-watch",
title="Competitor news watch",
description="Track named companies for material news — launches, "
"pricing, funding, filings — with a cited digest.",
category="general",
schedule_template="{minute} {hour} * * {dow}",
prompt_template=(
"Load the competitor-news-monitor skill and run the tick for this "
"watch: companies {companies}; event categories {categories}. "
"Collect incrementally from the last cutoff, deduplicate by "
"underlying event, score materiality against the watch contract, "
"and deliver a cited digest of material events only. If there are "
"no material events, respond with [SILENT]. On the first run, "
"execute the skill's setup phase first: freeze the watchlist, "
"build source coverage, and write the watch contract state file."
),
slots=[
BlueprintSlot(
name="companies", type="text", label="Which companies?",
default="two or three competitors, by canonical name",
help="canonical names and domains; aliases help dedup",
),
BlueprintSlot(
name="categories", type="text", label="Which events matter?",
default="product launches, pricing changes, funding, "
"partnerships, executive moves, incidents",
),
_TIME("09:00"),
BlueprintSlot(
name="recurrence", type="weekdays", label="Repeat on",
default="monday",
options=tuple(WEEKDAY_PRESETS.keys()),
),
_DELIVER,
],
skills=("competitor-news-monitor",),
tags=("competitors", "news", "monitor", "research"),
),
AutomationBlueprint(
key="habit-checkin",
title="Habit check-in",
description="A recurring nudge to keep a habit on track and reflect "
"on whether you did it.",
category="general",
schedule_template="{minute} {hour} * * {dow}",
prompt_template=(
"Nudge the user about their habit: {habit}. Ask whether they did it "
"today, keep it warm and non-judgmental, and offer a one-line word "
"of encouragement. One short message."
),
slots=[
BlueprintSlot(
name="habit", type="text", label="Which habit?",
default="20 minutes of reading",
),
_TIME("20:00"),
BlueprintSlot(
name="recurrence", type="weekdays", label="Repeat on",
default="everyday",
options=tuple(WEEKDAY_PRESETS.keys()),
),
_DELIVER,
],
tags=("habit", "wellbeing"),
),
AutomationBlueprint(
key="hydration-move",
title="Hydration & movement nudge",
description="A periodic nudge during the day to drink water, stand up, "
"and stretch.",
category="general",
# NOTE: cron minute-field steps (*/90) wrap per hour — */90 and */120
# both degrade to hourly. Use an hour-field step instead so the chosen
# cadence is what actually fires.
schedule_template="0 {start_hour}-{end_hour}/{interval_hours} * * 1-5",
prompt_template=(
"Send the user a brief, friendly nudge to drink some water, stand "
"up, and stretch for a moment. Vary the wording each time so it "
"doesn't feel robotic. One short line."
),
slots=[
BlueprintSlot(
name="interval_hours", type="enum", label="How often?",
default="1", options=("1", "2", "3"),
help="hours between nudges",
),
BlueprintSlot(
name="start_hour", type="enum", label="Start hour",
default="9", options=("7", "8", "9", "10"),
help="first hour of the active window (24h)",
),
BlueprintSlot(
name="end_hour", type="enum", label="End hour",
default="17", options=("16", "17", "18", "19"),
help="last hour of the active window (24h)",
),
_DELIVER,
],
tags=("wellbeing", "focus"),
),
AutomationBlueprint(
key="meal-plan",
title="Weekly meal plan",
description="A weekly meal plan plus a consolidated grocery list, "
"tuned to your diet and how much time you have to cook.",
category="weekly",
schedule_template="{minute} {hour} * * {dow}",
prompt_template=(
"Build the user a meal plan for the coming week: {meals} per day, "
"suited to a {diet} diet and roughly {effort} cooking effort. "
"Include a consolidated grocery list grouped by aisle. Keep blueprints "
"simple and skimmable."
),
slots=[
BlueprintSlot(
name="diet", type="enum", label="Diet?",
default="no restrictions",
options=("no restrictions", "vegetarian", "vegan",
"high-protein", "low-carb"),
),
BlueprintSlot(
name="meals", type="enum", label="Meals per day?",
default="dinner only",
options=("dinner only", "lunch and dinner", "all three"),
),
BlueprintSlot(
name="effort", type="enum", label="Cooking effort?",
default="quick", options=("quick", "medium", "ambitious"),
),
_TIME("17:00"),
BlueprintSlot(
name="day", type="enum", label="Which day?",
default="sunday",
options=("sunday", "monday", "friday", "saturday"),
),
_DELIVER,
],
tags=("weekly", "food"),
),
AutomationBlueprint(
key="learn-daily",
title="Daily learning drip",
description="One bite-sized lesson a day on a topic you want to learn, "
"building progressively over time.",
category="daily",
schedule_template="{minute} {hour} * * {dow}",
prompt_template=(
"Teach the user one bite-sized lesson about: {topic}. Build on "
"earlier lessons so it progresses rather than repeating. Keep it to "
"a couple of short paragraphs with one concrete example, and end "
"with a single question to check understanding."
),
slots=[
BlueprintSlot(
name="topic", type="text", label="Learn about…",
default="Spanish vocabulary",
),
_TIME("08:30"),
BlueprintSlot(
name="recurrence", type="weekdays", label="Repeat on",
default="weekdays",
options=tuple(WEEKDAY_PRESETS.keys()),
),
_DELIVER,
],
tags=("learning", "daily"),
),
AutomationBlueprint(
key="gratitude-journal",
title="Gratitude & reflection prompt",
description="A gentle evening prompt to reflect on the day and note "
"what went well.",
category="general",
schedule_template="{minute} {hour} * * {dow}",
prompt_template=(
"Send the user a short, warm reflection prompt for the end of the "
"day — invite them to note one thing that went well, one thing they "
"are grateful for, and one small win. If they reply, acknowledge it "
"kindly. One message."
),
slots=[
_TIME("21:30"),
BlueprintSlot(
name="recurrence", type="weekdays", label="Repeat on",
default="everyday",
options=tuple(WEEKDAY_PRESETS.keys()),
),
_DELIVER,
],
tags=("wellbeing", "reflection"),
),
AutomationBlueprint(
key="on-this-day",
title="On-this-day discovery",
description="A daily dose of curiosity: a notable historical event, "
"fact, or word for the day.",
category="daily",
schedule_template="{minute} {hour} * * *",
prompt_template=(
"Give the user one interesting '{flavor}' item for today — keep it "
"short, surprising, and genuinely interesting. One or two sentences, "
"no filler."
),
slots=[
BlueprintSlot(
name="flavor", type="enum", label="What kind?",
default="on this day in history",
options=("on this day in history", "word of the day",
"science fact", "quote of the day"),
),
_TIME("07:30"),
_DELIVER,
],
tags=("daily", "curiosity"),
),
]
_CATALOG_BY_KEY = {r.key: r for r in CATALOG}
def get_blueprint(key: str) -> Optional[AutomationBlueprint]:
return _CATALOG_BY_KEY.get(key)
# ---------------------------------------------------------------------------
# Renderers
# ---------------------------------------------------------------------------
def blueprint_form_schema(blueprint: AutomationBlueprint) -> Dict[str, Any]:
"""Emit the JSON a form renderer (dashboard / GUI) needs for this blueprint."""
return {
"key": blueprint.key,
"title": blueprint.title,
"description": blueprint.description,
"category": blueprint.category,
"tags": list(blueprint.tags),
"fields": [
{
"name": s.name,
"type": s.type,
"label": s.label,
"default": s.default,
"options": list(s.options),
"optional": s.optional,
"strict": s.strict,
"help": s.help,
}
for s in blueprint.slots
],
}
def blueprint_slash_command(blueprint: AutomationBlueprint, values: Optional[Dict[str, Any]] = None) -> str:
"""Build the flattened ``/blueprint <key> slot=val …`` command string.
Uses each slot's default when ``values`` is omitted, so the docs/dashboard
can show a ready-to-paste command. Free-text slots are quoted.
"""
values = values or {}
parts = [f"/blueprint {blueprint.key}"]
for s in blueprint.slots:
val = values.get(s.name, s.default)
if val is None or val == "":
if s.optional:
continue
val = ""
sval = str(val)
if s.type == "text" or " " in sval:
sval = '"' + sval.replace('"', '\\"') + '"'
parts.append(f"{s.name}={sval}")
return " ".join(parts)
def blueprint_deeplink(blueprint: AutomationBlueprint, values: Optional[Dict[str, Any]] = None) -> str:
"""Build the ``hermes://blueprint/<key>?slot=val`` deep-link URL."""
from urllib.parse import quote, urlencode
values = values or {}
query = {}
for s in blueprint.slots:
val = values.get(s.name, s.default)
if val not in (None, ""):
query[s.name] = str(val)
qs = ("?" + urlencode(query)) if query else ""
return f"hermes://blueprint/{quote(blueprint.key)}{qs}"
def _humanize_schedule(blueprint: AutomationBlueprint) -> str:
"""A short human-readable description of when a blueprint runs (defaults)."""
sched = blueprint.schedule_template
if sched.startswith("*/"):
iv = next((s for s in blueprint.slots if s.name == "interval_min"), None)
every = (iv.default if iv else None) or sched.split("/")[1].split()[0]
return f"every {every} minutes"
if "{interval_hours}" in sched:
iv = next((s for s in blueprint.slots if s.name == "interval_hours"), None)
every = str((iv.default if iv else None) or "1")
scope = "weekdays, " if "* * 1-5" in sched else ""
return f"{scope}every hour" if every == "1" else f"{scope}every {every} hours"
time_slot = next((s for s in blueprint.slots if s.type == "time"), None)
when = time_slot.default if time_slot else None
if "* * 1-5" in sched:
return f"weekdays at {when}" if when else "every weekday"
if "{dow}" in sched:
day_slot = next((s for s in blueprint.slots if s.name in ("day", "recurrence")), None)
scope = (day_slot.default if day_slot else "") or ""
if scope and when:
return f"{scope} at {when}"
return f"at {when}" if when else "on a schedule"
if when:
return f"daily at {when}"
return "on a schedule"
def blueprint_catalog_entry(blueprint: AutomationBlueprint) -> Dict[str, Any]:
"""Unified serializable shape for a blueprint — used by the docs generator
and the dashboard API. Combines the form schema, the ready-to-paste slash
command, the deep-link URL, and a human-readable schedule.
"""
return {
**blueprint_form_schema(blueprint),
"schedule": blueprint.schedule_template,
"scheduleHuman": _humanize_schedule(blueprint),
"command": blueprint_slash_command(blueprint),
"appUrl": blueprint_deeplink(blueprint),
}
# ---------------------------------------------------------------------------
# Fill + validate + translate to a create_job spec
# ---------------------------------------------------------------------------
_TIME_RE = re.compile(r"^([01]?\d|2[0-3]):([0-5]\d)$")
_DAY_TO_DOW = {
"sunday": "0", "monday": "1", "tuesday": "2", "wednesday": "3",
"thursday": "4", "friday": "5", "saturday": "6",
}
def _resolve_schedule(blueprint: AutomationBlueprint, values: Dict[str, Any]) -> str:
"""Fill the schedule_template placeholders from resolved slot values."""
sched = blueprint.schedule_template
# A free-text `schedule` slot passes through verbatim (full flexibility).
if "schedule" in values and values["schedule"]:
return str(values["schedule"])
repl: Dict[str, str] = {}
# time -> minute/hour
time_val = values.get("time")
if "{minute}" in sched or "{hour}" in sched:
if not time_val:
raise BlueprintFillError("a time is required")
m = _TIME_RE.match(str(time_val).strip())
if not m:
raise BlueprintFillError(f"invalid time {time_val!r} — use HH:MM (24h)")
repl["hour"] = str(int(m.group(1)))
repl["minute"] = str(int(m.group(2)))
# weekday set -> dow
if "{dow}" in sched:
if "recurrence" in values:
preset = str(values.get("recurrence", "everyday")).lower()
if preset not in WEEKDAY_PRESETS:
raise BlueprintFillError(
f"unknown recurrence {preset!r} — one of {', '.join(WEEKDAY_PRESETS)}"
)
repl["dow"] = WEEKDAY_PRESETS[preset]
elif "day" in values:
day = str(values.get("day", "")).lower()
if day not in _DAY_TO_DOW:
raise BlueprintFillError(f"unknown day {day!r}")
repl["dow"] = _DAY_TO_DOW[day]
else:
repl["dow"] = "*"
# interval (minutes) for */N schedules
if "{interval_min}" in sched:
iv = str(values.get("interval_min", "")).strip()
if not iv.isdigit() or int(iv) <= 0:
raise BlueprintFillError(f"invalid interval {iv!r} — minutes as a positive integer")
repl["interval_min"] = iv
# Any remaining {slot} placeholders are filled verbatim from validated
# enum/text slot values (e.g. an hour-range window). Enum options have
# already been checked in fill_blueprint, so these are safe to interpolate.
for name in re.findall(r"\{(\w+)\}", sched):
if name not in repl and name in values:
repl[name] = str(values[name])
try:
return sched.format(**repl)
except KeyError as e: # pragma: no cover - template/slot mismatch is a dev error
raise BlueprintFillError(f"schedule template missing value for {e}") from e
def fill_blueprint(
blueprint: AutomationBlueprint,
values: Dict[str, Any],
*,
origin: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""Validate ``values`` and return ``cron.jobs.create_job`` kwargs.
Missing required (non-optional) slots raise BlueprintFillError naming the
slot, so a form can show field errors and the agent knows what to ask.
Unknown slot names are rejected (a typo'd ``tiem=07:15`` must not silently
create a job with the default time). Enum values are checked against their
options. The result is passed straight to ``create_job`` — no second schema.
"""
known = {s.name for s in blueprint.slots}
unknown = sorted(set(values) - known)
if unknown:
raise BlueprintFillError(
f"unknown slot{'s' if len(unknown) > 1 else ''}: "
f"{', '.join(unknown)} — valid: {', '.join(s.name for s in blueprint.slots)}"
)
resolved: Dict[str, Any] = {}
for s in blueprint.slots:
raw = values.get(s.name, s.default)
if raw in (None, ""):
if s.optional:
continue
raise BlueprintFillError(f"missing required value: {s.name} ({s.label})")
if s.type == "enum" and s.strict and s.options and str(raw) not in {str(o) for o in s.options}:
raise BlueprintFillError(
f"{s.name}={raw!r} not allowed — one of {', '.join(map(str, s.options))}"
)
resolved[s.name] = raw
schedule = _resolve_schedule(blueprint, resolved)
# Render the prompt with whatever slots it references.
try:
prompt = blueprint.prompt_template.format(**resolved)
except KeyError as e:
raise BlueprintFillError(f"blueprint prompt missing value for {e}") from e
spec: Dict[str, Any] = {
"prompt": prompt,
"schedule": schedule,
"name": blueprint.title,
"deliver": resolved.get("deliver", blueprint.deliver_default),
}
if blueprint.skills:
spec["skills"] = list(blueprint.skills)
if origin is not None:
spec["origin"] = origin
return spec
cron/delivery_queue.py
+379
View File
@@ -0,0 +1,379 @@
"""Profile-local durable handoff for cron delivery through live gateway adapters.
A restart-safe cron worker executes outside the gateway cgroup. It cannot own
relay/E2EE adapter objects, so it queues the final send here. A gateway claims
each row at most once. If that gateway dies after claiming, the outcome is
marked unknown and never retried: losing a delivery is safer than duplicating a
possibly-completed send.
"""
from __future__ import annotations
import json
import logging
import os
import sqlite3
import threading
import time
import uuid
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Callable, Iterator, Optional
from agent.redact import redact_sensitive_text
from cron.executions import _owner_is_live, _process_start_time
from hermes_cli.sqlite_util import add_column_if_missing
from hermes_constants import get_hermes_home
from hermes_time import now as _hermes_now
logger = logging.getLogger(__name__)
DELIVERY_DB: Optional[Path] = None
_PROCESS_ID = uuid.uuid4().hex
_lock = threading.RLock()
_ACTIVE_DELIVERIES: set[str] = set()
_TERMINAL = ("delivered", "failed", "unknown")
MAX_TERMINAL_DELIVERIES = 1000
DEFAULT_DELIVERY_WAIT_TIMEOUT_SECONDS = 300.0
def _prune_terminal_unlocked(conn: sqlite3.Connection) -> None:
"""Redact terminal payloads and retain only bounded outcome metadata."""
conn.execute(
"""UPDATE deliveries SET job_json='{}', content=''
WHERE status IN ('delivered','failed','unknown')
AND (job_json != '{}' OR content != '')"""
)
keep = max(0, int(MAX_TERMINAL_DELIVERIES))
terminal_count = int(
conn.execute(
"SELECT COUNT(*) FROM deliveries "
"WHERE status IN ('delivered','failed','unknown')"
).fetchone()[0]
)
excess = terminal_count - keep
if excess > 0:
conn.execute(
"""INSERT OR IGNORE INTO delivery_tombstones
(execution_id, terminal_status, finished_at)
SELECT execution_id, status, finished_at FROM deliveries
WHERE status IN ('delivered','failed','unknown')
ORDER BY finished_at, created_at, execution_id
LIMIT ?""",
(excess,),
)
conn.execute(
"""DELETE FROM deliveries WHERE execution_id IN (
SELECT execution_id FROM deliveries
WHERE status IN ('delivered','failed','unknown')
ORDER BY finished_at, created_at, execution_id
LIMIT ?
)""",
(excess,),
)
def _path() -> Path:
return DELIVERY_DB or (get_hermes_home().resolve() / "cron" / "deliveries.db")
@contextmanager
def _transaction() -> Iterator[sqlite3.Connection]:
with _lock:
path = _path()
path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(path, timeout=5)
try:
path.chmod(0o600)
except OSError:
pass
conn.row_factory = sqlite3.Row
try:
from hermes_state import apply_wal_with_fallback
conn.execute("PRAGMA busy_timeout=5000")
apply_wal_with_fallback(conn, db_label="cron/deliveries.db")
conn.execute("PRAGMA synchronous=FULL")
conn.execute(
"""CREATE TABLE IF NOT EXISTS deliveries (
execution_id TEXT PRIMARY KEY,
job_json TEXT NOT NULL,
content TEXT NOT NULL,
for_failure INTEGER NOT NULL DEFAULT 0,
status TEXT NOT NULL CHECK(status IN
('pending','delivering','delivered','failed','unknown')),
owner_process_id TEXT,
owner_pid INTEGER,
owner_started_at INTEGER,
created_at TEXT NOT NULL,
finished_at TEXT,
error TEXT
)"""
)
conn.execute(
"""CREATE TABLE IF NOT EXISTS delivery_tombstones (
execution_id TEXT PRIMARY KEY,
terminal_status TEXT NOT NULL CHECK(terminal_status IN
('delivered','failed','unknown')),
finished_at TEXT
)"""
)
add_column_if_missing(
conn, "deliveries", "for_failure",
"for_failure INTEGER NOT NULL DEFAULT 0",
)
# Pruning is done explicitly by the paths that create terminal
# rows (_finish / recover_abandoned / _terminalize_wait_timeout);
# read-only polls must not pay for a full-table UPDATE + COUNT.
with conn:
yield conn
finally:
conn.close()
def enqueue(
execution_id: str,
job: dict,
content: str,
*,
for_failure: bool = False,
) -> dict:
"""Persist one idempotent delivery request before the worker waits."""
with _transaction() as conn:
tombstone = conn.execute(
"SELECT terminal_status, finished_at FROM delivery_tombstones "
"WHERE execution_id=?",
(str(execution_id),),
).fetchone()
if tombstone is not None:
return {
"execution_id": str(execution_id),
"status": tombstone["terminal_status"],
"finished_at": tombstone["finished_at"],
}
conn.execute(
"""INSERT OR IGNORE INTO deliveries
(execution_id, job_json, content, for_failure, status, created_at)
VALUES (?, ?, ?, ?, 'pending', ?)""",
(
str(execution_id),
json.dumps(job, ensure_ascii=False, sort_keys=True),
str(content),
int(bool(for_failure)),
_hermes_now().isoformat(),
),
)
row = conn.execute(
"SELECT * FROM deliveries WHERE execution_id=?", (str(execution_id),)
).fetchone()
return dict(row)
def get_status(execution_id: str) -> Optional[dict]:
with _transaction() as conn:
row = conn.execute(
"SELECT * FROM deliveries WHERE execution_id=?", (str(execution_id),)
).fetchone()
if row is not None:
return dict(row)
tombstone = conn.execute(
"SELECT execution_id, terminal_status, finished_at "
"FROM delivery_tombstones WHERE execution_id=?",
(str(execution_id),),
).fetchone()
if tombstone is None:
return None
return {
"execution_id": tombstone["execution_id"],
"status": tombstone["terminal_status"],
"finished_at": tombstone["finished_at"],
"error": None,
}
def claim_next() -> Optional[dict]:
"""Atomically claim one pending send before touching the transport."""
pid = os.getpid()
started = _process_start_time(pid)
with _transaction() as conn:
row = conn.execute(
"SELECT execution_id FROM deliveries WHERE status='pending' "
"ORDER BY created_at, execution_id LIMIT 1"
).fetchone()
if row is None:
return None
cur = conn.execute(
"""UPDATE deliveries SET status='delivering', owner_process_id=?,
owner_pid=?, owner_started_at=?
WHERE execution_id=? AND status='pending'""",
(_PROCESS_ID, pid, started, row["execution_id"]),
)
if cur.rowcount != 1:
return None
claimed = conn.execute(
"SELECT * FROM deliveries WHERE execution_id=?", (row["execution_id"],)
).fetchone()
_ACTIVE_DELIVERIES.add(row["execution_id"])
result = dict(claimed)
result["job"] = json.loads(result.pop("job_json"))
return result
def _finish(execution_id: str, *, error: Optional[str]) -> bool:
status = "failed" if error else "delivered"
safe_error = (
redact_sensitive_text(str(error), force=True, redact_url_credentials=True)
if error
else None
)
with _transaction() as conn:
cur = conn.execute(
"""UPDATE deliveries SET status=?, finished_at=?, error=?
WHERE execution_id=? AND status='delivering'
AND owner_process_id=? AND owner_pid=?""",
(
status,
_hermes_now().isoformat(),
safe_error,
execution_id,
_PROCESS_ID,
os.getpid(),
),
)
_prune_terminal_unlocked(conn)
return cur.rowcount == 1
def recover_abandoned() -> int:
"""Fence dead delivery owners as unknown; never replay uncertain sends."""
changed = 0
with _transaction() as conn:
rows = conn.execute(
"SELECT execution_id, owner_process_id, owner_pid, owner_started_at "
"FROM deliveries WHERE status='delivering'"
).fetchall()
for row in rows:
same_process = row["owner_process_id"] == _PROCESS_ID
if same_process:
with _lock:
if row["execution_id"] in _ACTIVE_DELIVERIES:
continue
elif _owner_is_live(int(row["owner_pid"]), row["owner_started_at"]):
continue
error = (
"Gateway finished delivery but could not persist its outcome; "
"send was not retried."
if same_process
else "Gateway exited during delivery; send outcome is unknown and was not retried."
)
cur = conn.execute(
"""UPDATE deliveries SET status='unknown', finished_at=?, error=?
WHERE execution_id=? AND status='delivering'""",
(
_hermes_now().isoformat(),
error,
row["execution_id"],
),
)
changed += cur.rowcount
_prune_terminal_unlocked(conn)
return changed
def drain(
send: Callable[[dict, str, bool], Optional[str]], *, limit: int = 20
) -> int:
"""Deliver pending rows through *send*, terminalizing every claimed row."""
recover_abandoned()
processed = 0
for _ in range(max(0, limit)):
row = claim_next()
if row is None:
break
with _lock:
_ACTIVE_DELIVERIES.add(row["execution_id"])
try:
try:
error = send(
row["job"], row["content"], bool(row["for_failure"])
)
except BaseException as exc:
error = f"{type(exc).__name__}: {exc}"
_finish(row["execution_id"], error=error)
finally:
with _lock:
_ACTIVE_DELIVERIES.discard(row["execution_id"])
processed += 1
return processed
def _terminalize_wait_timeout(execution_id: str) -> str:
"""Fence a delivery whose worker can no longer wait for confirmation.
A row still ``pending`` was provably never attempted, so it is left queued
for whichever gateway comes up next (a restart that includes an update can
easily exceed the worker's wait budget). That is a deferral, not a
failure: report success so the job is not recorded ``delivery_failed`` for
a message the drain will still send. Only a row caught mid-send is
uncertain and gets fenced ``unknown``.
"""
now = _hermes_now().isoformat()
uncertain_error = (
"timed out while gateway delivery was in progress; outcome is unknown and "
"was not retried"
)
with _transaction() as conn:
row = conn.execute(
"SELECT status FROM deliveries WHERE execution_id=?",
(str(execution_id),),
).fetchone()
if row is not None and row["status"] == "pending":
logger.warning(
"Cron delivery %s: no live gateway within the wait budget; "
"left queued for the next gateway",
execution_id,
)
return ""
conn.execute(
"""UPDATE deliveries SET status='unknown', finished_at=?, error=?
WHERE execution_id=? AND status='delivering'""",
(now, uncertain_error, str(execution_id)),
)
row = conn.execute(
"SELECT status, error FROM deliveries WHERE execution_id=?",
(str(execution_id),),
).fetchone()
_prune_terminal_unlocked(conn)
if row is None:
return "timed out waiting for live gateway delivery"
if row["status"] == "delivered":
return ""
return str(row["error"] or f"delivery {row['status']}")
def enqueue_and_wait(
execution_id: str,
job: dict,
content: str,
*,
for_failure: bool = False,
timeout: Optional[float] = None,
) -> Optional[str]:
"""Queue delivery and wait for a gateway's terminal at-most-once outcome."""
queued = enqueue(execution_id, job, content, for_failure=for_failure)
if queued["status"] in _TERMINAL:
return None if queued["status"] == "delivered" else str(
queued.get("error") or f"delivery {queued['status']}"
)
wait_timeout = (
DEFAULT_DELIVERY_WAIT_TIMEOUT_SECONDS if timeout is None else max(0.0, timeout)
)
deadline = time.monotonic() + wait_timeout
while time.monotonic() < deadline:
row = get_status(execution_id)
if row and row["status"] in _TERMINAL:
return None if row["status"] == "delivered" else str(
row.get("error") or f"delivery {row['status']}"
)
time.sleep(1.0)
return _terminalize_wait_timeout(execution_id) or None
cron/executions.py
+377
View File
@@ -0,0 +1,377 @@
"""Profile-local durable audit ledger for cron execution attempts.
The ledger records what is known about each attempt; it is not a retry queue.
Interrupted attempts become ``unknown`` only after their exact owner process is
proved gone. Terminal states are immutable.
"""
from __future__ import annotations
import os
import sqlite3
import threading
import time
import uuid
from contextlib import contextmanager
from typing import Any, Dict, Iterator, List, Optional
from hermes_constants import get_hermes_home
from hermes_time import now as _hermes_now
# Optional test override. Production resolves the path at transaction time so
# dashboard operations that temporarily enter another profile cannot leak that
# profile's execution records into the import-time home.
EXECUTIONS_FILE: Optional[Path] = None
MAX_TERMINAL_EXECUTIONS = 1000
HANDOFF_ADOPTION_GRACE_SECONDS = 30.0
_TERMINAL_STATES = ("completed", "failed", "unknown")
_lock = threading.RLock()
_PROCESS_ID = uuid.uuid4().hex
def _connect() -> sqlite3.Connection:
from cron.jobs import _ensure_cron_dir
path = EXECUTIONS_FILE or (get_hermes_home().resolve() / "cron" / "executions.db")
_ensure_cron_dir(path.parent)
return sqlite3.connect(path, timeout=5)
def _initialize_schema(conn: sqlite3.Connection) -> None:
from hermes_state import apply_wal_with_fallback
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA busy_timeout=5000")
apply_wal_with_fallback(conn, db_label="cron/executions.db")
conn.execute("PRAGMA synchronous=FULL")
conn.execute(
"""CREATE TABLE IF NOT EXISTS executions (
id TEXT PRIMARY KEY,
job_id TEXT NOT NULL,
source TEXT NOT NULL,
process_id TEXT NOT NULL,
pid INTEGER NOT NULL,
process_started_at INTEGER,
status TEXT NOT NULL CHECK(status IN
('claimed','running','completed','failed','unknown')),
handoff_pending INTEGER NOT NULL DEFAULT 0,
handoff_started_at REAL,
claimed_at TEXT NOT NULL,
started_at TEXT,
finished_at TEXT,
error TEXT
)"""
)
from hermes_cli.sqlite_util import add_column_if_missing
add_column_if_missing(
conn, "executions", "handoff_pending",
"handoff_pending INTEGER NOT NULL DEFAULT 0",
)
add_column_if_missing(
conn, "executions", "handoff_started_at", "handoff_started_at REAL"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_executions_job_claimed "
"ON executions(job_id, claimed_at DESC, id DESC)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_executions_status_claimed "
"ON executions(status, claimed_at DESC, id DESC)"
)
@contextmanager
def _transaction() -> Iterator[sqlite3.Connection]:
"""Open a connection, commit/rollback on exit, always close.
``sqlite3.Connection.__enter__``/``__exit__`` only commit or roll back
the transaction; it does not close the connection. Relying on that alone
leaks a connection (and its WAL/SHM file descriptors) on every call,
since closing then depends on the garbage collector. Schema init runs
inside the ``try`` too, so a PRAGMA/DDL failure after a successful
``connect()`` still closes the connection instead of leaking it.
"""
with _lock:
conn = _connect()
try:
_initialize_schema(conn)
with conn:
yield conn
finally:
conn.close()
def _record(row: Optional[sqlite3.Row]) -> Optional[Dict[str, Any]]:
return dict(row) if row is not None else None
def _emit_execution_state(
record: Optional[Dict[str, Any]], *, delivery_outcome: Optional[str] = None
) -> None:
"""Project durable state to monitoring without affecting ledger behavior."""
try:
from agent.monitoring.cron_health import emit_execution_state
emit_execution_state(record, delivery_outcome=delivery_outcome)
except Exception:
pass
def _process_start_time(pid: int) -> Optional[int]:
try:
from gateway.status import get_process_start_time
return get_process_start_time(pid)
except Exception:
return None
def _owner_is_live(pid: int, started_at: Optional[int]) -> bool:
try:
from gateway.status import _pid_exists
if not _pid_exists(pid):
return False
except Exception:
return True # fail safe: inability to prove death must not rewrite state
if started_at is None:
return pid == os.getpid()
current = _process_start_time(pid)
return current is not None and current == started_at
def _prune_unlocked(conn: sqlite3.Connection) -> None:
limit = max(0, int(MAX_TERMINAL_EXECUTIONS))
conn.execute(
"""DELETE FROM executions WHERE id IN (
SELECT id FROM executions
WHERE status IN ('completed','failed','unknown')
ORDER BY finished_at DESC, claimed_at DESC, id DESC LIMIT -1 OFFSET ?
)""",
(limit,),
)
def create_execution(job_id: str, *, source: str) -> Dict[str, Any]:
"""Persist a claimed attempt before executor/provider dispatch."""
now = _hermes_now().isoformat()
execution_id = uuid.uuid4().hex
pid = os.getpid()
with _transaction() as conn:
conn.execute(
"""INSERT INTO executions
(id, job_id, source, process_id, pid, process_started_at,
status, claimed_at)
VALUES (?, ?, ?, ?, ?, ?, 'claimed', ?)""",
(execution_id, str(job_id), str(source), _PROCESS_ID, pid,
_process_start_time(pid), now),
)
row = conn.execute(
"SELECT * FROM executions WHERE id=?", (execution_id,)
).fetchone()
record = _record(row)
_emit_execution_state(record)
return record # type: ignore[return-value]
def mark_execution_handoff_pending(execution_id: str) -> Optional[Dict[str, Any]]:
"""Fence restart recovery while an external worker is adopting a claim."""
with _transaction() as conn:
cur = conn.execute(
"""UPDATE executions
SET handoff_pending=1, handoff_started_at=?
WHERE id=? AND status='claimed'
AND process_id=? AND pid=?""",
(time.time(), execution_id, _PROCESS_ID, os.getpid()),
)
if cur.rowcount != 1:
return None
record = _record(conn.execute(
"SELECT * FROM executions WHERE id=?", (execution_id,)
).fetchone())
_emit_execution_state(record)
return record
def adopt_claimed_execution(execution_id: str) -> Optional[Dict[str, Any]]:
"""Atomically transfer and start an attempt in its worker process.
The dispatching gateway creates the row before spawning a restart-safe
worker. Adoption is the single ``claimed`` → ``running`` gate: only the
winner may acknowledge ownership or run side effects.
"""
pid = os.getpid()
process_started_at = _process_start_time(pid)
now = _hermes_now().isoformat()
with _transaction() as conn:
cur = conn.execute(
"""UPDATE executions
SET process_id=?, pid=?, process_started_at=?,
status='running', started_at=?, handoff_pending=0,
handoff_started_at=NULL
WHERE id=? AND status='claimed' AND handoff_pending=1""",
(_PROCESS_ID, pid, process_started_at, now, execution_id),
)
if cur.rowcount != 1:
return None
record = _record(conn.execute(
"SELECT * FROM executions WHERE id=?", (execution_id,)
).fetchone())
_emit_execution_state(record)
return record
def mark_execution_running(execution_id: str) -> Optional[Dict[str, Any]]:
"""Transition one claimed attempt to running exactly once."""
now = _hermes_now().isoformat()
with _transaction() as conn:
cur = conn.execute(
"""UPDATE executions
SET status='running', started_at=?, handoff_pending=0,
handoff_started_at=NULL
WHERE id=? AND status='claimed' AND handoff_pending=0
AND process_id=? AND pid=?""",
(now, execution_id, _PROCESS_ID, os.getpid()),
)
if cur.rowcount != 1:
return None
record = _record(conn.execute(
"SELECT * FROM executions WHERE id=?", (execution_id,)
).fetchone())
_emit_execution_state(record)
return record
def finish_execution(
execution_id: str, *, success: bool, error: Optional[str] = None,
delivery_outcome: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
"""Write a terminal result once; terminal attempts cannot be rewritten."""
now = _hermes_now().isoformat()
status = "completed" if success else "failed"
detail = None if success else (str(error) if error else "unknown failure")
with _transaction() as conn:
cur = conn.execute(
"""UPDATE executions
SET status=?, finished_at=?, error=?, handoff_pending=0,
handoff_started_at=NULL
WHERE id=? AND status IN ('claimed','running')
AND process_id=? AND pid=?""",
(status, now, detail, execution_id, _PROCESS_ID, os.getpid()),
)
if cur.rowcount != 1:
return None
_prune_unlocked(conn)
record = _record(conn.execute(
"SELECT * FROM executions WHERE id=?", (execution_id,)
).fetchone())
_emit_execution_state(record, delivery_outcome=delivery_outcome)
return record
def recover_interrupted_executions() -> int:
"""Mark provably abandoned attempts unknown without scheduling retries."""
now = _hermes_now().isoformat()
changed = 0
recovered: List[Dict[str, Any]] = []
with _transaction() as conn:
rows = conn.execute(
"""SELECT id, status, process_id, pid, process_started_at,
handoff_pending, handoff_started_at
FROM executions
WHERE status IN ('claimed','running')"""
).fetchall()
for row in rows:
if row["process_id"] == _PROCESS_ID:
continue
if _owner_is_live(int(row["pid"]), row["process_started_at"]):
continue
handoff_started_at = row["handoff_started_at"]
if (
row["handoff_pending"]
and handoff_started_at is not None
and time.time() - float(handoff_started_at)
< HANDOFF_ADOPTION_GRACE_SECONDS
):
continue
cur = conn.execute(
"""UPDATE executions
SET status='unknown', finished_at=?, error=?,
handoff_pending=0, handoff_started_at=NULL
WHERE id=? AND status=? AND process_id=? AND pid=?
AND handoff_pending=?
AND handoff_started_at IS ?""",
(now,
"Scheduler restarted after this execution's owner exited before a durable "
"terminal state; whether side effects ran is unknown.",
row["id"], row["status"], row["process_id"], row["pid"],
row["handoff_pending"], row["handoff_started_at"]),
)
changed += cur.rowcount
if cur.rowcount:
record = _record(conn.execute(
"SELECT * FROM executions WHERE id=?", (row["id"],)
).fetchone())
if record is not None:
recovered.append(record)
if changed:
_prune_unlocked(conn)
for record in recovered:
_emit_execution_state(record)
return changed
def list_executions(
*, job_id: Optional[str] = None, limit: int = 50,
before_claimed_at: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""Return indexed, newest-first execution history with cursor pagination."""
clauses: List[str] = []
params: List[Any] = []
if job_id is not None:
clauses.append("job_id=?")
params.append(str(job_id))
if before_claimed_at is not None:
clauses.append("claimed_at < ?")
params.append(str(before_claimed_at))
where = " WHERE " + " AND ".join(clauses) if clauses else ""
params.append(max(1, min(int(limit), 500)))
with _transaction() as conn:
rows = conn.execute(
"SELECT * FROM executions" + where
+ " ORDER BY claimed_at DESC, id DESC LIMIT ?",
params,
).fetchall()
return [dict(row) for row in rows]
def get_execution(execution_id: str) -> Optional[Dict[str, Any]]:
"""Return one exact execution attempt, or ``None`` when it is absent."""
with _transaction() as conn:
row = conn.execute(
"SELECT * FROM executions WHERE id=?",
(str(execution_id),),
).fetchone()
return dict(row) if row is not None else None
def latest_execution(job_id: str) -> Optional[Dict[str, Any]]:
rows = list_executions(job_id=job_id, limit=1)
return rows[0] if rows else None
def latest_executions(job_ids: List[str]) -> Dict[str, Dict[str, Any]]:
"""Load latest execution for many jobs in one indexed query."""
clean = [str(job_id) for job_id in dict.fromkeys(job_ids) if job_id]
if not clean:
return {}
placeholders = ",".join("?" for _ in clean)
with _transaction() as conn:
rows = conn.execute(
f"""SELECT e.* FROM executions e
WHERE e.job_id IN ({placeholders})
AND e.id=(SELECT e2.id FROM executions e2
WHERE e2.job_id=e.job_id
ORDER BY e2.claimed_at DESC, e2.id DESC LIMIT 1)""",
clean,
).fetchall()
return {row["job_id"]: dict(row) for row in rows}
cron/incidents.py
+302
View File
@@ -0,0 +1,302 @@
"""Durable cron failure incidents with signature dedup and ack.
The executions ledger (``cron.executions``) records every attempt; this module
groups the *failures* into durable incidents keyed by ``(job_id, error
signature)`` so the same job failing with the same error does not re-ping the
operator every run once they have acknowledged it.
Lifecycle: ``detected`` → ``alerted`` → ``closed``. Closing
(acking) an incident is per-signature: the same job + same normalized error
keeps resolving to the SAME incident id, so a closed incident stays closed (no
re-alert) until the error text changes, which mints a brand-new incident.
``detected`` means the failure was recorded; ``alerted`` means at least one
failure ping for the signature actually reached the operator. Richer states
(e.g. a dv9.6 ``reviewed``) are deliberately NOT reserved here — state
validity lives in ``INCIDENT_STATES`` (Python), not a SQLite CHECK, exactly
so a future slice can add states without a table rebuild.
Incidents live in the SAME ``cron/executions.db`` as ``cron.executions`` so
there is one durable cron store per profile. The schema is lazily created on
connect and a missing database never raises (directories are created).
"""
from __future__ import annotations
import hashlib
import re
import sqlite3
import threading
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional
from hermes_constants import get_hermes_home
from hermes_time import now as _hermes_now
# Optional test override (mirrors ``cron.executions.EXECUTIONS_FILE``).
EXECUTIONS_FILE: Optional[Path] = None
INCIDENT_STATES = ("detected", "alerted", "closed")
_FAILURE_TYPE_ORDER = (
("rate_limit", (r"\b429\b", "rate limit", "usage limit", "quota")),
("timeout", ("timeout", "timed out")),
("auth", (r"\b401\b", "unauthorized", "authentication", "auth")),
("delivery", ("delivery", "deliver", "delivering")),
("config", ("config", "configuration", "validation")),
("script", ("script", "no_agent")),
("agent", ("agent", "model", "provider", "inference")),
)
MAX_ERROR_CHARS = 500
_MAX_SIGNATURE_ERROR_CHARS = 200
_lock = threading.RLock()
def _connect() -> sqlite3.Connection:
from cron.jobs import _ensure_cron_dir
path = _db_path()
_ensure_cron_dir(path.parent)
return sqlite3.connect(path, timeout=5)
def _db_path() -> Path:
"""Resolve the shared cron DB path.
Prefer the ``cron.executions`` override when one is installed so an
operator/test that redirects the executions ledger also redirects the
incident table — they must stay in the SAME database. Falls back to this
module's own override, then the canonical profile home.
"""
try:
from cron.executions import EXECUTIONS_FILE as _EXEC_OVERRIDE
if _EXEC_OVERRIDE is not None:
return Path(_EXEC_OVERRIDE)
except Exception:
pass
if EXECUTIONS_FILE is not None:
return Path(EXECUTIONS_FILE)
return get_hermes_home().resolve() / "cron" / "executions.db"
def _initialize_schema(conn: sqlite3.Connection) -> None:
from hermes_state import apply_wal_with_fallback
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA busy_timeout=5000")
apply_wal_with_fallback(conn, db_label="cron/executions.db")
conn.execute("PRAGMA synchronous=FULL")
conn.execute(
"""CREATE TABLE IF NOT EXISTS cron_incidents (
id TEXT PRIMARY KEY,
job_id TEXT NOT NULL,
error_sig TEXT NOT NULL,
state TEXT NOT NULL,
failure_type TEXT NOT NULL DEFAULT 'unknown',
first_seen_at TEXT NOT NULL,
last_seen_at TEXT NOT NULL,
acked_at TEXT,
closed_at TEXT,
error TEXT NOT NULL,
output_file TEXT
)"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_cron_incidents_job "
"ON cron_incidents(job_id)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_cron_incidents_state "
"ON cron_incidents(state)"
)
@contextmanager
def _transaction() -> Iterator[sqlite3.Connection]:
"""Open a connection, commit/rollback on exit, always close.
Mirrors ``cron.executions._transaction``: schema init runs inside the
``try`` so a PRAGMA/DDL failure after a successful ``connect()`` still
closes the connection instead of leaking it.
"""
with _lock:
conn = _connect()
try:
_initialize_schema(conn)
with conn:
yield conn
finally:
conn.close()
def _normalize_error(error: str) -> str:
"""Strip whitespace and lowercase before signing (dedup normalization)."""
return re.sub(r"\s+", " ", str(error or "")).strip().lower()
def _redact_error(error: str) -> str:
"""Redact secrets then bound the stored error length."""
text = str(error or "")
try:
from agent.redact import redact_sensitive_text
text = redact_sensitive_text(text)
except Exception:
# Redaction is best-effort; the scheduler path never fails on it.
pass
return text[:MAX_ERROR_CHARS]
def _error_signature(job_id: str, error: str) -> str:
"""Dedup key: stable for same job + same normalized error prefix."""
normalized = _normalize_error(error)[:_MAX_SIGNATURE_ERROR_CHARS]
digest = hashlib.sha256(job_id.encode() + normalized.encode()).hexdigest()
return digest[:12]
def _incident_id(job_id: str, error_sig: str) -> str:
return f"{job_id[:6]}_{error_sig}"
def _classify_failure_type(error: str) -> str:
"""Classify a failure from error-text keywords; ``unknown`` is the default."""
text = _normalize_error(error)
if not text:
return "unknown"
for kind, patterns in _FAILURE_TYPE_ORDER:
for pattern in patterns:
if pattern.startswith("\\b") and pattern.endswith("\\b"):
if re.search(pattern, text):
return kind
elif pattern in text:
return kind
return "unknown"
def upsert_incident(
job_id: str,
error: str,
*,
job_name: Optional[str] = None,
failure_type: Optional[str] = None,
output_file: Optional[str] = None,
) -> tuple[str, bool]:
"""Record (or refresh) the incident for ``job_id`` + ``error``.
Returns ``(incident_id, is_new)``. A row for the same signature already
existing refreshes ``last_seen_at``/``error``/``output_file`` and keeps its
current state — a ``closed`` (acked) incident stays closed for the same
signature. A changed error text mints a new incident automatically.
"""
job_id = str(job_id or "")
sig = _error_signature(job_id, error)
stored_error = _redact_error(error)
incident_id = _incident_id(job_id, sig)
now = _hermes_now().isoformat()
failure_type = failure_type or _classify_failure_type(error)
output_file = str(output_file) if output_file is not None else None
with _transaction() as conn:
row = conn.execute(
"SELECT id FROM cron_incidents WHERE id=?", (incident_id,)
).fetchone()
if row is not None:
conn.execute(
"""UPDATE cron_incidents
SET last_seen_at=?, error=?, output_file=?
WHERE id=?""",
(now, stored_error, output_file, incident_id),
)
return incident_id, False
conn.execute(
"""INSERT INTO cron_incidents
(id, job_id, error_sig, state, failure_type,
first_seen_at, last_seen_at, error, output_file)
VALUES (?, ?, ?, 'detected', ?, ?, ?, ?, ?)""",
(incident_id, job_id, sig, failure_type, now, now,
stored_error, output_file),
)
return incident_id, True
def set_incident_state(incident_id: str, state: str) -> bool:
"""Transition an incident's lifecycle state; return whether it changed.
``closed`` is terminal for that signature: no transition (including back
to ``alerted``) leaves it — re-open happens by the error changing and
minting a NEW incident. Unknown states are rejected (no-op, ``False``).
"""
if state not in INCIDENT_STATES:
return False
now = _hermes_now().isoformat()
with _transaction() as conn:
row = conn.execute(
"SELECT state FROM cron_incidents WHERE id=?", (incident_id,)
).fetchone()
if row is None or row["state"] == state:
return False
if row["state"] == "closed":
return False
if state == "closed":
conn.execute(
"""UPDATE cron_incidents
SET state='closed', closed_at=?, acked_at=?
WHERE id=? AND state != 'closed'""",
(now, now, incident_id),
)
else:
conn.execute(
"UPDATE cron_incidents SET state=? WHERE id=?",
(state, incident_id),
)
return True
def ack_incident(incident_id: str) -> bool:
"""Acknowledge (close) an incident; return whether the state changed.
A no-op (``False``) when the incident does not exist or is already closed.
"""
return set_incident_state(incident_id, "closed")
def list_incidents(state: Optional[str] = None) -> List[Dict[str, Any]]:
"""Return incidents, newest-activity first, optionally filtered by state."""
if state is not None and state not in INCIDENT_STATES:
return []
with _transaction() as conn:
if state is None:
rows = conn.execute(
"SELECT * FROM cron_incidents "
"ORDER BY last_seen_at DESC, id DESC"
).fetchall()
else:
rows = conn.execute(
"SELECT * FROM cron_incidents WHERE state=? "
"ORDER BY last_seen_at DESC, id DESC",
(state,),
).fetchall()
return [dict(row) for row in rows]
def get_incident(incident_id: str) -> Optional[Dict[str, Any]]:
with _transaction() as conn:
row = conn.execute(
"SELECT * FROM cron_incidents WHERE id=?", (incident_id,)
).fetchone()
return dict(row) if row is not None else None
def count_incidents(state: Optional[str] = None) -> int:
if state is not None and state not in INCIDENT_STATES:
return 0
with _transaction() as conn:
if state is None:
row = conn.execute("SELECT COUNT(*) AS n FROM cron_incidents").fetchone()
else:
row = conn.execute(
"SELECT COUNT(*) AS n FROM cron_incidents WHERE state=?",
(state,),
).fetchone()
return int(row["n"]) if row is not None else 0
cron/jobs.py
+4671
View File
File diff suppressed because it is too large Load Diff
cron/lifecycle_guard.py
+1414
View File
File diff suppressed because it is too large Load Diff
cron/monitor.py
+213
View File
@@ -0,0 +1,213 @@
"""Monitor-mode cron support — hash-suppressed change detection.
A monitor job attaches a cheap *monitor source* (``monitor_script`` or
``monitor_url``) to an ordinary LLM cron job. Each tick the scheduler runs
the source FIRST and compares a hash of its exact output bytes against the
hash stored from the last agent-triggering tick:
* unchanged → the agent run is suppressed entirely (no LLM, no delivery);
the tick is recorded as a silent ``no_change`` run.
* changed (or first run) → a "MONITOR CHANGE DETECTED" context block —
unified diff of old vs new output (capped) plus the new output — is
injected into the prompt and the agent runs normally.
* source failure → treated as an ERROR, never as a change. The stored hash
is left untouched so a source that recovers to its previous output still
suppresses.
Output is compared as EXACT BYTES — no timestamp stripping or whitespace
normalization. Monitor scripts should emit stable output (sort results,
omit "generated at" lines) or every tick will look like a change.
State lives in two places, both durable across scheduler restarts:
* ``job["monitor_state"]`` in jobs.json — ``last_output_hash`` +
``last_changed_at`` (additive JSON fields, no migration needed);
* ``OUTPUT_DIR/<job_id>/monitor_last_output.txt`` — the previous output
text, kept only so the next change can render a diff.
Inspired by: ChatGPT Work monitor tasks (idea-level, docs-only);
enabler: #80774.
"""
from __future__ import annotations
import difflib
import hashlib
import logging
from dataclasses import dataclass
from typing import Optional
logger = logging.getLogger(__name__)
# Cap for the unified diff injected into the prompt.
MAX_DIFF_CHARS = 4000
# Cap for the new-output block injected into the prompt (mirrors the 8k
# context_from truncation in cron/scheduler.py).
MAX_OUTPUT_CHARS = 8000
# Bounded GET limits for monitor_url sources.
URL_TIMEOUT_SECONDS = 30
MAX_URL_BYTES = 262_144 # 256 KiB
_SNAPSHOT_FILENAME = "monitor_last_output.txt"
@dataclass
class MonitorOutcome:
"""Result of one monitor-source evaluation."""
ok: bool
changed: bool = False
first_run: bool = False
context_block: Optional[str] = None
error: Optional[str] = None
def hash_monitor_output(output: str) -> str:
"""Hash the monitor output as exact UTF-8 bytes (no normalization)."""
return hashlib.sha256(output.encode("utf-8", errors="replace")).hexdigest()
def build_monitor_diff(old: str, new: str) -> str:
"""Unified diff of old vs new monitor output, capped at MAX_DIFF_CHARS."""
diff = "\n".join(
difflib.unified_diff(
old.splitlines(),
new.splitlines(),
fromfile="previous",
tofile="current",
lineterm="",
)
)
if len(diff) > MAX_DIFF_CHARS:
diff = diff[:MAX_DIFF_CHARS] + "\n... [diff truncated]"
return diff
def _snapshot_path(job_id: str):
from cron.jobs import _job_output_dir
return _job_output_dir(job_id) / _SNAPSHOT_FILENAME
def _read_last_output(job_id: str) -> str:
try:
path = _snapshot_path(job_id)
if path.exists():
return path.read_text(encoding="utf-8")
except Exception as exc:
logger.warning("Monitor: failed to read last output for %r: %s", job_id, exc)
return ""
def _write_last_output(job_id: str, output: str) -> None:
try:
path = _snapshot_path(job_id)
from cron.jobs import _ensure_cron_dir
_ensure_cron_dir(path.parent)
path.write_text(output, encoding="utf-8")
except Exception as exc:
logger.warning("Monitor: failed to persist last output for %r: %s", job_id, exc)
def _fetch_monitor_url(url: str) -> tuple[bool, str]:
"""Bounded GET of a monitor URL. Returns (ok, body-or-error)."""
import urllib.request
if not str(url).lower().startswith(("http://", "https://")):
return False, f"monitor_url must be http(s): {url!r}"
try:
req = urllib.request.Request(url, headers={"User-Agent": "hermes-cron-monitor"})
with urllib.request.urlopen(req, timeout=URL_TIMEOUT_SECONDS) as resp: # nosec B310 — scheme checked above
body = resp.read(MAX_URL_BYTES + 1)
if len(body) > MAX_URL_BYTES:
body = body[:MAX_URL_BYTES]
return True, body.decode("utf-8", errors="replace")
except Exception as exc:
return False, f"monitor_url fetch failed: {exc}"
def _run_monitor_source(job: dict) -> tuple[bool, str]:
"""Run the job's monitor source (script or URL). Returns (ok, output)."""
monitor_script = (job.get("monitor_script") or "").strip()
if monitor_script:
# Same containment + interpreter rules as the existing `script` field.
from cron.scheduler import _run_job_script
workdir = (job.get("workdir") or "").strip() or None
return _run_job_script(monitor_script, workdir=workdir)
monitor_url = (job.get("monitor_url") or "").strip()
if monitor_url:
return _fetch_monitor_url(monitor_url)
return False, "monitor job has neither monitor_script nor monitor_url"
def job_has_monitor(job: dict) -> bool:
return bool((job.get("monitor_script") or "").strip() or (job.get("monitor_url") or "").strip())
def check_monitor(job: dict) -> MonitorOutcome:
"""Run the monitor source and decide whether the agent should run.
On change (or first run) the new hash + snapshot are persisted BEFORE
the agent runs — detection time is the state boundary, so a failed
agent run doesn't re-alert on the same content forever.
On failure nothing is persisted.
"""
job_id = str(job.get("id") or "")
ok, output = _run_monitor_source(job)
if not ok:
return MonitorOutcome(ok=False, error=output)
new_hash = hash_monitor_output(output)
raw_state = job.get("monitor_state")
state = raw_state if isinstance(raw_state, dict) else {}
last_hash = state.get("last_output_hash")
if last_hash is not None and new_hash == last_hash:
return MonitorOutcome(ok=True, changed=False)
first_run = last_hash is None
old_output = "" if first_run else _read_last_output(job_id)
shown_output = output
if len(shown_output) > MAX_OUTPUT_CHARS:
shown_output = shown_output[:MAX_OUTPUT_CHARS] + "\n... [output truncated]"
if first_run:
context_block = (
"## Monitor Baseline (first run)\n\n"
"This is the first observation of the monitored source — there is "
"no previous output to diff against.\n\n"
f"### Current output\n\n```\n{shown_output}\n```"
)
else:
diff = build_monitor_diff(old_output, output)
context_block = (
"## MONITOR CHANGE DETECTED\n\n"
"The monitored source's output changed since the last run.\n\n"
f"### Diff (previous → current)\n\n```diff\n{diff}\n```\n\n"
f"### Current output\n\n```\n{shown_output}\n```"
)
_persist_monitor_state(job_id, new_hash, output)
return MonitorOutcome(
ok=True, changed=True, first_run=first_run, context_block=context_block
)
def _persist_monitor_state(job_id: str, new_hash: str, output: str) -> None:
from cron.jobs import _hermes_now, update_job
_write_last_output(job_id, output)
try:
update_job(
job_id,
{
"monitor_state": {
"last_output_hash": new_hash,
"last_changed_at": _hermes_now().isoformat(),
}
},
)
except Exception as exc:
logger.warning("Monitor: failed to persist state for %r: %s", job_id, exc)
cron/notepad.py
+200
View File
@@ -0,0 +1,200 @@
"""Per-job durable notepad for cron jobs.
A tiny KV scratchpad each cron job can use to carry state across scheduled
wake-ups (cursors, watermarks, watchlists). Stored in its own profile-local
SQLite file next to the executions ledger, following the same
connection/pragma pattern as ``cron/executions.py``.
Size caps (documented contract):
- ``MAX_VALUE_BYTES`` (16 KB): per-key value cap, measured in UTF-8 bytes.
- ``MAX_JOB_TOTAL_BYTES`` (64 KB): per-job cap over the sum of key+value
bytes. Oversized writes raise ``ValueError`` and leave the store
untouched — the notepad is prompt-injected each run, so unbounded growth
would bloat every wake-up's prompt.
Write path is the CLI (``hermes cron notepad <job_id> set <key> <value>``),
which the running agent invokes via its terminal tool; no model tool is
added.
Inspired by: Amp (Sourcegraph) cron notepad (idea-level, proprietary — zero
code).
"""
from __future__ import annotations
import sqlite3
import threading
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional
from hermes_constants import get_hermes_home
from hermes_time import now as _hermes_now
# Optional test override. Production resolves the path at transaction time so
# multiplexed profile ticks (set_hermes_home_override) cannot leak one
# profile's notepad rows into the import-time home — and remove_job's
# clear_notepad cannot wipe the wrong profile's DB (#86519). Same pattern as
# cron/executions.py.
NOTEPAD_FILE: Optional[Path] = None
MAX_VALUE_BYTES = 16 * 1024
MAX_KEY_CHARS = 128
MAX_JOB_TOTAL_BYTES = 64 * 1024
_lock = threading.RLock()
def _current_notepad_file() -> Path:
return NOTEPAD_FILE or (get_hermes_home().resolve() / "cron" / "notepad.db")
def _connect() -> sqlite3.Connection:
from cron.jobs import _ensure_cron_dir
path = _current_notepad_file()
_ensure_cron_dir(path.parent)
return sqlite3.connect(path, timeout=5)
def _initialize_schema(conn: sqlite3.Connection) -> None:
from hermes_state import apply_wal_with_fallback
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA busy_timeout=5000")
apply_wal_with_fallback(conn, db_label="cron/notepad.db")
conn.execute(
"""CREATE TABLE IF NOT EXISTS cron_notepad (
job_id TEXT NOT NULL,
key TEXT NOT NULL,
value TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY (job_id, key)
)"""
)
@contextmanager
def _transaction() -> Iterator[sqlite3.Connection]:
"""Open a connection, commit/rollback on exit, always close.
Mirrors ``cron.executions._transaction``: schema init runs inside the
``try`` so a PRAGMA/DDL failure still closes the connection instead of
leaking it.
"""
with _lock:
conn = _connect()
try:
_initialize_schema(conn)
with conn:
yield conn
finally:
conn.close()
def _validate(job_id: str, key: str, value: str) -> None:
if not str(job_id):
raise ValueError("job_id must be non-empty")
if not key:
raise ValueError("key must be non-empty")
if len(key) > MAX_KEY_CHARS:
raise ValueError(f"key too long (max {MAX_KEY_CHARS} characters)")
if len(value.encode("utf-8")) > MAX_VALUE_BYTES:
raise ValueError(
f"value too large (max {MAX_VALUE_BYTES} bytes per key)"
)
def set_note(job_id: str, key: str, value: str) -> Dict[str, Any]:
"""Upsert one key. Raises ValueError when a size cap would be exceeded."""
job_id, key, value = str(job_id), str(key), str(value)
_validate(job_id, key, value)
now = _hermes_now().isoformat()
with _transaction() as conn:
row = conn.execute(
"""SELECT COALESCE(SUM(LENGTH(CAST(key AS BLOB))
+ LENGTH(CAST(value AS BLOB))), 0)
FROM cron_notepad WHERE job_id=? AND key<>?""",
(job_id, key),
).fetchone()
other_bytes = int(row[0])
entry_bytes = len(key.encode("utf-8")) + len(value.encode("utf-8"))
if other_bytes + entry_bytes > MAX_JOB_TOTAL_BYTES:
raise ValueError(
f"notepad full: job '{job_id}' would exceed "
f"{MAX_JOB_TOTAL_BYTES} bytes total; delete unused keys first"
)
conn.execute(
"""INSERT INTO cron_notepad (job_id, key, value, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(job_id, key)
DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at""",
(job_id, key, value, now),
)
return {"job_id": job_id, "key": key, "value": value, "updated_at": now}
def get_note(job_id: str, key: str) -> Optional[str]:
with _transaction() as conn:
row = conn.execute(
"SELECT value FROM cron_notepad WHERE job_id=? AND key=?",
(str(job_id), str(key)),
).fetchone()
return None if row is None else row["value"]
def delete_note(job_id: str, key: str) -> bool:
with _transaction() as conn:
cur = conn.execute(
"DELETE FROM cron_notepad WHERE job_id=? AND key=?",
(str(job_id), str(key)),
)
return cur.rowcount > 0
def list_notes(job_id: str) -> List[Dict[str, Any]]:
"""All entries for one job, sorted by key."""
with _transaction() as conn:
rows = conn.execute(
"SELECT job_id, key, value, updated_at FROM cron_notepad "
"WHERE job_id=? ORDER BY key",
(str(job_id),),
).fetchall()
return [dict(row) for row in rows]
def clear_notepad(job_id: str) -> int:
"""Delete every key for one job (e.g. on job removal). Returns row count.
Called from ``cron.jobs.remove_job`` so deleted jobs don't orphan their
rows. No-ops without creating the DB when no notepad file exists yet.
"""
if not _current_notepad_file().exists():
return 0
with _transaction() as conn:
cur = conn.execute(
"DELETE FROM cron_notepad WHERE job_id=?", (str(job_id),)
)
return cur.rowcount
def render_notepad_section(job_id: str) -> str:
"""Render a job's notepad as a prompt section, or '' when empty/unavailable.
Empty notepad MUST return the empty string so jobs that never use the
feature get a byte-identical prompt (prompt-cache + drift safety).
"""
try:
notes = list_notes(job_id)
except Exception:
return ""
if not notes:
return ""
lines = [f"- {note['key']}: {note['value']}" for note in notes]
return (
"## Job notepad (persistent across runs)\n"
"This durable scratchpad survives between scheduled runs of this "
"job. Update it via the CLI, e.g.:\n"
f"`hermes cron notepad {job_id} set <key> <value>` "
f"(also: get/delete/list; `hermes cron notepad {job_id} delete "
"<key>` removes an entry).\n\n" + "\n".join(lines) + "\n\n"
)
cron/scheduler.py
+9006
View File
File diff suppressed because it is too large Load Diff
cron/scheduler_provider.py
+865
View File
@@ -0,0 +1,865 @@
"""CronScheduler provider interface (Axis B — the trigger).
⚠️ EXPERIMENTAL — this interface is validated by exactly ONE consumer (the
built-in) until an external provider (Chronos, Phase 4) shakes it out. Until
then the module path, method signatures, and start() kwargs MAY change without
a deprecation cycle. Once a second provider validates the shape it becomes
stable. Any growth MUST be additive (new optional method with a default), never
a changed signature on start() or a new abstractmethod.
A CronScheduler decides *when* a due job fires. It does NOT decide what firing
means: execution + delivery stay in cron.scheduler.run_job / _deliver_result,
shared by all providers. Providers must never reimplement agent construction or
delivery.
The built-in InProcessCronScheduler runs the historical 60s daemon-thread
ticker. Alternative providers (e.g. Chronos, a NAS-mediated managed-cron
provider for scale-to-zero deployments) live under plugins/cron_providers/<name>/ and are
selected via the `cron.provider` config key (empty = built-in).
"""
from __future__ import annotations
import inspect
import threading
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any
# Cap for the exponential tick backoff applied while consecutive ticks fail
# with fd exhaustion (EMFILE/ENFILE, #87644). Base is the tick interval
# (60s by default); each consecutive EMFILE failure doubles the wait, capped
# here so a still-alive-but-exhausted gateway never sleeps longer than this
# between recovery attempts.
_EMFILE_BACKOFF_MAX_SECONDS = 15 * 60 # 15 minutes
def _backoff_wait_seconds(interval: float, consecutive_failures: int) -> float:
"""Exponential tick backoff shared by both ticker loops (#87644).
Returns the plain ``interval`` while healthy; doubles per consecutive
fd-exhaustion failure, capped at ``_EMFILE_BACKOFF_MAX_SECONDS``.
"""
if consecutive_failures <= 0:
return interval
return min(
interval * (2 ** (consecutive_failures - 1)),
_EMFILE_BACKOFF_MAX_SECONDS,
)
def _note_tick_failure(exc: BaseException, consecutive_failures: int) -> int:
"""Classify one failed tick and return the updated failure counter.
Shared by both ticker loops (#87644): on fd exhaustion, attempt
reclamation (gc.collect + raise the soft nofile limit) so the NEXT tick
can succeed, and bump the counter so ``_backoff_wait_seconds`` backs off
exponentially while the process has no chance of making progress. Any
other failure resets the counter — backoff is reserved for the
self-inflicted EMFILE storm, not transient errors.
"""
from cron.scheduler import _is_fd_exhaustion, _reclaim_fds_best_effort
if _is_fd_exhaustion(exc):
_reclaim_fds_best_effort()
return consecutive_failures + 1
return 0
def _existing_profile_homes(profile_homes: list) -> list:
"""Drop profile homes whose directory no longer exists on disk.
The multiplex ticker's ``profile_homes`` is a snapshot taken at startup
(``web_server.py`` calls ``profiles_to_serve(multiplex=True)`` once, and
the gateway multiplex path does the same). If a profile is deleted while
the ticker runs — via ``hermes profile delete``, the desktop's DELETE
``/api/profiles/<name>`` route, or any other path that removes the home
directory — that stale entry stays in the list.
Ticking or heartbeating a deleted home recreates its ``cron/`` workspace
(``record_ticker_heartbeat`` -> ``ensure_dirs`` -> ``mkdir(parents=True)``)
on every 60s cycle, so the "deleted" profile silently comes back on disk
and in ``hermes profile list`` (#47368). Filtering on directory existence
leaves a deleted profile's home untouched, which is the correct invariant:
a home that does not exist cannot hold jobs to fire.
"""
live = []
for entry in profile_homes:
home = entry[1] if isinstance(entry, tuple) else entry
if Path(home).is_dir():
live.append(entry)
return live
class CronScheduler(ABC):
"""Axis-B trigger provider. Decides WHEN a due cron job fires.
Required surface is intentionally minimal: ``name`` + ``start``. ``stop``
and ``is_available`` carry safe defaults. The three Phase-4 hooks
(``on_jobs_changed`` / ``fire_due`` / ``reconcile``) are added later as
NON-abstract methods so the built-in keeps satisfying the ABC without
overriding them — see ``test_abc_growth_stays_additive``.
"""
@property
@abstractmethod
def name(self) -> str:
"""Short identifier, e.g. 'builtin', 'chronos'."""
def is_available(self) -> bool:
"""Whether this provider can run in the current environment.
MUST NOT make network calls. The built-in is always available; an
external provider checks for configured endpoint/credentials. When a
named provider returns False, the resolver falls back to the built-in.
"""
return True
@abstractmethod
def start(
self,
stop_event: threading.Event,
*,
adapters: Any = None,
loop: Any = None,
interval: int = 60,
) -> None:
"""Begin firing due jobs.
For the built-in this BLOCKS in the 60s loop until stop_event is set
(it is run inside a daemon thread by the caller, exactly as today).
An external provider may register a schedule/webhook and return
immediately; in that case it must still honor stop_event for teardown.
"""
def stop(self) -> None:
"""Optional eager teardown hook. Default no-op; setting the stop_event
is the primary stop signal. Override for providers holding external
resources (queue consumers, HTTP servers)."""
return None
# --- Optional hooks for external providers (added Phase 4). --------------
# All default-safe so the built-in inherits working behavior without
# overriding. Keep these NON-abstract — see test_abc_growth_stays_additive.
def on_jobs_changed(self) -> None:
"""Called after a successful store mutation (create/update/remove/
pause/resume). External providers reconcile their registry here (e.g.
Chronos re-provisions/cancels the affected one-shot via NAS).
Built-in: no-op (it re-reads jobs.json on every tick)."""
return None
def register_job(self, job: dict[str, Any]) -> None:
"""Register the first external trigger for one newly persisted job.
The built-in provider reads the local store on every tick, so its
default is a no-op. External providers override this when creating a
job requires a remote registration before callers can honestly report
that the job is scheduled.
"""
return None
def recover_interrupted(self) -> int:
"""Run profile-local attempt recovery for every provider lifecycle."""
from cron.executions import recover_interrupted_executions
return recover_interrupted_executions()
@property
def supports_force_fire(self) -> bool:
"""Whether ``fire_due`` accepts the additive ``force`` keyword.
Signature detection keeps providers written before ``force`` was added
source-compatible. Providers accepting ``**kwargs`` are compatible.
"""
return provider_supports_force_fire(self)
def fire_due(
self,
job_id: str,
*,
adapters: Any = None,
loop: Any = None,
force: bool = False,
) -> bool:
"""Run a single job NOW via the shared orchestrator. Called by the
inbound fire webhook when an external scheduler signals a job is due.
The default claims the job with a store-level compare-and-set
(multi-machine at-most-once), then runs it via the shared
``run_one_job`` body. Built-in never calls this (it has its own tick
loop); an external provider routes its inbound fire here.
Returns True if THIS caller claimed and processed the attempt, even if
the job itself failed. Returns False only if the claim was lost
(another machine/retry won it) or the job no longer exists.
"""
claimed_job = self.claim_fire(job_id, force=force)
if claimed_job is None:
return False
return self.fire_claimed(claimed_job, adapters=adapters, loop=loop)
def claim_fire(self, job_id: str, *, force: bool = False) -> dict | None:
"""Durably claim one fire and create its audit attempt before dispatch.
Webhook transports call this synchronously before acknowledging the
external scheduler, then pass the exact owner-bearing snapshot to
``fire_claimed`` in tracked background work.
"""
from cron.executions import create_execution, finish_execution
from cron.jobs import claim_job_for_fire
execution = create_execution(job_id, source=self.name)
claim_kwargs = {"return_job": True}
if force:
claim_kwargs["force"] = True
try:
claimed_job = claim_job_for_fire(job_id, **claim_kwargs)
except BaseException as exc:
finish_execution(
execution["id"],
success=False,
error=f"Fire claim failed before dispatch: {type(exc).__name__}: {exc}",
)
raise
if not isinstance(claimed_job, dict):
finish_execution(
execution["id"],
success=False,
error="Fire claim was not acquired",
)
return None
claimed_job["execution_id"] = execution["id"]
return claimed_job
def fire_claimed(
self,
claimed_job: dict,
*,
adapters: Any = None,
loop: Any = None,
cancel_event: Any = None,
) -> bool:
"""Run an exact snapshot returned by ``claim_fire``.
``cancel_event``: optional transport-owned ``threading.Event`` (or
compatible) that lets the caller stop this execution cooperatively
— e.g. the dashboard lifespan drain signalling pending webhook
fires before the event loop shuts down.
"""
from cron.scheduler import run_one_job
run_one_job(
claimed_job,
adapters=adapters,
loop=loop,
cancel_event=cancel_event,
)
return True
def reconcile(self) -> None:
"""Converge the external registry toward jobs.json (the desired state):
arm missing one-shots, cancel orphaned ones, re-arm changed times.
Built-in: no-op."""
return None
def provider_supports_force_fire(provider: Any) -> bool:
"""Return whether a provider can safely receive ``fire_due(force=...)``."""
try:
parameters = inspect.signature(provider.fire_due).parameters.values()
except (TypeError, ValueError):
return False
return any(
parameter.kind is inspect.Parameter.VAR_KEYWORD
or (
parameter.name == "force"
and parameter.kind
in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY)
)
for parameter in parameters
)
def provider_supports_split_fire(provider: Any) -> bool:
"""Return whether a provider implements the two-phase fire contract.
The webhook admission path uses ``claim_fire`` + ``fire_claimed`` so the
202 response is backed by a durable, owner-fenced claim. A legacy
third-party provider that overrides the documented single-phase
``fire_due`` hook (custom claim/re-arm/telemetry behavior) but inherits
the base ``claim_fire`` must keep being driven through its own
``fire_due`` — silently routing around its override would drop that
behavior. Providers that customize ``claim_fire`` itself are already
split-aware and keep the two-phase path.
"""
cls = type(provider)
fire_due_impl = getattr(cls, "fire_due", None)
claim_fire_impl = getattr(cls, "claim_fire", None)
fire_claimed_impl = getattr(cls, "fire_claimed", None)
if claim_fire_impl is not None and claim_fire_impl is not CronScheduler.claim_fire:
return True
# Overriding the second phase is also proof of split-awareness (the
# provider composes with the inherited claim path) — e.g. Chronos keeps
# its re-arm logic in ``fire_claimed`` only.
if fire_claimed_impl is not None and fire_claimed_impl is not CronScheduler.fire_claimed:
return True
if fire_due_impl is None or fire_due_impl is CronScheduler.fire_due:
return True
return False
def provider_supports_fire_cancel(provider: Any) -> bool:
"""Return whether ``fire_claimed`` accepts a ``cancel_event`` kwarg."""
try:
parameters = inspect.signature(provider.fire_claimed).parameters.values()
except (TypeError, ValueError):
return False
return any(
parameter.kind is inspect.Parameter.VAR_KEYWORD
or (
parameter.name == "cancel_event"
and parameter.kind
in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY)
)
for parameter in parameters
)
DEFAULT_MISFIRE_GRACE_MINUTES = 10
def _misfire_grace_minutes() -> float:
"""Resolve the misfire catch-up grace window from config.
``cron.misfire_grace_minutes`` (number, default
``DEFAULT_MISFIRE_GRACE_MINUTES``). A non-positive value disables the
catch-up sweep entirely.
"""
try:
from hermes_cli.config import cfg_get, load_config
return float(
cfg_get(
load_config(),
"cron",
"misfire_grace_minutes",
default=DEFAULT_MISFIRE_GRACE_MINUTES,
)
)
except Exception:
return float(DEFAULT_MISFIRE_GRACE_MINUTES)
def fire_overdue_jobs(
provider: "CronScheduler",
*,
adapters: Any = None,
loop: Any = None,
now: Any = None,
) -> int:
"""Fire jobs whose scheduled time passed without an external fire arriving.
The misfire catch-up half of the hosted fire path. External providers
(Chronos) deliver scheduled fires over HTTP to this process's api_server
adapter; when that hop is down at fire time (gateway restart window,
api_server not bound, scheduler retry budget exhausted), the job's
``next_run_at`` stays parked in the past and — because external providers
have no local tick loop — nothing ever runs it. The day is silently lost
even though the gateway may be healthy again minutes later.
Called from the gateway housekeeping loop. Deliberately:
- **No-op for the built-in provider.** Its tick loop already picks up
past-due jobs via ``get_due_jobs`` — local scheduling self-heals.
- **Routes through the provider's own two-phase fire path** — a
synchronous ``claim_fire`` (store CAS, so a late external retry
landing concurrently is de-duplicated) and then ``fire_claimed`` in
a daemon thread, mirroring the webhook admission pattern. The
housekeeping loop that calls this must never block for the length
of an agent run. Provider-specific re-arm logic (Chronos NAS
one-shots) runs exactly as for a normal fire.
- **Waits out a grace window** (``cron.misfire_grace_minutes``, default
10, non-positive disables) so the external scheduler's own retry
backoff gets first right to deliver — catch-up is the backstop, not
a race.
- **Operates on the process-global cron store only** — same profile
scoping as the external provider's reconcile.
Returns the number of jobs this sweep claimed and dispatched.
"""
import logging
import threading
from datetime import datetime
logger = logging.getLogger("cron.scheduler_provider")
if isinstance(provider, InProcessCronScheduler):
return 0
grace_minutes = _misfire_grace_minutes()
if grace_minutes <= 0:
return 0
from cron.jobs import _ensure_aware, _hermes_now, is_job_runnable, load_jobs
if now is None:
now = _hermes_now()
fired = 0
for job in load_jobs():
if not is_job_runnable(job):
continue
next_run_at = job.get("next_run_at")
if not next_run_at:
continue
try:
due_dt = _ensure_aware(datetime.fromisoformat(next_run_at))
except (ValueError, TypeError):
continue
overdue_seconds = (now - due_dt).total_seconds()
if overdue_seconds < grace_minutes * 60:
continue
job_id = str(job.get("id") or "")
# One-shot jobs share the module-wide policy: more than
# ONESHOT_GRACE_SECONDS past their run time means "will never fire"
# (create/update/resume/recovery and, since #89571, the due-scan all
# enforce it). The misfire backstop must not resurrect them hours
# late after downtime — that's #93526.
schedule = job.get("schedule") or {}
if str(schedule.get("kind") or "") == "once":
from cron.jobs import ONESHOT_GRACE_SECONDS
if overdue_seconds > ONESHOT_GRACE_SECONDS:
logger.warning(
"Misfire catch-up: one-shot job %s (%s) was due %s "
"(%.0f min overdue) — outside the %ss one-shot grace "
"window, not firing.",
job_id,
job.get("name") or "unnamed",
next_run_at,
overdue_seconds / 60,
ONESHOT_GRACE_SECONDS,
)
continue
logger.warning(
"Misfire catch-up: job %s (%s) was due %s (%.0f min overdue) and "
"no external fire arrived — firing locally.",
job_id,
job.get("name") or "unnamed",
next_run_at,
overdue_seconds / 60,
)
try:
# Two-phase, webhook-style: claim synchronously (fast store
# CAS — losing means an external retry beat us, which is
# fine), then run the job off-thread so the caller's loop is
# never blocked for the length of an agent run.
claimed = provider.claim_fire(job_id)
if claimed is None:
continue
threading.Thread(
target=provider.fire_claimed,
args=(claimed,),
kwargs={"adapters": adapters, "loop": loop},
daemon=True,
name=f"cron-misfire-{job_id[:12]}",
).start()
fired += 1
except Exception as exc:
logger.warning(
"Misfire catch-up failed for job %s: %s: %s",
job_id, type(exc).__name__, exc,
)
return fired
def resolve_cron_scheduler() -> "CronScheduler":
"""Return the active cron scheduler provider.
Reads ``cron.provider`` from config. Empty/absent → built-in. A named
provider that is missing, fails to load, or reports ``is_available() ==
False`` falls back to the built-in with a warning — cron must never be left
without a trigger.
"""
import logging
logger = logging.getLogger("cron.scheduler_provider")
name = ""
try:
from hermes_cli.config import cfg_get, load_config
name = (cfg_get(load_config(), "cron", "provider", default="") or "").strip()
except Exception:
pass
if not name or name in ("builtin", "in-process", "inprocess"):
return InProcessCronScheduler()
try:
from plugins.cron_providers import load_cron_scheduler
provider = load_cron_scheduler(name)
if provider is None:
logger.warning("cron.provider '%s' not found; using built-in ticker", name)
return InProcessCronScheduler()
if not provider.is_available():
logger.warning("cron.provider '%s' not available; using built-in ticker", name)
return InProcessCronScheduler()
logger.info("Using cron scheduler provider: %s", provider.name)
return provider
except Exception as e:
logger.warning(
"Failed to load cron.provider '%s' (%s); using built-in ticker", name, e
)
return InProcessCronScheduler()
def scheduler_for_profile_mode(
provider: "CronScheduler", *, multiplex_profiles: bool
) -> "CronScheduler":
"""Return a scheduler that can safely serve the gateway's profile mode.
External providers currently own one unscoped remote registry/client and
therefore cannot safely reconcile several profile stores from one process.
Fail closed to the built-in multiplex ticker until the provider API carries
explicit profile identity through lifecycle and webhook calls.
"""
if not multiplex_profiles or isinstance(provider, InProcessCronScheduler):
return provider
import logging
logging.getLogger("cron.scheduler_provider").warning(
"cron.provider '%s' does not support multiplex_profiles; using built-in ticker",
provider.name,
)
return InProcessCronScheduler()
class InProcessCronScheduler(CronScheduler):
"""Default provider: the historical in-process 60s ticker.
``start()`` blocks in the tick loop until ``stop_event`` is set, identical
to the pre-refactor ``_start_cron_ticker`` core loop. The caller runs it in
a daemon thread. ``can_dispatch`` is an optional synchronous gate supplied
by GatewayRunner during external drain; skipped ticks leave due jobs intact
for the next allowed tick.
"""
@property
def name(self) -> str:
return "builtin"
def start(
self,
stop_event,
*,
adapters=None,
loop=None,
interval=60,
can_dispatch=None,
profile_homes=None,
profile_adapters=None,
default_profile=None,
profile_gate=None,
):
import logging
from cron.scheduler import CronTickYielded
from cron.scheduler import tick as cron_tick
from cron.jobs import (
clear_ticker_error,
record_ticker_error,
record_ticker_heartbeat,
)
logger = logging.getLogger("cron.scheduler_provider")
logger.info("In-process cron scheduler started (interval=%ds)", interval)
# ── Multiplex profiles ────────────────────────────────────────────
# When profile_homes is set (multiplex_profiles on), tick EACH profile's
# cron store on every tick cycle so secondary-profile jobs actually fire
# instead of languishing in a store no ticker owns (#69377). Without this,
# only the process-global HERMES_HOME (the default profile) is ticked.
# Heartbeats and recovery are also scoped per profile so `hermes cron
# status` reflects liveness for every profile independently.
if profile_homes:
self._start_multiplex(
stop_event,
profile_homes=profile_homes,
adapters=adapters,
loop=loop,
interval=interval,
can_dispatch=can_dispatch,
profile_adapters=profile_adapters,
default_profile=default_profile,
profile_gate=profile_gate,
)
return
# ── Single-profile (legacy) path ──────────────────────────────────
recovered = self.recover_interrupted()
if recovered:
logger.warning(
"Marked %d interrupted cron execution(s) unknown after restart",
recovered,
)
# Heartbeat once before the first sleep so `hermes cron status` sees a
# live ticker immediately after startup, not only after the first tick.
record_ticker_heartbeat()
# Exponential backoff for consecutive tick failures — most importantly
# fd exhaustion (EMFILE/ENFILE, #87644). While FDs stay exhausted the
# ticker must NOT hammer the store every 60s; once they free (leak
# fixed, reclamation ran) the next tick succeeds and the backoff
# resets, so the scheduler self-heals without a gateway restart.
consecutive_failures = 0
while not stop_event.is_set():
ok = False
try:
if can_dispatch is not None and not can_dispatch():
logger.debug("Cron dispatch paused while gateway drains existing work")
else:
cron_tick(
verbose=False,
adapters=adapters,
loop=loop,
sync=False,
can_dispatch=can_dispatch,
)
ok = True
except BaseException as e:
# Catch BaseException (not just Exception) so a SystemExit from
# a misbehaving provider SDK / agent retry path does not kill
# the ticker thread silently (#32612). KeyboardInterrupt is
# intentionally caught here too — gateway shutdown is driven by
# stop_event (set by the main thread's signal handler), not by
# an exception in this daemon thread, so swallowing it and
# re-checking stop_event keeps shutdown clean.
if isinstance(e, CronTickYielded):
# Expected while this process is stale and a fresh gateway
# owns the runtime lock: not an error to debug, but it IS
# recorded below so status shows why ticks aren't firing
# from here. tick() already logged it once per episode.
logger.info("Cron tick yielded: %s", e)
else:
logger.error("Cron tick error: %s", e, exc_info=True)
# Persist the failure reason next to the heartbeat markers so
# `hermes cron status`/`list` (separate processes) can show
# WHY ticks fail, not just that the success marker is stale —
# e.g. a root-rewritten jobs.json locking out the ticker's
# uid went unnoticed for ~14h with the reason buried in the
# gateway log (#68483).
record_ticker_error(f"{type(e).__name__}: {e}")
# EMFILE: reclaim fds + back off exponentially so the
# exhausted process stops hammering the store while it has no
# chance of making progress (#87644).
consecutive_failures = _note_tick_failure(e, consecutive_failures)
# Record liveness every iteration; bump the success marker only on a
# clean tick, so status can tell "alive but failing every tick" from
# "actually firing jobs" (#32612, #32895).
record_ticker_heartbeat(success=ok)
if ok:
clear_ticker_error()
consecutive_failures = 0
stop_event.wait(_backoff_wait_seconds(interval, consecutive_failures))
def _start_multiplex(
self,
stop_event,
*,
profile_homes,
adapters=None,
loop=None,
interval=60,
can_dispatch=None,
profile_adapters=None,
default_profile=None,
profile_gate=None,
):
"""Tick every served profile's cron store when multiplex_profiles is on.
Each profile uses ``set_hermes_home_override()`` + ``use_cron_store()``
to scope its tick, heartbeat, recovery, lock file, config/.env, and
agent execution to that profile's home — mirroring how
``_profile_runtime_scope`` scopes the multiplexed inbound path and
``web_server.py`` scopes per-profile cron API calls.
``profile_gate(name, home) -> bool``, when given, is consulted every
cycle; a profile it rejects is neither ticked nor heartbeated that
cycle (the desktop ticker uses it to stand down for profiles whose
own gateway is running, #100489).
"""
import logging
from cron.scheduler import tick as cron_tick
from cron.scheduler import (
CronTickYielded,
SharedRouteAdapters,
_is_fd_exhaustion,
_primary_profile_routes_for_current_home,
)
from cron.jobs import (
clear_ticker_error,
record_ticker_error,
record_ticker_heartbeat,
use_cron_store,
)
from hermes_constants import set_hermes_home_override, reset_hermes_home_override
logger = logging.getLogger("cron.scheduler_provider")
logger.info(
"Multiplex cron scheduler started for %d profile(s): %s",
len(profile_homes),
[p[0] if isinstance(p, tuple) else p for p in profile_homes],
)
# Recovery + initial heartbeat for every profile.
# A profile may have been deleted since this snapshot was taken;
# never recreate a deleted home's cron workspace via the heartbeat
# below (#47368).
# One profile's broken store (corrupt executions.db, unreadable
# cron dir) must not abort startup for every other profile (#74878).
for entry in _existing_profile_homes(profile_homes):
home = entry[1] if isinstance(entry, tuple) else entry
home_token = set_hermes_home_override(str(home))
try:
with use_cron_store(home):
recovered = self.recover_interrupted()
if recovered:
logger.warning(
"Marked %d interrupted cron execution(s) for profile at %s",
recovered,
home,
)
record_ticker_heartbeat()
except BaseException as e:
logger.error(
"Cron startup recovery error for profile at %s: %s",
home,
e,
exc_info=True,
)
finally:
reset_hermes_home_override(home_token)
consecutive_failures = 0
while not stop_event.is_set():
ok = False
_tick_error = None
_profile_errors: dict[str, str] = {}
# Worst per-profile failure this cycle (fd exhaustion wins) so the
# #87644 backoff/reclaim is applied once per cycle, not per profile.
_cycle_exc: BaseException | None = None
cycle_homes = _existing_profile_homes(profile_homes)
if profile_gate is not None:
cycle_homes = [
entry
for entry in cycle_homes
if profile_gate(
entry[0] if isinstance(entry, tuple) else None,
entry[1] if isinstance(entry, tuple) else entry,
)
]
try:
if can_dispatch is not None and not can_dispatch():
logger.debug("Cron dispatch paused while gateway drains existing work")
else:
for entry in cycle_homes:
_pname = entry[0] if isinstance(entry, tuple) else None
home = entry[1] if isinstance(entry, tuple) else entry
home_token = set_hermes_home_override(str(home))
try:
with use_cron_store(home):
# Deliver each profile's cron via ITS OWN adapters.
# The shared `adapters` set belongs to the default
# profile only. A secondary profile uses its own map
# in profile_adapters[name], which is populated only
# once that profile's bot connects. A secondary must
# NEVER fall back to the default profile's `adapters`
# (that ships its cron output through the wrong bot),
# so before its adapter connects — map absent or empty
# — it simply does not deliver this tick.
if _pname is None or _pname == default_profile:
_tick_adapters = adapters
else:
_tick_adapters = (profile_adapters or {}).get(_pname) or {}
if not _tick_adapters and adapters:
# Credentialless satellite under
# gateway.profile_routes: no bot of its
# own, so its output may ride the
# PRIMARY adapter — but only for
# targets an exact enabled primary
# route maps to this profile
# (#101113). Unmatched targets still
# fail closed; this is not a default
# fallback.
_tick_adapters = SharedRouteAdapters(
adapters,
_primary_profile_routes_for_current_home(),
)
cron_tick(
verbose=False,
adapters=_tick_adapters,
loop=loop,
sync=False,
can_dispatch=can_dispatch,
)
except CronTickYielded as e:
# This profile is served stale and a fresh
# gateway owns its runtime lock: record the yield
# for THIS profile's status only, and keep
# ticking the remaining profiles — one profile's
# fresh gateway must not cancel another profile's
# only ticker in the same cycle.
logger.info("Cron tick yielded for profile at %s: %s", home, e)
_profile_errors[str(home)] = f"{type(e).__name__}: {e}"
except BaseException as e:
# Any other failure is THIS profile's failure
# (#74878): record it against this profile's
# status and keep ticking the remaining profiles.
# BaseException for the same reason as the
# single-profile loop (#32612).
logger.error(
"Cron tick error for profile at %s: %s",
home,
e,
exc_info=True,
)
_profile_errors[str(home)] = f"{type(e).__name__}: {e}"
if _cycle_exc is None or _is_fd_exhaustion(e):
_cycle_exc = e
finally:
reset_hermes_home_override(home_token)
ok = not _profile_errors
if _cycle_exc is not None:
consecutive_failures = _note_tick_failure(_cycle_exc, consecutive_failures)
except BaseException as e:
logger.error("Cron tick error: %s", e, exc_info=True)
_tick_error = f"{type(e).__name__}: {e}"
# EMFILE: reclaim fds + exponential backoff (#87644).
consecutive_failures = _note_tick_failure(e, consecutive_failures)
# Record per-profile heartbeat after each tick cycle. Distinguish
# a COMPLETED cycle (``_tick_error`` unset) — where each profile's
# beat reflects its own outcome, so a yielding profile does not
# darken healthy siblings — from an aborted one (exception), where
# no profile completed and all beats are unsuccessful (#32612).
for entry in cycle_homes:
home = entry[1] if isinstance(entry, tuple) else entry
home_token = set_hermes_home_override(str(home))
try:
with use_cron_store(home):
_home_ok = (
_tick_error is None and str(home) not in _profile_errors
)
record_ticker_heartbeat(success=_home_ok)
# Surface the failure reason (or clear it) per profile
# so `hermes cron status` can show WHY ticks fail
# (#68483).
if _home_ok:
clear_ticker_error()
elif str(home) in _profile_errors:
record_ticker_error(_profile_errors[str(home)])
elif _tick_error:
record_ticker_error(_tick_error)
finally:
reset_hermes_home_override(home_token)
if ok:
consecutive_failures = 0
stop_event.wait(_backoff_wait_seconds(interval, consecutive_failures))
cron/scripts/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Scripts shipped with the cron subsystem (runnable via ``python3 -m cron.scripts.<name>``)."""
cron/scripts/classify_items.py
+226
View File
@@ -0,0 +1,226 @@
#!/usr/bin/env python3
"""Classify candidate items by urgency/importance and emit only the urgent ones.
The proactive-monitor pattern: a fetch step (a watcher script, an inbox dump, a
feed) produces a list of candidate items; this script scores each with a cheap
LLM and prints ONLY the items at or above a threshold. Below-threshold runs
print nothing, so a cron job wrapping this stays silent unless something
actually matters -- the classic urgency-monitor pattern (fetch -> classify
urgency -> surface only what's above the bar).
Design choices:
* Uses Hermes' auxiliary client with task="monitor", so the classifier model
is configured once in config.yaml (auxiliary.monitor.{provider,model}) and
can be a cheap fast model independent of the main chat model.
* Reads items as JSON (a list of objects) from stdin or --input-file.
* One LLM call scores the whole batch (cheap, single round-trip) and returns
structured scores; we filter locally.
* Empty result -> empty stdout -> the cron job's [SILENT]/empty-stdout path
suppresses delivery. No spam on quiet intervals.
Usage (standalone):
cat items.json | python classify_items.py --threshold 7 \
--criteria "Urgent if it needs a reply today or is from my manager/family"
Usage (wired to a watcher via cron, agent mode):
Ask the agent: "Every 10 minutes, run watch_http_json.py for my inbox feed,
pipe its JSON into classify_items.py with my urgency criteria, and deliver
whatever it prints. Stay silent if it prints nothing."
Item schema (flexible): each item is an object; the classifier sees the whole
object. A "title"/"subject"/"summary"/"text" field helps it judge. An "id"
field (any of id/guid/message_id/url) is echoed back so duplicates can be
deduped upstream.
"""
from __future__ import annotations
import argparse
import json
import sys
from typing import Any, Dict, List, Optional
def _eprint(*args: Any) -> None:
print(*args, file=sys.stderr)
def _load_items(input_file: Optional[str]) -> List[Dict[str, Any]]:
raw = ""
if input_file:
with open(input_file, encoding="utf-8") as f:
raw = f.read()
else:
raw = sys.stdin.read()
raw = raw.strip()
if not raw:
return []
try:
data = json.loads(raw)
except json.JSONDecodeError as e:
_eprint(f"classify_items: input is not valid JSON: {e}")
sys.exit(2)
if isinstance(data, dict):
# Allow {"items": [...]} or a single object.
if isinstance(data.get("items"), list):
return data["items"]
return [data]
if isinstance(data, list):
return [x for x in data if isinstance(x, dict)]
_eprint("classify_items: expected a JSON list or {items: [...]}")
sys.exit(2)
def _item_id(item: Dict[str, Any], index: int) -> str:
for key in ("id", "guid", "message_id", "url", "link"):
val = item.get(key)
if val:
return str(val)
return f"item-{index}"
_CLASSIFY_INSTRUCTIONS = (
"You are an urgency classifier for a proactive assistant. You will be given "
"a numbered list of items and the user's importance criteria. Score EACH "
"item from 0 (ignore entirely) to 10 (interrupt the user now). Return ONLY a "
"JSON array, one object per item, in the same order: "
'[{"index": <int>, "score": <int 0-10>, "reason": "<short>"}]. '
"No prose, no markdown fences. Be conservative: most items should score low. "
"Only score high when the item clearly meets the user's criteria."
)
def _build_prompt(items: List[Dict[str, Any]], criteria: str) -> str:
lines = [f"USER IMPORTANCE CRITERIA:\n{criteria}\n", "ITEMS:"]
for i, item in enumerate(items):
# Show a compact view; the model sees the salient fields.
view = {
k: item[k]
for k in ("title", "subject", "summary", "text", "body", "from", "sender", "url")
if k in item
}
if not view:
view = item # fall back to the whole object
lines.append(f"[{i}] {json.dumps(view, ensure_ascii=False)[:1200]}")
lines.append(
"\nReturn the JSON array of scores now (one object per item, same order)."
)
return "\n".join(lines)
def _parse_scores(content: str, n_items: int) -> Dict[int, Dict[str, Any]]:
text = (content or "").strip()
# Tolerate accidental markdown fences.
if text.startswith("```"):
text = text.strip("`")
if "\n" in text:
text = text.split("\n", 1)[1]
try:
arr = json.loads(text)
except json.JSONDecodeError:
# Last-ditch: find the first [...] block.
start = text.find("[")
end = text.rfind("]")
if start >= 0 and end > start:
try:
arr = json.loads(text[start : end + 1])
except json.JSONDecodeError:
_eprint("classify_items: could not parse classifier output")
return {}
else:
_eprint("classify_items: classifier returned no JSON array")
return {}
out: Dict[int, Dict[str, Any]] = {}
if isinstance(arr, list):
for obj in arr:
if not isinstance(obj, dict):
continue
idx = obj.get("index")
if isinstance(idx, int) and 0 <= idx < n_items:
out[idx] = obj
return out
def main() -> int:
parser = argparse.ArgumentParser(description="Classify items by urgency; emit only urgent ones.")
parser.add_argument("--criteria", required=True, help="Plain-language importance criteria.")
parser.add_argument("--threshold", type=int, default=7, help="Minimum score (0-10) to surface. Default 7.")
parser.add_argument("--input-file", default=None, help="Read items JSON from this file instead of stdin.")
parser.add_argument("--format", choices=["text", "json"], default="text", help="Output format for surfaced items.")
args = parser.parse_args()
items = _load_items(args.input_file)
if not items:
# Nothing to classify -> silent. This is the common quiet-interval case.
return 0
# Import here so --help works without the package importable.
try:
from agent.auxiliary_client import call_llm
except Exception as e: # pragma: no cover - import guard
_eprint(f"classify_items: cannot import auxiliary client: {e}")
return 3
prompt = _build_prompt(items, args.criteria)
try:
resp = call_llm(
task="monitor",
messages=[{"role": "user", "content": prompt}],
max_tokens=1024,
temperature=0,
)
content = resp.choices[0].message.content
if not isinstance(content, str):
content = str(content) if content else ""
except Exception as e:
# Classification failure is NOT silent -- surface it so a broken monitor
# doesn't quietly swallow important items. Non-zero exit -> cron alerts.
_eprint(f"classify_items: classifier call failed: {e}")
return 4
scores = _parse_scores(content, len(items))
surfaced = []
for i, item in enumerate(items):
s = scores.get(i)
score = s.get("score") if isinstance(s, dict) else None
if isinstance(score, int) and score >= args.threshold:
surfaced.append((i, item, s))
if not surfaced:
# Below threshold -> silent. Empty stdout; cron suppresses delivery.
return 0
if args.format == "json":
out = [
{
"id": _item_id(item, i),
"score": s.get("score"),
"reason": s.get("reason", ""),
"item": item,
}
for (i, item, s) in surfaced
]
print(json.dumps(out, ensure_ascii=False, indent=2))
else:
blocks = []
for (i, item, s) in surfaced:
title = (
item.get("title")
or item.get("subject")
or item.get("summary")
or _item_id(item, i)
)
url = item.get("url") or item.get("link") or ""
reason = s.get("reason", "")
block = f"## [{s.get('score')}/10] {title}"
if url:
block += f"\n{url}"
if reason:
block += f"\n_{reason}_"
blocks.append(block)
print("\n\n".join(blocks))
return 0
if __name__ == "__main__":
sys.exit(main())
cron/suggestion_catalog.py
+154
View File
@@ -0,0 +1,154 @@
"""Curated catalog of starter cron-job suggestions.
These are the built-in automations Hermes can offer a new user out of the box —
the ``catalog`` source of the unified suggestion surface. Each entry is a
ready-to-run ``cron.jobs.create_job`` spec wrapped as a suggestion; the user
accepts via ``/suggestions``. Nothing here auto-schedules.
The "important-mail monitor" entry is where the old proactive-monitor engine
lives now: its ``classify_items.py`` (poll a source -> LLM-score urgency ->
surface only above-threshold) is ONE catalog automation, not a standalone
feature.
Adding a catalog entry: append a CatalogEntry. Keep prompts self-contained
(cron jobs run with no chat context) and schedules sensible. The ``job_spec``
is passed verbatim to ``create_job`` on accept.
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
__all__ = ["CatalogEntry", "CATALOG", "seed_catalog_suggestions", "classify_items_script_path"]
def classify_items_script_path() -> str:
"""Absolute path to the urgency classifier script shipped with cron/."""
return str((Path(__file__).resolve().parent / "scripts" / "classify_items.py"))
@dataclass(frozen=True)
class CatalogEntry:
"""A curated starter automation offered as a suggestion."""
key: str # stable dedup key (never re-offered once dismissed)
title: str
description: str
job_spec: Dict[str, Any] # kwargs for cron.jobs.create_job
# The curated set. Schedules use the cron/interval syntax create_job accepts.
CATALOG: List[CatalogEntry] = [
CatalogEntry(
key="catalog:daily-briefing",
title="Daily briefing",
description="Every morning at 8am, a short briefing: today's calendar, "
"weather, and anything urgent waiting on you.",
job_spec={
"prompt": (
"Produce a concise morning briefing for the user: today's "
"calendar events, the local weather, and any urgent items "
"(unread important email, due tasks). Keep it short and "
"scannable. If you have no connected data sources, give a brief "
"general good-morning with the date and offer to connect "
"calendar/email."
),
"schedule": "0 8 * * *",
"name": "Daily briefing",
"deliver": "origin",
},
),
CatalogEntry(
key="catalog:important-mail-monitor",
title="Important-mail monitor",
description="Check your inbox periodically and ping you ONLY about mail "
"that actually needs attention — never the newsletters.",
job_spec={
"prompt": (
"Check the user's inbox for new messages since the last run. "
"For each candidate, judge urgency against this rule: surface "
"only mail that needs a reply today, is from a manager/family "
"member, or mentions a deadline. Pipe candidates through the "
"urgency classifier (run `python3 -m cron.scripts.classify_items "
"--threshold 7 --criteria ...` from the hermes-agent install — "
"resolve the script path at run time, do not assume a fixed "
"location) and deliver ONLY what it returns. If nothing "
"clears the bar, respond with [SILENT] so the user is not "
"pinged. Requires a connected mail source; if none is "
"configured, explain how to connect one and then stop."
),
"schedule": "every 30m",
"name": "Important-mail monitor",
"deliver": "origin",
},
),
CatalogEntry(
key="catalog:weekly-review",
title="Weekly review",
description="Every Sunday evening, a recap of the week: what got done, "
"what's still open, and what's coming up next week.",
job_spec={
"prompt": (
"Produce a weekly review for the user: summarize what was "
"accomplished this week, list still-open items, and preview "
"next week's calendar. Pull from whatever sources are connected "
"(calendar, task tools, recent conversations). Keep it tight."
),
"schedule": "0 18 * * 0",
"name": "Weekly review",
"deliver": "origin",
},
),
CatalogEntry(
key="catalog:standup-reminder",
title="Workday start reminder",
description="A weekday nudge at 9am with your day's agenda and top "
"priorities, so you start focused.",
job_spec={
"prompt": (
"Give the user a brief weekday start-of-day nudge: their "
"calendar for today and the 1-3 highest-priority things to "
"focus on, inferred from recent context and any task tools. "
"Encouraging, short, one message."
),
"schedule": "0 9 * * 1-5",
"name": "Workday start reminder",
"deliver": "origin",
},
),
]
def seed_catalog_suggestions(
*,
add_fn: Optional[Callable[..., Optional[Dict[str, Any]]]] = None,
keys: Optional[List[str]] = None,
) -> List[Dict[str, Any]]:
"""Register catalog entries as pending suggestions.
``add_fn`` defaults to ``cron.suggestions.add_suggestion`` (injectable for
tests). ``keys`` restricts to specific catalog entries; omit to seed all.
Entries already dismissed/accepted (by dedup key) or beyond the pending cap
are skipped by the store, so re-seeding is safe and idempotent. Returns the
list of suggestion records actually created.
"""
if add_fn is None:
from cron.suggestions import add_suggestion as add_fn # type: ignore[assignment]
wanted = set(keys) if keys else None
created: List[Dict[str, Any]] = []
for entry in CATALOG:
if wanted is not None and entry.key not in wanted:
continue
rec = add_fn(
title=entry.title,
description=entry.description,
source="catalog",
job_spec=dict(entry.job_spec),
dedup_key=entry.key,
)
if rec is not None:
created.append(rec)
return created
cron/suggestions.py
+281
View File
@@ -0,0 +1,281 @@
"""Suggested cron jobs — proposed automations the user accepts with one tap.
A *suggestion* is a ready-to-run cron job spec that Hermes surfaces to the
user, who accepts it (creates the real cron job) or dismisses it (latched so
it is never re-offered). This is the single surface every automation proposal
flows through, regardless of where it came from:
* ``catalog`` — a curated starter automation (daily briefing, important-mail
monitor, weekly digest, ...).
* ``blueprint`` — the user installed a skill that carries a ``blueprint:`` block
(see ``tools/blueprints.py``); installing it registers a
suggestion instead of auto-scheduling.
* ``usage`` — the background self-improvement review noticed a recurring
ask that a scheduled job would serve.
* ``integration`` — the user connected an account (Gmail, GitHub, ...) and
the obvious automations for that surface are offered.
Accepting a suggestion just calls the existing ``cron.jobs.create_job`` with
the stored ``job_spec`` — there is NO second job engine. Suggestions never
auto-create jobs; acceptance is always explicit (consent-first). Dismissed
suggestions latch by a stable ``dedup_key`` so the same proposal is not
re-offered after the user says no.
Storage mirrors ``cron/jobs.py``: ``~/.hermes/cron/suggestions.json``, atomic
writes, an in-process lock, and 0600 perms.
"""
from __future__ import annotations
import json
import logging
import os
import tempfile
import threading
import uuid
from pathlib import Path
from typing import Any, Dict, List, Optional
from hermes_constants import get_hermes_home
from hermes_time import now as _hermes_now
from utils import atomic_replace
logger = logging.getLogger(__name__)
# Per-profile by design (issue #4707): suggestions live alongside the active
# profile's cron store. Anchor on get_hermes_home() (profile home), not the
# shared default root. See cron/jobs.py for the full rationale.
#
# Optional test override. Production resolves the path at call time so
# multiplexed profile ticks (set_hermes_home_override) cannot leak one
# profile's suggestions into the import-time home (#86519). Same pattern as
# cron/executions.py.
SUGGESTIONS_FILE: Optional[Path] = None
def _current_suggestions_file() -> Path:
return SUGGESTIONS_FILE or (get_hermes_home().resolve() / "cron" / "suggestions.json")
# In-process lock protecting load->modify->save cycles (the background review
# fork and the main agent can both write).
_suggestions_lock = threading.Lock()
# Cap pending suggestions so the list never becomes a nag wall. When full,
# new suggestions are dropped (the user should clear the backlog first).
MAX_PENDING = 5
VALID_SOURCES = frozenset({"catalog", "blueprint", "usage", "integration"})
_STATUS_PENDING = "pending"
_STATUS_ACCEPTED = "accepted"
_STATUS_DISMISSED = "dismissed"
def _secure_file(path: Path) -> None:
try:
os.chmod(path, 0o600)
except OSError:
pass
def _ensure_dir() -> None:
from cron.jobs import _ensure_cron_dir
_ensure_cron_dir(_current_suggestions_file().parent)
def _load_raw() -> Dict[str, Any]:
suggestions_file = _current_suggestions_file()
if not suggestions_file.exists():
return {"suggestions": []}
try:
with open(suggestions_file, "r", encoding="utf-8") as f:
data = json.load(f)
except (json.JSONDecodeError, OSError) as e:
logger.warning("suggestions.json unreadable (%s); starting empty", e)
return {"suggestions": []}
if isinstance(data, dict) and isinstance(data.get("suggestions"), list):
return data
if isinstance(data, list):
return {"suggestions": data}
logger.warning("suggestions.json malformed; starting empty")
return {"suggestions": []}
def _save_raw(suggestions: List[Dict[str, Any]]) -> None:
_ensure_dir()
suggestions_file = _current_suggestions_file()
fd, tmp_path = tempfile.mkstemp(dir=str(suggestions_file.parent), suffix=".tmp", prefix=".sugg_")
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(
{"suggestions": suggestions, "updated_at": _hermes_now().isoformat()},
f,
indent=2,
)
f.flush()
os.fsync(f.fileno())
atomic_replace(tmp_path, suggestions_file)
_secure_file(suggestions_file)
except BaseException:
try:
os.unlink(tmp_path)
except OSError:
pass
raise
def load_suggestions() -> List[Dict[str, Any]]:
"""Return all suggestion records (any status)."""
return _load_raw().get("suggestions", [])
def list_pending() -> List[Dict[str, Any]]:
"""Return pending suggestions in creation order (oldest first)."""
return [s for s in load_suggestions() if s.get("status") == _STATUS_PENDING]
def add_suggestion(
*,
title: str,
description: str,
source: str,
job_spec: Dict[str, Any],
dedup_key: str,
) -> Optional[Dict[str, Any]]:
"""Register a pending suggestion. Returns the record, or None if skipped.
Skipped when: the source is unknown, the same ``dedup_key`` was already
dismissed or accepted (never re-offer), an identical pending suggestion
exists, or the pending list is full (``MAX_PENDING``).
``job_spec`` is a dict of kwargs for ``cron.jobs.create_job`` — accepting
the suggestion passes it straight through, so there is no second schema to
keep in sync.
"""
if source not in VALID_SOURCES:
raise ValueError(f"unknown suggestion source: {source!r}")
if not title.strip() or not dedup_key.strip():
raise ValueError("title and dedup_key are required")
with _suggestions_lock:
suggestions = _load_raw().get("suggestions", [])
# Never re-offer something the user already saw and decided on, and
# never duplicate a still-pending proposal.
for existing in suggestions:
if existing.get("dedup_key") == dedup_key:
if existing.get("status") in (_STATUS_DISMISSED, _STATUS_ACCEPTED):
return None
if existing.get("status") == _STATUS_PENDING:
return None
pending_count = sum(1 for s in suggestions if s.get("status") == _STATUS_PENDING)
if pending_count >= MAX_PENDING:
logger.info("Suggestion backlog full (%d); dropping %r", MAX_PENDING, title)
return None
record = {
"id": uuid.uuid4().hex[:12],
"title": title.strip(),
"description": description.strip(),
"source": source,
"job_spec": job_spec,
"dedup_key": dedup_key.strip(),
"status": _STATUS_PENDING,
"created_at": _hermes_now().isoformat(),
}
suggestions.append(record)
_save_raw(suggestions)
return record
def get_suggestion(ref: str) -> Optional[Dict[str, Any]]:
"""Resolve a suggestion by id, 1-based pending index, or title (exact)."""
suggestions = load_suggestions()
# By id.
for s in suggestions:
if s.get("id") == ref:
return s
# By 1-based pending index.
if ref.isdigit():
pending = [s for s in suggestions if s.get("status") == _STATUS_PENDING]
idx = int(ref) - 1
if 0 <= idx < len(pending):
return pending[idx]
# By exact title (case-insensitive).
for s in suggestions:
if s.get("title", "").lower() == ref.lower():
return s
return None
def _set_status(suggestion_id: str, status: str) -> bool:
with _suggestions_lock:
suggestions = _load_raw().get("suggestions", [])
changed = False
for s in suggestions:
if s.get("id") == suggestion_id:
s["status"] = status
s["resolved_at"] = _hermes_now().isoformat()
changed = True
break
if changed:
_save_raw(suggestions)
return changed
def dismiss_suggestion(ref: str) -> bool:
"""Dismiss a suggestion (latched — never re-offered for its dedup_key)."""
s = get_suggestion(ref)
if not s:
return False
return _set_status(s["id"], _STATUS_DISMISSED)
def accept_suggestion(ref: str, *, origin: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
"""Accept a suggestion: create the real cron job from its ``job_spec``.
Returns the created cron job dict, or None if the suggestion isn't found /
not pending. The job_spec is passed straight to ``cron.jobs.create_job``;
an ``origin`` (platform/chat) is merged so "origin" delivery routes back to
the chat where the user accepted.
"""
s = get_suggestion(ref)
if not s or s.get("status") != _STATUS_PENDING:
return None
from cron.scheduler import (
CronSchedulerRegistrationError,
create_job_with_scheduler_registration,
)
spec = dict(s.get("job_spec") or {})
if origin is not None and "origin" not in spec:
spec["origin"] = origin
try:
job = create_job_with_scheduler_registration(**spec)
except CronSchedulerRegistrationError:
# The job is already durable. Resolve the suggestion so retrying the
# same acceptance cannot create another local copy.
_set_status(s["id"], _STATUS_ACCEPTED)
raise
_set_status(s["id"], _STATUS_ACCEPTED)
return job
def clear_resolved() -> int:
"""Drop accepted/dismissed records from disk. Returns the count removed.
Pending suggestions and the dedup memory of dismissed ones are the only
things that matter long-term, but dismissed records must be RETAINED for
their dedup_key (so they aren't re-offered). This only prunes ACCEPTED
records, which have served their purpose once the job exists.
"""
with _suggestions_lock:
suggestions = _load_raw().get("suggestions", [])
kept = [s for s in suggestions if s.get("status") != _STATUS_ACCEPTED]
removed = len(suggestions) - len(kept)
if removed:
_save_raw(kept)
return removed