Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,356 @@
|
||||
"""Cron scheduler provider plugin discovery.
|
||||
|
||||
Scans two directories for cron scheduler provider plugins:
|
||||
|
||||
1. Bundled providers: ``plugins/cron_providers/<name>/`` (shipped with hermes-agent)
|
||||
2. User-installed providers: ``$HERMES_HOME/plugins/<name>/``
|
||||
|
||||
Each subdirectory must contain ``__init__.py`` with a class implementing the
|
||||
``CronScheduler`` ABC (``cron/scheduler_provider.py``). On name collisions,
|
||||
bundled providers take precedence.
|
||||
|
||||
This is a near-verbatim clone of ``plugins/memory/__init__.py`` — the same
|
||||
discovery/loader machinery, retargeted at ``CronScheduler``. The built-in
|
||||
``InProcessCronScheduler`` is NOT discovered here: it is core (lives in
|
||||
``cron/scheduler_provider.py``) so the fallback can never be accidentally
|
||||
removed. Only NON-default providers (e.g. "chronos") live under this directory.
|
||||
|
||||
Only ONE provider can be active at a time, selected via ``cron.provider`` in
|
||||
config.yaml (empty = built-in). See ``cron.scheduler_provider.resolve_cron_scheduler``.
|
||||
|
||||
Usage:
|
||||
from plugins.cron_providers import discover_cron_schedulers, load_cron_scheduler
|
||||
|
||||
available = discover_cron_schedulers() # [(name, desc, available), ...]
|
||||
provider = load_cron_scheduler("chronos") # CronScheduler instance
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CRON_PLUGINS_DIR = Path(__file__).parent
|
||||
|
||||
# Synthetic parent package for user-installed providers, so they don't
|
||||
# collide with bundled providers in sys.modules.
|
||||
_USER_NAMESPACE = "_hermes_user_cron"
|
||||
|
||||
|
||||
def _register_synthetic_package(name: str, search_locations: List[str]) -> None:
|
||||
"""Register an empty package shell in sys.modules.
|
||||
|
||||
User-installed providers import as ``_hermes_user_cron.<name>``, a dotted
|
||||
name whose parents exist nowhere on disk. Unless those parents are present
|
||||
in ``sys.modules``, any relative import inside the plugin
|
||||
(``from . import config``) fails with
|
||||
``ModuleNotFoundError: No module named '_hermes_user_cron'`` — the same
|
||||
reason the loader already registers ``plugins`` and ``plugins.cron_providers`` for
|
||||
bundled providers.
|
||||
"""
|
||||
if name in sys.modules:
|
||||
return
|
||||
spec = importlib.machinery.ModuleSpec(name, None, is_package=True)
|
||||
spec.submodule_search_locations = search_locations
|
||||
sys.modules[name] = importlib.util.module_from_spec(spec)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Directory helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_user_plugins_dir() -> Optional[Path]:
|
||||
"""Return ``$HERMES_HOME/plugins/`` or None if unavailable."""
|
||||
try:
|
||||
from hermes_constants import get_hermes_home
|
||||
d = get_hermes_home() / "plugins"
|
||||
return d if d.is_dir() else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _is_cron_provider_dir(path: Path) -> bool:
|
||||
"""Heuristic: does *path* look like a cron scheduler provider plugin?
|
||||
|
||||
Checks for ``register_cron_scheduler`` or ``CronScheduler`` in the
|
||||
``__init__.py`` source. Cheap text scan — no import needed.
|
||||
"""
|
||||
init_file = path / "__init__.py"
|
||||
if not init_file.exists():
|
||||
return False
|
||||
try:
|
||||
source = init_file.read_text(errors="replace", encoding="utf-8")[:8192]
|
||||
return "register_cron_scheduler" in source or "CronScheduler" in source
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _iter_provider_dirs() -> List[Tuple[str, Path]]:
|
||||
"""Yield ``(name, path)`` for all discovered provider directories.
|
||||
|
||||
Scans bundled first, then user-installed. Bundled takes precedence on
|
||||
name collisions (first-seen wins via ``seen`` set).
|
||||
"""
|
||||
seen: set = set()
|
||||
dirs: List[Tuple[str, Path]] = []
|
||||
|
||||
# 1. Bundled providers (plugins/cron_providers/<name>/)
|
||||
if _CRON_PLUGINS_DIR.is_dir():
|
||||
for child in sorted(_CRON_PLUGINS_DIR.iterdir()):
|
||||
if not child.is_dir() or child.name.startswith(("_", ".")):
|
||||
continue
|
||||
if not (child / "__init__.py").exists():
|
||||
continue
|
||||
seen.add(child.name)
|
||||
dirs.append((child.name, child))
|
||||
|
||||
# 2. User-installed providers ($HERMES_HOME/plugins/<name>/)
|
||||
user_dir = _get_user_plugins_dir()
|
||||
if user_dir:
|
||||
for child in sorted(user_dir.iterdir()):
|
||||
if not child.is_dir() or child.name.startswith(("_", ".")):
|
||||
continue
|
||||
if child.name in seen:
|
||||
continue # bundled takes precedence
|
||||
if not _is_cron_provider_dir(child):
|
||||
continue # skip non-cron plugins
|
||||
dirs.append((child.name, child))
|
||||
|
||||
return dirs
|
||||
|
||||
|
||||
def find_provider_dir(name: str) -> Optional[Path]:
|
||||
"""Resolve a provider name to its directory.
|
||||
|
||||
Checks bundled first, then user-installed.
|
||||
"""
|
||||
# Bundled
|
||||
bundled = _CRON_PLUGINS_DIR / name
|
||||
if bundled.is_dir() and (bundled / "__init__.py").exists():
|
||||
return bundled
|
||||
# User-installed
|
||||
user_dir = _get_user_plugins_dir()
|
||||
if user_dir:
|
||||
user = user_dir / name
|
||||
if user.is_dir() and _is_cron_provider_dir(user):
|
||||
return user
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def discover_cron_schedulers() -> List[Tuple[str, str, bool]]:
|
||||
"""Scan bundled and user-installed directories for available providers.
|
||||
|
||||
Returns list of (name, description, is_available) tuples. May be empty —
|
||||
the built-in is core, not discovered here, so a fresh checkout with no
|
||||
bundled non-default provider returns []. Bundled providers take precedence
|
||||
on name collisions.
|
||||
"""
|
||||
results = []
|
||||
|
||||
for name, child in _iter_provider_dirs():
|
||||
# Read description from plugin.yaml if available
|
||||
desc = ""
|
||||
yaml_file = child / "plugin.yaml"
|
||||
if yaml_file.exists():
|
||||
try:
|
||||
import yaml
|
||||
with open(yaml_file, encoding="utf-8-sig") as f:
|
||||
meta = yaml.safe_load(f) or {}
|
||||
desc = meta.get("description", "")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Quick availability check — try loading and calling is_available()
|
||||
available = True
|
||||
try:
|
||||
provider = _load_provider_from_dir(child)
|
||||
if provider:
|
||||
available = provider.is_available()
|
||||
else:
|
||||
available = False
|
||||
except Exception:
|
||||
available = False
|
||||
|
||||
results.append((name, desc, available))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def load_cron_scheduler(name: str) -> Optional["CronScheduler"]: # noqa: F821
|
||||
"""Load and return a CronScheduler instance by name.
|
||||
|
||||
Checks both bundled (``plugins/cron_providers/<name>/``) and user-installed
|
||||
(``$HERMES_HOME/plugins/<name>/``) directories. Bundled takes precedence
|
||||
on name collisions.
|
||||
|
||||
Returns None if the provider is not found or fails to load.
|
||||
"""
|
||||
provider_dir = find_provider_dir(name)
|
||||
if not provider_dir:
|
||||
logger.debug("Cron provider '%s' not found in bundled or user plugins", name)
|
||||
return None
|
||||
|
||||
try:
|
||||
provider = _load_provider_from_dir(provider_dir)
|
||||
if provider:
|
||||
return provider
|
||||
logger.warning("Cron provider '%s' loaded but no provider instance found", name)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load cron provider '%s': %s", name, e)
|
||||
return None
|
||||
|
||||
|
||||
def _load_provider_from_dir(provider_dir: Path) -> Optional["CronScheduler"]: # noqa: F821
|
||||
"""Import a provider module and extract the CronScheduler instance.
|
||||
|
||||
The module must have either:
|
||||
- A register(ctx) function (plugin-style) — we simulate a ctx
|
||||
- A top-level class that extends CronScheduler — we instantiate it
|
||||
"""
|
||||
name = provider_dir.name
|
||||
# Use a separate namespace for user-installed plugins so they don't
|
||||
# collide with bundled providers in sys.modules.
|
||||
_is_bundled = _CRON_PLUGINS_DIR in provider_dir.parents or provider_dir.parent == _CRON_PLUGINS_DIR
|
||||
module_name = f"plugins.cron_providers.{name}" if _is_bundled else f"{_USER_NAMESPACE}.{name}"
|
||||
init_file = provider_dir / "__init__.py"
|
||||
|
||||
if not init_file.exists():
|
||||
return None
|
||||
|
||||
# Check if already loaded. A synthetic package shell has no __file__;
|
||||
# only reuse modules that were actually loaded from disk.
|
||||
cached = sys.modules.get(module_name)
|
||||
if cached is not None and getattr(cached, "__file__", None):
|
||||
mod = cached
|
||||
else:
|
||||
# Ensure the parent packages are registered (for relative imports)
|
||||
for parent in ("plugins", "plugins.cron_providers"):
|
||||
if parent not in sys.modules:
|
||||
parent_path = Path(__file__).parent
|
||||
if parent == "plugins":
|
||||
parent_path = parent_path.parent
|
||||
parent_init = parent_path / "__init__.py"
|
||||
if parent_init.exists():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
parent, str(parent_init),
|
||||
submodule_search_locations=[str(parent_path)]
|
||||
)
|
||||
if spec:
|
||||
parent_mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules[parent] = parent_mod
|
||||
try:
|
||||
spec.loader.exec_module(parent_mod)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# User-installed plugins need their synthetic parent registered the
|
||||
# same way, or relative imports inside the plugin cannot resolve.
|
||||
if not _is_bundled:
|
||||
_register_synthetic_package(_USER_NAMESPACE, [])
|
||||
|
||||
# Now load the provider module
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
module_name, str(init_file),
|
||||
submodule_search_locations=[str(provider_dir)]
|
||||
)
|
||||
if not spec:
|
||||
return None
|
||||
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules[module_name] = mod
|
||||
loaded_submodules = []
|
||||
|
||||
# Register submodules so relative imports work
|
||||
# e.g., "from ._nas_client import NasCronClient" in the chronos plugin
|
||||
for sub_file in provider_dir.glob("*.py"):
|
||||
if sub_file.name == "__init__.py":
|
||||
continue
|
||||
sub_name = sub_file.stem
|
||||
full_sub_name = f"{module_name}.{sub_name}"
|
||||
if full_sub_name not in sys.modules:
|
||||
sub_spec = importlib.util.spec_from_file_location(
|
||||
full_sub_name, str(sub_file)
|
||||
)
|
||||
if sub_spec:
|
||||
sub_mod = importlib.util.module_from_spec(sub_spec)
|
||||
sys.modules[full_sub_name] = sub_mod
|
||||
try:
|
||||
sub_spec.loader.exec_module(sub_mod)
|
||||
loaded_submodules.append((sub_name, sub_mod))
|
||||
except Exception as e:
|
||||
logger.debug("Failed to load submodule %s: %s", full_sub_name, e)
|
||||
|
||||
try:
|
||||
spec.loader.exec_module(mod)
|
||||
except Exception as e:
|
||||
logger.debug("Failed to exec_module %s: %s", module_name, e)
|
||||
sys.modules.pop(module_name, None)
|
||||
return None
|
||||
|
||||
# Manual importlib loading bypasses the normal import machinery that
|
||||
# binds child modules onto their parent packages. Restore that shape so
|
||||
# later dotted imports and pytest monkeypatch paths resolve normally.
|
||||
parent_name, child_name = module_name.rsplit(".", 1)
|
||||
parent_mod = sys.modules.get(parent_name)
|
||||
if parent_mod is not None:
|
||||
setattr(parent_mod, child_name, mod)
|
||||
for sub_name, sub_mod in loaded_submodules:
|
||||
setattr(mod, sub_name, sub_mod)
|
||||
|
||||
# Try register(ctx) pattern first (how our plugins are written)
|
||||
if hasattr(mod, "register"):
|
||||
collector = _ProviderCollector()
|
||||
try:
|
||||
mod.register(collector)
|
||||
if collector.provider:
|
||||
return collector.provider
|
||||
except Exception as e:
|
||||
logger.debug("register() failed for %s: %s", name, e)
|
||||
|
||||
# Fallback: find a CronScheduler subclass and instantiate it
|
||||
from cron.scheduler_provider import CronScheduler
|
||||
for attr_name in dir(mod):
|
||||
attr = getattr(mod, attr_name, None)
|
||||
if (isinstance(attr, type) and issubclass(attr, CronScheduler)
|
||||
and attr is not CronScheduler):
|
||||
try:
|
||||
return attr()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class _ProviderCollector:
|
||||
"""Fake plugin context that captures register_cron_scheduler calls."""
|
||||
|
||||
def __init__(self):
|
||||
self.provider = None
|
||||
|
||||
def register_cron_scheduler(self, provider):
|
||||
self.provider = provider
|
||||
|
||||
# No-op for other registration methods
|
||||
def register_tool(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def register_hook(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def register_memory_provider(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def register_cli_command(self, *args, **kwargs):
|
||||
pass
|
||||
@@ -0,0 +1,267 @@
|
||||
"""Chronos — NAS-mediated managed cron provider (scale-to-zero).
|
||||
|
||||
Chronos (the Greek god of time, alongside Hermes) is the first non-default
|
||||
``CronScheduler``. It lets a hosted gateway scale to zero while idle and still
|
||||
fire cron jobs: instead of a 60s in-process ticker, it asks NAS to arm exactly
|
||||
one external one-shot per job at that job's real next-fire time. NAS calls the
|
||||
agent back at fire time over an authenticated webhook (``/api/cron/fire``); the
|
||||
agent runs the job via the shared ``run_one_job`` body and re-arms the next
|
||||
one-shot.
|
||||
|
||||
The external scheduler NAS uses is an internal NAS implementation detail —
|
||||
Chronos names no vendor, holds no scheduler credentials, and speaks only to
|
||||
NAS's ``agent-cron`` endpoints with the agent's existing Nous token.
|
||||
|
||||
Design constraints (see the plan's DQ-1):
|
||||
- start() arms all enabled jobs and RETURNS; it never blocks and never spawns
|
||||
a periodic wake. Between fires the machine is truly at zero.
|
||||
- reconcile runs only on a warm process (start / on_jobs_changed / piggybacked
|
||||
on a fire), never as a periodic wake of a sleeping machine.
|
||||
|
||||
Inert unless ``cron.provider: chronos``. ``resolve_cron_scheduler`` falls back
|
||||
to the built-in if Chronos is unavailable, so cron never loses its trigger.
|
||||
|
||||
Wire contract: ``docs/chronos-managed-cron-contract.md``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from cron.scheduler_provider import CronScheduler
|
||||
|
||||
logger = logging.getLogger("cron.chronos")
|
||||
|
||||
|
||||
def _cfg(*keys: str, default: Any = "") -> Any:
|
||||
"""Read a cron.chronos.* config value (no network)."""
|
||||
try:
|
||||
from hermes_cli.config import cfg_get, load_config
|
||||
return cfg_get(load_config(), *keys, default=default)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
class ChronosCronScheduler(CronScheduler):
|
||||
"""NAS-mediated external cron provider."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# In-memory map of job_id → fire_at we've asked NAS to arm. Best-effort
|
||||
# cache; reconcile rebuilds desired state from jobs.json, so a cold
|
||||
# process simply re-arms (idempotent via dedup_key).
|
||||
self._armed: Dict[str, str] = {}
|
||||
self._lock = threading.Lock()
|
||||
self._client = None # lazily constructed (no network in is_available)
|
||||
|
||||
# -- identity / availability -----------------------------------------
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "chronos"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""Config presence only — NO network.
|
||||
|
||||
Chronos needs a portal base URL, the agent's own publicly-reachable
|
||||
callback URL (for NAS→agent fires), and a usable Nous token (the agent
|
||||
is logged into the portal). If any is missing, resolve_cron_scheduler
|
||||
falls back to the built-in ticker.
|
||||
"""
|
||||
if not (_cfg("cron", "chronos", "portal_url") and _cfg("cron", "chronos", "callback_url")):
|
||||
return False
|
||||
return self._have_nous_token()
|
||||
|
||||
def _have_nous_token(self) -> bool:
|
||||
"""True if the agent has a Nous Portal login (no network call).
|
||||
|
||||
Checks the stored auth state for a Nous access token — does NOT refresh
|
||||
or hit the network (is_available must stay offline). The actual
|
||||
refresh-aware token is resolved lazily at provision time.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.auth import get_provider_auth_state
|
||||
state = get_provider_auth_state("nous") or {}
|
||||
return bool(state.get("access_token"))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# -- client -----------------------------------------------------------
|
||||
|
||||
def _get_client(self):
|
||||
if self._client is None:
|
||||
from ._nas_client import NasCronClient
|
||||
self._client = NasCronClient(_cfg("cron", "chronos", "portal_url"))
|
||||
return self._client
|
||||
|
||||
def _callback_url(self) -> str:
|
||||
return str(_cfg("cron", "chronos", "callback_url") or "")
|
||||
|
||||
# -- lifecycle --------------------------------------------------------
|
||||
|
||||
def start(self, stop_event, *, adapters=None, loop=None, interval=60):
|
||||
"""Arm all enabled jobs via NAS, then RETURN immediately.
|
||||
|
||||
Does NOT block and does NOT spawn a 60s wake (DQ-1) — that is the whole
|
||||
point of scale-to-zero. The machine wakes only on a NAS→agent fire.
|
||||
"""
|
||||
# A new provider lifecycle cannot prove what an interrupted prior
|
||||
# process did. Classify those attempts unknown for audit only; do not
|
||||
# requeue them here.
|
||||
self.recover_interrupted()
|
||||
try:
|
||||
self.reconcile()
|
||||
except Exception as e:
|
||||
logger.warning("Chronos start() reconcile failed: %s", e)
|
||||
# Intentionally return — no loop, no periodic wake.
|
||||
|
||||
def stop(self) -> None:
|
||||
return None
|
||||
|
||||
def on_jobs_changed(self) -> None:
|
||||
"""A job was created/updated/removed/paused/resumed — reconcile the NAS
|
||||
registry so the affected one-shot is (re-)armed or cancelled."""
|
||||
try:
|
||||
self.reconcile()
|
||||
except Exception as e:
|
||||
logger.debug("Chronos on_jobs_changed reconcile failed: %s", e)
|
||||
|
||||
def register_job(self, job: Dict[str, Any]) -> None:
|
||||
"""Arm the first one-shot for a newly persisted job.
|
||||
|
||||
Unlike full reconciliation, this operation is allowed to raise so the
|
||||
creation surface can report that the local job exists but its external
|
||||
trigger was not registered.
|
||||
"""
|
||||
self._arm_one_shot(job)
|
||||
|
||||
# -- arming -----------------------------------------------------------
|
||||
|
||||
def _arm_one_shot(self, job: Dict[str, Any]) -> None:
|
||||
"""Ask NAS to arm exactly one one-shot at the job's next_run_at.
|
||||
|
||||
The agent computes the time; NAS+its scheduler are the dumb executor.
|
||||
Idempotent per (job_id, fire_at) via dedup_key, so re-arming the same
|
||||
fire is a no-op NAS-side.
|
||||
"""
|
||||
job_id = job["id"]
|
||||
fire_at = job.get("next_run_at")
|
||||
if not fire_at:
|
||||
return
|
||||
dedup_key = f"{job_id}:{fire_at}"
|
||||
self._get_client().provision(
|
||||
job_id=job_id,
|
||||
fire_at=fire_at,
|
||||
agent_callback_url=self._callback_url(),
|
||||
dedup_key=dedup_key,
|
||||
)
|
||||
with self._lock:
|
||||
self._armed[job_id] = fire_at
|
||||
|
||||
def _cancel(self, job_id: str) -> None:
|
||||
try:
|
||||
self._get_client().cancel(job_id=job_id)
|
||||
finally:
|
||||
with self._lock:
|
||||
self._armed.pop(job_id, None)
|
||||
|
||||
def _list_armed(self) -> Dict[str, str]:
|
||||
"""Observed armed one-shots: job_id → fire_at.
|
||||
|
||||
Prefer the in-memory map (warm process); on a cold/empty map, ask NAS
|
||||
(best-effort). If NAS list fails, return what we have — reconcile then
|
||||
re-arms desired jobs idempotently.
|
||||
"""
|
||||
with self._lock:
|
||||
if self._armed:
|
||||
return dict(self._armed)
|
||||
try:
|
||||
observed = {
|
||||
item["job_id"]: item.get("fire_at", "")
|
||||
for item in self._get_client().list_armed()
|
||||
if item.get("job_id")
|
||||
}
|
||||
with self._lock:
|
||||
self._armed.update(observed)
|
||||
return observed
|
||||
except Exception as e:
|
||||
logger.debug("Chronos _list_armed failed (will re-arm idempotently): %s", e)
|
||||
return {}
|
||||
|
||||
# -- reconcile --------------------------------------------------------
|
||||
|
||||
def reconcile(self) -> None:
|
||||
"""Converge the NAS-armed one-shots toward jobs.json (desired state):
|
||||
arm missing / re-arm changed-time, cancel orphaned."""
|
||||
from cron.jobs import load_jobs
|
||||
|
||||
desired: Dict[str, str] = {
|
||||
j["id"]: j["next_run_at"]
|
||||
for j in load_jobs()
|
||||
if j.get("enabled") and j.get("next_run_at") and j.get("state") != "paused"
|
||||
}
|
||||
observed = self._list_armed()
|
||||
|
||||
# Arm missing or changed-time.
|
||||
for job_id, fire_at in desired.items():
|
||||
if observed.get(job_id) != fire_at:
|
||||
# Re-fetch the full job dict to arm (need the whole record).
|
||||
from cron.jobs import get_job
|
||||
job = get_job(job_id)
|
||||
if job:
|
||||
try:
|
||||
self._arm_one_shot(job)
|
||||
except Exception as e:
|
||||
logger.warning("Chronos failed to arm job %s: %s", job_id, e)
|
||||
|
||||
# Cancel orphans (armed but no longer desired).
|
||||
for job_id in list(observed.keys()):
|
||||
if job_id not in desired:
|
||||
try:
|
||||
self._cancel(job_id)
|
||||
except Exception as e:
|
||||
logger.warning("Chronos failed to cancel orphan %s: %s", job_id, e)
|
||||
|
||||
# -- fire -------------------------------------------------------------
|
||||
|
||||
# NOTE: no ``fire_due`` override on purpose. The base implementation
|
||||
# virtually dispatches through ``self.claim_fire``/``self.fire_claimed``,
|
||||
# and ``provider_supports_split_fire`` treats ANY ``fire_due`` override
|
||||
# (even a pure ``super()`` delegate) as the legacy single-phase signal —
|
||||
# overriding it here would silently opt Chronos out of claim admission,
|
||||
# duplicate detection, and the cancel-aware drain on the fire webhook.
|
||||
|
||||
def fire_claimed(
|
||||
self,
|
||||
claimed_job: dict,
|
||||
*,
|
||||
adapters: Any = None,
|
||||
loop: Any = None,
|
||||
cancel_event: Any = None,
|
||||
) -> bool:
|
||||
job_id = claimed_job["id"]
|
||||
ran = super().fire_claimed(
|
||||
claimed_job,
|
||||
adapters=adapters,
|
||||
loop=loop,
|
||||
cancel_event=cancel_event,
|
||||
)
|
||||
if ran:
|
||||
from cron.jobs import get_job
|
||||
job = get_job(job_id)
|
||||
if job and job.get("enabled") and job.get("next_run_at"):
|
||||
try:
|
||||
self._arm_one_shot(job)
|
||||
except Exception as e:
|
||||
logger.warning("Chronos failed to re-arm job %s after fire: %s", job_id, e)
|
||||
return ran
|
||||
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Plugin entrypoint — register the Chronos provider with the loader.
|
||||
|
||||
Mirrors the memory-plugin shape; plugins/cron_providers discovery calls this and
|
||||
collects the provider via register_cron_scheduler.
|
||||
"""
|
||||
ctx.register_cron_scheduler(ChronosCronScheduler())
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Thin HTTP client for the agent → NAS ``agent-cron`` endpoints (Chronos).
|
||||
|
||||
The Chronos provider speaks ONLY to NAS — it names no scheduler vendor and
|
||||
holds no scheduler credentials. NAS owns the external scheduler (an internal
|
||||
implementation detail) and that scheduler's account; the agent just asks NAS to
|
||||
"arm a one-shot at time T" / "cancel" / "list", authenticated with the agent's
|
||||
existing Nous Portal access token (the same token it already uses to call the
|
||||
portal — no new secret).
|
||||
|
||||
Wire contract: ``docs/chronos-managed-cron-contract.md``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger("cron.chronos")
|
||||
|
||||
# Endpoint paths under the portal base URL.
|
||||
_PROVISION_PATH = "/api/agent-cron/provision"
|
||||
_CANCEL_PATH = "/api/agent-cron/cancel"
|
||||
_LIST_PATH = "/api/agent-cron/list"
|
||||
|
||||
|
||||
class NasCronClientError(RuntimeError):
|
||||
"""Raised when a NAS agent-cron call fails (non-2xx or transport error)."""
|
||||
|
||||
|
||||
class NasCronClient:
|
||||
"""Minimal client for the agent→NAS provision/cancel/list endpoints.
|
||||
|
||||
Uses the agent's refresh-aware Nous access token for auth. No scheduler
|
||||
vendor, no scheduler creds — NAS hides all of that behind these three calls.
|
||||
"""
|
||||
|
||||
def __init__(self, portal_url: str, *, timeout_seconds: float = 15.0) -> None:
|
||||
self.portal_url = portal_url.rstrip("/")
|
||||
self.timeout_seconds = timeout_seconds
|
||||
|
||||
# -- auth -------------------------------------------------------------
|
||||
|
||||
def _access_token(self) -> str:
|
||||
"""The agent's existing Nous Portal access token (refresh-aware)."""
|
||||
from hermes_cli.auth import resolve_nous_access_token
|
||||
return resolve_nous_access_token()
|
||||
|
||||
def _headers(self) -> Dict[str, str]:
|
||||
return {
|
||||
"Authorization": f"Bearer {self._access_token()}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# -- HTTP -------------------------------------------------------------
|
||||
|
||||
def _post(self, path: str, body: Dict[str, Any]) -> Dict[str, Any]:
|
||||
import requests # lazy: agent already depends on requests
|
||||
|
||||
url = f"{self.portal_url}{path}"
|
||||
try:
|
||||
resp = requests.post(
|
||||
url, json=body, headers=self._headers(), timeout=self.timeout_seconds
|
||||
)
|
||||
except Exception as e:
|
||||
raise NasCronClientError(f"POST {path} failed: {e}") from e
|
||||
if resp.status_code // 100 != 2:
|
||||
raise NasCronClientError(
|
||||
f"POST {path} returned {resp.status_code}: {resp.text[:200]}"
|
||||
)
|
||||
try:
|
||||
return resp.json() if resp.content else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def _get(self, path: str, params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
import requests
|
||||
|
||||
url = f"{self.portal_url}{path}"
|
||||
try:
|
||||
resp = requests.get(
|
||||
url, params=params, headers=self._headers(), timeout=self.timeout_seconds
|
||||
)
|
||||
except Exception as e:
|
||||
raise NasCronClientError(f"GET {path} failed: {e}") from e
|
||||
if resp.status_code // 100 != 2:
|
||||
raise NasCronClientError(
|
||||
f"GET {path} returned {resp.status_code}: {resp.text[:200]}"
|
||||
)
|
||||
try:
|
||||
return resp.json() if resp.content else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
# -- endpoints --------------------------------------------------------
|
||||
|
||||
def provision(self, *, job_id: str, fire_at: str, agent_callback_url: str,
|
||||
dedup_key: str) -> Dict[str, Any]:
|
||||
"""Ask NAS to arm a one-shot for ``job_id`` at ``fire_at`` (ISO 8601).
|
||||
|
||||
``dedup_key`` (``{job_id}:{fire_at}``) makes re-arming the same fire
|
||||
idempotent NAS-side. Returns the NAS response (e.g. ``{schedule_id}``).
|
||||
"""
|
||||
return self._post(_PROVISION_PATH, {
|
||||
"job_id": job_id,
|
||||
"fire_at": fire_at,
|
||||
"agent_callback_url": agent_callback_url,
|
||||
"dedup_key": dedup_key,
|
||||
})
|
||||
|
||||
def cancel(self, *, job_id: str) -> Dict[str, Any]:
|
||||
"""Ask NAS to cancel any armed one-shot for ``job_id``."""
|
||||
return self._post(_CANCEL_PATH, {"job_id": job_id})
|
||||
|
||||
def list_armed(self) -> List[Dict[str, Any]]:
|
||||
"""List the one-shots NAS currently has armed for this agent.
|
||||
|
||||
Returns a list of ``{job_id, fire_at, schedule_id}``. Best-effort: used
|
||||
by reconcile to find orphaned arms on a cold process; on error the
|
||||
caller falls back to idempotent re-arm of all desired jobs.
|
||||
"""
|
||||
data = self._get(_LIST_PATH, {})
|
||||
items = data.get("armed") if isinstance(data, dict) else None
|
||||
return items if isinstance(items, list) else []
|
||||
@@ -0,0 +1,9 @@
|
||||
name: chronos
|
||||
description: >-
|
||||
Chronos — NAS-mediated managed cron provider for scale-to-zero hosted agents.
|
||||
Delegates the "wake me at time T" trigger to Nous infrastructure so an idle
|
||||
gateway can scale to zero and still fire cron jobs. The agent computes each
|
||||
job's next-fire time and asks NAS to arm a one-shot; NAS calls the agent back
|
||||
at fire time over an authenticated webhook. Inert unless cron.provider=chronos.
|
||||
version: 1.0.0
|
||||
author: Nous Research
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Inbound cron-fire token verification for Chronos (Phase 4E.1).
|
||||
|
||||
When NAS relays an external scheduler fire to the agent, it POSTs
|
||||
``/api/cron/fire`` with a short-lived NAS-minted JWT. This module verifies that
|
||||
JWT before any job runs — the security boundary for remotely-triggered job
|
||||
execution.
|
||||
|
||||
We verify a NAS-minted JWT (the trust path the agent already has) rather than
|
||||
let an external scheduler call the agent directly: the scheduler signs with
|
||||
NAS's keys, which the agent doesn't (and shouldn't) hold. See the plan's DQ-4.
|
||||
|
||||
The verifier is pluggable (``get_fire_verifier``) so the escape-hatch mode
|
||||
(direct per-job cron-key) can swap in later with no handler change.
|
||||
|
||||
Crypto is delegated to PyJWT (already a declared dependency) — we do NOT
|
||||
hand-roll JWT verification.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
logger = logging.getLogger("cron.chronos.verify")
|
||||
|
||||
# The purpose claim that scopes a token to the fire endpoint. A general agent
|
||||
# JWT (without this claim) must NOT be replayable against /api/cron/fire.
|
||||
_FIRE_PURPOSE = "cron_fire"
|
||||
|
||||
# Process-wide cache of PyJWKClient instances, keyed by JWKS URL.
|
||||
#
|
||||
# WHY THIS EXISTS: a PyJWKClient caches the fetched JWKS (signing keys) on the
|
||||
# INSTANCE. Constructing a fresh client per fire therefore threw that cache
|
||||
# away and forced a synchronous JWKS HTTP GET to the portal on EVERY fire. Under
|
||||
# a burst of concurrent fires (an instance with several cron jobs firing in the
|
||||
# same window) that fanned out into N simultaneous JWKS fetches, which the
|
||||
# portal rate-limited (HTTP 403) — verification then failed and the agent
|
||||
# answered 401. When a fetch was merely slow rather than rate-limited, it blocked
|
||||
# the event loop long enough that the fire webhook could not return its 202
|
||||
# before the relay's 30s timeout (observed in prod as relay 504s concentrated on
|
||||
# high-job-count instances). Reusing one client per URL keeps the signing keys
|
||||
# cached (NAS keys rotate rarely), so the steady state is zero JWKS fetches per
|
||||
# fire. See docs/chronos-managed-cron-contract.md and the betterstack triage.
|
||||
_JWK_CLIENTS: Dict[str, Any] = {}
|
||||
_JWK_CLIENTS_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _get_jwk_client(jwks_url: str) -> Any:
|
||||
"""Return a process-cached PyJWKClient for ``jwks_url`` (one per URL).
|
||||
|
||||
PyJWKClient does its own key caching internally (``cache_keys``/``lifespan``);
|
||||
the whole point here is to reuse the SAME instance across fires so that cache
|
||||
is actually hit instead of discarded. Double-checked-locked so concurrent
|
||||
fires resolve to a single shared client without racing.
|
||||
"""
|
||||
client = _JWK_CLIENTS.get(jwks_url)
|
||||
if client is not None:
|
||||
return client
|
||||
with _JWK_CLIENTS_LOCK:
|
||||
client = _JWK_CLIENTS.get(jwks_url)
|
||||
if client is None:
|
||||
from jwt import PyJWKClient
|
||||
|
||||
# Explicit Accept + User-Agent so the JWKS fetch isn't blocked by the
|
||||
# NAS portal's WAF, which 403s the default Python-urllib fingerprint
|
||||
# (same fix as the dashboard-auth nous/self_hosted providers).
|
||||
client = PyJWKClient(
|
||||
jwks_url,
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "HermesAgent/1.0",
|
||||
},
|
||||
)
|
||||
_JWK_CLIENTS[jwks_url] = client
|
||||
return client
|
||||
|
||||
|
||||
def verify_nas_fire_token(
|
||||
*,
|
||||
token: str,
|
||||
expected_audience: str,
|
||||
jwks_or_key: Optional[str] = None,
|
||||
issuer: Optional[str] = None,
|
||||
leeway_seconds: int = 30,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Verify a NAS-minted cron-fire JWT. Return decoded claims, or None.
|
||||
|
||||
Checks (all must pass):
|
||||
- signature against the NAS JWKS (``jwks_or_key`` is a JWKS URL) — RS256
|
||||
family; symmetric secrets are rejected (NAS signs asymmetrically).
|
||||
- ``aud`` == ``expected_audience`` (this agent: ``agent:{instance_id}``).
|
||||
- ``exp`` / ``nbf`` within ``leeway_seconds``.
|
||||
- ``iss`` == ``issuer`` when an issuer is configured.
|
||||
- ``purpose`` == ``"cron_fire"`` — so a general agent JWT can't be
|
||||
replayed against the fire endpoint.
|
||||
|
||||
Returns None (never raises) on any failure, so the handler can answer 401
|
||||
without leaking which check failed.
|
||||
"""
|
||||
if not token or not expected_audience:
|
||||
return None
|
||||
if not jwks_or_key:
|
||||
# No verification key configured → cannot verify → refuse. We never
|
||||
# fall back to unsigned decode for a security boundary.
|
||||
logger.warning("cron fire: no JWKS/key configured; refusing token")
|
||||
return None
|
||||
|
||||
try:
|
||||
import jwt
|
||||
|
||||
# Resolve the signing key from the JWKS endpoint by the token's kid.
|
||||
signing_key = None
|
||||
if jwks_or_key.startswith("http://") or jwks_or_key.startswith("https://"):
|
||||
# Reuse a process-cached client so the JWKS fetch is amortised across
|
||||
# fires (a fresh client per fire re-fetched the JWKS every time and,
|
||||
# under concurrent fires, tripped the portal's rate limit → 403 →
|
||||
# 401, or blocked the event loop past the relay's 30s timeout → 504).
|
||||
jwk_client = _get_jwk_client(jwks_or_key)
|
||||
signing_key = jwk_client.get_signing_key_from_jwt(token).key
|
||||
else:
|
||||
# A PEM public key passed inline (test / pinned-key deployments).
|
||||
signing_key = jwks_or_key
|
||||
|
||||
options = {"require": ["exp", "aud"]}
|
||||
decode_kwargs: Dict[str, Any] = dict(
|
||||
algorithms=["RS256", "RS384", "RS512", "ES256", "ES384"],
|
||||
audience=expected_audience,
|
||||
leeway=leeway_seconds,
|
||||
options=options,
|
||||
)
|
||||
if issuer:
|
||||
decode_kwargs["issuer"] = issuer
|
||||
|
||||
claims = jwt.decode(token, signing_key, **decode_kwargs)
|
||||
except Exception as e:
|
||||
logger.warning("cron fire: token verification failed: %s", e)
|
||||
return None
|
||||
|
||||
if claims.get("purpose") != _FIRE_PURPOSE:
|
||||
logger.warning("cron fire: token missing/!=%s purpose claim", _FIRE_PURPOSE)
|
||||
return None
|
||||
|
||||
return claims
|
||||
|
||||
|
||||
def get_fire_verifier() -> Callable[..., Optional[Dict[str, Any]]]:
|
||||
"""Return the active inbound-fire verifier.
|
||||
|
||||
Default = the NAS-JWT verifier. The DQ-4 escape hatch (direct per-job
|
||||
cron-key) would return a cron-key verifier here instead, selected by config
|
||||
— so the webhook handler never changes when the auth mode is swapped.
|
||||
"""
|
||||
return verify_nas_fire_token
|
||||
Reference in New Issue
Block a user