Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,291 @@
|
||||
"""Idle deferral for background reviews on the managed local runtime.
|
||||
|
||||
The post-turn review fork replays the whole conversation on the review
|
||||
runtime. On a cloud provider that costs seconds and runs concurrently
|
||||
with whatever the user does next. When the review runtime IS the managed
|
||||
llama-server, the same fork monopolizes the GPU the user's next prompt
|
||||
needs, for minutes — and the next live turn cancels it, so an active
|
||||
session tends to pay the decode cost AND lose the learning.
|
||||
|
||||
This module keeps the decision to learn exactly where it was (turn end,
|
||||
nudge intervals, full-strength model, full transcript) and moves only
|
||||
the execution moment: reviews bound for the managed local endpoint are
|
||||
queued and dispatched when the machine is quiet. Everything else runs
|
||||
immediately, as before.
|
||||
|
||||
Policy (auxiliary.background_review.defer):
|
||||
auto (default) — defer exactly when the resolved review runtime
|
||||
targets the managed local server.
|
||||
never — old behavior everywhere.
|
||||
Explicit /refine (focus set) never defers: an explicit ask runs now,
|
||||
matching its bypass of the enabled gate.
|
||||
|
||||
Queue semantics:
|
||||
- One slot per session, newest snapshot wins. A review replays the whole
|
||||
conversation, so a newer snapshot strictly supersedes an older one —
|
||||
coalescing is deduplication, not loss.
|
||||
- Preempted (cancelled-by-live-turn) reviews are requeued by the spawn
|
||||
wrapper observing the run token's cancel flag, not killed-and-forgotten.
|
||||
- Aged-out events (defer_max_age_s, default 30 min) dispatch regardless
|
||||
of idleness — deferral may delay learning, never lose it.
|
||||
- In-memory, best-effort: dropped on process exit, the same durability
|
||||
contract the immediate daemon-thread fork always had.
|
||||
|
||||
Idle truth comes from the supervisor's /slots (machine-level: it sees
|
||||
every client of the managed server, including other Hermes profiles) and
|
||||
must hold for a settle window so a review is not launched into the gap
|
||||
between two quick prompts. Local in-process turn liveness is tracked via
|
||||
note_turn_started/note_turn_finished from run_conversation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Sustained-quiet window before dispatch. Long enough that "typed two
|
||||
# prompts back to back" does not look idle; short enough that walking
|
||||
# away for coffee runs the queue.
|
||||
_IDLE_SETTLE_S = 15.0
|
||||
# Poll cadence while the queue is non-empty. The thread parks when empty.
|
||||
_POLL_INTERVAL_S = 5.0
|
||||
# Age at which a queued review dispatches regardless of idleness.
|
||||
_MAX_AGE_DEFAULT_S = 30.0 * 60.0
|
||||
|
||||
|
||||
def defer_mode(task_cfg: Optional[Dict[str, Any]]) -> str:
|
||||
"""'auto' (default) or 'never' from auxiliary.background_review.defer."""
|
||||
raw = str((task_cfg or {}).get("defer", "auto")).strip().lower()
|
||||
return raw if raw in ("auto", "never") else "auto"
|
||||
|
||||
|
||||
def defer_max_age_s(task_cfg: Optional[Dict[str, Any]]) -> float:
|
||||
raw = (task_cfg or {}).get("defer_max_age_s", _MAX_AGE_DEFAULT_S)
|
||||
try:
|
||||
value = float(raw)
|
||||
except (TypeError, ValueError):
|
||||
return _MAX_AGE_DEFAULT_S
|
||||
return value if value > 0 else _MAX_AGE_DEFAULT_S
|
||||
|
||||
|
||||
def review_targets_managed_local(agent: Any,
|
||||
task_cfg: Optional[Dict[str, Any]]) -> bool:
|
||||
"""Would this review fork decode on the llama-server WE manage?
|
||||
|
||||
Resolves the review runtime the same way the fork itself will and
|
||||
exact-matches its netloc against the supervisor state file — the
|
||||
matcher that cannot false-positive on external local servers. Any
|
||||
failure reads False: immediate spawn is always the safe default.
|
||||
|
||||
Order matters: the netloc probe (one TTL-cached state-file read)
|
||||
runs FIRST, so machines with no managed server — every cloud-only
|
||||
install — return False without resolving the review runtime at all.
|
||||
This wrapper runs on the turn's tail; runtime resolution belongs on
|
||||
that path only when a managed server actually exists.
|
||||
"""
|
||||
try:
|
||||
from agent.auxiliary_client import (
|
||||
_is_managed_local_endpoint,
|
||||
_managed_local_netloc,
|
||||
)
|
||||
|
||||
if not _managed_local_netloc():
|
||||
return False
|
||||
from agent.background_review import _resolve_review_runtime
|
||||
|
||||
runtime = _resolve_review_runtime(agent, task_cfg)
|
||||
return _is_managed_local_endpoint(runtime.get("base_url"))
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
|
||||
class _PendingReview:
|
||||
__slots__ = ("agent", "kwargs", "enqueued_at", "session_key")
|
||||
|
||||
def __init__(self, agent: Any, session_key: str, kwargs: Dict[str, Any]):
|
||||
self.agent = agent
|
||||
self.session_key = session_key
|
||||
self.kwargs = kwargs
|
||||
self.enqueued_at = time.monotonic()
|
||||
|
||||
|
||||
class ReviewIdleQueue:
|
||||
"""Session-coalescing queue + idle-gated dispatcher thread."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._pending: Dict[str, _PendingReview] = {}
|
||||
self._wake = threading.Event()
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._live_turns = 0
|
||||
self._quiet_since: Optional[float] = None
|
||||
# Test seams — replaced by unit tests, never in production.
|
||||
self._now: Callable[[], float] = time.monotonic
|
||||
self._server_idle: Callable[[], bool] = _managed_server_idle
|
||||
|
||||
# ── turn liveness (this process) ────────────────────────────
|
||||
|
||||
def note_turn_started(self) -> None:
|
||||
with self._lock:
|
||||
self._live_turns += 1
|
||||
self._quiet_since = None
|
||||
|
||||
def note_turn_finished(self) -> None:
|
||||
with self._lock:
|
||||
self._live_turns = max(0, self._live_turns - 1)
|
||||
if self._live_turns == 0:
|
||||
self._quiet_since = self._now()
|
||||
self._wake.set()
|
||||
|
||||
# ── queue ────────────────────────────────────────────────────
|
||||
|
||||
def enqueue(self, agent: Any, session_key: str,
|
||||
kwargs: Dict[str, Any]) -> None:
|
||||
"""Add (or replace — newest snapshot wins) a session's pending review."""
|
||||
with self._lock:
|
||||
existing = self._pending.get(session_key)
|
||||
item = _PendingReview(agent, session_key, kwargs)
|
||||
# Stamp through the queue's clock (test seam); keep the ORIGINAL
|
||||
# enqueue time on coalesce so a busy session cannot push its
|
||||
# review's age-out forever.
|
||||
item.enqueued_at = (existing.enqueued_at if existing is not None
|
||||
else self._now())
|
||||
self._pending[session_key] = item
|
||||
self._ensure_thread()
|
||||
self._wake.set()
|
||||
logger.info("Background review deferred (session=%s, queued=%d)",
|
||||
session_key[-12:], len(self._pending))
|
||||
|
||||
def pending_count(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._pending)
|
||||
|
||||
# ── dispatcher ───────────────────────────────────────────────
|
||||
|
||||
def _ensure_thread(self) -> None:
|
||||
with self._lock:
|
||||
if self._thread is None or not self._thread.is_alive():
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, daemon=True, name="bg-review-idle-queue")
|
||||
self._thread.start()
|
||||
|
||||
def _quiet_for(self) -> float:
|
||||
"""Seconds this process has been turn-free (0 while a turn runs)."""
|
||||
with self._lock:
|
||||
if self._live_turns > 0 or self._quiet_since is None:
|
||||
return 0.0
|
||||
return self._now() - self._quiet_since
|
||||
|
||||
def _pop_dispatchable(self) -> Optional[_PendingReview]:
|
||||
"""Oldest aged-out item, else any item once quiet+idle hold."""
|
||||
with self._lock:
|
||||
if not self._pending:
|
||||
return None
|
||||
items = sorted(self._pending.values(),
|
||||
key=lambda p: p.enqueued_at)
|
||||
aged = [p for p in items
|
||||
if self._now() - p.enqueued_at
|
||||
>= defer_max_age_s(p.kwargs.get("task_cfg"))]
|
||||
candidate = aged[0] if aged else None
|
||||
if candidate is None:
|
||||
if self._quiet_for() < _IDLE_SETTLE_S:
|
||||
return None
|
||||
if not self._server_idle():
|
||||
return None
|
||||
with self._lock:
|
||||
if not self._pending:
|
||||
return None
|
||||
candidate = min(self._pending.values(),
|
||||
key=lambda p: p.enqueued_at)
|
||||
with self._lock:
|
||||
return self._pending.pop(candidate.session_key, None)
|
||||
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
self._wake.wait()
|
||||
with self._lock:
|
||||
if not self._pending:
|
||||
self._wake.clear()
|
||||
continue
|
||||
item = None
|
||||
try:
|
||||
item = self._pop_dispatchable()
|
||||
if item is not None:
|
||||
if not self._still_enabled(item):
|
||||
logger.info(
|
||||
"Deferred background review dropped: reviews "
|
||||
"were disabled while it was queued (session=%s)",
|
||||
item.session_key[-12:])
|
||||
continue
|
||||
logger.info(
|
||||
"Dispatching deferred background review "
|
||||
"(session=%s, waited=%.0fs, queued=%d)",
|
||||
item.session_key[-12:],
|
||||
self._now() - item.enqueued_at,
|
||||
self.pending_count())
|
||||
item.agent._spawn_background_review_now(**item.kwargs)
|
||||
except Exception: # noqa: BLE001 — dispatcher must survive anything
|
||||
logger.warning("Deferred review dispatch failed",
|
||||
exc_info=True)
|
||||
if item is None:
|
||||
time.sleep(_POLL_INTERVAL_S)
|
||||
|
||||
@staticmethod
|
||||
def _still_enabled(item: _PendingReview) -> bool:
|
||||
"""Re-check the enabled gate at DISPATCH time.
|
||||
|
||||
The entry wrapper gates at enqueue time, but minutes may pass in
|
||||
the queue — a user who sets background_review.enabled: false while
|
||||
a review waits means it, and the dispatch must not resurrect it.
|
||||
Fail-open like the gate itself (a broken config never silently
|
||||
disables reviews)."""
|
||||
try:
|
||||
from agent.background_review import load_background_review_settings
|
||||
|
||||
enabled, _ = load_background_review_settings()
|
||||
return enabled
|
||||
except Exception: # noqa: BLE001
|
||||
return True
|
||||
|
||||
|
||||
def _managed_server_idle() -> bool:
|
||||
"""Machine-level idle: no processing slot on any loaded model of the
|
||||
managed router. Unreachable/no state file reads idle (nothing to
|
||||
contend with). One /models + one /slots call per loaded model."""
|
||||
try:
|
||||
from hermes_cli.local_runtime.supervisor import state_path
|
||||
|
||||
state = json.loads(state_path().read_text(encoding="utf-8"))
|
||||
base = str(state.get("base_url", "")).rsplit("/v1", 1)[0]
|
||||
key = str(state.get("api_key", ""))
|
||||
if not base:
|
||||
return True
|
||||
headers = {"Authorization": f"Bearer {key}"}
|
||||
req = urllib.request.Request(f"{base}/models", headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=3) as r:
|
||||
models = json.loads(r.read())
|
||||
loaded = [m["id"] for m in models.get("data", [])
|
||||
if (m.get("status") or {}).get("value") in ("loaded", "ready")]
|
||||
from urllib.parse import quote
|
||||
|
||||
for mid in loaded:
|
||||
req = urllib.request.Request(f"{base}/slots?model={quote(mid)}",
|
||||
headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=3) as r:
|
||||
slots = json.loads(r.read())
|
||||
if any(s.get("is_processing") for s in slots
|
||||
if isinstance(s, dict)):
|
||||
return False
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
return True
|
||||
|
||||
|
||||
# Module singleton — one queue per process, like the load-progress watcher.
|
||||
QUEUE = ReviewIdleQueue()
|
||||
Reference in New Issue
Block a user