"""Supervision of one llama-server in router mode. The router process is ours (restart with backoff on crash); router children are its problem — child failures surface via GET /models exit_code, never auto-retried here. Readiness rules (each learned the hard way on real hardware): - health-200 is NOT readiness; every readiness claim requires a touch generation (temp-0, expected token, generous budget, reasoning_content scanned). - Always dial 127.0.0.1 — resolving localhost adds ~2s per request on Windows via IPv6 fallback. - /metrics is opt-in (--metrics) and carries no KV-usage metric; idleness = requests_processing == 0 and no slot is_processing. - The router's LRU eviction has no pin for the primary model: until an upstream pin exists, keep_primary_loaded re-touches the primary after any other model load. """ from __future__ import annotations import json import logging import secrets import socket import subprocess import threading import time import urllib.error import urllib.request from pathlib import Path from hermes_cli.local_runtime.binaries import server_binary, runtimes_root logger = logging.getLogger(__name__) TOUCH_PROMPT = "Reply with exactly one word: the capital of France." TOUCH_EXPECT = "paris" _RESTART_BACKOFF_S = (1, 5, 15, 60) def state_path() -> Path: """Endpoint state for other Hermes processes (provider resolution reads this to route llamacpp-alias requests at the managed server).""" return runtimes_root() / "server.json" def _free_port() -> int: with socket.socket() as s: s.bind(("127.0.0.1", 0)) return s.getsockname()[1] # Default port for the managed server, chosen once and reused across # restarts. Sessions persist the resolved base_url; an ephemeral port # would strand every resumed session on a dead endpoint after each # restart. Deliberately NOT 8080 so we never collide with a user's own # llama-server/Ollama-adjacent stack. _DEFAULT_PORT = 18434 def _stable_port() -> int: """The stable default port, falling back to an ephemeral one only when something else already listens there (and it isn't a leftover managed server, which stop() would have cleaned up).""" try: with socket.socket() as s: s.bind(("127.0.0.1", _DEFAULT_PORT)) return _DEFAULT_PORT except OSError: logger.warning( "port %d busy; managed llama-server falling back to an ephemeral " "port — existing sessions may need a model re-pick", _DEFAULT_PORT) return _free_port() def _stable_api_key() -> str: """One key for the life of the install, persisted beside the runtimes. Endpoint identity must survive restarts as a UNIT — sessions persist the resolved base_url + api_key, so a per-boot key strands every resumed session on HTTP 401 exactly the way a per-boot port would strand them on connection errors. Rotating it buys nothing: the key exists to stop other loopback processes free-riding, and it lives on the same disk as the state file that would leak it. Delete the file to rotate manually. """ key_path = runtimes_root() / ".api_key" try: existing = key_path.read_text(encoding="utf-8").strip() if len(existing) >= 16: return existing except OSError: pass key = secrets.token_urlsafe(24) try: key_path.parent.mkdir(parents=True, exist_ok=True) key_path.write_text(key, encoding="utf-8") except OSError as exc: logger.warning("could not persist api key (%s); sessions will need " "a re-pick after restart", exc) return key class LlamaServerSupervisor: """Own one llama-server router process for the life of a Hermes session. Usage:: sup = LlamaServerSupervisor(install_dir, models_dir) sup.start() # spawn + wait healthy sup.ensure_model_ready(name) # load + touch-generate ... sup.base_url is the /v1 endpoint, sup.api_key its key ... sup.stop() """ def __init__(self, install_dir: Path, models_dir: Path, *, models_max: int = 4, port: int | None = None, extra_args: list[str] | None = None, log_path: Path | None = None, preset_path: Path | None = None): self.install_dir = Path(install_dir) self.models_dir = Path(models_dir) self.models_max = models_max self.port = port or _stable_port() self.api_key = _stable_api_key() self.extra_args = list(extra_args or []) self.log_path = log_path or (self.models_dir.parent / "logs" / "llama-server.log") self.preset_path = preset_path self.proc: subprocess.Popen | None = None self.primary_model: str | None = None self._restarts = 0 self._stopping = False self._watchdog: threading.Thread | None = None self._log_handle = None self._idle_since: dict[str, float] = {} # ── endpoints ──────────────────────────────────────────── @property def base_url(self) -> str: return f"http://127.0.0.1:{self.port}/v1" def _url(self, route: str) -> str: return f"http://127.0.0.1:{self.port}{route}" def _request(self, route: str, body: dict | None = None, timeout_s: int = 30) -> dict: req = urllib.request.Request( self._url(route), data=json.dumps(body).encode() if body is not None else None, headers={"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"}, ) with urllib.request.urlopen(req, timeout=timeout_s) as r: raw = r.read() return json.loads(raw) if raw else {} # ── lifecycle ──────────────────────────────────────────── def _spawn(self) -> None: exe = server_binary(self.install_dir) cmd = [ str(exe), "--host", "127.0.0.1", "--port", str(self.port), "--api-key", self.api_key, "--models-dir", str(self.models_dir), "--models-max", str(self.models_max), # The residency contract at the layer that sees every message: # a chat request to a staged-but-unloaded model loads it (slow # first token) instead of failing with 'model not found' — # without this flag, chat after an eject is a bare 400/404. "--models-autoload", "--metrics", # opt-in flag; supervisor telemetry needs it "--slots", # /slots endpoint is also opt-in; is_idle reads it "--no-webui", "--jinja", # Direct I/O on model load: bypasses the page cache, so a # multi-GB load doesn't evict half the OS cache — measured # faster loads on NVMe, and our router bounces (download/ # delete/activate) reload models often enough to care. "-dio", ] if self.preset_path and self.preset_path.exists(): cmd += ["--models-preset", str(self.preset_path)] cmd += [ *self.extra_args, ] self.log_path.parent.mkdir(parents=True, exist_ok=True) if self._log_handle is not None: # The crash-restart loop calls _spawn repeatedly; without # closing the prior handle each restart leaks one fd. try: self._log_handle.close() except Exception: # noqa: BLE001 — best-effort pass self._log_handle = open(self.log_path, "a", encoding="utf-8", errors="replace") self._log_handle.write(f"\n# spawn: {cmd}\n") self._log_handle.flush() # list-args, never a shell: spaced paths (user homes) must survive. self.proc = subprocess.Popen(cmd, stdout=self._log_handle, stderr=subprocess.STDOUT, cwd=str(exe.parent)) logger.info("llama-server router spawned pid=%s port=%s", self.proc.pid, self.port) # State goes down at SPAWN, not after health: endpoint resolution # treats a live-pid-but-not-yet-healthy server as "starting" rather # than "unconfigured", so a readiness probe racing the boot doesn't # throw the app back to onboarding (observed on first restart test). self._write_state() def start(self, timeout_s: int = 120) -> None: self._stopping = False self._spawn() self._wait_health(timeout_s) self._write_state() self._watchdog = threading.Thread(target=self._watch, daemon=True, name="llamacpp-supervisor") self._watchdog.start() def _write_state(self) -> None: path = state_path() path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps({ "base_url": self.base_url, "api_key": self.api_key, "pid": self.proc.pid if self.proc else None, }), encoding="utf-8") def _wait_health(self, timeout_s: int) -> None: deadline = time.monotonic() + timeout_s while time.monotonic() < deadline: if self.proc and self.proc.poll() is not None: raise RuntimeError( f"llama-server exited rc={self.proc.returncode} during startup " f"(log: {self.log_path})") try: with urllib.request.urlopen(self._url("/health"), timeout=3) as r: if r.status == 200: return except (urllib.error.URLError, OSError, TimeoutError): pass time.sleep(1) raise TimeoutError(f"llama-server not healthy after {timeout_s}s (log: {self.log_path})") def _watch(self) -> None: """Restart the router (not its children) on crash, with backoff.""" while not self._stopping: proc = self.proc if proc is None: return rc = proc.poll() if rc is None: time.sleep(2) continue if self._stopping: return backoff = _RESTART_BACKOFF_S[min(self._restarts, len(_RESTART_BACKOFF_S) - 1)] logger.warning("llama-server exited rc=%s; restart #%s in %ss", rc, self._restarts + 1, backoff) time.sleep(backoff) self._restarts += 1 try: self._reap_orphaned_children() self._spawn() self._wait_health(120) if self.primary_model: self.ensure_model_ready(self.primary_model) except Exception as exc: # noqa: BLE001 logger.error("llama-server restart failed: %s", exc) def stop(self) -> None: self._stopping = True state_path().unlink(missing_ok=True) if self.proc and self.proc.poll() is None: self._terminate_tree(self.proc) if self._log_handle: self._log_handle.close() self._log_handle = None @staticmethod def _terminate_tree(proc: subprocess.Popen) -> None: """Terminate the router AND its model children. The router spawns one child llama-server per loaded model, each holding gigabytes of VRAM. Terminating only the router (on Windows, TerminateProcess — no signal handlers, no cleanup pass) orphans those children: the port goes quiet but the weights stay resident, and the next spawn re-loads models alongside a ghost still holding the memory. Enumerate children FIRST (the parent must be alive to walk them), then terminate parent and children together, escalating to kill for stragglers. """ children: list = [] try: import psutil children = psutil.Process(proc.pid).children(recursive=True) except Exception: # noqa: BLE001 — no psutil view; still stop the router children = [] proc.terminate() for child in children: try: child.terminate() except Exception: # noqa: BLE001 pass try: proc.wait(timeout=15) except subprocess.TimeoutExpired: proc.kill() for child in children: try: if child.is_running(): child.kill() except Exception: # noqa: BLE001 pass def _reap_orphaned_children(self) -> None: """Kill model children orphaned by a router crash, before respawn. A crashed router can't clean up its children, and a dead parent can't be walked — so match by identity instead: any process running OUR llama-server binary whose parent is gone is an orphan of a previous router. Their VRAM must come back before the new router loads models next to the ghosts. External llama-servers (different binary path) never match. """ try: import psutil exe = str(server_binary(self.install_dir)) except Exception: # noqa: BLE001 return for p in psutil.process_iter(["exe", "ppid"]): try: if p.info.get("exe") != exe: continue if self.proc is not None and p.pid == self.proc.pid: continue ppid = p.info.get("ppid") or 0 if ppid and psutil.pid_exists(ppid): continue logger.warning("reaping orphaned llama-server child pid=%s", p.pid) p.kill() except (psutil.NoSuchProcess, psutil.AccessDenied): continue # ── model management (router endpoints) ────────────────── def models(self) -> dict: """{model_id: status_value} from GET /models.""" data = self._request("/models") return {m["id"]: m.get("status", {}).get("value", "unknown") for m in data.get("data", [])} def model_failures(self) -> dict: """{model_id: exit_code} for children that died — surfaced to the UI, never auto-retried (design: router children are its problem).""" data = self._request("/models") out = {} for m in data.get("data", []): status = m.get("status", {}) if status.get("value") == "failed" or status.get("exit_code"): out[m["id"]] = status.get("exit_code") return out def load_model(self, model_id: str, timeout_s: int = 600) -> None: self._request("/models/load", {"model": model_id}, timeout_s=timeout_s) def unload_model(self, model_id: str) -> None: """Free the child's VRAM now. Route existence verified empirically on b10290 (POST /models/unload; bogus name -> 400 'model is not found'). Momentary action: never touches primary_model — the declaration is durable, an eject is not (residency design). Settle before returning: for a few seconds after unload returns, the router still routes to the dying child and answers chat with 500 'proxy error: Could not establish connection' (probed on b10362). Waiting for the model to report unloaded means the next message autoloads cleanly instead of racing the teardown. """ self._request("/models/unload", {"model": model_id}, timeout_s=120) deadline = time.monotonic() + 15 while time.monotonic() < deadline: try: if self.models().get(model_id) not in ("loaded", "ready", "unloading"): return except Exception: # noqa: BLE001 return time.sleep(0.3) # ── idle residency (non-primary models) ────────────────── # A model that has gone quiet gets its VRAM back after this long. A # constant, not a knob: long enough that an active conversation never # trips it, short enough that a wandered-off session frees ~20 GiB # within the hour. No exemptions (residency v2): demand reloads # anything the user comes back to. IDLE_UNLOAD_S = 15 * 60 def sweep_idle(self, now: float | None = None) -> list[str]: """Unload models idle past IDLE_UNLOAD_S. Returns the model ids unloaded. Idle means no busy slots and no queued work, tracked per model across calls; a model seen busy resets its clock.""" now = time.monotonic() if now is None else now unloaded: list[str] = [] try: statuses = self.models() except Exception: # noqa: BLE001 return unloaded for model_id, status in statuses.items(): if status not in ("loaded", "ready"): self._idle_since.pop(model_id, None) continue if not self.is_idle(model_id): self._idle_since.pop(model_id, None) continue first_idle = self._idle_since.setdefault(model_id, now) if now - first_idle >= self.IDLE_UNLOAD_S: try: self.unload_model(model_id) self._idle_since.pop(model_id, None) unloaded.append(model_id) logger.info("idle-unloaded %s (idle %ds)", model_id, int(now - first_idle)) except Exception as exc: # noqa: BLE001 logger.warning("idle unload of %s failed: %s", model_id, exc) return unloaded def touch_generate(self, model_id: str, timeout_s: int = 300) -> bool: """The readiness proof. Generous budget + reasoning_content scan — small token budgets false-fail reasoning models, which spend their first tokens thinking.""" try: resp = self._request("/v1/chat/completions", { "model": model_id, "messages": [{"role": "user", "content": TOUCH_PROMPT}], "max_tokens": 512, "temperature": 0, }, timeout_s=timeout_s) msg = resp["choices"][0]["message"] blob = (msg.get("content") or "") + " " + (msg.get("reasoning_content") or "") return TOUCH_EXPECT in blob.lower() except Exception as exc: # noqa: BLE001 logger.warning("touch generation failed for %s: %s", model_id, exc) return False def ensure_model_ready(self, model_id: str, timeout_s: int = 600) -> bool: """Load if needed, then prove readiness with a touch generation.""" status = self.models().get(model_id) if status is None: raise KeyError(f"model {model_id} not present in models dir") if status not in ("loaded", "ready"): self.load_model(model_id, timeout_s=timeout_s) return self.touch_generate(model_id) def actual_n_ctx(self, model_id: str) -> int | None: """/props reconciliation: the granted window as the child reports it — the compressor's budget and the picker's 'running at 87K of 262K' both read THIS value, never the request (design step 4).""" try: props = self._request(f"/props?model={model_id}") return props.get("default_generation_settings", {}).get("n_ctx") except Exception: # noqa: BLE001 return None def keep_primary_loaded(self) -> None: """The router's LRU eviction has no pin, so after any other load re-touch the primary to keep it most-recently-used. Best-effort under bursty multi-model load — replaced when an upstream pin exists.""" if self.primary_model and self.models().get(self.primary_model) in ( "loaded", "ready"): self.touch_generate(self.primary_model, timeout_s=60) # ── telemetry ──────────────────────────────────────────── def is_idle(self, model_id: str | None = None) -> bool: """No processing requests and no busy slots. Router quirk: /slots and /metrics are per-child and require ?model= (bare calls 400), and no KV-usage metric exists. With ``model_id`` checks that one child; without, every loaded child.""" try: if model_id is not None: loaded = [model_id] else: loaded = [m for m, status in self.models().items() if status in ("loaded", "ready")] for mid in loaded: slots = self._request(f"/slots?model={mid}") if any(s.get("is_processing") for s in slots): return False req = urllib.request.Request( self._url(f"/metrics?model={mid}"), headers={"Authorization": f"Bearer {self.api_key}"}) with urllib.request.urlopen(req, timeout=10) as r: text = r.read().decode() for line in text.splitlines(): if line.startswith("llamacpp:requests_processing"): if float(line.split()[-1]) != 0.0: return False return True except Exception: # noqa: BLE001 return False