Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,55 @@
|
||||
"""Managed llama.cpp runtime.
|
||||
|
||||
Hermes downloads, verifies, supervises, and updates one llama-server, and
|
||||
decides per machine which model build and context window to run. Key
|
||||
modules:
|
||||
|
||||
- ``binaries`` — resolve/download/verify official llama.cpp release zips
|
||||
into ``$HERMES_HOME/runtimes/llamacpp/<tag>/``.
|
||||
- ``supervisor``— spawn and supervise one llama-server in router mode;
|
||||
readiness is a touch generation, never health-200 alone.
|
||||
- ``detect`` — find an already-running llama-server (external or ours).
|
||||
- ``estimator`` / ``context_policy`` / ``growth`` — price context memory
|
||||
per architecture and run the window ladder (zero-spill start, grow
|
||||
toward native max, compress only at the top).
|
||||
- ``catalog`` / ``presets`` — the curated model list and the per-model
|
||||
launch flags that carry policy decisions to the router.
|
||||
|
||||
Everything is driven by the ``local_runtime`` section of config.yaml.
|
||||
"""
|
||||
|
||||
from hermes_cli.local_runtime.binaries import ( # noqa: F401
|
||||
BinaryResolutionError,
|
||||
ensure_runtime_installed,
|
||||
resolve_assets,
|
||||
select_backend,
|
||||
)
|
||||
from hermes_cli.local_runtime.bootstrap import ( # noqa: F401
|
||||
ensure_local_runtime,
|
||||
shutdown_local_runtime,
|
||||
)
|
||||
from hermes_cli.local_runtime.context_policy import ( # noqa: F401
|
||||
FLOOR,
|
||||
growth_decision,
|
||||
initial_window,
|
||||
ladder,
|
||||
launch_args,
|
||||
)
|
||||
from hermes_cli.local_runtime.growth import ( # noqa: F401
|
||||
clear_window_override,
|
||||
load_window_overrides,
|
||||
maybe_grow_window,
|
||||
save_window_override,
|
||||
)
|
||||
from hermes_cli.local_runtime.detect import detect_server # noqa: F401
|
||||
from hermes_cli.local_runtime.endpoint import resolve_llamacpp_endpoint # noqa: F401
|
||||
from hermes_cli.local_runtime.estimator import ( # noqa: F401
|
||||
HardwareBudget,
|
||||
ctx_bytes,
|
||||
physics_check,
|
||||
profile_from_gguf,
|
||||
)
|
||||
from hermes_cli.local_runtime.gguf import read_gguf_header # noqa: F401
|
||||
from hermes_cli.local_runtime.hardware import probe_budget # noqa: F401
|
||||
from hermes_cli.local_runtime.presets import generate_presets # noqa: F401
|
||||
from hermes_cli.local_runtime.supervisor import LlamaServerSupervisor # noqa: F401
|
||||
@@ -0,0 +1,351 @@
|
||||
"""Binary acquisition for the managed llama.cpp runtime.
|
||||
|
||||
llama.cpp publishes per-tag assets (rolling ``bNNNN`` tags, no semver).
|
||||
Backends are dlopen'd plugins, so a runtime = CPU/base zip + backend zip
|
||||
extracted into one directory, plus the cudart runtime zip on Windows CUDA
|
||||
(end users have no CUDA toolkit). We pin the tag in config, sha256-verify
|
||||
every download, and keep the previous tag for rollback (N-1).
|
||||
|
||||
Layout: ``$HERMES_HOME/runtimes/llamacpp/<tag>/<backend>/<binaries>``
|
||||
with a ``manifest.json`` recording zips, sha256s, and the verified
|
||||
llama-server version string.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
RELEASE_URL = "https://github.com/ggml-org/llama.cpp/releases/download/{tag}/{asset}"
|
||||
|
||||
# Windows CUDA zips ship per CUDA major; the runtime zip must be paired with
|
||||
# its cudart zip so end users need no toolkit. 13.3 verified on 13.1 and
|
||||
# 13.2 drivers.
|
||||
_WIN_CUDA_VERSION = "13.3"
|
||||
# arm64 Windows CUDA prebuilts landed upstream (~b1036x) on CUDA 13.4 —
|
||||
# verified against live asset lists (b10362, b10630, b10679). Tags at or before
|
||||
# b10290 don't have them; resolution succeeds and the download 404s
|
||||
# honestly on such tags, which only arises if a user pins backward.
|
||||
_WIN_CUDA_VERSION_ARM64 = "13.4"
|
||||
|
||||
|
||||
# Fallback when the config section is missing entirely (deep-merge normally
|
||||
# guarantees the key). Single source: DEFAULT_CONFIG owns the shipped tag.
|
||||
def default_tag() -> str:
|
||||
from hermes_cli.config_defaults import DEFAULT_CONFIG
|
||||
|
||||
return DEFAULT_CONFIG["local_runtime"]["tag"]
|
||||
|
||||
|
||||
class BinaryResolutionError(RuntimeError):
|
||||
"""No usable asset combination for this platform/backend."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class AssetPlan:
|
||||
"""The exact zips one runtime install needs, in extraction order."""
|
||||
|
||||
tag: str
|
||||
backend: str # cuda | metal | vulkan | hip | cpu
|
||||
assets: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def install_dir(self) -> Path:
|
||||
return runtimes_root() / self.tag / self.backend
|
||||
|
||||
|
||||
def runtimes_root() -> Path:
|
||||
"""Machine-scoped, deliberately NOT profile-scoped. Engine binaries,
|
||||
presets, and server state describe this machine's hardware and its one
|
||||
managed server (stable port) — a second profile re-downloading the
|
||||
engine or fighting over the port would be the bug. Profile-scoped
|
||||
things (which model is the default, enabled) live in each profile's
|
||||
config.yaml as ever."""
|
||||
from hermes_constants import get_default_hermes_root
|
||||
|
||||
return get_default_hermes_root() / "runtimes" / "llamacpp"
|
||||
|
||||
|
||||
def installed_tags() -> list[str]:
|
||||
"""Tags with a verified install (manifest carries verified_version),
|
||||
newest first by release number. The boot ladder and the update check
|
||||
both read installed-ness from here — one resolver, every caller."""
|
||||
root = runtimes_root()
|
||||
if not root.exists():
|
||||
return []
|
||||
found: list[str] = []
|
||||
for entry in root.iterdir():
|
||||
if not entry.is_dir() or entry.name == "downloads":
|
||||
continue
|
||||
for manifest in entry.glob("*/manifest.json"):
|
||||
try:
|
||||
if json.loads(manifest.read_text(encoding="utf-8")).get("verified_version"):
|
||||
found.append(entry.name)
|
||||
break
|
||||
except (json.JSONDecodeError, OSError):
|
||||
continue
|
||||
|
||||
def _release_number(tag: str) -> int:
|
||||
digits = "".join(ch for ch in tag if ch.isdigit())
|
||||
return int(digits) if digits else 0
|
||||
|
||||
return sorted(set(found), key=_release_number, reverse=True)
|
||||
|
||||
|
||||
def _host_os_arch() -> tuple[str, str]:
|
||||
"""(os, arch) normalized to release-asset vocabulary.
|
||||
|
||||
PITFALL: PROCESSOR_ARCHITECTURE lies under x64 emulation on
|
||||
ARM64 Windows. platform.machine() reads the same env on some Pythons, so
|
||||
on Windows prefer PROCESSOR_IDENTIFIER's text when present.
|
||||
"""
|
||||
system = platform.system().lower()
|
||||
os_name = {"windows": "win", "darwin": "macos", "linux": "ubuntu"}.get(system, system)
|
||||
machine = platform.machine().lower()
|
||||
arch = "arm64" if machine in ("arm64", "aarch64") else "x64"
|
||||
if os_name == "win":
|
||||
import os as _os
|
||||
ident = _os.environ.get("PROCESSOR_IDENTIFIER", "")
|
||||
if "armv8" in ident.lower() or "arm " in ident.lower():
|
||||
arch = "arm64"
|
||||
return os_name, arch
|
||||
|
||||
|
||||
def select_backend(gpu_vendor: str | None, os_name: str | None = None) -> str:
|
||||
"""Backend choice per design: CUDA if NVIDIA, Metal on macOS, Vulkan if
|
||||
a non-NVIDIA GPU is present, else CPU. ``--list-devices`` validates the
|
||||
choice post-install; the supervisor's touch generation is ground truth."""
|
||||
if os_name is None:
|
||||
os_name, _ = _host_os_arch()
|
||||
if os_name == "macos":
|
||||
return "metal"
|
||||
vendor = (gpu_vendor or "").lower()
|
||||
if "nvidia" in vendor:
|
||||
return "cuda"
|
||||
if vendor in ("amd", "intel") or "radeon" in vendor or "arc" in vendor:
|
||||
return "vulkan"
|
||||
return "cpu"
|
||||
|
||||
|
||||
def resolve_assets(tag: str, backend: str, os_name: str | None = None,
|
||||
arch: str | None = None) -> AssetPlan:
|
||||
"""Compose the asset list for (tag, backend, platform).
|
||||
|
||||
Raises BinaryResolutionError for combinations the release does not ship
|
||||
(a platform/backend pair upstream publishes no artifact for). Callers
|
||||
fall back down the backend ladder: cuda -> vulkan -> cpu.
|
||||
"""
|
||||
host_os, host_arch = _host_os_arch()
|
||||
os_name = os_name or host_os
|
||||
arch = arch or host_arch
|
||||
plan = AssetPlan(tag=tag, backend=backend)
|
||||
|
||||
if os_name == "macos":
|
||||
# macOS tarballs are unified (Metal built in).
|
||||
plan.assets = [f"llama-{tag}-bin-macos-{arch}.tar.gz"]
|
||||
return plan
|
||||
|
||||
if os_name == "ubuntu":
|
||||
if backend == "cuda":
|
||||
# No prebuilt Linux CUDA zips at current tags — Linux CUDA users
|
||||
# build from source or use vulkan; resolver is honest about it.
|
||||
raise BinaryResolutionError(
|
||||
f"no prebuilt linux CUDA asset at {tag}; use vulkan/cpu or a source build")
|
||||
suffix = {"vulkan": f"vulkan-{arch}", "hip": f"rocm-7.2-{arch}",
|
||||
"cpu": arch}.get(backend)
|
||||
if suffix is None:
|
||||
raise BinaryResolutionError(f"unsupported linux backend {backend}")
|
||||
plan.assets = [f"llama-{tag}-bin-ubuntu-{suffix}.tar.gz"]
|
||||
return plan
|
||||
|
||||
if os_name == "win":
|
||||
if backend == "cuda":
|
||||
cuda_ver = _WIN_CUDA_VERSION_ARM64 if arch == "arm64" else _WIN_CUDA_VERSION
|
||||
plan.assets = [
|
||||
f"llama-{tag}-bin-win-cuda-{cuda_ver}-{arch}.zip",
|
||||
f"cudart-llama-bin-win-cuda-{cuda_ver}-{arch}.zip",
|
||||
]
|
||||
elif backend == "vulkan":
|
||||
if arch == "arm64":
|
||||
raise BinaryResolutionError(f"no win-vulkan-arm64 asset at {tag}")
|
||||
plan.assets = [f"llama-{tag}-bin-win-vulkan-x64.zip"]
|
||||
elif backend == "hip":
|
||||
plan.assets = [f"llama-{tag}-bin-win-hip-radeon-x64.zip"]
|
||||
elif backend == "cpu":
|
||||
plan.assets = [f"llama-{tag}-bin-win-cpu-{arch}.zip"]
|
||||
else:
|
||||
raise BinaryResolutionError(f"unsupported windows backend {backend}")
|
||||
return plan
|
||||
|
||||
raise BinaryResolutionError(f"unsupported platform {os_name}-{arch}")
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 22), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def _download(url: str, dest: Path,
|
||||
progress: "Callable[[int, int], None] | None" = None) -> None:
|
||||
"""Stream url -> dest. ``progress(done_bytes, total_bytes)`` ticks per
|
||||
chunk (total 0 when the server sends no Content-Length) — a several-
|
||||
hundred-MB archive on a slow line must never look hung."""
|
||||
logger.info("downloading %s", url)
|
||||
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||
with urllib.request.urlopen(url, timeout=120) as r, open(tmp, "wb") as f:
|
||||
total = int(r.headers.get("Content-Length") or 0)
|
||||
done = 0
|
||||
while True:
|
||||
chunk = r.read(1 << 20)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
done += len(chunk)
|
||||
if progress is not None:
|
||||
progress(done, total)
|
||||
tmp.replace(dest)
|
||||
|
||||
|
||||
def _extract(archive: Path, dest: Path,
|
||||
progress: "Callable[[int, int], None] | None" = None) -> None:
|
||||
"""Extract member by member so ``progress(done, total)`` can tick in
|
||||
uncompressed bytes — big archives take real time on laptop disks."""
|
||||
if archive.name.endswith(".zip"):
|
||||
with zipfile.ZipFile(archive) as z:
|
||||
members = z.infolist()
|
||||
total = sum(m.file_size for m in members)
|
||||
done = 0
|
||||
for m in members:
|
||||
z.extract(m, dest)
|
||||
done += m.file_size
|
||||
if progress is not None:
|
||||
progress(done, total)
|
||||
else:
|
||||
import tarfile
|
||||
with tarfile.open(archive) as t:
|
||||
members = t.getmembers()
|
||||
total = sum(m.size for m in members)
|
||||
done = 0
|
||||
for m in members:
|
||||
t.extract(m, dest, filter="data")
|
||||
done += m.size
|
||||
if progress is not None:
|
||||
progress(done, total)
|
||||
|
||||
|
||||
def server_binary(install_dir: Path) -> Path:
|
||||
"""Locate llama-server within an extracted runtime (zips differ in
|
||||
whether they nest a build/bin directory)."""
|
||||
names = ("llama-server.exe", "llama-server")
|
||||
for name in names:
|
||||
direct = install_dir / name
|
||||
if direct.exists():
|
||||
return direct
|
||||
for name in names:
|
||||
hits = sorted(install_dir.rglob(name))
|
||||
if hits:
|
||||
return hits[0]
|
||||
raise BinaryResolutionError(f"llama-server not found under {install_dir}")
|
||||
|
||||
|
||||
def verify_install(install_dir: Path, tag: str) -> str:
|
||||
"""Run --version; require the tag's build number in the output.
|
||||
(The binary prints the tag WITHOUT the 'b' prefix.)"""
|
||||
exe = server_binary(install_dir)
|
||||
out = subprocess.run([str(exe), "--version"], capture_output=True,
|
||||
text=True, encoding="utf-8", errors="replace",
|
||||
timeout=60, cwd=str(exe.parent))
|
||||
text = (out.stdout + out.stderr).strip()
|
||||
if tag.lstrip("b") not in text:
|
||||
raise BinaryResolutionError(
|
||||
f"version check failed for {exe}: expected {tag}, got: {text[:120]}")
|
||||
return text.splitlines()[0] if text else ""
|
||||
|
||||
|
||||
def prune_old_tags(keep: list[str]) -> None:
|
||||
"""Retain only the tags in ``keep`` (current + previous — N-1 rollback).
|
||||
The shared ``downloads/`` archive cache is not a tag and always survives."""
|
||||
root = runtimes_root()
|
||||
if not root.exists():
|
||||
return
|
||||
for entry in root.iterdir():
|
||||
if entry.is_dir() and entry.name != "downloads" and entry.name not in keep:
|
||||
shutil.rmtree(entry, ignore_errors=True)
|
||||
logger.info("pruned old runtime %s", entry.name)
|
||||
|
||||
|
||||
def ensure_runtime_installed(tag: str, backend: str,
|
||||
expected_sha256: dict[str, str] | None = None,
|
||||
progress: "Callable[[str, int, int, str], None] | None" = None) -> Path:
|
||||
"""Idempotent: resolve, download, verify, extract, version-check.
|
||||
|
||||
``expected_sha256`` maps asset name -> hash when the catalog pins them;
|
||||
without pins the computed hash is recorded in the manifest (trust on
|
||||
first download, verified on every reinstall).
|
||||
``progress(stage, done_bytes, total_bytes, label)`` ticks through the
|
||||
slow parts — stage is "download" | "extract" | "verify", label is the
|
||||
asset counter ("1/2") when the plan has several archives.
|
||||
Returns the install directory containing llama-server.
|
||||
"""
|
||||
plan = resolve_assets(tag, backend)
|
||||
install_dir = plan.install_dir
|
||||
manifest_path = install_dir / "manifest.json"
|
||||
if manifest_path.exists():
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if manifest.get("verified_version"):
|
||||
return install_dir
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass # damaged manifest -> reinstall
|
||||
|
||||
install_dir.mkdir(parents=True, exist_ok=True)
|
||||
downloads = runtimes_root() / "downloads"
|
||||
downloads.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
recorded: dict[str, str] = {}
|
||||
n_assets = len(plan.assets)
|
||||
for i, asset in enumerate(plan.assets, 1):
|
||||
label = f"{i}/{n_assets}" if n_assets > 1 else ""
|
||||
archive = downloads / asset
|
||||
if not archive.exists():
|
||||
_download(RELEASE_URL.format(tag=tag, asset=asset), archive,
|
||||
progress=(lambda d, t, _l=label: progress("download", d, t, _l))
|
||||
if progress is not None else None)
|
||||
if progress is not None:
|
||||
progress("verify", 0, 0, label)
|
||||
digest = _sha256(archive)
|
||||
expected = (expected_sha256 or {}).get(asset)
|
||||
if expected and digest != expected:
|
||||
archive.unlink(missing_ok=True)
|
||||
raise BinaryResolutionError(
|
||||
f"sha256 mismatch for {asset}: expected {expected}, got {digest}")
|
||||
recorded[asset] = digest
|
||||
_extract(archive, install_dir,
|
||||
progress=(lambda d, t, _l=label: progress("extract", d, t, _l))
|
||||
if progress is not None else None)
|
||||
|
||||
if progress is not None:
|
||||
progress("verify", 0, 0, "")
|
||||
version = verify_install(install_dir, tag)
|
||||
manifest_path.write_text(json.dumps({
|
||||
"tag": tag, "backend": plan.backend, "assets": recorded,
|
||||
"verified_version": version,
|
||||
}, indent=2), encoding="utf-8")
|
||||
logger.info("installed llama.cpp %s (%s): %s", tag, backend, version)
|
||||
return install_dir
|
||||
@@ -0,0 +1,337 @@
|
||||
"""Bootstrap for the managed runtime: config -> installed binaries ->
|
||||
running supervised server.
|
||||
|
||||
One public call, ``ensure_local_runtime(config)``, safe to call at any
|
||||
session start:
|
||||
- disabled or already-running (state file answers /health) -> no-op
|
||||
- enabled -> install binaries if missing (idempotent), spawn supervisor
|
||||
|
||||
Kept import-light: callers gate on config before importing this module so
|
||||
sessions with local_runtime disabled never pay the import.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from hermes_constants import get_hermes_home # noqa: F401 — config paths
|
||||
|
||||
from hermes_cli.local_runtime.binaries import runtimes_root
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SUPERVISOR = None # process-wide singleton; one router per Hermes process
|
||||
|
||||
|
||||
def _detect_gpu_vendor() -> str | None:
|
||||
"""Best-effort GPU vendor for backend selection. NVIDIA via nvidia-smi
|
||||
(resolved by the hardware probe's PATH-independent ladder — a stripped
|
||||
service PATH must not demote an NVIDIA box to vulkan/cpu); anything
|
||||
else defers to select_backend's fallback ladder."""
|
||||
from hermes_cli.local_runtime.hardware import _nvidia_smi_path
|
||||
|
||||
smi = _nvidia_smi_path()
|
||||
if smi is None:
|
||||
return None
|
||||
try:
|
||||
out = subprocess.run(
|
||||
[smi, "--query-gpu=name", "--format=csv,noheader"],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
if out.returncode == 0 and out.stdout.strip():
|
||||
return "nvidia " + out.stdout.strip().splitlines()[0]
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def models_dir() -> Path:
|
||||
"""Machine-scoped, deliberately NOT profile-scoped: a 20 GB GGUF is a
|
||||
machine asset, and every profile shares the one managed server that
|
||||
serves it. See runtimes_root() for the same rule on the engine."""
|
||||
from hermes_constants import get_default_hermes_root
|
||||
|
||||
return get_default_hermes_root() / "models"
|
||||
|
||||
|
||||
def assets_dir() -> Path:
|
||||
"""Non-model companion files (mmproj vision projectors, spec-decode
|
||||
draft models). A subdirectory so the router's model listing — and our
|
||||
staged_models() — never mistakes an asset for a servable model."""
|
||||
return models_dir() / "assets"
|
||||
|
||||
|
||||
def staged_models() -> "list[Path]":
|
||||
"""Servable staged models: single-file GGUFs count when present; a
|
||||
split GGUF counts once, by its first part, and only when EVERY part
|
||||
is on disk — a mid-download split is not servable and must not
|
||||
surface anywhere as a model. Continuation parts and assets/ never
|
||||
count."""
|
||||
import re
|
||||
|
||||
part = re.compile(r"-(\d{5})-of-(\d{5})\.gguf$")
|
||||
files = sorted(models_dir().glob("*.gguf"))
|
||||
names = {p.name for p in files}
|
||||
out = []
|
||||
for p in files:
|
||||
m = part.search(p.name)
|
||||
if m is None:
|
||||
out.append(p)
|
||||
continue
|
||||
if m.group(1) != "00001":
|
||||
continue
|
||||
stem = p.name[: m.start()]
|
||||
total = int(m.group(2))
|
||||
if all(f"{stem}-{i:05d}-of-{m.group(2)}.gguf" in names
|
||||
for i in range(2, total + 1)):
|
||||
out.append(p)
|
||||
return out
|
||||
|
||||
|
||||
def staged_model_ids() -> "list[str]":
|
||||
import re
|
||||
|
||||
return [re.sub(r"-\d{5}-of-\d{5}$", "", p.stem) for p in staged_models()]
|
||||
|
||||
|
||||
def _presets_stale() -> bool:
|
||||
"""True when a staged model has no section in the preset INI — it
|
||||
would autoload with stock fit instead of a policy decision."""
|
||||
try:
|
||||
from hermes_cli.local_runtime.presets import read_preset_decisions
|
||||
|
||||
known = set(read_preset_decisions())
|
||||
return any(mid not in known for mid in staged_model_ids())
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
|
||||
def _stop_state_server(state: dict) -> None:
|
||||
"""Best-effort stop of the server the state file points at (an
|
||||
incumbent this process doesn't supervise). The state pid is ours by
|
||||
contract — the file only ever describes the managed server."""
|
||||
from hermes_cli.local_runtime.endpoint import _pid_alive
|
||||
|
||||
pid = state.get("pid")
|
||||
try:
|
||||
pid = int(pid)
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
if pid <= 0:
|
||||
return
|
||||
try:
|
||||
import signal
|
||||
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except (OSError, ValueError):
|
||||
return
|
||||
# Give it a moment to release the port and the GPU. Liveness via
|
||||
# psutil — on Windows os.kill(pid, 0) TERMINATES the process, it is
|
||||
# not a probe (the endpoint.py pitfall note; #local-models review).
|
||||
for _ in range(50):
|
||||
if not _pid_alive(pid):
|
||||
return
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
def refresh_local_runtime() -> bool:
|
||||
"""Restart the managed server so it rescans the models directory.
|
||||
|
||||
The router's model list is SPAWN-ONLY: a GGUF added after start is
|
||||
invisible to GET /models and 400s on completion, so anything that
|
||||
changes the staged set while the server runs must bounce it. Covers
|
||||
both ownership shapes: a supervised server restarts in-process; an
|
||||
ADOPTED server (started by a previous backend session — the normal
|
||||
shape after any restart) is stopped via its state-file pid and
|
||||
replaced with a supervised boot. Without the adopted branch, every
|
||||
download/delete in a post-restart session silently no-ops the bounce
|
||||
and the router serves a stale catalog. Returns False when there is
|
||||
nothing to refresh (no server anywhere; next boot scans fresh).
|
||||
"""
|
||||
global _SUPERVISOR
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
if _SUPERVISOR is None:
|
||||
from hermes_cli.local_runtime.endpoint import _state_endpoint
|
||||
|
||||
state = _state_endpoint()
|
||||
if state is None:
|
||||
return False
|
||||
logger.info("bouncing adopted llama-server (pid=%s) to rescan models",
|
||||
state.get("pid"))
|
||||
_stop_state_server(state)
|
||||
else:
|
||||
shutdown_local_runtime()
|
||||
return ensure_local_runtime(load_config(), force=True) is not None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("local runtime refresh failed: %s", exc)
|
||||
return False
|
||||
|
||||
|
||||
def ensure_local_runtime(config: dict, force: bool = False) -> "object | None":
|
||||
"""Idempotent boot of the managed runtime. Returns the supervisor (or
|
||||
None when disabled/unavailable). Never raises into a session start —
|
||||
failures log and return None; chat falls back to configured providers.
|
||||
|
||||
``force=True`` skips the enabled gate — used by the explicit "Use this
|
||||
model" action, where the click IS the opt-in (the caller records it in
|
||||
config so future boots auto-start).
|
||||
"""
|
||||
global _SUPERVISOR
|
||||
section = (config or {}).get("local_runtime") or {}
|
||||
if not force and not section.get("enabled"):
|
||||
return None
|
||||
if _SUPERVISOR is not None:
|
||||
return _SUPERVISOR
|
||||
|
||||
# Residency: no staged models means nothing to serve — don't boot an
|
||||
# empty server. The walked-away story handled with zero configuration
|
||||
# (delete your last model and boots stop); Use force-boots as ever.
|
||||
if not force and not staged_models():
|
||||
logger.info("local runtime enabled but no models staged; not booting")
|
||||
return None
|
||||
|
||||
# Another Hermes process may already be supervising — reuse via state,
|
||||
# but ONLY while its launch policy still covers every staged model. A
|
||||
# server whose preset file predates a download serves the new model
|
||||
# with no policy at all (--models-autoload + stock fit: f16 KV at max
|
||||
# context, no placement — the silent-demotion busy-wait on WDDM). A
|
||||
# stale incumbent gets stopped and replaced by a fresh boot with
|
||||
# regenerated presets; sessions ride through exactly like any other
|
||||
# supervised restart (stable port + persisted key).
|
||||
from hermes_cli.local_runtime.endpoint import _state_endpoint
|
||||
|
||||
state = _state_endpoint()
|
||||
if state is not None:
|
||||
if not _presets_stale():
|
||||
logger.info("managed llama-server already running (another process)")
|
||||
return None
|
||||
logger.info("running server's presets predate the staged models; "
|
||||
"replacing it so every model launches with a policy")
|
||||
_stop_state_server(state)
|
||||
|
||||
try:
|
||||
from hermes_cli.local_runtime.binaries import (
|
||||
ensure_runtime_installed,
|
||||
select_backend,
|
||||
)
|
||||
from hermes_cli.local_runtime.hardware import probe_budget
|
||||
from hermes_cli.local_runtime.presets import generate_presets
|
||||
from hermes_cli.local_runtime.supervisor import LlamaServerSupervisor
|
||||
|
||||
backend = section.get("backend", "auto")
|
||||
if backend == "auto":
|
||||
backend = select_backend(_detect_gpu_vendor())
|
||||
# Boot ladder: serve what is INSTALLED, never download here. The
|
||||
# configured tag (config root-of-trust; deep-merge supplies the
|
||||
# Hermes-release default when unpinned) is preferred; when it isn't
|
||||
# installed yet, the newest installed tag serves and the status
|
||||
# endpoint reports the pending update — the download is a deliberate
|
||||
# button click in the pane, not a boot-path surprise (a multi-minute
|
||||
# inline download here is exactly how the onboarding bounce returns).
|
||||
from hermes_cli.local_runtime.binaries import default_tag, installed_tags
|
||||
|
||||
tag = section.get("tag") or default_tag()
|
||||
have = installed_tags()
|
||||
if tag not in have:
|
||||
if not have:
|
||||
logger.info("local runtime enabled but no build installed; "
|
||||
"install happens in the Local Models pane")
|
||||
return None
|
||||
logger.info("configured tag %s not installed; serving %s "
|
||||
"(update is a click in Local Models)", tag, have[0])
|
||||
tag = have[0]
|
||||
install_dir = ensure_runtime_installed(tag, backend)
|
||||
|
||||
mdir = models_dir()
|
||||
mdir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Context policy: one launch decision per staged model, carried to
|
||||
# the router via the preset INI. Priced against CAPACITY, not live
|
||||
# free VRAM: this runs while the outgoing server instance may still
|
||||
# hold the card (restart, refresh after a download), and its memory
|
||||
# is freed before the new instance loads anything. Pricing against
|
||||
# live-free here once pinned a fitting model's weights to CPU
|
||||
# because the probe saw the predecessor's VRAM as gone.
|
||||
preset_path = runtimes_root() / "presets.ini"
|
||||
try:
|
||||
entries = generate_presets(mdir, probe_budget(planning=True), preset_path)
|
||||
for entry in entries:
|
||||
if entry.refusal:
|
||||
logger.warning("model refused by physics check: %s", entry.refusal)
|
||||
except Exception as exc: # noqa: BLE001 — policy failure must not block serving
|
||||
# Degradation ladder: a STALE policy still beats no policy —
|
||||
# stock fit (f16 KV at max context, no placement) is the
|
||||
# silent-busy-wait failure on Windows. Keep serving with the
|
||||
# previous INI when one exists; only a first boot with no INI
|
||||
# at all falls to stock fit.
|
||||
if preset_path.exists():
|
||||
logger.error("preset generation failed (%s); serving with the "
|
||||
"PREVIOUS launch policies — models staged since "
|
||||
"the last successful generation run unpoliced "
|
||||
"until this is fixed", exc)
|
||||
else:
|
||||
logger.error("preset generation failed (%s) and no previous "
|
||||
"policy file exists; router runs stock fit", exc)
|
||||
preset_path = None
|
||||
|
||||
sup = LlamaServerSupervisor(
|
||||
install_dir, mdir,
|
||||
models_max=int(section.get("models_max", 4)),
|
||||
port=int(section.get("port", 0)) or None,
|
||||
preset_path=preset_path,
|
||||
)
|
||||
try:
|
||||
sup.start()
|
||||
except Exception:
|
||||
# start() can fail after the router process exists (health
|
||||
# timeout, spawn error): leaving it running unsupervised
|
||||
# strands its VRAM behind a port nothing will clean up.
|
||||
try:
|
||||
sup.stop()
|
||||
except Exception: # noqa: BLE001 — cleanup is best-effort
|
||||
pass
|
||||
raise
|
||||
_SUPERVISOR = sup
|
||||
logger.info("managed llama-server up at %s (backend=%s tag=%s)",
|
||||
sup.base_url, backend, tag)
|
||||
_start_idle_sweeper(sup)
|
||||
return sup
|
||||
except Exception as exc: # noqa: BLE001 — never break session start
|
||||
logger.warning("managed local runtime unavailable: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def shutdown_local_runtime() -> None:
|
||||
global _SUPERVISOR
|
||||
if _SUPERVISOR is not None:
|
||||
_SUPERVISOR.stop()
|
||||
_SUPERVISOR = None
|
||||
|
||||
|
||||
def get_supervisor():
|
||||
"""The process-local supervisor, or None (server may still be running
|
||||
under another process — check the state file)."""
|
||||
return _SUPERVISOR
|
||||
|
||||
|
||||
def _start_idle_sweeper(sup) -> None:
|
||||
"""Idle-residency loop: every couple of minutes, unload non-primary
|
||||
models idle past the supervisor's threshold. Daemon thread tied to the
|
||||
supervisor's lifetime — exits when the server stops."""
|
||||
import threading
|
||||
|
||||
def _loop():
|
||||
while sup.proc is not None and sup.proc.poll() is None:
|
||||
time.sleep(120)
|
||||
try:
|
||||
sup.sweep_idle()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("idle sweep skipped: %s", exc)
|
||||
|
||||
threading.Thread(target=_loop, daemon=True,
|
||||
name="local-runtime-idle-sweep").start()
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Capability answers for models served by the managed runtime.
|
||||
|
||||
Capability lookups (vision, and whatever comes next) consult cloud-shaped
|
||||
catalogs that have never heard of a local GGUF, so a vision-capable local
|
||||
model reads as text-only and images detour to an auxiliary cloud model —
|
||||
the wrong behavior twice over for a local-first user (broken feature, and
|
||||
a screenshot silently leaving the machine).
|
||||
|
||||
The managed runtime can answer from ground truth instead, best source
|
||||
first:
|
||||
|
||||
1. The RUNNING child's /props: llama-server reports a ``modalities`` block
|
||||
when a vision projector is loaded. The server that will receive the
|
||||
image says whether it can see — no inference, no catalog.
|
||||
2. The catalog entry's declared capability (the ``vision`` tag + mmproj
|
||||
asset) for staged-but-unloaded models: what the model WILL support once
|
||||
its projector loads beside it.
|
||||
3. None — not one of ours, or nothing known; the caller falls through to
|
||||
its other sources.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import urllib.request
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LLAMACPP_ALIASES = frozenset({"llamacpp", "llama.cpp", "llama-cpp"})
|
||||
|
||||
# Image formats the managed server's decoder actually handles. llama.cpp
|
||||
# decodes with stb_image: PNG/JPEG/GIF/BMP yes, WebP NO — and a WebP part
|
||||
# fails SILENTLY (no HTTP error, no log line; the model just never sees an
|
||||
# image and confabulates a description). Anything outside this set must be
|
||||
# transcoded before the request. Measured against the live server: the
|
||||
# same red square answered 'Red' as PNG and 'Unseen' as WebP.
|
||||
ACCEPTED_IMAGE_MIMES = frozenset({"image/png", "image/jpeg"})
|
||||
|
||||
|
||||
def is_managed_provider(provider: str, base_url: str = "") -> bool:
|
||||
"""True when this provider/base_url pair points at the managed server.
|
||||
``custom`` only counts when the base_url IS the managed endpoint —
|
||||
background lookups must never claim someone else's custom server."""
|
||||
p = (provider or "").strip().lower()
|
||||
if p in _LLAMACPP_ALIASES:
|
||||
return True
|
||||
if p == "custom" and base_url:
|
||||
try:
|
||||
from hermes_cli.local_runtime.growth import is_managed_endpoint
|
||||
|
||||
return is_managed_endpoint(base_url)
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _props_modalities(model_id: str) -> "bool | None":
|
||||
"""Ask the running server whether this loaded child sees images.
|
||||
None when the server is down, the model isn't loaded, or the build
|
||||
doesn't report modalities."""
|
||||
try:
|
||||
from hermes_cli.local_runtime.endpoint import _state_endpoint
|
||||
|
||||
state = _state_endpoint()
|
||||
if state is None:
|
||||
return None
|
||||
base = state["base_url"].rsplit("/v1", 1)[0]
|
||||
req = urllib.request.Request(
|
||||
f"{base}/props?model={model_id}",
|
||||
headers={"Authorization": f"Bearer {state.get('api_key', '')}"})
|
||||
with urllib.request.urlopen(req, timeout=3) as r:
|
||||
props = json.load(r)
|
||||
modalities = props.get("modalities")
|
||||
if isinstance(modalities, dict) and "vision" in modalities:
|
||||
return bool(modalities["vision"])
|
||||
return None
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
def managed_model_supports_vision(model_id: str) -> "bool | None":
|
||||
"""Ground-truth vision capability for a staged model, or None when the
|
||||
model isn't ours / nothing is known (caller keeps falling through)."""
|
||||
if not model_id:
|
||||
return None
|
||||
|
||||
# Only answer for models actually staged with us.
|
||||
try:
|
||||
from hermes_cli.local_runtime.bootstrap import staged_model_ids
|
||||
|
||||
if model_id not in staged_model_ids():
|
||||
return None
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
live = _props_modalities(model_id)
|
||||
if live is not None:
|
||||
return live
|
||||
|
||||
# Staged but not loaded (or an older server build): the catalog knows
|
||||
# whether this model ships a vision projector.
|
||||
try:
|
||||
from hermes_cli.local_runtime.bootstrap import assets_dir
|
||||
from hermes_cli.local_runtime.catalog import find_entry_for_model
|
||||
|
||||
hit = find_entry_for_model(model_id)
|
||||
if hit is None:
|
||||
return None
|
||||
entry = hit[0]
|
||||
if entry.mmproj is None:
|
||||
return False
|
||||
# Capability requires the projector to actually be on disk — a
|
||||
# model downloaded before its mmproj (partial delete, old layout)
|
||||
# genuinely cannot see.
|
||||
return (assets_dir() / entry.mmproj.local_name).exists()
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
@@ -0,0 +1,174 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"models": [
|
||||
{
|
||||
"id": "qwen3.8-27b",
|
||||
"display_name": "Qwen3.8 27B",
|
||||
"description": "Best all-round agent model; sees images; long context stays fast",
|
||||
"repo": "unsloth/Qwen3.8-27B-GGUF",
|
||||
"variants": [
|
||||
{
|
||||
"quant": "UD-Q4_K_M",
|
||||
"files": [
|
||||
{
|
||||
"path": "Qwen3.8-27B-UD-Q4_K_M.gguf",
|
||||
"size_bytes": 16464440224
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"n_ctx_train": 262144,
|
||||
"full_layers": 16,
|
||||
"recurrent_layers": 48,
|
||||
"per_layer_f16": 4096,
|
||||
"n_vocab": 248320,
|
||||
"mmproj": {
|
||||
"path": "mmproj-BF16.gguf",
|
||||
"size_bytes": 931146432,
|
||||
"local": "mmproj-Qwen3.8-27B-BF16.gguf"
|
||||
},
|
||||
"mtp": true,
|
||||
"mtp_draft_depth": 2,
|
||||
"sampling": {
|
||||
"temp": "1.0",
|
||||
"top-p": "0.95",
|
||||
"top-k": "20",
|
||||
"min-p": "0.0"
|
||||
},
|
||||
"quality": 90,
|
||||
"decode_fraction": 1.0
|
||||
},
|
||||
{
|
||||
"id": "qwen3.8-flash-next",
|
||||
"display_name": "Qwen3.8 Flash Next",
|
||||
"description": "Frontier-scale model; needs a very large GPU to run well",
|
||||
"repo": "unsloth/Qwen3.8-Flash-Next-GGUF",
|
||||
"variants": [
|
||||
{
|
||||
"quant": "UD-Q4_K_XL",
|
||||
"files": [
|
||||
{
|
||||
"path": "UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00001-of-00004.gguf",
|
||||
"size_bytes": 10946624
|
||||
},
|
||||
{
|
||||
"path": "UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00002-of-00004.gguf",
|
||||
"size_bytes": 49859583136
|
||||
},
|
||||
{
|
||||
"path": "UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00003-of-00004.gguf",
|
||||
"size_bytes": 49376141504
|
||||
},
|
||||
{
|
||||
"path": "UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00004-of-00004.gguf",
|
||||
"size_bytes": 12087983520
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"n_ctx_train": 262144,
|
||||
"full_layers": 12,
|
||||
"recurrent_layers": 36,
|
||||
"per_layer_f16": 2048,
|
||||
"moe": true,
|
||||
"n_vocab": 248320,
|
||||
"mmproj": {
|
||||
"path": "mmproj-BF16.gguf",
|
||||
"size_bytes": 907542944,
|
||||
"local": "mmproj-Qwen3.8-Flash-Next-BF16.gguf"
|
||||
},
|
||||
"min_engine": "b10678",
|
||||
"quality": 95,
|
||||
"decode_fraction": 0.08
|
||||
},
|
||||
{
|
||||
"id": "qwen3.6-35b-a3b",
|
||||
"display_name": "Qwen3.6 35B-A3B",
|
||||
"description": "Bigger mixture-of-experts with multi-token prediction; sees images",
|
||||
"repo": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF",
|
||||
"variants": [
|
||||
{
|
||||
"quant": "UD-Q4_K_M",
|
||||
"files": [
|
||||
{
|
||||
"path": "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf",
|
||||
"size_bytes": 22663387424
|
||||
}
|
||||
],
|
||||
"validated": true
|
||||
}
|
||||
],
|
||||
"n_ctx_train": 262144,
|
||||
"full_layers": 10,
|
||||
"recurrent_layers": 30,
|
||||
"per_layer_f16": 2048,
|
||||
"moe": true,
|
||||
"mtp": true,
|
||||
"n_vocab": 248320,
|
||||
"mtp_draft_depth": 2,
|
||||
"mmproj": {
|
||||
"path": "mmproj-BF16.gguf",
|
||||
"size_bytes": 902822528,
|
||||
"local": "mmproj-Qwen3.6-35B-A3B-BF16.gguf"
|
||||
},
|
||||
"sampling": {
|
||||
"temp": "1.0",
|
||||
"top-p": "0.95",
|
||||
"top-k": "20",
|
||||
"min-p": "0.0"
|
||||
},
|
||||
"quality": 80,
|
||||
"decode_fraction": 0.15
|
||||
},
|
||||
{
|
||||
"id": "deepseek-v4-flash",
|
||||
"display_name": "DeepSeek V4 Flash",
|
||||
"description": "Frontier-class model for machines with 128GB+ memory",
|
||||
"repo": "unsloth/DeepSeek-V4-Flash-0731-GGUF",
|
||||
"variants": [
|
||||
{
|
||||
"quant": "UD-Q4_K_XL",
|
||||
"files": [
|
||||
{
|
||||
"path": "UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00001-of-00005.gguf",
|
||||
"size_bytes": 5257408
|
||||
},
|
||||
{
|
||||
"path": "UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00002-of-00005.gguf",
|
||||
"size_bytes": 48935523072
|
||||
},
|
||||
{
|
||||
"path": "UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00003-of-00005.gguf",
|
||||
"size_bytes": 48980787136
|
||||
},
|
||||
{
|
||||
"path": "UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00004-of-00005.gguf",
|
||||
"size_bytes": 49999168416
|
||||
},
|
||||
{
|
||||
"path": "UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00005-of-00005.gguf",
|
||||
"size_bytes": 7174505088
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"n_ctx_train": 1048576,
|
||||
"full_layers": 43,
|
||||
"recurrent_layers": 0,
|
||||
"per_layer_f16": 1152,
|
||||
"moe": true,
|
||||
"n_vocab": 163840,
|
||||
"draft": {
|
||||
"path": "dspark-DeepSeek-V4-Flash-0731-Q8_0.gguf",
|
||||
"size_bytes": 10896057440
|
||||
},
|
||||
"sampling": {
|
||||
"temp": "1.0",
|
||||
"top-p": "0.95",
|
||||
"min-p": "0.01"
|
||||
},
|
||||
"quality": 85,
|
||||
"decode_fraction": 0.1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,464 @@
|
||||
"""Curated starter catalog for the managed local runtime.
|
||||
|
||||
Small and honest: every entry carries the estimator inputs (measured on
|
||||
real GGUFs) so the picker can price a model BEFORE the user downloads
|
||||
gigabytes. Once a file is on disk, profile_from_gguf() is the authority
|
||||
and the catalog numbers are only used for the download decision. Entries
|
||||
whose base config is gated upstream carry a same-family conservative
|
||||
prior (commented) — the GGUF header corrects it at load time.
|
||||
|
||||
Each model ships ONE build, Q4-class (UD-Q4_K_M where the repo has it,
|
||||
UD-Q4_K_XL elsewhere). Q4 is the quant class current engines optimize
|
||||
for and the sweet spot of the size/quality curve, so there is no quant
|
||||
ladder: headroom buys a bigger context window, never a bigger quant,
|
||||
and every machine runs the same well-tested build. Below Q4 the quality
|
||||
loss is too severe to ship as someone's first local-AI experience; the
|
||||
fit policy prices the build honestly (zero-spill, spilled, or refused by
|
||||
the physics check).
|
||||
|
||||
Validation lifecycle: builds proven end-to-end on real hardware are
|
||||
marked validated. Day-0 entries ship before that proof (they simply lack
|
||||
the validated flag) — ensure_model_ready's touch generation still gates
|
||||
every first load at runtime.
|
||||
|
||||
Multi-file models: variants may carry split-GGUF parts (llama-server loads
|
||||
from the first part; all parts download together). Entries may carry an
|
||||
mmproj (vision projector) and a speculative-decode draft model — both
|
||||
download alongside the weights. MTP-integrated models run spec decode
|
||||
wherever they load; a separate draft model attaches only when the launch
|
||||
decision spills, where its speedup is largest.
|
||||
|
||||
File sizes come from HF LFS metadata and feed the estimator, the fit
|
||||
pills, and download progress. There is no download-time integrity check
|
||||
by design: a corrupt or truncated file surfaces as a llama.cpp
|
||||
load error at first use, and the reachability test catches upstream
|
||||
re-uploads by size drift before users do.
|
||||
|
||||
This is deliberately not a live registry feed: entries are reviewed like a
|
||||
version bump (the same policy governs vendor recipe ingestion — parsed
|
||||
data, never executed commands).
|
||||
|
||||
Vendor recipes overlay: a per-SKU recipes repo may SUPPLEMENT these
|
||||
entries where applicable — vendor SKUs only, never the base layer for
|
||||
other platforms. A recipe may enrich identity (GGUF/quant/sha), perf
|
||||
hints (-b/-ub, spec-decode), and sampling defaults; it never carries
|
||||
context/slots/placement/serving flags (the fit policy owns those).
|
||||
Resolution: exact SKU -> GPU-class bucket -> fit-only. Snapshot-synced,
|
||||
reviewed like a tag bump.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import PurePosixPath
|
||||
|
||||
from hermes_cli.local_runtime.context_policy import (
|
||||
FLOOR,
|
||||
RUNTIME_OVERHEAD_BYTES,
|
||||
TARGET_WINDOW,
|
||||
ub_logits_bytes,
|
||||
)
|
||||
from hermes_cli.local_runtime.estimator import (
|
||||
HardwareBudget,
|
||||
LayerKind,
|
||||
ModelProfile,
|
||||
ctx_bytes,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_GIB = 1 << 30
|
||||
_PART_SUFFIX = re.compile(r"-\d{5}-of-\d{5}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AssetFile:
|
||||
"""One downloadable file: repo-relative path and exact bytes (the size
|
||||
feeds the estimator and the download progress bar; there is no
|
||||
download-time integrity check by design — a corrupt file surfaces as a
|
||||
llama.cpp load error). ``local`` overrides the on-disk name (repos
|
||||
reuse generic names like mmproj-BF16.gguf across models). Non-model
|
||||
extras live under the models dir's assets/ subdirectory so the router
|
||||
never lists them."""
|
||||
|
||||
path: str # repo-relative (may include a subdir)
|
||||
size_bytes: int
|
||||
local: str | None = None
|
||||
|
||||
@property
|
||||
def local_name(self) -> str:
|
||||
return self.local or PurePosixPath(self.path).name
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QuantVariant:
|
||||
"""One downloadable build of a model. Split GGUFs list every part in
|
||||
files; the model loads from the first part."""
|
||||
|
||||
quant: str # e.g. "UD-Q4_K_M"
|
||||
files: tuple # AssetFile, first = the load target
|
||||
validated: bool = False # proven end-to-end on real hardware
|
||||
|
||||
@property
|
||||
def model_id(self) -> str:
|
||||
stem = PurePosixPath(self.files[0].path).name.removesuffix(".gguf")
|
||||
return _PART_SUFFIX.sub("", stem)
|
||||
|
||||
@property
|
||||
def size_bytes(self) -> int:
|
||||
return sum(f.size_bytes for f in self.files)
|
||||
|
||||
@property
|
||||
def weights_bytes(self) -> int:
|
||||
"""Pre-download weights estimate: GGUF bytes ≈ tensor bytes + a
|
||||
small header (<2%) — a safe, slightly conservative stand-in until
|
||||
profile_from_gguf reads the real table."""
|
||||
return self.size_bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CatalogEntry:
|
||||
id: str # stable family id (variant-independent)
|
||||
display_name: str
|
||||
description: str # one line, plain language
|
||||
repo: str # HF repo
|
||||
variants: tuple # QuantVariant (exactly one, Q4-class)
|
||||
# Estimator inputs (measured or config-derived; quant changes weights,
|
||||
# never KV). Entries with gated upstream configs carry a conservative
|
||||
# same-family prior — the GGUF header is the authority after download.
|
||||
n_ctx_train: int
|
||||
full_layers: int
|
||||
recurrent_layers: int
|
||||
per_layer_f16: int # KV bytes/token per full-attention layer
|
||||
swa_layers: int = 0
|
||||
swa_window: int = 0
|
||||
moe: bool = False
|
||||
mtp: bool = False # ships MTP heads (spec decode when loaded)
|
||||
# Speculative draft depth for MTP models. Per-model and measured:
|
||||
# deeper drafting pays only while draft acceptance holds, and the
|
||||
# break-even depth differs by model.
|
||||
mtp_draft_depth: int = 3
|
||||
# Vocab size prices the GPU logits buffers (ubatch x vocab x fp32,
|
||||
# doubled under MTP backend sampling) — a multi-GiB term at large
|
||||
# vocab sizes that a weights-only fit would miss.
|
||||
n_vocab: int = 0
|
||||
mmproj: "AssetFile | None" = None # vision projector, downloads with model
|
||||
draft: "AssetFile | None" = None # spec-decode draft model (e.g. DSpark)
|
||||
sampling: dict = field(default_factory=dict) # INI long-form launch defaults
|
||||
# Oldest llama.cpp release tag that can load this model (day-0
|
||||
# architectures need the release where their support landed). Empty
|
||||
# means any installed engine. The pane gates download/activate on it.
|
||||
min_engine: str = ""
|
||||
# Editorial quality ordering (higher = smarter), authored once,
|
||||
# globally, at catalog-authoring time — Artificial Analysis-informed
|
||||
# where they cover the model (scripts/aa_quality_sync.py proposes,
|
||||
# the commit decides), editorial elsewhere. Ranks entries for the
|
||||
# per-machine recommendation; never displayed as a score (it grades
|
||||
# the full-precision model, not our Q4 build).
|
||||
quality: int = 0
|
||||
# Fraction of the build's bytes read per decoded token: 1.0 for dense
|
||||
# models (every weight streams every token), the active slice for MoE
|
||||
# (attention + shared + routed experts over total). With memory
|
||||
# bandwidth this predicts decode speed — the physics half of the
|
||||
# recommendation.
|
||||
decode_fraction: float = 1.0
|
||||
|
||||
def profile(self, variant: QuantVariant) -> ModelProfile:
|
||||
layers = ([(LayerKind.FULL, self.per_layer_f16)] * self.full_layers
|
||||
+ [(LayerKind.SWA, self.per_layer_f16)] * self.swa_layers
|
||||
+ [(LayerKind.RECURRENT, 0)] * self.recurrent_layers)
|
||||
return ModelProfile(
|
||||
name=variant.model_id, weights_bytes=variant.weights_bytes,
|
||||
embd_table_bytes=0, n_ctx_train=self.n_ctx_train,
|
||||
layers=layers, swa_window=self.swa_window, moe=self.moe,
|
||||
n_vocab=self.n_vocab,
|
||||
kv_scale=1.2 if self.mtp else 1.0)
|
||||
|
||||
def download_files(self, variant: QuantVariant) -> tuple:
|
||||
"""Everything a download job fetches for this variant, in order."""
|
||||
extras = tuple(a for a in (self.mmproj, self.draft) if a is not None)
|
||||
return tuple(variant.files) + extras
|
||||
|
||||
def download_bytes(self, variant: QuantVariant) -> int:
|
||||
return sum(f.size_bytes for f in self.download_files(variant))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VariantChoice:
|
||||
"""Selection result: which build this machine should download and why.
|
||||
reason_key is a UI-copy discriminator, not display text."""
|
||||
|
||||
variant: QuantVariant
|
||||
zero_spill: bool
|
||||
reason_key: str # "best-large-window" | "best-fits" | "smallest-fits-spilled"
|
||||
|
||||
|
||||
def select_variant(entry: CatalogEntry, budget: HardwareBudget) -> VariantChoice | None:
|
||||
"""Fit the entry's one build (Q4-class) to this machine.
|
||||
|
||||
Every entry ships exactly one variant (see the module docstring for
|
||||
why there is no quant ladder); headroom buys a bigger window, never
|
||||
a bigger quant. The fit shapes:
|
||||
|
||||
- "best-large-window": zero-spills at TARGET_WINDOW
|
||||
- "best-fits": zero-spills at the 64K floor
|
||||
- "smallest-fits-spilled": weights spill to host RAM, priced honestly
|
||||
- None: even spilled, physics refuses (the machine can't run it)
|
||||
"""
|
||||
overhead = (RUNTIME_OVERHEAD_BYTES
|
||||
+ (entry.mmproj.size_bytes if entry.mmproj else 0)
|
||||
+ ub_logits_bytes(entry.n_vocab, mtp_capable=entry.mtp))
|
||||
native = entry.n_ctx_train or FLOOR
|
||||
variant = entry.variants[-1]
|
||||
profile = entry.profile(variant)
|
||||
need = variant.weights_bytes + overhead
|
||||
if (need + ctx_bytes(profile, min(TARGET_WINDOW, native))
|
||||
<= budget.usable_vram_bytes):
|
||||
return VariantChoice(variant=variant, zero_spill=True,
|
||||
reason_key="best-large-window")
|
||||
floor_kv = ctx_bytes(profile, min(FLOOR, native))
|
||||
if need + floor_kv <= budget.usable_vram_bytes:
|
||||
return VariantChoice(variant=variant, zero_spill=True,
|
||||
reason_key="best-fits")
|
||||
if need + floor_kv <= budget.usable_vram_bytes + budget.ram_available_bytes:
|
||||
return VariantChoice(variant=variant, zero_spill=False,
|
||||
reason_key="smallest-fits-spilled")
|
||||
return None
|
||||
|
||||
|
||||
# ── recommendation: best quality that fits and isn't miserably slow ──
|
||||
#
|
||||
# Two axes, each living where it belongs. QUALITY is a judgment made once,
|
||||
# globally, at authoring time (entry.quality — AA-informed, editorially
|
||||
# owned). SPEED is physics computed per machine: decode is memory-bound,
|
||||
# so predicted tok/s ≈ bandwidth / bytes-read-per-token, and the bytes per
|
||||
# token are the build's size scaled by its decode fraction (dense reads
|
||||
# everything; MoE reads the active slice). The pick: highest quality among
|
||||
# entries that run resident and clear a pleasant speed floor; else the
|
||||
# fastest resident entry; else the least-painful spilled one.
|
||||
#
|
||||
# The bandwidth axis is the `uma` flag for now: every discrete card that
|
||||
# matters is 900+ GB/s GDDR while the unified-memory class measures ~1/5th
|
||||
# of that, so the flag IS the high/low split. A measured per-machine
|
||||
# bandwidth (one cached memcpy probe) can replace these class constants
|
||||
# without touching the rule; predictions order candidates and gate the
|
||||
# floor — they are not display values.
|
||||
|
||||
_DISCRETE_BANDWIDTH_GB_S = 1000.0 # representative GDDR6X/GDDR7 class
|
||||
_UMA_BANDWIDTH_GB_S = 210.0 # measured on unified-memory NVIDIA
|
||||
_HOST_BANDWIDTH_GB_S = 80.0 # spilled weights stream over host DRAM
|
||||
|
||||
# The one editorial constant in the tree: below this predicted decode
|
||||
# speed a model stops feeling pleasant for agentic use (roughly reading
|
||||
# speed with headroom for tool-call bursts). Distinct from the growth
|
||||
# policy's 6 tok/s compress floor, which marks unusable, not unpleasant.
|
||||
PLEASANT_FLOOR_TOK_S = 20.0
|
||||
|
||||
|
||||
def predicted_decode_tok_s(entry: CatalogEntry, variant: QuantVariant,
|
||||
budget: HardwareBudget, *,
|
||||
spilled: bool = False) -> float:
|
||||
"""Memory-bound decode prediction for ordering and floor-gating."""
|
||||
bandwidth = (_HOST_BANDWIDTH_GB_S if spilled
|
||||
else _UMA_BANDWIDTH_GB_S if budget.uma
|
||||
else _DISCRETE_BANDWIDTH_GB_S)
|
||||
bytes_per_token = max(1.0, variant.size_bytes * entry.decode_fraction)
|
||||
return bandwidth * 1e9 / bytes_per_token
|
||||
|
||||
|
||||
def recommended_entry(budget: HardwareBudget,
|
||||
entries: "tuple[CatalogEntry, ...] | None" = None
|
||||
) -> "tuple[CatalogEntry, str] | None":
|
||||
"""The catalog's default pick for THIS machine, with its reason.
|
||||
|
||||
Callers pass pre-filtered entries when some are ineligible for
|
||||
reasons the catalog can't know (engine too old); default is the full
|
||||
catalog. Returns (entry, reason) — the reason is a key the UI turns
|
||||
into the Recommended badge's tooltip, so the rationale shown to the
|
||||
user is the branch that actually fired, never a parallel explanation
|
||||
that can drift:
|
||||
|
||||
best-quality-resident quality won among resident entries that
|
||||
clear the pleasant floor
|
||||
speed-gated-quality same, but the floor eliminated a HIGHER
|
||||
quality candidate — the exact 'why not the
|
||||
big model?' a unified-memory owner asks
|
||||
fastest-resident nothing resident clears the floor; the
|
||||
quickest resident entry wins
|
||||
least-painful-spilled nothing runs resident; fastest from host
|
||||
memory (MoE by construction)
|
||||
|
||||
Returns None only when nothing fits at all.
|
||||
"""
|
||||
pool = CATALOG if entries is None else entries
|
||||
fitting: list[tuple[CatalogEntry, VariantChoice]] = []
|
||||
for entry in pool:
|
||||
choice = select_variant(entry, budget)
|
||||
if choice is not None:
|
||||
fitting.append((entry, choice))
|
||||
if not fitting:
|
||||
return None
|
||||
|
||||
resident = [(e, c) for e, c in fitting if c.zero_spill]
|
||||
pleasant = [
|
||||
(e, c) for e, c in resident
|
||||
if predicted_decode_tok_s(e, c.variant, budget) >= PLEASANT_FLOOR_TOK_S
|
||||
]
|
||||
if pleasant:
|
||||
pick = max(pleasant, key=lambda t: (t[0].quality, -t[1].variant.size_bytes))[0]
|
||||
floor_gated = any(e.quality > pick.quality for e, _ in resident)
|
||||
return (pick, "speed-gated-quality" if floor_gated
|
||||
else "best-quality-resident")
|
||||
if resident:
|
||||
pick = max(resident,
|
||||
key=lambda t: predicted_decode_tok_s(t[0], t[1].variant, budget))[0]
|
||||
return (pick, "fastest-resident")
|
||||
# Everything spills: take the least painful — fastest predicted decode
|
||||
# from host memory (MoE wins here by construction; a dense spill
|
||||
# streams every weight over the host bus).
|
||||
pick = max(fitting,
|
||||
key=lambda t: predicted_decode_tok_s(t[0], t[1].variant, budget,
|
||||
spilled=True))[0]
|
||||
return (pick, "least-painful-spilled")
|
||||
|
||||
|
||||
def recommended_id(budget: HardwareBudget,
|
||||
entries: "tuple[CatalogEntry, ...] | None" = None) -> str | None:
|
||||
picked = recommended_entry(budget, entries)
|
||||
return picked[0].id if picked is not None else None
|
||||
|
||||
|
||||
# ── catalog data: packaged JSON, refreshed from GitHub in memory ─
|
||||
#
|
||||
# The catalog DATA lives in catalog.json (checked in beside this module
|
||||
# and shipped as package data); this module keeps all policy. At import
|
||||
# we load the packaged copy — no network on the import path. A TTL-gated
|
||||
# background refresh fetches the same file from the repo's main branch
|
||||
# and swaps it in memory only: nothing on disk changes, so a git
|
||||
# checkout never sees a dirty tracked file and the packaged copy remains
|
||||
# the offline truth. A reverted commit on main heals every install on
|
||||
# its next fetch, and day-0 entries reach users without an app release.
|
||||
|
||||
_CATALOG_URL = ("https://raw.githubusercontent.com/NousResearch/hermes-agent"
|
||||
"/main/hermes_cli/local_runtime/catalog.json")
|
||||
_SCHEMA_VERSION = 1
|
||||
_REFRESH_TTL_S = 6 * 3600
|
||||
_refresh_lock = threading.Lock()
|
||||
_last_refresh_attempt = 0.0
|
||||
|
||||
|
||||
def _asset_from(d: "dict | None") -> "AssetFile | None":
|
||||
if not d:
|
||||
return None
|
||||
return AssetFile(path=d["path"], size_bytes=int(d["size_bytes"]),
|
||||
local=d.get("local"))
|
||||
|
||||
|
||||
def _load_catalog(doc: dict) -> "tuple[CatalogEntry, ...]":
|
||||
"""Parse a catalog document into entries. Unknown fields are ignored
|
||||
(newer catalogs stay readable by older apps); a major schema bump is
|
||||
the signal that they wouldn't be, and the caller skips the document."""
|
||||
if int(doc.get("schema_version", 0)) != _SCHEMA_VERSION:
|
||||
raise ValueError(f"catalog schema {doc.get('schema_version')!r} "
|
||||
f"(this build reads {_SCHEMA_VERSION})")
|
||||
entries = []
|
||||
for m in doc["models"]:
|
||||
variants = tuple(
|
||||
QuantVariant(quant=v["quant"],
|
||||
files=tuple(_asset_from(f) for f in v["files"]),
|
||||
validated=bool(v.get("validated")))
|
||||
for v in m["variants"])
|
||||
entries.append(CatalogEntry(
|
||||
id=m["id"], display_name=m["display_name"],
|
||||
description=m["description"], repo=m["repo"], variants=variants,
|
||||
n_ctx_train=int(m["n_ctx_train"]),
|
||||
full_layers=int(m["full_layers"]),
|
||||
recurrent_layers=int(m["recurrent_layers"]),
|
||||
per_layer_f16=int(m["per_layer_f16"]),
|
||||
swa_layers=int(m.get("swa_layers", 0)),
|
||||
swa_window=int(m.get("swa_window", 0)),
|
||||
moe=bool(m.get("moe")), mtp=bool(m.get("mtp")),
|
||||
mtp_draft_depth=int(m.get("mtp_draft_depth", 3)),
|
||||
n_vocab=int(m.get("n_vocab", 0)),
|
||||
mmproj=_asset_from(m.get("mmproj")),
|
||||
draft=_asset_from(m.get("draft")),
|
||||
sampling=dict(m.get("sampling", {})),
|
||||
min_engine=str(m.get("min_engine", "")),
|
||||
quality=int(m.get("quality", 0)),
|
||||
decode_fraction=float(m.get("decode_fraction", 1.0)),
|
||||
))
|
||||
return tuple(entries)
|
||||
|
||||
|
||||
def _packaged_catalog() -> "tuple[CatalogEntry, ...]":
|
||||
from importlib.resources import files
|
||||
|
||||
raw = files("hermes_cli.local_runtime").joinpath("catalog.json").read_text(
|
||||
encoding="utf-8")
|
||||
return _load_catalog(json.loads(raw))
|
||||
|
||||
|
||||
CATALOG: "tuple[CatalogEntry, ...]" = _packaged_catalog()
|
||||
|
||||
|
||||
def refresh_catalog(force: bool = False) -> bool:
|
||||
"""Fetch the current catalog from the repo and swap it in memory.
|
||||
|
||||
Best-effort by design: any failure (offline, GitHub down, unreadable
|
||||
schema) leaves the running catalog untouched and retries after the
|
||||
TTL. Returns True when a fetched document replaced the catalog."""
|
||||
global CATALOG, _last_refresh_attempt
|
||||
|
||||
now = time.monotonic()
|
||||
with _refresh_lock:
|
||||
if not force and now - _last_refresh_attempt < _REFRESH_TTL_S:
|
||||
return False
|
||||
_last_refresh_attempt = now
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
_CATALOG_URL, headers={"User-Agent": "hermes-local-runtime"})
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
fetched = _load_catalog(json.load(r))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("catalog refresh skipped: %s", exc)
|
||||
return False
|
||||
if fetched != CATALOG:
|
||||
logger.info("catalog refreshed from repo (%d models)", len(fetched))
|
||||
CATALOG = fetched
|
||||
return True
|
||||
|
||||
|
||||
def refresh_catalog_soon() -> None:
|
||||
"""TTL-gated background refresh; returns immediately. The caller's
|
||||
current request serves the catalog it already has — the refresh
|
||||
lands for the next one."""
|
||||
if time.monotonic() - _last_refresh_attempt < _REFRESH_TTL_S:
|
||||
return
|
||||
threading.Thread(target=refresh_catalog, daemon=True,
|
||||
name="catalog-refresh").start()
|
||||
|
||||
|
||||
def catalog_by_id() -> dict[str, CatalogEntry]:
|
||||
return {entry.id: entry for entry in CATALOG}
|
||||
|
||||
|
||||
def find_variant(entry_id: str, model_id: str) -> QuantVariant | None:
|
||||
entry = catalog_by_id().get(entry_id)
|
||||
if entry is None:
|
||||
return None
|
||||
return next((v for v in entry.variants if v.model_id == model_id), None)
|
||||
|
||||
|
||||
def find_entry_for_model(model_id: str) -> "tuple[CatalogEntry, QuantVariant] | None":
|
||||
"""Locate the entry + variant that owns a staged model id."""
|
||||
for entry in CATALOG:
|
||||
for variant in entry.variants:
|
||||
if variant.model_id == model_id:
|
||||
return entry, variant
|
||||
return None
|
||||
@@ -0,0 +1,286 @@
|
||||
"""Context policy — the window ladder for managed local models.
|
||||
|
||||
One contract: any model runs at any window up to its native max; hardware
|
||||
and session depth only change tokens/s. Constants, not knobs — nothing in
|
||||
this module reads config.
|
||||
|
||||
The policy encodes behavior measured on real hardware (llama.cpp,
|
||||
discrete NVIDIA GPUs on Windows/WDDM, and unified-memory devices):
|
||||
|
||||
- Windows never over-allocates VRAM ahead of need. On WDDM, allocating
|
||||
past residency slows decode roughly 9x even at identical conversation
|
||||
depth — the driver silently demotes pages instead of failing. Every
|
||||
window grant therefore re-fits against live memory at grant time.
|
||||
- Models launch at the largest window that fits entirely in GPU memory
|
||||
(zero-spill) and grow toward their native max as the session needs
|
||||
room, at request boundaries only.
|
||||
- Growth re-prefills the conversation into the larger window. Measured
|
||||
cost is comparable to save/restore on discrete GPUs, and recurrent or
|
||||
hybrid-attention models cannot rewind mid-sequence anyway, so
|
||||
re-prefill is the only mechanism that works for every architecture.
|
||||
- Every recommended model gets at least a 64K window. When weights alone
|
||||
exceed VRAM, the fit deliberately spills weights to host RAM to
|
||||
protect that floor (measured: an explicit context size makes the fit
|
||||
spill weights and hold the window rather than shrink it).
|
||||
- Below ~6 tok/s decode, growth stops and compression becomes the
|
||||
default; deeper context is an explicit per-session choice. The deepest
|
||||
measured host-spilled configuration bottomed out near this rate.
|
||||
- Spilled mixture-of-experts configs pin expert/FFN weights to host so
|
||||
attention and KV stay GPU-resident — measured ~1.75x faster than
|
||||
spilling layers naively at the same host byte count.
|
||||
- Speculative decoding (MTP) defaults on only for spilled configs, where
|
||||
its speedup is largest (measured 1.43x spilled vs 1.35x resident).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from hermes_cli.local_runtime.estimator import (
|
||||
HardwareBudget,
|
||||
ModelProfile,
|
||||
PhysicsRefusal,
|
||||
ctx_bytes,
|
||||
physics_check,
|
||||
)
|
||||
|
||||
FLOOR = 64 * 1024 # = target; one internal constant
|
||||
_LADDER_GROWTH = 1.5
|
||||
_GROW_AT_OCCUPANCY = 0.85 # of the current window, at turn boundary
|
||||
SPEED_FLOOR_TOK_S = 6.0 # deepest measured spill bottomed near this
|
||||
_EARLY_COST_CTX_FRACTION = 0.15 # bounded early cost when weights spill
|
||||
|
||||
# TARGET_WINDOW: the smallest ladder rung at which compression becomes the
|
||||
# exception rather than the routine. Measured over 161 real agentic
|
||||
# sessions: 66% complete uncompressed in 64K, 82% in 96K, 91% in 144K —
|
||||
# and the marginal gain past 144K (+6 points for 216K) falls below the
|
||||
# quality cost of stepping down another quant. Quant selection prefers
|
||||
# the best build that reaches this; the FLOOR remains the guarantee.
|
||||
TARGET_WINDOW = 144 * 1024
|
||||
|
||||
# What a load really costs beyond weights + KV: CUDA contexts and compute
|
||||
# buffers at the DEFAULT microbatch (-ub 512, no MTP). Measured on a
|
||||
# 32 GiB card: a model estimated at 29.3 GiB (weights+KV) loaded at
|
||||
# ~31.2 GiB resident and the server's own fit still shaved a layer to
|
||||
# CPU. Microbatch/MTP logits buffers are priced separately per model
|
||||
# (ub_logits_bytes — they scale with the model's vocab and doubled once
|
||||
# packed a card 3.9 GiB past this constant). Callers add mmproj bytes on
|
||||
# top.
|
||||
RUNTIME_OVERHEAD_BYTES = int(1.5 * (1 << 30))
|
||||
|
||||
|
||||
def ladder(native: int) -> list[int]:
|
||||
"""64K -> 96K -> 128K -> ... -> native (native always the last rung)."""
|
||||
rungs: list[int] = []
|
||||
step = float(FLOOR)
|
||||
while step < native:
|
||||
rungs.append(int(step))
|
||||
step *= _LADDER_GROWTH
|
||||
rungs.append(native)
|
||||
return rungs
|
||||
|
||||
|
||||
@dataclass
|
||||
class WindowDecision:
|
||||
window: int
|
||||
spill_bytes: int # weights displaced to host at this window
|
||||
kv_on_gpu: bool
|
||||
reasons: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def spilled(self) -> bool:
|
||||
return self.spill_bytes > 0
|
||||
|
||||
|
||||
def initial_window(profile: ModelProfile, budget: HardwareBudget,
|
||||
*, flash_attention: bool = True,
|
||||
overhead_bytes: int = 0) -> WindowDecision | PhysicsRefusal:
|
||||
"""The launch decision: largest cheap rung, never below the floor.
|
||||
|
||||
Zero-spill rung: weights + ctx + overhead fit usable VRAM entirely.
|
||||
Bounded-early-cost rung: weights already exceed VRAM; take the largest
|
||||
rung whose ctx stays <= ~15% of usable VRAM.
|
||||
Floor everywhere, capped at native.
|
||||
|
||||
``overhead_bytes``: runtime cost beyond weights+KV (RUNTIME_OVERHEAD
|
||||
plus the vision projector when one loads). Zero keeps this function
|
||||
pure physics for decision-table tests; production callers pass it.
|
||||
"""
|
||||
refusal = physics_check(profile, budget, FLOOR, flash_attention=flash_attention)
|
||||
if refusal:
|
||||
return refusal
|
||||
|
||||
native = profile.n_ctx_train or FLOOR
|
||||
rungs = ladder(native)
|
||||
|
||||
reasons: list[str] = []
|
||||
best_zero_spill: int | None = None
|
||||
for rung in rungs:
|
||||
need = (profile.weights_bytes + overhead_bytes
|
||||
+ ctx_bytes(profile, rung, flash_attention=flash_attention))
|
||||
if need <= budget.usable_vram_bytes:
|
||||
best_zero_spill = rung
|
||||
else:
|
||||
break
|
||||
|
||||
if best_zero_spill is not None and best_zero_spill >= min(FLOOR, native):
|
||||
window = best_zero_spill
|
||||
reasons.append(f"largest zero-spill rung ({window // 1024}K)")
|
||||
else:
|
||||
# Weights spill from turn one (steep-curve model on a small card) —
|
||||
# hold the floor, bound the early ctx cost.
|
||||
cap = int(budget.usable_vram_bytes * _EARLY_COST_CTX_FRACTION)
|
||||
window = min(FLOOR, native)
|
||||
for rung in rungs:
|
||||
if rung < window:
|
||||
continue
|
||||
if ctx_bytes(profile, rung, flash_attention=flash_attention) <= cap:
|
||||
window = rung
|
||||
else:
|
||||
break
|
||||
reasons.append(f"floor held at {window // 1024}K; weights spill (deliberate price of the guarantee)")
|
||||
|
||||
kv = ctx_bytes(profile, window, flash_attention=flash_attention)
|
||||
spill = max(0, profile.weights_bytes + kv - budget.usable_vram_bytes)
|
||||
return WindowDecision(window=window, spill_bytes=spill,
|
||||
kv_on_gpu=kv <= budget.usable_vram_bytes,
|
||||
reasons=reasons)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GrowthDecision:
|
||||
action: str # "grow" | "hold" | "compress-default"
|
||||
next_window: int | None = None
|
||||
reason: str = ""
|
||||
|
||||
|
||||
def growth_decision(profile: ModelProfile, budget: HardwareBudget, *,
|
||||
current_window: int, session_tokens: int,
|
||||
measured_decode_tok_s: float | None,
|
||||
server_idle: bool,
|
||||
flash_attention: bool = True,
|
||||
occupancy_confirmed: bool = False) -> GrowthDecision:
|
||||
"""One growth evaluation, END-OF-TURN ONLY (caller guarantees the turn
|
||||
boundary; recurrent state cannot rewind mid-sequence).
|
||||
|
||||
Gate ordering:
|
||||
1. occupancy (~85%) — nothing to do before the edge;
|
||||
2. native cap — the contract tops out at trained context;
|
||||
3. idleness — growth re-grants only on an otherwise-idle
|
||||
server (concurrency design);
|
||||
4. speed floor — below it, compression becomes the default and deeper
|
||||
is an explicit user choice;
|
||||
5. re-fit against LIVE free memory (the rung must fit residency
|
||||
NOW, not at launch time — over-allocation is the slow path).
|
||||
|
||||
``occupancy_confirmed``: the caller has independently established that
|
||||
the session is at its window's edge (the agent's compression gate fired
|
||||
on its own threshold). Skips gate 1 so two separately-derived edge
|
||||
definitions can't deadlock into compress-before-grow.
|
||||
"""
|
||||
if not occupancy_confirmed and session_tokens < current_window * _GROW_AT_OCCUPANCY:
|
||||
return GrowthDecision("hold", reason="session below growth occupancy")
|
||||
|
||||
native = profile.n_ctx_train or current_window
|
||||
if current_window >= native:
|
||||
return GrowthDecision("compress-default",
|
||||
reason="at native window; compression is the only move")
|
||||
|
||||
if not server_idle:
|
||||
return GrowthDecision("hold", reason="server busy; re-grant deferred to idle")
|
||||
|
||||
if measured_decode_tok_s is not None and measured_decode_tok_s < SPEED_FLOOR_TOK_S:
|
||||
return GrowthDecision(
|
||||
"compress-default",
|
||||
reason=(f"decode {measured_decode_tok_s:.1f} tok/s below the "
|
||||
f"~{SPEED_FLOOR_TOK_S:.0f} tok/s floor; growth is now an "
|
||||
"explicit per-session choice"))
|
||||
|
||||
next_rung = next((r for r in ladder(native) if r > current_window), native)
|
||||
|
||||
# Re-fit against live free memory: allocation beyond residency is the
|
||||
# slow path, so a rung that no longer fits doesn't get granted.
|
||||
kv = ctx_bytes(profile, next_rung, flash_attention=flash_attention)
|
||||
total_need = profile.weights_bytes + kv
|
||||
if total_need > budget.usable_vram_bytes + budget.ram_available_bytes:
|
||||
return GrowthDecision("compress-default",
|
||||
reason="next rung exceeds physics; compression instead")
|
||||
|
||||
return GrowthDecision("grow", next_window=next_rung,
|
||||
reason=f"rung {current_window // 1024}K -> {next_rung // 1024}K")
|
||||
|
||||
|
||||
def spill_overrides(profile: ModelProfile) -> list[str]:
|
||||
"""-ot placement for spilled configs: expert/FFN weights to host so
|
||||
attention + KV stay GPU-resident. MoE gets the expert pattern;
|
||||
hybrids push recurrent-layer FFNs (their n_head_kv==0 layers carry no
|
||||
KV worth protecting)."""
|
||||
if profile.moe:
|
||||
return ["-ot", r"blk\.\d+\.ffn_.*_exps\.weight=CPU"]
|
||||
if profile.recurrent_layer_count:
|
||||
return ["-ot", r"blk\.\d+\.ffn_.*\.weight=CPU"]
|
||||
return [] # dense: fit's back-to-front layer cut is the only axis
|
||||
|
||||
|
||||
def launch_args(profile: ModelProfile, decision: WindowDecision, *,
|
||||
flash_attention: bool = True,
|
||||
mtp_capable: bool = False,
|
||||
mtp_draft_depth: int = 3,
|
||||
uma: bool = False,
|
||||
mtp_prefill: bool = False) -> list[str]:
|
||||
"""Per-model launch flags from a window decision. Explicit -c puts fit
|
||||
into spill-weights-and-hold-ctx; q8 KV cache wherever flash attention
|
||||
exists; -ot placement on spilled configs — DISCRETE cards only.
|
||||
|
||||
``uma``: on unified memory there is no bus to protect tensors from —
|
||||
"CPU" and "GPU" are the same silicon, and pinning FFN weights to the
|
||||
host path just forces CPU compute (measured well over 2x slower than
|
||||
letting the allocator place everything). The discrete
|
||||
~1.75x win the -ot pattern encodes does not transfer; a spilled UMA
|
||||
config runs unpinned.
|
||||
|
||||
MTP and the large prefill microbatch both win, and whether they may
|
||||
STACK is a fit question, not a rule: backend sampling keeps a
|
||||
ubatch x vocab x fp32 logits buffer on the GPU and MTP's draft
|
||||
context doubles it, so the stacked posture costs a few GiB extra at
|
||||
large vocab. Where it fits, it measures best on both axes (Qwen3.8
|
||||
Q4 on a 32 GiB card: 93.3 tok/s decode vs 89.5 at ub512, prefill
|
||||
slightly better too); where it doesn't, ub512 keeps the decode win
|
||||
without packing the card. ``mtp_prefill`` is that fit verdict —
|
||||
presets decide it against the priced margin, and ub_logits_bytes()
|
||||
prices the same choice so the flag and its cost travel together."""
|
||||
args = ["-c", str(decision.window)]
|
||||
if mtp_capable:
|
||||
args += ["--spec-type", "draft-mtp",
|
||||
"--spec-draft-n-max", str(mtp_draft_depth),
|
||||
"--backend-sampling", "--spec-draft-backend-sampling"]
|
||||
if mtp_prefill:
|
||||
args += ["-b", "4096", "-ub", "2048"]
|
||||
else:
|
||||
args += ["-b", "2048", "-ub", "2048"]
|
||||
if flash_attention:
|
||||
args += ["-ctk", "q8_0", "-ctv", "q8_0", "-fa", "on"]
|
||||
if decision.spilled and not uma:
|
||||
args += spill_overrides(profile)
|
||||
return args
|
||||
|
||||
|
||||
def ub_logits_bytes(n_vocab: int, *, mtp_capable: bool,
|
||||
mtp_prefill: bool = False) -> int:
|
||||
"""GPU logits/compute-buffer cost of the microbatch posture chosen by
|
||||
launch_args, priced from the model's own vocab and calibrated against
|
||||
measured server RSS (Qwen3.8 Q4, both postures, three windows):
|
||||
|
||||
stacked (MTP + ub2048): ubatch x vocab x fp32 x 1.5 (~2.9 GiB at
|
||||
248K vocab; fitted 2.5, rounded up)
|
||||
decode (MTP + ub512): ubatch x vocab x fp32 x 2 (~1.0 GiB)
|
||||
plain (ub2048): ubatch x vocab x fp32 (~1.9 GiB)
|
||||
|
||||
Callers add this to RUNTIME_OVERHEAD per model — the flag and its
|
||||
price travel together or the fit lies."""
|
||||
v = max(0, int(n_vocab))
|
||||
if mtp_capable and mtp_prefill:
|
||||
return int(2048 * v * 4 * 1.5)
|
||||
if mtp_capable:
|
||||
return 512 * v * 4 * 2
|
||||
return 2048 * v * 4
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Detection of running llama-server instances.
|
||||
|
||||
Probes well-known local roots and fingerprints genuine llama-server via
|
||||
/props (build_info + model fields — Ollama and LM Studio answer /v1/models
|
||||
but not /props). The credential is reachability; detection never needs a
|
||||
key, but honors one if the probed server requires it (401 -> detected,
|
||||
auth_required=True).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Always 127.0.0.1 — resolving localhost costs ~2s/request on Windows.
|
||||
DEFAULT_PROBE_PORTS = (8080,) # llama-server default; managed port comes from config
|
||||
|
||||
|
||||
@dataclass
|
||||
class DetectedServer:
|
||||
base_url: str # OpenAI-compatible /v1 root
|
||||
build_info: str # e.g. "b10290-c8e03ce81"
|
||||
model_path: str # currently loaded model (may be empty in router mode)
|
||||
n_ctx: int | None
|
||||
router_mode: bool # GET /models answered -> router management available
|
||||
auth_required: bool
|
||||
|
||||
|
||||
def _get(url: str, timeout_s: int = 3) -> tuple[int, dict | None]:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=timeout_s) as r:
|
||||
raw = r.read()
|
||||
return r.status, (json.loads(raw) if raw else None)
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, None
|
||||
except (urllib.error.URLError, OSError, TimeoutError, json.JSONDecodeError):
|
||||
return 0, None
|
||||
|
||||
|
||||
def probe_port(port: int) -> DetectedServer | None:
|
||||
"""One port: /props fingerprint, then /models for router capability."""
|
||||
root = f"http://127.0.0.1:{port}"
|
||||
status, props = _get(f"{root}/props")
|
||||
if status == 401:
|
||||
return DetectedServer(base_url=f"{root}/v1", build_info="", model_path="",
|
||||
n_ctx=None, router_mode=False, auth_required=True)
|
||||
if status != 200 or not isinstance(props, dict):
|
||||
return None
|
||||
build = str(props.get("build_info", ""))
|
||||
if not build:
|
||||
return None # answers /props but isn't llama-server
|
||||
n_ctx = None
|
||||
dgs = props.get("default_generation_settings")
|
||||
if isinstance(dgs, dict):
|
||||
n_ctx = dgs.get("n_ctx")
|
||||
models_status, models = _get(f"{root}/models")
|
||||
return DetectedServer(
|
||||
base_url=f"{root}/v1",
|
||||
build_info=build,
|
||||
model_path=str(props.get("model_path", "")),
|
||||
n_ctx=n_ctx,
|
||||
router_mode=(models_status == 200 and isinstance(models, dict)
|
||||
and "data" in models),
|
||||
auth_required=False,
|
||||
)
|
||||
|
||||
|
||||
def detect_server(extra_ports: tuple[int, ...] = ()) -> DetectedServer | None:
|
||||
"""First hit across default + extra ports (managed port, config port)."""
|
||||
seen = set()
|
||||
for port in (*DEFAULT_PROBE_PORTS, *extra_ports):
|
||||
if port in seen:
|
||||
continue
|
||||
seen.add(port)
|
||||
hit = probe_port(port)
|
||||
if hit:
|
||||
return hit
|
||||
return None
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Endpoint resolution for llamacpp-alias requests (provider integration).
|
||||
|
||||
The seam between the existing provider mechanism and the managed runtime:
|
||||
``provider: llamacpp`` with no explicit base_url resolves, in order, to
|
||||
|
||||
1. the managed server this Hermes is supervising (state file written by
|
||||
LlamaServerSupervisor.start, removed on stop, staleness-checked), or
|
||||
2. a detected external llama-server.
|
||||
|
||||
Returns None when neither exists — the caller falls through to the normal
|
||||
custom-provider path and its own error reporting.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
LLAMACPP_ALIASES = frozenset({"llamacpp", "llama.cpp", "llama-cpp"})
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _pid_alive(pid: int) -> bool:
|
||||
"""Liveness for the state file's supervisor-child pid.
|
||||
|
||||
psutil when available; otherwise fall back to True (optimistic) — on
|
||||
Windows ``os.kill(pid, 0)`` TERMINATES the process, so it must never be
|
||||
used as a probe (windows-git-bash interop pitfall).
|
||||
"""
|
||||
if not pid or pid < 0:
|
||||
return False
|
||||
try:
|
||||
import psutil # type: ignore
|
||||
|
||||
return psutil.pid_exists(pid)
|
||||
except Exception: # noqa: BLE001
|
||||
return True
|
||||
|
||||
|
||||
def _state_endpoint() -> dict | None:
|
||||
from hermes_cli.local_runtime.supervisor import state_path
|
||||
|
||||
path = state_path()
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
state = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
base_url = state.get("base_url", "")
|
||||
if not base_url:
|
||||
return None
|
||||
endpoint = {"base_url": base_url, "api_key": state.get("api_key", "")}
|
||||
# Ownership proof: the stable port means a SECOND install (different
|
||||
# HERMES_HOME — a scratch profile, say) can own 127.0.0.1:18434 with a
|
||||
# different api key while this install's state file still points there.
|
||||
# /health is a public route, so it answers 200 for ANYONE's server —
|
||||
# trusting it alone sent every chat request and the load-progress
|
||||
# watcher at a server that 401s our key, silently. The recorded
|
||||
# supervisor pid is the tiebreaker: health-200 from a server whose
|
||||
# recorded child is DEAD is someone else's server, never a starting one.
|
||||
pid_ok = _pid_alive(int(state.get("pid") or 0))
|
||||
# Healthy server: done (when it's ours).
|
||||
try:
|
||||
health = base_url.rsplit("/v1", 1)[0] + "/health"
|
||||
with urllib.request.urlopen(health, timeout=3) as r:
|
||||
if r.status == 200:
|
||||
return endpoint if pid_ok else None
|
||||
except (urllib.error.URLError, OSError, TimeoutError):
|
||||
pass
|
||||
# Not healthy YET: a live supervisor child is a STARTING server (state
|
||||
# is written at spawn; llama-server takes seconds to listen). Resolve
|
||||
# optimistically so readiness probes racing the boot see a configured
|
||||
# provider, not missing credentials. A dead pid is a crashed-without-
|
||||
# cleanup leftover — ignore it so requests don't blackhole.
|
||||
if pid_ok:
|
||||
return endpoint
|
||||
return None
|
||||
|
||||
|
||||
def resolve_llamacpp_endpoint(config: dict | None = None,
|
||||
wait_for_boot_s: float = 8.0) -> dict | None:
|
||||
"""Managed-first, detection-second endpoint for llamacpp aliases.
|
||||
|
||||
Returns {"base_url", "api_key"} or None. api_key is empty for keyless
|
||||
external servers (callers substitute the SDK placeholder).
|
||||
|
||||
Boot-race rung: on a fresh backend start there is NO state file yet —
|
||||
the lifespan boot thread is still spawning the server (config load +
|
||||
preset generation + spawn ≈ 1-3 s) while the desktop's readiness probe
|
||||
fires the moment the WebSocket connects. When the runtime is enabled
|
||||
and installed, a missing endpoint means BOOTING, not unconfigured:
|
||||
poll briefly for the state file instead of failing the probe (twice
|
||||
observed as 'no usable credentials' → onboarding on restart).
|
||||
"""
|
||||
managed = _state_endpoint()
|
||||
if managed:
|
||||
return managed
|
||||
|
||||
from hermes_cli.local_runtime.detect import detect_server
|
||||
|
||||
extra = ()
|
||||
if config:
|
||||
ports = (config.get("local_runtime") or {}).get("detect_ports") or []
|
||||
extra = tuple(int(p) for p in ports)
|
||||
hit = detect_server(extra_ports=extra)
|
||||
if hit and not hit.auth_required:
|
||||
return {"base_url": hit.base_url, "api_key": ""}
|
||||
|
||||
if wait_for_boot_s > 0 and _boot_in_flight(config):
|
||||
_kick_managed_boot(config)
|
||||
deadline = time.monotonic() + wait_for_boot_s
|
||||
while time.monotonic() < deadline:
|
||||
time.sleep(0.25)
|
||||
managed = _state_endpoint()
|
||||
if managed:
|
||||
return managed
|
||||
return None
|
||||
|
||||
|
||||
_KICK_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _kick_managed_boot(config: dict | None) -> None:
|
||||
"""Actively start the managed server when resolution finds it missing.
|
||||
|
||||
The wait loop above assumes some OTHER thread is bringing the server
|
||||
up — true only at backend start (the lifespan boot thread). A router
|
||||
that dies LATER leaves no boot in flight: the backend process was
|
||||
killed with the router as part of its tree, or another install took
|
||||
the stable port and the ownership guard rightly refused it. In those
|
||||
states the wait just expired and agent init failed with 'no provider
|
||||
configured', even though the fix is the same idempotent ensure call
|
||||
the lifespan makes. Kick it here, off-thread (the resolver's wait
|
||||
stays bounded; ensure's own state checks make a concurrent lifespan
|
||||
boot harmless) and non-reentrant (racing resolutions kick once).
|
||||
"""
|
||||
if not _KICK_LOCK.acquire(blocking=False):
|
||||
return # a kick is already in flight
|
||||
|
||||
def _boot() -> None:
|
||||
try:
|
||||
cfg = config
|
||||
if cfg is None:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
cfg = load_config()
|
||||
from hermes_cli.local_runtime.bootstrap import ensure_local_runtime
|
||||
|
||||
ensure_local_runtime(cfg)
|
||||
except Exception: # noqa: BLE001 — best-effort; resolution falls back
|
||||
logger.warning("on-demand managed-server boot failed", exc_info=True)
|
||||
finally:
|
||||
_KICK_LOCK.release()
|
||||
|
||||
threading.Thread(target=_boot, daemon=True,
|
||||
name="lr-on-demand-boot").start()
|
||||
|
||||
|
||||
def _boot_in_flight(config: dict | None) -> bool:
|
||||
"""True when the managed runtime is enabled and installed — the state
|
||||
a lifespan boot thread is (or is about to be) bringing up.
|
||||
|
||||
Installed-ness is a verified-manifest scan under runtimes_root(), NOT a
|
||||
server_binary() call — that helper requires an install_dir argument, and
|
||||
calling it bare made this gate throw-and-return-False forever, silently
|
||||
disabling the boot wait (the regression
|
||||
test had monkeypatched this function instead of exercising it).
|
||||
"""
|
||||
try:
|
||||
if config is None:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
config = load_config()
|
||||
if not ((config or {}).get("local_runtime") or {}).get("enabled"):
|
||||
return False
|
||||
import json as _json
|
||||
|
||||
from hermes_cli.local_runtime.binaries import runtimes_root
|
||||
|
||||
for manifest in runtimes_root().glob("*/*/manifest.json"):
|
||||
try:
|
||||
if _json.loads(manifest.read_text(encoding="utf-8")).get("verified_version"):
|
||||
return True
|
||||
except (ValueError, OSError):
|
||||
continue
|
||||
return False
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Per-layer context-memory estimator + physics check.
|
||||
|
||||
The whole-model dense formula misprices 1M-context hybrids by ~100x; the
|
||||
per-layer walk fixes that, and every column is measured on real GGUFs:
|
||||
|
||||
- full-attention layer: linear in T (B1: 144.0 KiB/tok on Qwen3-4B
|
||||
f16 — formula-exact)
|
||||
- SWA layer: capped at the sliding window
|
||||
- recurrent layer (n_head_kv == 0): constant (state is ~context-free)
|
||||
- q8_0 KV = exactly 34/64 of f16 (holds on CUDA and CPU)
|
||||
- weights: exact from the tensor table (within 0.01% of the loader)
|
||||
|
||||
The estimator is ADVISORY: fit's allocation is authoritative at launch and
|
||||
the touch generation is ground truth after it. Unknown shapes round UP
|
||||
(never underestimate memory).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
from hermes_cli.local_runtime.gguf import GGUFHeader
|
||||
|
||||
# q8_0: 34-byte blocks of 32 f16-equivalent elements (exact).
|
||||
_Q8_BYTES_PER_ELEM = 34 / 32
|
||||
_F16_BYTES_PER_ELEM = 2.0
|
||||
|
||||
# Architectures with a known SWA layer pattern: arch -> fraction of layers
|
||||
# that are sliding-window. Unknown SWA archs conservatively treat every
|
||||
# layer as full attention (overestimate; safe direction).
|
||||
_SWA_LAYER_FRACTION = {"gemma3": 5 / 6, "gemma2": 1 / 2}
|
||||
|
||||
# Per-recurrent-layer state allowance (bytes/seq). Deliberately generous —
|
||||
# Measured: an entire hybrid slot state is ~99 MB including 8K tokens of
|
||||
# full-attn KV, so tens of MiB total is the right order; unknown SSM shapes
|
||||
# must never underestimate.
|
||||
_RECURRENT_STATE_PER_LAYER = 4 << 20
|
||||
|
||||
|
||||
class LayerKind(Enum):
|
||||
FULL = "full"
|
||||
SWA = "swa"
|
||||
RECURRENT = "recurrent"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelProfile:
|
||||
"""Everything the policy needs, decoupled from GGUF parsing so the
|
||||
decision-table tests can construct profiles directly (design's
|
||||
verification plan)."""
|
||||
|
||||
name: str
|
||||
weights_bytes: int
|
||||
embd_table_bytes: int
|
||||
n_ctx_train: int
|
||||
layers: list[tuple[LayerKind, int]] # (kind, kv_bytes_per_token_f16);
|
||||
# SWA/recurrent reuse the same
|
||||
# per-token figure, capped/ignored
|
||||
swa_window: int = 0
|
||||
moe: bool = False
|
||||
architecture: str = ""
|
||||
n_vocab: int = 0 # prices logits buffers (ubatch x vocab)
|
||||
# Context-cost multiplier. MTP spec decode keeps a small draft
|
||||
# context beside the main one. Calibrated against four measured
|
||||
# server-RSS points on Qwen3.8 Q4 (128K/221K/256K, both postures):
|
||||
# the draft adds ~17% to per-token KV; 1.2 rounds up so the error
|
||||
# stays on the safe side (+250 MiB at 256K, never negative).
|
||||
kv_scale: float = 1.0
|
||||
|
||||
@property
|
||||
def per_token_kv_f16(self) -> int:
|
||||
"""Uncapped per-token KV cost (full + SWA share)."""
|
||||
return sum(b for kind, b in self.layers if kind != LayerKind.RECURRENT)
|
||||
|
||||
@property
|
||||
def recurrent_layer_count(self) -> int:
|
||||
return sum(1 for kind, _ in self.layers if kind == LayerKind.RECURRENT)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HardwareBudget:
|
||||
"""Memory the physics check may budget against.
|
||||
|
||||
Budget-source rule: discrete cards may trust the device query
|
||||
(measured honest); unified-memory devices must budget from OS free
|
||||
physical memory minus headroom — their device queries have been
|
||||
observed off by 3x. Callers construct
|
||||
this accordingly; the estimator just consumes it.
|
||||
"""
|
||||
|
||||
usable_vram_bytes: int # live free (discrete) / derived (UMA)
|
||||
total_device_bytes: int
|
||||
ram_available_bytes: int
|
||||
uma: bool = False
|
||||
|
||||
|
||||
def profile_from_gguf(header: GGUFHeader) -> ModelProfile:
|
||||
kv_heads = header.head_counts_kv()
|
||||
dk, dv = header.head_dim_k, header.head_dim_v
|
||||
swa_fraction = _SWA_LAYER_FRACTION.get(header.architecture, 0.0)
|
||||
has_swa = header.sliding_window > 0 and swa_fraction > 0
|
||||
|
||||
layers: list[tuple[LayerKind, int]] = []
|
||||
n_attn_seen = 0
|
||||
n_attn_total = sum(1 for h in kv_heads if h > 0)
|
||||
n_swa = round(n_attn_total * swa_fraction) if has_swa else 0
|
||||
for heads in kv_heads:
|
||||
if heads == 0:
|
||||
layers.append((LayerKind.RECURRENT, 0))
|
||||
continue
|
||||
per_token = round(heads * (dk + dv) * _F16_BYTES_PER_ELEM)
|
||||
# Distribute the SWA share across the first n_swa attention layers;
|
||||
# only the full/SWA SPLIT matters to the totals, not which indexes.
|
||||
kind = LayerKind.SWA if n_attn_seen < n_swa else LayerKind.FULL
|
||||
layers.append((kind, per_token))
|
||||
n_attn_seen += 1
|
||||
|
||||
return ModelProfile(
|
||||
name=header.path,
|
||||
weights_bytes=header.tensor_bytes,
|
||||
embd_table_bytes=header.embd_table_bytes,
|
||||
n_ctx_train=header.n_ctx_train,
|
||||
layers=layers,
|
||||
swa_window=header.sliding_window,
|
||||
moe=header.expert_count > 0,
|
||||
architecture=header.architecture,
|
||||
n_vocab=header.n_vocab,
|
||||
)
|
||||
|
||||
|
||||
def kv_dtype_factor(flash_attention: bool) -> float:
|
||||
"""q8_0 with FA (every backend we ship); f16 on exotic non-FA fallbacks
|
||||
— the 64K guarantee stands either way, the physics check just prices
|
||||
the doubled KV (design: KV dtype is behavior, not config)."""
|
||||
return (_Q8_BYTES_PER_ELEM / _F16_BYTES_PER_ELEM) if flash_attention else 1.0
|
||||
|
||||
|
||||
def ctx_bytes(profile: ModelProfile, window: int, *,
|
||||
flash_attention: bool = True) -> int:
|
||||
"""Context memory for one window: full layers linear in T, SWA layers
|
||||
capped at the sliding window, recurrent layers constant. Scaled by
|
||||
profile.kv_scale (MTP draft context)."""
|
||||
factor = kv_dtype_factor(flash_attention)
|
||||
total = 0.0
|
||||
for kind, per_token_f16 in profile.layers:
|
||||
if kind == LayerKind.RECURRENT:
|
||||
total += _RECURRENT_STATE_PER_LAYER
|
||||
elif kind == LayerKind.SWA:
|
||||
total += per_token_f16 * factor * min(window, profile.swa_window)
|
||||
else:
|
||||
total += per_token_f16 * factor * window
|
||||
return int(total * profile.kv_scale)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PhysicsRefusal:
|
||||
"""The only true refusal: weights + floor-KV + state exceed VRAM + RAM.
|
||||
The remedy is a smaller quant, never a smaller window."""
|
||||
|
||||
needed_bytes: int
|
||||
available_bytes: int
|
||||
message: str
|
||||
|
||||
|
||||
def physics_check(profile: ModelProfile, budget: HardwareBudget,
|
||||
floor: int, *, flash_attention: bool = True) -> PhysicsRefusal | None:
|
||||
needed = (profile.weights_bytes
|
||||
+ ctx_bytes(profile, min(floor, profile.n_ctx_train or floor),
|
||||
flash_attention=flash_attention))
|
||||
available = budget.usable_vram_bytes + budget.ram_available_bytes
|
||||
if needed > available:
|
||||
gib = 1 << 30
|
||||
return PhysicsRefusal(
|
||||
needed_bytes=needed, available_bytes=available,
|
||||
message=(f"{profile.name}: needs ~{needed / gib:.1f} GiB at the "
|
||||
f"{floor // 1024}K floor but only ~{available / gib:.1f} GiB "
|
||||
"of VRAM+RAM exist — try a smaller quant (UD-Q3/Q2)"))
|
||||
return None
|
||||
@@ -0,0 +1,220 @@
|
||||
"""GGUF metadata + tensor-table reader (stdlib only).
|
||||
|
||||
Feeds the per-layer context estimator: architecture, layer count, per-layer
|
||||
KV head counts (0 = recurrent layer — the hybrid discriminator), head dims,
|
||||
sliding-window config, trained context, and exact weight bytes summed from
|
||||
the tensor table (validated to within 0.01% of the loader's buffer).
|
||||
|
||||
Reads the header only (metadata + tensor infos); never touches tensor data,
|
||||
so it is fast enough to run at picker time on multi-GB files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
_GGUF_MAGIC = b"GGUF"
|
||||
|
||||
# ggml tensor type sizes: type_id -> (block_bytes, block_elems).
|
||||
# IQ-family sizes verified against ggml-common.h.
|
||||
_GGML_TYPE_SIZES = {
|
||||
0: (4, 1), 1: (2, 1), 2: (18, 32), 3: (20, 32), 6: (22, 32), 7: (24, 32),
|
||||
8: (34, 32), 9: (36, 32), 10: (84, 256), 11: (110, 256), 12: (144, 256),
|
||||
13: (176, 256), 14: (210, 256), 15: (292, 256), 16: (66, 256),
|
||||
17: (74, 256), 18: (98, 256), 19: (50, 256), 20: (18, 32),
|
||||
21: (110, 256), 22: (82, 256), 23: (136, 256), 24: (1, 1), 25: (2, 1),
|
||||
26: (4, 1), 27: (8, 1), 28: (8, 1), 29: (56, 256), 30: (2, 1),
|
||||
}
|
||||
|
||||
# GGUF metadata value types.
|
||||
_V_UINT8, _V_INT8, _V_UINT16, _V_INT16 = 0, 1, 2, 3
|
||||
_V_UINT32, _V_INT32, _V_FLOAT32, _V_BOOL = 4, 5, 6, 7
|
||||
_V_STRING, _V_ARRAY, _V_UINT64, _V_INT64, _V_FLOAT64 = 8, 9, 10, 11, 12
|
||||
|
||||
_SCALAR_FMT = {
|
||||
_V_UINT8: "<B", _V_INT8: "<b", _V_UINT16: "<H", _V_INT16: "<h",
|
||||
_V_UINT32: "<I", _V_INT32: "<i", _V_FLOAT32: "<f", _V_BOOL: "<?",
|
||||
_V_UINT64: "<Q", _V_INT64: "<q", _V_FLOAT64: "<d",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class GGUFHeader:
|
||||
path: str
|
||||
version: int
|
||||
metadata: dict = field(default_factory=dict)
|
||||
n_tensors: int = 0
|
||||
tensor_bytes: int = 0 # exact sum over the tensor table
|
||||
embd_table_bytes: int = 0 # token_embd.weight (duplicated host-side
|
||||
# when fully offloaded)
|
||||
|
||||
# ── typed accessors ──────────────────────────────────────
|
||||
|
||||
@property
|
||||
def architecture(self) -> str:
|
||||
return str(self.metadata.get("general.architecture", ""))
|
||||
|
||||
def _arch_key(self, suffix: str):
|
||||
return self.metadata.get(f"{self.architecture}.{suffix}")
|
||||
|
||||
@property
|
||||
def n_layer(self) -> int:
|
||||
return int(self._arch_key("block_count") or 0)
|
||||
|
||||
@property
|
||||
def n_vocab(self) -> int:
|
||||
"""Vocabulary size: prices the GPU logits buffers (they scale
|
||||
ubatch x vocab). vocab_size metadata when present, else the
|
||||
tokenizer list length."""
|
||||
v = self._arch_key("vocab_size")
|
||||
if v:
|
||||
return int(v)
|
||||
toks = self.metadata.get("tokenizer.ggml.tokens")
|
||||
return len(toks) if isinstance(toks, list) else 0
|
||||
|
||||
@property
|
||||
def n_ctx_train(self) -> int:
|
||||
return int(self._arch_key("context_length") or 0)
|
||||
|
||||
@property
|
||||
def sampling_defaults(self) -> dict:
|
||||
"""Upstream's recommended sampling, when the file carries it.
|
||||
|
||||
Model publishers bake general.sampling.* keys into the GGUF
|
||||
(llama-server reads them as that model's default generation
|
||||
settings), so the file itself is the source of truth for how its
|
||||
publisher wants it run — it arrives with the download and updates
|
||||
with every re-upload, no catalog required. Returned as preset INI
|
||||
keys; empty when the file carries none.
|
||||
"""
|
||||
ini_key = {"temp": "temp", "temperature": "temp", "top_p": "top-p",
|
||||
"top_k": "top-k", "min_p": "min-p",
|
||||
"repeat_penalty": "repeat-penalty",
|
||||
"presence_penalty": "presence-penalty"}
|
||||
out = {}
|
||||
for key, value in self.metadata.items():
|
||||
if not key.startswith("general.sampling."):
|
||||
continue
|
||||
name = ini_key.get(key.rsplit(".", 1)[-1])
|
||||
if name is not None and isinstance(value, (int, float)):
|
||||
num = round(float(value), 4)
|
||||
out[name] = str(int(num)) if num == int(num) else str(num)
|
||||
return out
|
||||
|
||||
@property
|
||||
def n_embd(self) -> int:
|
||||
return int(self._arch_key("embedding_length") or 0)
|
||||
|
||||
@property
|
||||
def n_head(self) -> int:
|
||||
v = self._arch_key("attention.head_count")
|
||||
if isinstance(v, list):
|
||||
return int(max(v))
|
||||
return int(v or 0)
|
||||
|
||||
@property
|
||||
def full_attention_interval(self) -> int:
|
||||
"""GDN-hybrid discriminator (qwen35 family): every Nth layer is full
|
||||
attention, the rest are linear/recurrent. 0 = not present."""
|
||||
return int(self._arch_key("full_attention_interval") or 0)
|
||||
|
||||
def head_counts_kv(self) -> list[int]:
|
||||
"""Per-layer KV head counts; 0 marks a recurrent/linear layer (the
|
||||
n_head_kv == 0 discriminator).
|
||||
|
||||
Three GGUF shapes, each verified against real files:
|
||||
- per-layer array (nemotron_h_moe): use as-is;
|
||||
- scalar + full_attention_interval (qwen35): the scalar applies to
|
||||
every INTERVAL-th layer (1-indexed: layers where (i+1) % N == 0),
|
||||
zero elsewhere — pricing all layers as attention was a 4x
|
||||
overestimate on Qwen3.6-27B;
|
||||
- plain scalar (dense): broadcast to every layer.
|
||||
"""
|
||||
v = self._arch_key("attention.head_count_kv")
|
||||
if isinstance(v, list):
|
||||
return [int(x) for x in v]
|
||||
scalar = int(v or 0)
|
||||
interval = self.full_attention_interval
|
||||
if interval > 1:
|
||||
return [scalar if (i + 1) % interval == 0 else 0
|
||||
for i in range(self.n_layer)]
|
||||
return [scalar] * self.n_layer
|
||||
|
||||
@property
|
||||
def head_dim_k(self) -> int:
|
||||
v = self._arch_key("attention.key_length")
|
||||
if v:
|
||||
return int(v)
|
||||
return self.n_embd // self.n_head if self.n_head else 0
|
||||
|
||||
@property
|
||||
def head_dim_v(self) -> int:
|
||||
v = self._arch_key("attention.value_length")
|
||||
if v:
|
||||
return int(v)
|
||||
return self.head_dim_k
|
||||
|
||||
@property
|
||||
def sliding_window(self) -> int:
|
||||
return int(self._arch_key("attention.sliding_window") or 0)
|
||||
|
||||
@property
|
||||
def expert_count(self) -> int:
|
||||
return int(self._arch_key("expert_count") or 0)
|
||||
|
||||
|
||||
def read_gguf_header(path: str | Path) -> GGUFHeader:
|
||||
path = Path(path)
|
||||
|
||||
def read_str(f) -> str:
|
||||
(n,) = struct.unpack("<Q", f.read(8))
|
||||
return f.read(n).decode("utf-8", errors="replace")
|
||||
|
||||
def read_value(f, vtype: int):
|
||||
if vtype == _V_STRING:
|
||||
return read_str(f)
|
||||
if vtype == _V_ARRAY:
|
||||
(etype,) = struct.unpack("<I", f.read(4))
|
||||
(n,) = struct.unpack("<Q", f.read(8))
|
||||
return [read_value(f, etype) for _ in range(n)]
|
||||
fmt = _SCALAR_FMT[vtype]
|
||||
(value,) = struct.unpack(fmt, f.read(struct.calcsize(fmt)))
|
||||
return value
|
||||
|
||||
with open(path, "rb") as f:
|
||||
if f.read(4) != _GGUF_MAGIC:
|
||||
raise ValueError(f"not a GGUF file: {path}")
|
||||
(version,) = struct.unpack("<I", f.read(4))
|
||||
n_tensors, n_kv = struct.unpack("<QQ", f.read(16))
|
||||
|
||||
metadata: dict = {}
|
||||
for _ in range(n_kv):
|
||||
key = read_str(f)
|
||||
(vtype,) = struct.unpack("<I", f.read(4))
|
||||
metadata[key] = read_value(f, vtype)
|
||||
|
||||
tensor_bytes = 0
|
||||
embd_bytes = 0
|
||||
for _ in range(n_tensors):
|
||||
name = read_str(f)
|
||||
(n_dims,) = struct.unpack("<I", f.read(4))
|
||||
dims = struct.unpack(f"<{n_dims}Q", f.read(8 * n_dims))
|
||||
(ttype,) = struct.unpack("<I", f.read(4))
|
||||
f.read(8) # offset
|
||||
size = _GGML_TYPE_SIZES.get(ttype)
|
||||
if size is None:
|
||||
raise ValueError(f"unknown ggml tensor type {ttype} in {path}")
|
||||
block_bytes, block_elems = size
|
||||
elems = 1
|
||||
for d in dims:
|
||||
elems *= d
|
||||
nbytes = (elems // block_elems) * block_bytes
|
||||
tensor_bytes += nbytes
|
||||
if name == "token_embd.weight":
|
||||
embd_bytes = nbytes
|
||||
|
||||
return GGUFHeader(path=str(path), version=version, metadata=metadata,
|
||||
n_tensors=n_tensors, tensor_bytes=tensor_bytes,
|
||||
embd_table_bytes=embd_bytes)
|
||||
@@ -0,0 +1,143 @@
|
||||
"""In-session context growth for the managed llama.cpp runtime.
|
||||
|
||||
The live half of the window ladder (context_policy.growth_decision): when a
|
||||
session reaches the edge of its granted window, Hermes grows the window
|
||||
toward the model's native max INSTEAD of compressing. Compression becomes
|
||||
what the design says it is — the move of last resort, once the window is at
|
||||
native (or the speed floor / physics say stop).
|
||||
|
||||
Mechanism: growth is re-prefill. A per-model window
|
||||
override is persisted, presets regenerate with the bigger window, the
|
||||
supervised server bounces, and the next request autoloads the model at the
|
||||
new window and re-prefills the conversation. Nothing about the Hermes
|
||||
conversation mutates — no prompt-cache or role-alternation risk; the whole
|
||||
operation is server-side.
|
||||
|
||||
Scope guard: only a server THIS process supervises grows. Detected external
|
||||
servers and other-process supervisors keep their own policies.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def window_overrides_path():
|
||||
from hermes_cli.local_runtime.binaries import runtimes_root
|
||||
|
||||
return runtimes_root() / "window_overrides.json"
|
||||
|
||||
|
||||
def load_window_overrides() -> dict:
|
||||
"""model_id -> granted window (int). Empty on any read problem."""
|
||||
try:
|
||||
with open(window_overrides_path(), encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
return {str(k): int(v) for k, v in data.items()}
|
||||
except Exception: # noqa: BLE001
|
||||
return {}
|
||||
|
||||
|
||||
def save_window_override(model_id: str, window: int) -> None:
|
||||
overrides = load_window_overrides()
|
||||
overrides[model_id] = int(window)
|
||||
path = window_overrides_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(overrides, indent=1), encoding="utf-8")
|
||||
|
||||
|
||||
def clear_window_override(model_id: str) -> None:
|
||||
"""Drop a model's growth state (delete/re-download paths)."""
|
||||
overrides = load_window_overrides()
|
||||
if model_id in overrides:
|
||||
del overrides[model_id]
|
||||
window_overrides_path().write_text(
|
||||
json.dumps(overrides, indent=1), encoding="utf-8")
|
||||
|
||||
|
||||
def is_managed_endpoint(base_url: str) -> bool:
|
||||
"""True when base_url is the server this process's state file points at."""
|
||||
try:
|
||||
from hermes_cli.local_runtime.endpoint import _state_endpoint
|
||||
|
||||
state = _state_endpoint()
|
||||
if state is None:
|
||||
return False
|
||||
return (base_url or "").rstrip("/") == str(
|
||||
state.get("base_url", "")).rstrip("/")
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
|
||||
def maybe_grow_window(model_id: str, *, base_url: str, session_tokens: int,
|
||||
current_window: int,
|
||||
measured_decode_tok_s: float | None = None) -> int | None:
|
||||
"""One growth evaluation + execution. Returns the NEW window when the
|
||||
ladder granted a bigger one, else None (hold / compress / not ours).
|
||||
|
||||
The caller sits at a request boundary by construction (the pre-API
|
||||
compression gate), so re-prefill growth is safe at any call: the next
|
||||
request rebuilds server state from scratch in the larger window —
|
||||
nothing rewinds.
|
||||
"""
|
||||
from hermes_cli.local_runtime.bootstrap import (
|
||||
get_supervisor,
|
||||
refresh_local_runtime,
|
||||
staged_models,
|
||||
)
|
||||
from hermes_cli.local_runtime.context_policy import growth_decision
|
||||
from hermes_cli.local_runtime.estimator import profile_from_gguf
|
||||
from hermes_cli.local_runtime.gguf import read_gguf_header
|
||||
from hermes_cli.local_runtime.hardware import probe_budget
|
||||
|
||||
sup = get_supervisor()
|
||||
if sup is None or not is_managed_endpoint(base_url):
|
||||
return None
|
||||
|
||||
gguf = next((p for p in staged_models()
|
||||
if p.stem.startswith(model_id) or model_id in p.stem), None)
|
||||
if gguf is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
profile = profile_from_gguf(read_gguf_header(gguf))
|
||||
except (ValueError, OSError) as exc:
|
||||
logger.debug("growth skip %s: unreadable gguf (%s)", model_id, exc)
|
||||
return None
|
||||
|
||||
try:
|
||||
server_idle = sup.is_idle(model_id)
|
||||
except Exception: # noqa: BLE001
|
||||
server_idle = False
|
||||
|
||||
decision = growth_decision(
|
||||
# Capacity budget, not live-free: growth executes via a server
|
||||
# bounce, so the grown instance loads onto a freed card. Live-free
|
||||
# here is distorted by the very model being grown — it reads its
|
||||
# own residency as unavailable and vetoes rungs that fit.
|
||||
profile, probe_budget(planning=True),
|
||||
current_window=current_window,
|
||||
session_tokens=session_tokens,
|
||||
measured_decode_tok_s=measured_decode_tok_s,
|
||||
server_idle=server_idle,
|
||||
# The caller IS the occupancy signal: this runs from the agent's
|
||||
# compression gate, which fired on its own threshold. Two
|
||||
# separately-derived edges must not deadlock into
|
||||
# compress-before-grow.
|
||||
occupancy_confirmed=True,
|
||||
)
|
||||
if decision.action != "grow" or not decision.next_window:
|
||||
logger.debug("growth %s: %s (%s)", model_id, decision.action, decision.reason)
|
||||
return None
|
||||
|
||||
logger.info("context growth %s: %s", model_id, decision.reason)
|
||||
save_window_override(model_id, decision.next_window)
|
||||
if not refresh_local_runtime():
|
||||
# The override still lands at the next boot; report no growth NOW
|
||||
# so the caller compresses instead of overflowing a stale window.
|
||||
logger.warning("growth %s: server refresh failed; compression proceeds", model_id)
|
||||
return None
|
||||
return decision.next_window
|
||||
@@ -0,0 +1,379 @@
|
||||
"""Live hardware budget probe.
|
||||
|
||||
Budget-source rule: discrete cards may trust the device query (measured
|
||||
honest within rounding); unified-memory devices must budget from OS free
|
||||
physical memory minus headroom — their device queries have been observed
|
||||
off by 3x in both directions. The probe classifies the device and
|
||||
constructs the right HardwareBudget for the estimator.
|
||||
|
||||
Vendor probe quirk (WDDM carve-out): on unified-memory NVIDIA devices
|
||||
under Windows, nvidia-smi answers from the legacy dedicated-VRAM
|
||||
carve-out — a fraction of the pool the CUDA allocator actually
|
||||
addresses uniformly at full bandwidth. The CUDA driver API is
|
||||
the tiebreaker: cuDeviceGetAttribute(INTEGRATED) is the vendor's own
|
||||
declaration and always wins — 1 budgets unified, 0 stays discrete no
|
||||
matter what any other number says. Only when the driver API is
|
||||
unreachable does the engine's --list-devices view apply, and then only
|
||||
behind two independent conditions no discrete card can meet.
|
||||
|
||||
Every probe here must work under a stripped PATH — gateway and service
|
||||
sessions don't inherit the interactive environment. nvcuda/libcuda load
|
||||
through the system loader (PATH plays no part), so classification never
|
||||
depends on PATH; nvidia-smi resolves through an explicit candidate
|
||||
ladder (PATH first, then the driver's known install locations) and its
|
||||
absence only softens the live number, never the verdict.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from hermes_cli.local_runtime.estimator import HardwareBudget
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_GIB = 1 << 30
|
||||
# Reserve carved off the card before any grant: the desktop's own
|
||||
# co-residents (compositor, browser, Electron) measure ~2-2.5 GiB on a
|
||||
# working machine, and a window granted into that space demotes silently
|
||||
# under WDDM. 7% covers big cards; the 2 GiB floor is what the margin's
|
||||
# old 512 MiB floor failed to cover in practice (a 221K grant measured
|
||||
# 31.9/32.6 GiB with the desktop running — 'fits' by the math, demoted
|
||||
# in reality). Small cards give up window to this; spill mode is their
|
||||
# path to big models regardless.
|
||||
_MARGIN_FLOOR = 2 << 30
|
||||
_MARGIN_FRACTION = 0.09
|
||||
# UMA headroom: on unified-memory machines (Apple Silicon, unified-memory
|
||||
# NVIDIA) the model shares physical memory with the OS and every app, so
|
||||
# budget from RAM minus this fraction.
|
||||
_UMA_HEADROOM_FRACTION = 0.20
|
||||
|
||||
# Engine-fallback gates for the unified-pool quirk — BOTH must hold, and
|
||||
# no discrete card can meet either: (1) the allocator's pool exceeds the
|
||||
# smi report by well past rounding/ECC slack (discrete cards agree within
|
||||
# ~2%; carve-out disagreement runs to whole multiples), and (2) the pool is
|
||||
# system-RAM-sized — a workstation card in a RAM-matched box fails (1)
|
||||
# because its smi and allocator AGREE, and a big discrete card in a
|
||||
# bigger box fails (2). The driver's INTEGRATED attribute, when
|
||||
# readable, bypasses both gates in whichever direction it points.
|
||||
_POOL_DISAGREEMENT_FACTOR = 1.5
|
||||
_POOL_RAM_FRACTION = 0.75
|
||||
|
||||
# cuDeviceGetAttribute enum: device is integrated with host memory.
|
||||
_CU_DEVICE_ATTRIBUTE_INTEGRATED = 18
|
||||
|
||||
# One probe per process once a device answers (silicon doesn't change);
|
||||
# a miss retries after this long so a runtime installed mid-session gets
|
||||
# picked up by the engine fallback.
|
||||
_POOL_NEGATIVE_TTL_S = 60.0
|
||||
_pool_probe_cache: tuple[float, "tuple[int, bool | None] | None"] | None = None
|
||||
|
||||
# ' CUDA0: NVIDIA Example Device (1234-core Example GPU) (46464 MiB, 46284 MiB free)'
|
||||
# — greedy .* pins the LAST parenthesized group, so device names carrying
|
||||
# their own parentheses parse correctly.
|
||||
_DEVICE_LINE_RE = re.compile(r"CUDA\d+:.*\((\d+)\s*MiB,\s*\d+\s*MiB free\)\s*$")
|
||||
|
||||
|
||||
def _ram_bytes() -> tuple[int, int]:
|
||||
"""(total, available) physical memory, cross-platform stdlib."""
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
class MEMORYSTATUSEX(ctypes.Structure):
|
||||
_fields_ = [("dwLength", ctypes.c_ulong),
|
||||
("dwMemoryLoad", ctypes.c_ulong),
|
||||
("ullTotalPhys", ctypes.c_ulonglong),
|
||||
("ullAvailPhys", ctypes.c_ulonglong),
|
||||
("ullTotalPageFile", ctypes.c_ulonglong),
|
||||
("ullAvailPageFile", ctypes.c_ulonglong),
|
||||
("ullTotalVirtual", ctypes.c_ulonglong),
|
||||
("ullAvailVirtual", ctypes.c_ulonglong),
|
||||
("ullAvailExtendedVirtual", ctypes.c_ulonglong)]
|
||||
|
||||
stat = MEMORYSTATUSEX()
|
||||
stat.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
|
||||
ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
|
||||
return stat.ullTotalPhys, stat.ullAvailPhys
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
if sys.platform == "darwin":
|
||||
# macOS getconf has no _PHYS_PAGES/_AVPHYS_PAGES (exit 64, "no such
|
||||
# configuration parameter") — the POSIX branch below returns (0, 0)
|
||||
# and every model reads unavailable. sysctl is the platform truth.
|
||||
try:
|
||||
total = int(subprocess.run(
|
||||
["/usr/sbin/sysctl", "-n", "hw.memsize"],
|
||||
capture_output=True, text=True, timeout=5).stdout.strip() or 0)
|
||||
if total <= 0:
|
||||
return 0, 0
|
||||
avail = total // 2 # conservative fallback
|
||||
try:
|
||||
out = subprocess.run(["/usr/bin/vm_stat"], capture_output=True,
|
||||
text=True, timeout=5).stdout
|
||||
page_m = re.search(r"page size of (\d+)", out)
|
||||
page = int(page_m.group(1)) if page_m else 16384
|
||||
pages = 0
|
||||
# free + inactive + purgeable ≈ reclaimable-on-demand; the
|
||||
# speculative pool is dropped by the OS under pressure too.
|
||||
for key in ("Pages free", "Pages inactive", "Pages purgeable",
|
||||
"Pages speculative"):
|
||||
m = re.search(rf"{key}:\s+(\d+)\.", out)
|
||||
if m:
|
||||
pages += int(m.group(1))
|
||||
if pages > 0:
|
||||
avail = pages * page
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return total, avail
|
||||
except (OSError, ValueError):
|
||||
return 0, 0
|
||||
# POSIX
|
||||
try:
|
||||
page = int(subprocess.run(["getconf", "PAGE_SIZE"], capture_output=True,
|
||||
text=True, timeout=5).stdout or 4096)
|
||||
total = int(subprocess.run(["getconf", "_PHYS_PAGES"], capture_output=True,
|
||||
text=True, timeout=5).stdout or 0) * page
|
||||
avail = total // 2 # conservative when _AVPHYS is unavailable
|
||||
try:
|
||||
avail = int(subprocess.run(["getconf", "_AVPHYS_PAGES"],
|
||||
capture_output=True, text=True,
|
||||
timeout=5).stdout or 0) * page or avail
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return total, avail
|
||||
except (OSError, ValueError):
|
||||
return 0, 0
|
||||
|
||||
|
||||
# nvidia-smi lives at a fixed path under the driver install; PATH presence
|
||||
# varies by session type (services and gateways often run with a minimal
|
||||
# environment) and by driver generation (legacy NVSMI dir was never on
|
||||
# PATH). Resolution result is cached: the driver doesn't move mid-process.
|
||||
_smi_path_cache: "tuple[str | None] | None" = None
|
||||
|
||||
|
||||
def _nvidia_smi_path() -> str | None:
|
||||
"""Absolute path to nvidia-smi, or None. PATH first (respects user
|
||||
overrides), then the driver's known install locations on Windows;
|
||||
on Linux/WSL the PATH lookup is the whole ladder."""
|
||||
global _smi_path_cache
|
||||
if _smi_path_cache is not None:
|
||||
return _smi_path_cache[0]
|
||||
found = shutil.which("nvidia-smi")
|
||||
if found is None and os.name == "nt":
|
||||
windir = os.environ.get("SystemRoot", r"C:\Windows")
|
||||
for candidate in (
|
||||
# DCH drivers (every modern install) place it in System32.
|
||||
Path(windir) / "System32" / "nvidia-smi.exe",
|
||||
# Legacy standalone drivers used NVSMI, never on PATH.
|
||||
Path(os.environ.get("ProgramFiles", r"C:\Program Files"))
|
||||
/ "NVIDIA Corporation" / "NVSMI" / "nvidia-smi.exe",
|
||||
):
|
||||
if candidate.exists():
|
||||
found = str(candidate)
|
||||
break
|
||||
_smi_path_cache = (found,)
|
||||
return found
|
||||
|
||||
|
||||
def _nvidia_vram() -> tuple[int, int] | None:
|
||||
"""(total, free) MiB->bytes from nvidia-smi, or None."""
|
||||
exe = _nvidia_smi_path()
|
||||
if exe is None:
|
||||
return None
|
||||
try:
|
||||
out = subprocess.run(
|
||||
[exe, "--query-gpu=memory.total,memory.free",
|
||||
"--format=csv,noheader,nounits"],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
if out.returncode != 0 or not out.stdout.strip():
|
||||
return None
|
||||
total_mib, free_mib = (int(x) for x in out.stdout.strip().splitlines()[0].split(","))
|
||||
return total_mib << 20, free_mib << 20
|
||||
except (OSError, ValueError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
|
||||
|
||||
def _cuda_driver_pool() -> "tuple[int, bool | None] | None":
|
||||
"""(allocator_total_bytes, integrated_or_None) from the CUDA driver
|
||||
API, or None when unreachable. ctypes against the driver's own DLL/SO
|
||||
— no toolkit, no subprocess, ~ms. INTEGRATED is the vendor's own
|
||||
unified-memory declaration; total is the pool the allocator will
|
||||
actually hand out (on carve-out devices, several times what
|
||||
nvidia-smi reports)."""
|
||||
import ctypes
|
||||
|
||||
for name in ("nvcuda.dll", "libcuda.so.1", "libcuda.so"):
|
||||
try:
|
||||
cuda = ctypes.CDLL(name)
|
||||
break
|
||||
except OSError:
|
||||
continue
|
||||
else:
|
||||
return None
|
||||
try:
|
||||
if cuda.cuInit(0) != 0:
|
||||
return None
|
||||
dev = ctypes.c_int()
|
||||
if cuda.cuDeviceGet(ctypes.byref(dev), 0) != 0:
|
||||
return None
|
||||
total = ctypes.c_size_t()
|
||||
getter = getattr(cuda, "cuDeviceTotalMem_v2", None) or cuda.cuDeviceTotalMem
|
||||
if getter(ctypes.byref(total), dev) != 0 or total.value <= 0:
|
||||
return None
|
||||
integrated: bool | None = None
|
||||
attr = ctypes.c_int()
|
||||
if cuda.cuDeviceGetAttribute(
|
||||
ctypes.byref(attr), _CU_DEVICE_ATTRIBUTE_INTEGRATED, dev) == 0:
|
||||
integrated = bool(attr.value)
|
||||
return total.value, integrated
|
||||
except (OSError, AttributeError):
|
||||
return None
|
||||
|
||||
|
||||
def _engine_device_pool() -> "tuple[int, bool | None] | None":
|
||||
"""(engine_total_bytes, None) from the installed runtime's own
|
||||
--list-devices, or None. The fallback truth source when the driver
|
||||
API is unreachable: asks the exact binary that will do the
|
||||
allocating. Carries no integrated verdict — callers must gate it."""
|
||||
try:
|
||||
from hermes_cli.local_runtime.binaries import (
|
||||
installed_tags,
|
||||
runtimes_root,
|
||||
server_binary,
|
||||
)
|
||||
|
||||
tags = installed_tags()
|
||||
if not tags:
|
||||
return None
|
||||
tag_dir = runtimes_root() / tags[0]
|
||||
backend_dirs = [d for d in tag_dir.iterdir() if d.is_dir()]
|
||||
if not backend_dirs:
|
||||
return None
|
||||
exe = server_binary(backend_dirs[0])
|
||||
out = subprocess.run([str(exe), "--list-devices"], capture_output=True,
|
||||
text=True, timeout=30, cwd=str(exe.parent))
|
||||
if out.returncode != 0:
|
||||
return None
|
||||
for line in (out.stdout + out.stderr).splitlines():
|
||||
m = _DEVICE_LINE_RE.search(line)
|
||||
if m:
|
||||
return int(m.group(1)) << 20, None
|
||||
return None
|
||||
except Exception: # noqa: BLE001 — a probe miss must never block budgeting
|
||||
return None
|
||||
|
||||
|
||||
def _device_pool_view() -> "tuple[int, bool | None] | None":
|
||||
"""Best available allocator-side view, cached: a hit is permanent for
|
||||
the process, a miss retries after a short TTL (the engine binary can
|
||||
appear mid-session via a pane install)."""
|
||||
global _pool_probe_cache
|
||||
now = time.monotonic()
|
||||
if _pool_probe_cache is not None:
|
||||
stamp, view = _pool_probe_cache
|
||||
if view is not None or now - stamp < _POOL_NEGATIVE_TTL_S:
|
||||
return view
|
||||
view = _cuda_driver_pool() or _engine_device_pool()
|
||||
_pool_probe_cache = (now, view)
|
||||
return view
|
||||
|
||||
|
||||
def _unified_pool_bytes(smi_total: int, ram_total: int) -> int | None:
|
||||
"""The real pool size when this NVIDIA device is unified memory behind
|
||||
a WDDM carve-out, else None (trust nvidia-smi as ever).
|
||||
|
||||
The driver's INTEGRATED attribute decides when readable — in BOTH
|
||||
directions (0 pins discrete even if the numbers look weird; a driver
|
||||
that declares integrated is believed even at modest pool sizes). Only
|
||||
an attribute-less view (engine fallback) needs the two numeric gates;
|
||||
both must hold and no discrete card meets either.
|
||||
"""
|
||||
view = _device_pool_view()
|
||||
if view is None:
|
||||
return None
|
||||
pool, integrated = view
|
||||
if integrated is False:
|
||||
return None
|
||||
if integrated is True:
|
||||
return pool
|
||||
if (smi_total > 0 and pool >= int(smi_total * _POOL_DISAGREEMENT_FACTOR)
|
||||
and ram_total > 0 and pool >= int(ram_total * _POOL_RAM_FRACTION)):
|
||||
return pool
|
||||
return None
|
||||
|
||||
|
||||
def probe_budget(*, planning: bool = False) -> HardwareBudget:
|
||||
"""Construct the budget per the source rules above.
|
||||
|
||||
``planning=False`` (default): LIVE budget — free VRAM right now. The
|
||||
right input for launch-time fit decisions and growth re-grants.
|
||||
|
||||
``planning=True``: CAPACITY budget — what this machine can run once
|
||||
the runtime manages placement (total device memory minus the margin).
|
||||
The right input for catalog pricing and quant selection: pricing
|
||||
against live-free while a model is already loaded made every row read
|
||||
'larger than your GPU memory' and degraded quant picks to Q2 on a
|
||||
32 GiB card. The managed server
|
||||
unloads/relaunches models itself, so at load time the capacity is
|
||||
genuinely available.
|
||||
"""
|
||||
ram_total, ram_avail = _ram_bytes()
|
||||
vram = _nvidia_vram()
|
||||
|
||||
# Unified-memory NVIDIA: the CUDA allocator pool is the real
|
||||
# capacity. Classification comes from the driver API/engine — it
|
||||
# must not require nvidia-smi (stripped-PATH sessions lose smi but
|
||||
# nvcuda loads via the system loader regardless). Crossing the
|
||||
# carve-out costs nothing (effective bandwidth is flat through the
|
||||
# boundary; smi's used/total merely saturate at it) — the carve-out
|
||||
# is an OS accounting knob, not a GPU limit. Deliberately NOT
|
||||
# clamped to OS RAM: carved-out memory is invisible to
|
||||
# GlobalMemoryStatusEx (the OS reports correspondingly less total
|
||||
# RAM), so a RAM clamp would throw away exactly the carved capacity.
|
||||
unified = _unified_pool_bytes(vram[0] if vram else 0, ram_total)
|
||||
if unified is not None:
|
||||
logger.info(
|
||||
"unified-memory NVIDIA device: allocator pool %.1f GiB "
|
||||
"(nvidia-smi carve-out: %s); budgeting from the pool",
|
||||
unified / _GIB,
|
||||
f"{vram[0] / _GIB:.1f} GiB" if vram else "unavailable")
|
||||
if planning:
|
||||
base = unified
|
||||
else:
|
||||
# Live: dedicated-free plus what the OS can still give. smi's
|
||||
# free saturates at the carve-out so this under-counts a bit —
|
||||
# the safe direction (the pool edge is a measured soft cliff:
|
||||
# decode collapses ~3.5x when concurrent demand hits it).
|
||||
# Without smi, OS-available alone is the honest floor.
|
||||
live = (vram[1] + ram_avail) if vram else ram_avail
|
||||
base = min(unified, live)
|
||||
usable = max(0, int(base * (1 - _UMA_HEADROOM_FRACTION)))
|
||||
return HardwareBudget(usable_vram_bytes=usable,
|
||||
total_device_bytes=unified,
|
||||
ram_available_bytes=0, uma=True)
|
||||
|
||||
if vram is None:
|
||||
# No NVIDIA device visible: Metal/Vulkan/CPU paths budget from RAM
|
||||
# as UMA (Apple Silicon) — conservative for discrete AMD until a
|
||||
# vendor probe lands (E3 hardware).
|
||||
base = ram_total if planning else ram_avail
|
||||
usable = max(0, int(base * (1 - _UMA_HEADROOM_FRACTION)))
|
||||
return HardwareBudget(usable_vram_bytes=usable,
|
||||
total_device_bytes=ram_total,
|
||||
ram_available_bytes=0, uma=True)
|
||||
|
||||
total, free = vram
|
||||
margin = max(_MARGIN_FLOOR, int(total * _MARGIN_FRACTION))
|
||||
base = total if planning else free
|
||||
return HardwareBudget(usable_vram_bytes=max(0, base - margin),
|
||||
total_device_bytes=total,
|
||||
ram_available_bytes=ram_avail if not planning else ram_total,
|
||||
uma=False)
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Browse Hugging Face for GGUF models the user can run.
|
||||
|
||||
The curated catalog is the front page; this module is the firehose behind
|
||||
it — day-0 models not yet in the catalog, community quants,
|
||||
anything. Three rules keep it safe and honest:
|
||||
|
||||
1. Acquisition only. Nothing here serves a model: a browsed download
|
||||
lands in the machine-scoped models dir and from that moment the
|
||||
normal machinery owns it — staleness bounce, preset generation from
|
||||
the real GGUF header, fit policy, placement pills.
|
||||
2. The fit verdict shown BEFORE download is a rough cut priced from file
|
||||
size alone (weights dominate; KV/overhead use conservative fill-ins).
|
||||
After download the GGUF header is the authority, as everywhere.
|
||||
3. HF is queried directly with short timeouts and a small in-process
|
||||
cache. No third-party proxy service; if HF rate limits ever bite at
|
||||
fleet scale, revisit with a caching proxy then.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_HF = "https://huggingface.co"
|
||||
_TIMEOUT_S = 15
|
||||
# Rough-fit fill-ins for pre-download pricing: a mid-size model's 64K-floor
|
||||
# KV plus runtime overhead. Deliberately round numbers — the verdict bands
|
||||
# are coarse (fits GPU / needs RAM / too big), not window grants.
|
||||
_ROUGH_KV_AND_OVERHEAD = 4 << 30
|
||||
|
||||
# Tiny TTL cache: the pane fires a search per keystroke pause and re-opens
|
||||
# repos the user flips between. Process-local, size-capped, no invalidation
|
||||
# subtleties — upstream truth changes slowly at this granularity.
|
||||
_CACHE: dict[str, tuple[float, object]] = {}
|
||||
_CACHE_TTL_S = 300
|
||||
_CACHE_MAX = 128
|
||||
|
||||
|
||||
def _get_json(url: str) -> object:
|
||||
now = time.monotonic()
|
||||
hit = _CACHE.get(url)
|
||||
if hit and now - hit[0] < _CACHE_TTL_S:
|
||||
return hit[1]
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "hermes-local-models"})
|
||||
with urllib.request.urlopen(req, timeout=_TIMEOUT_S) as r:
|
||||
data = json.load(r)
|
||||
if len(_CACHE) >= _CACHE_MAX:
|
||||
_CACHE.pop(min(_CACHE, key=lambda k: _CACHE[k][0]))
|
||||
_CACHE[url] = (now, data)
|
||||
return data
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HFModelHit:
|
||||
repo: str # e.g. "unsloth/Qwen3.8-27B-GGUF"
|
||||
downloads: int
|
||||
likes: int
|
||||
updated: str # ISO date from HF
|
||||
gated: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HFFileGroup:
|
||||
"""One downloadable quant: a single GGUF or all parts of a split one."""
|
||||
|
||||
label: str # e.g. "Q4_K_M" or the file stem
|
||||
paths: tuple[str, ...] # repo-relative, split parts in order
|
||||
total_bytes: int
|
||||
fit: str = "unknown" # fits-gpu | needs-ram | too-big | unknown
|
||||
|
||||
|
||||
_QUANT_RE = re.compile(
|
||||
r"(?:IQ|Q)\d[_A-Z0-9]*|F16|BF16|F32", re.IGNORECASE)
|
||||
_SPLIT_RE = re.compile(r"-(\d{5})-of-(\d{5})\.gguf$", re.IGNORECASE)
|
||||
|
||||
|
||||
def search_models(query: str, limit: int = 20) -> list[HFModelHit]:
|
||||
"""Full-text search over HF models that ship GGUF files, most
|
||||
downloaded first (the closest public signal to 'trending')."""
|
||||
q = urllib.parse.quote(query.strip())
|
||||
url = (f"{_HF}/api/models?search={q}&filter=gguf&sort=downloads"
|
||||
f"&direction=-1&limit={max(1, min(int(limit), 50))}")
|
||||
out: list[HFModelHit] = []
|
||||
for m in _get_json(url):
|
||||
out.append(HFModelHit(
|
||||
repo=str(m.get("id", "")),
|
||||
downloads=int(m.get("downloads") or 0),
|
||||
likes=int(m.get("likes") or 0),
|
||||
updated=str(m.get("lastModified") or ""),
|
||||
gated=bool(m.get("gated")),
|
||||
))
|
||||
return out
|
||||
|
||||
|
||||
def _quant_label(filename: str) -> str:
|
||||
m = _QUANT_RE.search(filename)
|
||||
return m.group(0).upper() if m else filename
|
||||
|
||||
|
||||
def repo_files(repo: str) -> list[HFFileGroup]:
|
||||
"""The servable GGUFs in a repo, grouped: split parts collapse into one
|
||||
entry (first part is what llama.cpp loads), mmproj/draft companions are
|
||||
excluded (they aren't standalone models). Largest quant first."""
|
||||
url = f"{_HF}/api/models/{urllib.parse.quote(repo)}/tree/main?recursive=true"
|
||||
files = _get_json(url)
|
||||
|
||||
singles: list[tuple[str, int]] = []
|
||||
splits: dict[str, list[tuple[int, str, int]]] = {}
|
||||
for f in files:
|
||||
path = str(f.get("path", ""))
|
||||
if not path.lower().endswith(".gguf"):
|
||||
continue
|
||||
name = path.rsplit("/", 1)[-1].lower()
|
||||
if name.startswith("mmproj") or name.startswith("dspark") or "draft" in name:
|
||||
continue
|
||||
size = int(f.get("size") or 0)
|
||||
m = _SPLIT_RE.search(path)
|
||||
if m:
|
||||
stem = path[: m.start()]
|
||||
splits.setdefault(stem, []).append((int(m.group(1)), path, size))
|
||||
else:
|
||||
singles.append((path, size))
|
||||
|
||||
groups: list[HFFileGroup] = []
|
||||
for path, size in singles:
|
||||
groups.append(HFFileGroup(label=_quant_label(path), paths=(path,),
|
||||
total_bytes=size))
|
||||
for stem, parts in splits.items():
|
||||
parts.sort()
|
||||
groups.append(HFFileGroup(
|
||||
label=_quant_label(stem),
|
||||
paths=tuple(p for _, p, _ in parts),
|
||||
total_bytes=sum(s for _, _, s in parts)))
|
||||
groups.sort(key=lambda g: g.total_bytes, reverse=True)
|
||||
return groups
|
||||
|
||||
|
||||
def rough_fit(total_bytes: int, budget) -> str:
|
||||
"""Coarse pre-download verdict from file size alone. The GGUF header
|
||||
refines this after download; bands match the catalog pills' language.
|
||||
File size ≈ in-memory weights for GGUF (mmap'd as-is)."""
|
||||
need = total_bytes + _ROUGH_KV_AND_OVERHEAD
|
||||
if need <= budget.usable_vram_bytes:
|
||||
return "fits-gpu"
|
||||
if need <= budget.usable_vram_bytes + budget.ram_available_bytes:
|
||||
return "needs-ram"
|
||||
return "too-big"
|
||||
|
||||
|
||||
def priced_repo_files(repo: str, budget) -> list[HFFileGroup]:
|
||||
from dataclasses import replace
|
||||
|
||||
return [replace(g, fit=rough_fit(g.total_bytes, budget))
|
||||
for g in repo_files(repo)]
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Live model-load progress from the managed llama-server router.
|
||||
|
||||
llama-server's child processes emit per-tensor load progress
|
||||
({stages, current, value}, throttled upstream to ~200ms) which the
|
||||
router relays ONLY over its /models/sse stream — GET /models carries
|
||||
just the coarse status string. This module owns one lazy background
|
||||
watcher on that stream and keeps an in-memory snapshot other code can
|
||||
poll cheaply:
|
||||
|
||||
get_loading_progress() -> {model_id: {"stage", "value", "percent"}}
|
||||
|
||||
"percent" is a composite across stages so a bar doesn't sprint 0->100
|
||||
once per stage: the text model dominates load time (its weights dwarf
|
||||
the mmproj/spec extras), so it gets the lion's share of the range and
|
||||
the extras split the remainder.
|
||||
|
||||
The watcher starts on first call, reconnects with backoff (the router
|
||||
bounces on model download/eject), and never raises into callers — no
|
||||
router, no state file, or no SSE support (older engines) all read as
|
||||
"nothing loading". Safe from any process on the machine: the endpoint
|
||||
comes from the supervisor's machine-scoped state file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_TEXT_STAGE_SHARE = 0.85 # composite range share for the text model
|
||||
_RECONNECT_DELAY_S = 3.0
|
||||
_STALE_ENTRY_TTL_S = 120.0 # a loading entry with no events this long is dead
|
||||
|
||||
_lock = threading.Lock()
|
||||
_watcher: threading.Thread | None = None
|
||||
_snapshot: dict[str, dict] = {}
|
||||
|
||||
|
||||
def _composite_percent(stages: list[str], current: str, value: float) -> int:
|
||||
"""Map (stage, in-stage value) onto one 0-100 range, text-heavy."""
|
||||
if not stages or current not in stages or len(stages) == 1:
|
||||
return max(0, min(100, round(value * 100)))
|
||||
extras = [s for s in stages if s != "text_model"]
|
||||
extra_share = (1.0 - _TEXT_STAGE_SHARE) / len(extras) if extras else 0.0
|
||||
offset = 0.0
|
||||
for stage in stages:
|
||||
share = _TEXT_STAGE_SHARE if stage == "text_model" else extra_share
|
||||
if stage == current:
|
||||
return max(0, min(100, round((offset + share * value) * 100)))
|
||||
offset += share
|
||||
return max(0, min(100, round(value * 100)))
|
||||
|
||||
|
||||
def _endpoint() -> "tuple[str, str] | None":
|
||||
"""(base_root, api_key) of the managed router, or None.
|
||||
|
||||
Resolved through the endpoint module's ownership-guarded reader, not
|
||||
a raw state-file read: on the shared stable port, a foreign install's
|
||||
server answers /health for anyone, and a raw read would attach this
|
||||
watcher to someone else's SSE stream (or spin on 401s against it).
|
||||
The guard's dead-pid check is the ownership proof."""
|
||||
try:
|
||||
from hermes_cli.local_runtime.endpoint import _state_endpoint
|
||||
|
||||
state = _state_endpoint()
|
||||
if state is None:
|
||||
return None
|
||||
base = str(state.get("base_url", "")).rsplit("/v1", 1)[0]
|
||||
return (base, str(state.get("api_key", ""))) if base else None
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
def _apply_event(model: str, event: str, data: dict) -> None:
|
||||
with _lock:
|
||||
status = str(data.get("status", ""))
|
||||
if event in ("status_change", "model_status") and status == "loading":
|
||||
progress = data.get("progress") or {}
|
||||
stages = [str(s) for s in (progress.get("stages") or [])]
|
||||
current = str(progress.get("current", ""))
|
||||
value = progress.get("value")
|
||||
entry = _snapshot.setdefault(model, {"stage": "", "value": 0.0,
|
||||
"percent": 0, "ts": 0.0})
|
||||
entry["ts"] = time.monotonic()
|
||||
if current and isinstance(value, (int, float)):
|
||||
entry["stage"] = current
|
||||
entry["value"] = float(value)
|
||||
entry["percent"] = _composite_percent(stages, current, float(value))
|
||||
elif event in ("status_change", "model_status", "model_remove"):
|
||||
# Any terminal status (loaded/unloaded/failed) ends the load.
|
||||
if status != "loading":
|
||||
_snapshot.pop(model, None)
|
||||
|
||||
|
||||
def _watch() -> None:
|
||||
while True:
|
||||
endpoint = _endpoint()
|
||||
if endpoint is None:
|
||||
with _lock:
|
||||
_snapshot.clear()
|
||||
time.sleep(_RECONNECT_DELAY_S)
|
||||
continue
|
||||
base, key = endpoint
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{base}/models/sse",
|
||||
headers={"Authorization": f"Bearer {key}",
|
||||
"Accept": "text/event-stream"})
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
buf = b""
|
||||
while True:
|
||||
chunk = r.read1(4096) if hasattr(r, "read1") else r.read(4096)
|
||||
if not chunk:
|
||||
break
|
||||
buf += chunk
|
||||
while b"\n" in buf:
|
||||
line, buf = buf.split(b"\n", 1)
|
||||
text = line.decode("utf-8", "replace").strip()
|
||||
if not text.startswith("data:"):
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(text[5:].strip())
|
||||
_apply_event(str(msg.get("model", "")),
|
||||
str(msg.get("event", "")),
|
||||
msg.get("data") or {})
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
except Exception as exc: # noqa: BLE001 — watcher must never die loud
|
||||
logger.debug("load-progress SSE reconnecting: %s", exc)
|
||||
# Stream ended (router bounce, timeout, error): loading entries from
|
||||
# the dead connection are unverifiable — drop rather than freeze.
|
||||
with _lock:
|
||||
_snapshot.clear()
|
||||
time.sleep(_RECONNECT_DELAY_S)
|
||||
|
||||
|
||||
def _ensure_watcher() -> None:
|
||||
global _watcher
|
||||
with _lock:
|
||||
if _watcher is None or not _watcher.is_alive():
|
||||
_watcher = threading.Thread(target=_watch, daemon=True,
|
||||
name="llamacpp-load-progress")
|
||||
_watcher.start()
|
||||
|
||||
|
||||
def get_loading_progress() -> dict[str, dict]:
|
||||
"""{model_id: {"stage", "value", "percent"}} for models loading right
|
||||
now. Empty when nothing is loading (or nothing is knowable)."""
|
||||
_ensure_watcher()
|
||||
now = time.monotonic()
|
||||
with _lock:
|
||||
return {m: {"stage": e["stage"], "value": e["value"],
|
||||
"percent": e["percent"]}
|
||||
for m, e in _snapshot.items()
|
||||
if now - e["ts"] < _STALE_ENTRY_TTL_S}
|
||||
|
||||
|
||||
def get_prefill_progress(model: str) -> "dict | None":
|
||||
"""{"processed": tokens} while the managed server is prompt-processing
|
||||
for ``model``, or None (idle, decoding, unreachable, or foreign server).
|
||||
|
||||
llama-server's /slots reports ``n_prompt_tokens_processed`` climbing in
|
||||
real time during prefill, but exposes no total — callers supply their
|
||||
own denominator (the request's estimated token count). Busiest
|
||||
processing slot wins when several are active: a parallel small request
|
||||
(title generation) freezes its counter during decode while a live
|
||||
prefill keeps climbing past it. One authenticated HTTP call per poll;
|
||||
every failure reads as "no prefill" — this is garnish, never load-
|
||||
bearing.
|
||||
"""
|
||||
ep = _endpoint()
|
||||
if ep is None:
|
||||
return None
|
||||
base, key = ep
|
||||
try:
|
||||
from urllib.parse import quote
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"{base}/slots?model={quote(model)}",
|
||||
headers={"Authorization": f"Bearer {key}"})
|
||||
with urllib.request.urlopen(req, timeout=2) as r:
|
||||
slots = json.loads(r.read())
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
best = 0
|
||||
for slot in slots if isinstance(slots, list) else []:
|
||||
if not slot.get("is_processing"):
|
||||
continue
|
||||
try:
|
||||
processed = int(slot.get("n_prompt_tokens_processed") or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
best = max(best, processed)
|
||||
return {"processed": best} if best > 0 else None
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Per-model preset generation (--models-preset INI) — the router-side
|
||||
carrier for context-policy launch decisions.
|
||||
|
||||
The INI shape is what the router itself generates per child: a
|
||||
[model-id] section whose keys are long-form
|
||||
llama-server flag names without the leading dashes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from hermes_cli.local_runtime.context_policy import (
|
||||
RUNTIME_OVERHEAD_BYTES,
|
||||
WindowDecision,
|
||||
initial_window,
|
||||
launch_args,
|
||||
ub_logits_bytes,
|
||||
)
|
||||
from hermes_cli.local_runtime.estimator import (
|
||||
HardwareBudget,
|
||||
PhysicsRefusal,
|
||||
profile_from_gguf,
|
||||
)
|
||||
from hermes_cli.local_runtime.gguf import read_gguf_header
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# args list -> INI keys. Flags the policy owns; everything else stays out
|
||||
# of the preset (recipe sampling defaults merge in a later pass).
|
||||
_FLAG_TO_KEY = {
|
||||
"-c": "ctx-size",
|
||||
"-b": "batch-size",
|
||||
"-ub": "ubatch-size",
|
||||
"-ctk": "cache-type-k",
|
||||
"-ctv": "cache-type-v",
|
||||
"-fa": "flash-attn",
|
||||
"-ot": "override-tensor",
|
||||
"--spec-type": "spec-type",
|
||||
"--spec-draft-n-max": "spec-draft-n-max",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class PresetEntry:
|
||||
model_id: str
|
||||
window: int
|
||||
spilled: bool
|
||||
refusal: str | None = None
|
||||
keys: dict[str, str] | None = None
|
||||
|
||||
|
||||
def _args_to_keys(args: list[str]) -> dict[str, str]:
|
||||
keys: dict[str, str] = {}
|
||||
i = 0
|
||||
while i < len(args):
|
||||
flag = args[i]
|
||||
key = _FLAG_TO_KEY.get(flag)
|
||||
if key is None:
|
||||
i += 1
|
||||
continue
|
||||
keys[key] = args[i + 1]
|
||||
i += 2
|
||||
return keys
|
||||
|
||||
|
||||
def generate_presets(models_dir: Path, budget: HardwareBudget,
|
||||
preset_path: Path,
|
||||
mtp_capable: set[str] | None = None) -> list[PresetEntry]:
|
||||
"""Walk the staged models, run the launch decision per model, and
|
||||
write one INI. Refused models get no section (the router simply won't
|
||||
have policy for them; the picker surfaces the refusal + smaller-quant
|
||||
suggestion from the returned entries).
|
||||
|
||||
Catalog-declared companions merge in here: sampling defaults (policy
|
||||
keys always win), the vision projector when present, and a spec-decode
|
||||
draft model iff the decision spilled — the rule: speculative
|
||||
decode is a spill amplifier, so a resident draft accelerates a spilled
|
||||
main model; a zero-spill model doesn't pay the draft's memory."""
|
||||
from hermes_cli.local_runtime.bootstrap import assets_dir
|
||||
from hermes_cli.local_runtime.catalog import find_entry_for_model
|
||||
|
||||
entries: list[PresetEntry] = []
|
||||
sections: list[str] = []
|
||||
for gguf in _staged_in(models_dir):
|
||||
model_id = _strip_part(gguf.stem)
|
||||
try:
|
||||
header = read_gguf_header(gguf)
|
||||
profile = profile_from_gguf(header)
|
||||
except (ValueError, OSError) as exc:
|
||||
logger.warning("preset skip %s: %s", gguf.name, exc)
|
||||
continue
|
||||
# Overhead beyond weights+KV: runtime buffers, the vision projector
|
||||
# when this model ships one, and the logits buffers of whichever
|
||||
# microbatch/MTP posture launch_args will choose — flag and price
|
||||
# decided together, from the same facts.
|
||||
hit = find_entry_for_model(model_id)
|
||||
entry = hit[0] if hit is not None else None
|
||||
is_mtp = (entry.mtp if entry is not None
|
||||
else model_id in (mtp_capable or set()))
|
||||
if is_mtp and profile.kv_scale == 1.0:
|
||||
# Header-derived profiles don't know about MTP's draft
|
||||
# context; apply the calibrated KV multiplier here so the
|
||||
# launch fit prices what the server will actually allocate.
|
||||
import dataclasses
|
||||
|
||||
profile = dataclasses.replace(profile, kv_scale=1.2)
|
||||
mmproj_bytes = 0
|
||||
if entry is not None and entry.mmproj is not None:
|
||||
mmproj_path = assets_dir() / entry.mmproj.local_name
|
||||
if mmproj_path.exists():
|
||||
mmproj_bytes = entry.mmproj.size_bytes
|
||||
# MTP posture ladder — window first, prefill second: price the
|
||||
# launch under both postures and keep whichever grants the larger
|
||||
# window (the stacked posture's bigger compute buffer buys ~3x
|
||||
# short-prompt prefill but costs ~2 GiB that would otherwise be
|
||||
# window; measured at 256K the ub512 posture still prefills at
|
||||
# 2.7K tok/s, so window wins ties only one way: never trade
|
||||
# context away for prefill). Same window -> stacked.
|
||||
mtp_prefill = False
|
||||
logits_bytes = ub_logits_bytes(profile.n_vocab, mtp_capable=is_mtp)
|
||||
if is_mtp:
|
||||
stacked_logits = ub_logits_bytes(profile.n_vocab, mtp_capable=True,
|
||||
mtp_prefill=True)
|
||||
stacked_probe = initial_window(
|
||||
profile, budget,
|
||||
overhead_bytes=(RUNTIME_OVERHEAD_BYTES + mmproj_bytes
|
||||
+ stacked_logits))
|
||||
plain_probe = initial_window(
|
||||
profile, budget,
|
||||
overhead_bytes=(RUNTIME_OVERHEAD_BYTES + mmproj_bytes
|
||||
+ logits_bytes))
|
||||
if (not isinstance(stacked_probe, PhysicsRefusal)
|
||||
and not stacked_probe.spilled
|
||||
and (isinstance(plain_probe, PhysicsRefusal)
|
||||
or stacked_probe.window >= plain_probe.window)):
|
||||
mtp_prefill = True
|
||||
logits_bytes = stacked_logits
|
||||
decision = initial_window(
|
||||
profile, budget,
|
||||
overhead_bytes=RUNTIME_OVERHEAD_BYTES + mmproj_bytes + logits_bytes)
|
||||
if isinstance(decision, PhysicsRefusal):
|
||||
entries.append(PresetEntry(model_id=model_id, window=0,
|
||||
spilled=False, refusal=decision.message))
|
||||
continue
|
||||
|
||||
# Session growth (growth.py): a persisted override lifts the launch
|
||||
# window to where the ladder last grew it — capped at native, and
|
||||
# only when physics still clears the bigger window on THIS boot's
|
||||
# budget (a smaller-VRAM day re-fits honestly back down).
|
||||
try:
|
||||
from hermes_cli.local_runtime.estimator import ctx_bytes
|
||||
from hermes_cli.local_runtime.growth import load_window_overrides
|
||||
|
||||
override = load_window_overrides().get(model_id)
|
||||
native = profile.n_ctx_train or decision.window
|
||||
if override and override > decision.window:
|
||||
target = min(int(override), native)
|
||||
kv = ctx_bytes(profile, target)
|
||||
need = (profile.weights_bytes + kv
|
||||
+ RUNTIME_OVERHEAD_BYTES + mmproj_bytes + logits_bytes)
|
||||
if need <= budget.usable_vram_bytes + budget.ram_available_bytes:
|
||||
spill = max(0, need - budget.usable_vram_bytes)
|
||||
decision = WindowDecision(
|
||||
window=target, spill_bytes=spill,
|
||||
kv_on_gpu=kv <= budget.usable_vram_bytes,
|
||||
reasons=[f"grown window restored ({target // 1024}K)"])
|
||||
except Exception as exc: # noqa: BLE001 — overrides are advisory
|
||||
logger.debug("window override skipped for %s: %s", model_id, exc)
|
||||
|
||||
# (entry and is_mtp resolved above, where the overhead was priced —
|
||||
# the launch flags below MUST match that pricing.)
|
||||
args = launch_args(profile, decision, mtp_capable=is_mtp,
|
||||
mtp_draft_depth=(entry.mtp_draft_depth
|
||||
if entry is not None else 3),
|
||||
uma=budget.uma, mtp_prefill=mtp_prefill)
|
||||
keys = _args_to_keys(args)
|
||||
|
||||
if entry is not None and is_mtp:
|
||||
# Integrated-MTP targets sample on the backend, and so does
|
||||
# the draft (pairing validated against the vendor's published
|
||||
# llama.cpp recipes for these models).
|
||||
keys["backend-sampling"] = "on"
|
||||
keys["spec-draft-backend-sampling"] = "on"
|
||||
|
||||
# Sampling deference ladder, under the policy keys (policy wins
|
||||
# on clash). The GGUF's own general.sampling.* metadata is the
|
||||
# publisher's recommendation — it arrives with the file, updates
|
||||
# with every re-upload, and covers models the catalog has never
|
||||
# heard of. Catalog sampling applies only where the file is
|
||||
# silent; a model carrying neither runs llama.cpp defaults.
|
||||
for k, v in header.sampling_defaults.items():
|
||||
keys.setdefault(k, v)
|
||||
if entry is not None:
|
||||
for k, v in (entry.sampling or {}).items():
|
||||
keys.setdefault(k, v)
|
||||
if entry.mmproj is not None:
|
||||
mmproj_path = assets_dir() / entry.mmproj.local_name
|
||||
if mmproj_path.exists():
|
||||
keys["mmproj"] = str(mmproj_path)
|
||||
if entry.draft is not None and decision.spilled:
|
||||
draft_path = assets_dir() / entry.draft.local_name
|
||||
if draft_path.exists():
|
||||
keys["model-draft"] = str(draft_path)
|
||||
keys["spec-type"] = "draft-dspark"
|
||||
# Unsloth's measured cliff: acceptance 83% at 2-3
|
||||
# drafts, collapses at 4.
|
||||
keys["spec-draft-n-max"] = "3"
|
||||
|
||||
entries.append(PresetEntry(model_id=model_id, window=decision.window,
|
||||
spilled=decision.spilled, keys=keys))
|
||||
body = "\n".join(f"{k} = {v}" for k, v in keys.items())
|
||||
sections.append(f"[{model_id}]\n{body}\n")
|
||||
|
||||
preset_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
preset_path.write_text("\n".join(sections), encoding="utf-8")
|
||||
logger.info("wrote %d preset sections to %s", len(sections), preset_path)
|
||||
return entries
|
||||
|
||||
|
||||
def read_preset_decisions(preset_path: Path | None = None) -> dict[str, PresetEntry]:
|
||||
"""The launch decisions the running server was actually given, read
|
||||
back from the preset INI (the INI is the record — it's what spawned
|
||||
the children). Missing/unparseable file returns {}."""
|
||||
import configparser
|
||||
|
||||
if preset_path is None:
|
||||
from hermes_cli.local_runtime.binaries import runtimes_root
|
||||
|
||||
preset_path = runtimes_root() / "presets.ini"
|
||||
out: dict[str, PresetEntry] = {}
|
||||
try:
|
||||
parser = configparser.ConfigParser()
|
||||
parser.read(preset_path, encoding="utf-8")
|
||||
for section in parser.sections():
|
||||
window = parser.getint(section, "ctx-size", fallback=0)
|
||||
spilled = parser.has_option(section, "override-tensor")
|
||||
out[section] = PresetEntry(model_id=section, window=window,
|
||||
spilled=spilled)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("preset read-back failed: %s", exc)
|
||||
return out
|
||||
|
||||
|
||||
def _strip_part(stem: str) -> str:
|
||||
import re
|
||||
|
||||
return re.sub(r"-\d{5}-of-\d{5}$", "", stem)
|
||||
|
||||
|
||||
def _staged_in(models_dir: Path) -> "list[Path]":
|
||||
"""Servable models in an arbitrary directory (split first-parts only) —
|
||||
the validation harness points at non-default dirs."""
|
||||
import re
|
||||
|
||||
part = re.compile(r"-(\d{5})-of-\d{5}\.gguf$")
|
||||
out = []
|
||||
for p in sorted(models_dir.glob("*.gguf")):
|
||||
m = part.search(p.name)
|
||||
if m and m.group(1) != "00001":
|
||||
continue
|
||||
out.append(p)
|
||||
return out
|
||||
@@ -0,0 +1,500 @@
|
||||
"""Supervision of one llama-server in router mode.
|
||||
|
||||
The router process is ours (restart with backoff on crash); router children
|
||||
are its problem — child failures surface via GET /models exit_code, never
|
||||
auto-retried here.
|
||||
|
||||
Readiness rules (each learned the hard way on real hardware):
|
||||
- health-200 is NOT readiness; every readiness claim requires a touch
|
||||
generation (temp-0, expected token, generous budget, reasoning_content
|
||||
scanned).
|
||||
- Always dial 127.0.0.1 — resolving localhost adds ~2s per request on
|
||||
Windows via IPv6 fallback.
|
||||
- /metrics is opt-in (--metrics) and carries no KV-usage metric;
|
||||
idleness = requests_processing == 0 and no slot is_processing.
|
||||
- The router's LRU eviction has no pin for the primary model: until an
|
||||
upstream pin exists, keep_primary_loaded re-touches the primary after
|
||||
any other model load.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
import socket
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from hermes_cli.local_runtime.binaries import server_binary, runtimes_root
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TOUCH_PROMPT = "Reply with exactly one word: the capital of France."
|
||||
TOUCH_EXPECT = "paris"
|
||||
_RESTART_BACKOFF_S = (1, 5, 15, 60)
|
||||
|
||||
|
||||
def state_path() -> Path:
|
||||
"""Endpoint state for other Hermes processes (provider resolution reads
|
||||
this to route llamacpp-alias requests at the managed server)."""
|
||||
return runtimes_root() / "server.json"
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket() as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
# Default port for the managed server, chosen once and reused across
|
||||
# restarts. Sessions persist the resolved base_url; an ephemeral port
|
||||
# would strand every resumed session on a dead endpoint after each
|
||||
# restart. Deliberately NOT 8080 so we never collide with a user's own
|
||||
# llama-server/Ollama-adjacent stack.
|
||||
_DEFAULT_PORT = 18434
|
||||
|
||||
|
||||
def _stable_port() -> int:
|
||||
"""The stable default port, falling back to an ephemeral one only when
|
||||
something else already listens there (and it isn't a leftover managed
|
||||
server, which stop() would have cleaned up)."""
|
||||
try:
|
||||
with socket.socket() as s:
|
||||
s.bind(("127.0.0.1", _DEFAULT_PORT))
|
||||
return _DEFAULT_PORT
|
||||
except OSError:
|
||||
logger.warning(
|
||||
"port %d busy; managed llama-server falling back to an ephemeral "
|
||||
"port — existing sessions may need a model re-pick", _DEFAULT_PORT)
|
||||
return _free_port()
|
||||
|
||||
|
||||
def _stable_api_key() -> str:
|
||||
"""One key for the life of the install, persisted beside the runtimes.
|
||||
|
||||
Endpoint identity must survive restarts as a UNIT — sessions persist the
|
||||
resolved base_url + api_key, so a per-boot key strands every resumed
|
||||
session on HTTP 401 exactly the way a per-boot port would strand them
|
||||
on connection errors. Rotating it buys nothing: the key exists to stop
|
||||
other loopback processes free-riding, and it lives on the same disk as
|
||||
the state file that would leak it. Delete the file to rotate manually.
|
||||
"""
|
||||
key_path = runtimes_root() / ".api_key"
|
||||
try:
|
||||
existing = key_path.read_text(encoding="utf-8").strip()
|
||||
if len(existing) >= 16:
|
||||
return existing
|
||||
except OSError:
|
||||
pass
|
||||
key = secrets.token_urlsafe(24)
|
||||
try:
|
||||
key_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
key_path.write_text(key, encoding="utf-8")
|
||||
except OSError as exc:
|
||||
logger.warning("could not persist api key (%s); sessions will need "
|
||||
"a re-pick after restart", exc)
|
||||
return key
|
||||
|
||||
|
||||
class LlamaServerSupervisor:
|
||||
"""Own one llama-server router process for the life of a Hermes session.
|
||||
|
||||
Usage::
|
||||
|
||||
sup = LlamaServerSupervisor(install_dir, models_dir)
|
||||
sup.start() # spawn + wait healthy
|
||||
sup.ensure_model_ready(name) # load + touch-generate
|
||||
... sup.base_url is the /v1 endpoint, sup.api_key its key ...
|
||||
sup.stop()
|
||||
"""
|
||||
|
||||
def __init__(self, install_dir: Path, models_dir: Path, *,
|
||||
models_max: int = 4, port: int | None = None,
|
||||
extra_args: list[str] | None = None,
|
||||
log_path: Path | None = None,
|
||||
preset_path: Path | None = None):
|
||||
self.install_dir = Path(install_dir)
|
||||
self.models_dir = Path(models_dir)
|
||||
self.models_max = models_max
|
||||
self.port = port or _stable_port()
|
||||
self.api_key = _stable_api_key()
|
||||
self.extra_args = list(extra_args or [])
|
||||
self.log_path = log_path or (self.models_dir.parent / "logs" / "llama-server.log")
|
||||
self.preset_path = preset_path
|
||||
self.proc: subprocess.Popen | None = None
|
||||
self.primary_model: str | None = None
|
||||
self._restarts = 0
|
||||
self._stopping = False
|
||||
self._watchdog: threading.Thread | None = None
|
||||
self._log_handle = None
|
||||
self._idle_since: dict[str, float] = {}
|
||||
|
||||
# ── endpoints ────────────────────────────────────────────
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
return f"http://127.0.0.1:{self.port}/v1"
|
||||
|
||||
def _url(self, route: str) -> str:
|
||||
return f"http://127.0.0.1:{self.port}{route}"
|
||||
|
||||
def _request(self, route: str, body: dict | None = None, timeout_s: int = 30) -> dict:
|
||||
req = urllib.request.Request(
|
||||
self._url(route),
|
||||
data=json.dumps(body).encode() if body is not None else None,
|
||||
headers={"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout_s) as r:
|
||||
raw = r.read()
|
||||
return json.loads(raw) if raw else {}
|
||||
|
||||
# ── lifecycle ────────────────────────────────────────────
|
||||
|
||||
def _spawn(self) -> None:
|
||||
exe = server_binary(self.install_dir)
|
||||
cmd = [
|
||||
str(exe),
|
||||
"--host", "127.0.0.1",
|
||||
"--port", str(self.port),
|
||||
"--api-key", self.api_key,
|
||||
"--models-dir", str(self.models_dir),
|
||||
"--models-max", str(self.models_max),
|
||||
# The residency contract at the layer that sees every message:
|
||||
# a chat request to a staged-but-unloaded model loads it (slow
|
||||
# first token) instead of failing with 'model not found' —
|
||||
# without this flag, chat after an eject is a bare 400/404.
|
||||
"--models-autoload",
|
||||
"--metrics", # opt-in flag; supervisor telemetry needs it
|
||||
"--slots", # /slots endpoint is also opt-in; is_idle reads it
|
||||
"--no-webui",
|
||||
"--jinja",
|
||||
# Direct I/O on model load: bypasses the page cache, so a
|
||||
# multi-GB load doesn't evict half the OS cache — measured
|
||||
# faster loads on NVMe, and our router bounces (download/
|
||||
# delete/activate) reload models often enough to care.
|
||||
"-dio",
|
||||
]
|
||||
if self.preset_path and self.preset_path.exists():
|
||||
cmd += ["--models-preset", str(self.preset_path)]
|
||||
cmd += [
|
||||
*self.extra_args,
|
||||
]
|
||||
self.log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if self._log_handle is not None:
|
||||
# The crash-restart loop calls _spawn repeatedly; without
|
||||
# closing the prior handle each restart leaks one fd.
|
||||
try:
|
||||
self._log_handle.close()
|
||||
except Exception: # noqa: BLE001 — best-effort
|
||||
pass
|
||||
self._log_handle = open(self.log_path, "a", encoding="utf-8", errors="replace")
|
||||
self._log_handle.write(f"\n# spawn: {cmd}\n")
|
||||
self._log_handle.flush()
|
||||
# list-args, never a shell: spaced paths (user homes) must survive.
|
||||
self.proc = subprocess.Popen(cmd, stdout=self._log_handle,
|
||||
stderr=subprocess.STDOUT, cwd=str(exe.parent))
|
||||
logger.info("llama-server router spawned pid=%s port=%s", self.proc.pid, self.port)
|
||||
# State goes down at SPAWN, not after health: endpoint resolution
|
||||
# treats a live-pid-but-not-yet-healthy server as "starting" rather
|
||||
# than "unconfigured", so a readiness probe racing the boot doesn't
|
||||
# throw the app back to onboarding (observed on first restart test).
|
||||
self._write_state()
|
||||
|
||||
def start(self, timeout_s: int = 120) -> None:
|
||||
self._stopping = False
|
||||
self._spawn()
|
||||
self._wait_health(timeout_s)
|
||||
self._write_state()
|
||||
self._watchdog = threading.Thread(target=self._watch, daemon=True,
|
||||
name="llamacpp-supervisor")
|
||||
self._watchdog.start()
|
||||
|
||||
def _write_state(self) -> None:
|
||||
path = state_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({
|
||||
"base_url": self.base_url,
|
||||
"api_key": self.api_key,
|
||||
"pid": self.proc.pid if self.proc else None,
|
||||
}), encoding="utf-8")
|
||||
|
||||
def _wait_health(self, timeout_s: int) -> None:
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
if self.proc and self.proc.poll() is not None:
|
||||
raise RuntimeError(
|
||||
f"llama-server exited rc={self.proc.returncode} during startup "
|
||||
f"(log: {self.log_path})")
|
||||
try:
|
||||
with urllib.request.urlopen(self._url("/health"), timeout=3) as r:
|
||||
if r.status == 200:
|
||||
return
|
||||
except (urllib.error.URLError, OSError, TimeoutError):
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"llama-server not healthy after {timeout_s}s (log: {self.log_path})")
|
||||
|
||||
def _watch(self) -> None:
|
||||
"""Restart the router (not its children) on crash, with backoff."""
|
||||
while not self._stopping:
|
||||
proc = self.proc
|
||||
if proc is None:
|
||||
return
|
||||
rc = proc.poll()
|
||||
if rc is None:
|
||||
time.sleep(2)
|
||||
continue
|
||||
if self._stopping:
|
||||
return
|
||||
backoff = _RESTART_BACKOFF_S[min(self._restarts, len(_RESTART_BACKOFF_S) - 1)]
|
||||
logger.warning("llama-server exited rc=%s; restart #%s in %ss",
|
||||
rc, self._restarts + 1, backoff)
|
||||
time.sleep(backoff)
|
||||
self._restarts += 1
|
||||
try:
|
||||
self._reap_orphaned_children()
|
||||
self._spawn()
|
||||
self._wait_health(120)
|
||||
if self.primary_model:
|
||||
self.ensure_model_ready(self.primary_model)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("llama-server restart failed: %s", exc)
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stopping = True
|
||||
state_path().unlink(missing_ok=True)
|
||||
if self.proc and self.proc.poll() is None:
|
||||
self._terminate_tree(self.proc)
|
||||
if self._log_handle:
|
||||
self._log_handle.close()
|
||||
self._log_handle = None
|
||||
|
||||
@staticmethod
|
||||
def _terminate_tree(proc: subprocess.Popen) -> None:
|
||||
"""Terminate the router AND its model children.
|
||||
|
||||
The router spawns one child llama-server per loaded model, each
|
||||
holding gigabytes of VRAM. Terminating only the router (on
|
||||
Windows, TerminateProcess — no signal handlers, no cleanup pass)
|
||||
orphans those children: the port goes quiet but the weights stay
|
||||
resident, and the next spawn re-loads models alongside a ghost
|
||||
still holding the memory. Enumerate children FIRST (the parent
|
||||
must be alive to walk them), then terminate parent and children
|
||||
together, escalating to kill for stragglers.
|
||||
"""
|
||||
children: list = []
|
||||
try:
|
||||
import psutil
|
||||
|
||||
children = psutil.Process(proc.pid).children(recursive=True)
|
||||
except Exception: # noqa: BLE001 — no psutil view; still stop the router
|
||||
children = []
|
||||
proc.terminate()
|
||||
for child in children:
|
||||
try:
|
||||
child.terminate()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
try:
|
||||
proc.wait(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
for child in children:
|
||||
try:
|
||||
if child.is_running():
|
||||
child.kill()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
def _reap_orphaned_children(self) -> None:
|
||||
"""Kill model children orphaned by a router crash, before respawn.
|
||||
|
||||
A crashed router can't clean up its children, and a dead parent
|
||||
can't be walked — so match by identity instead: any process
|
||||
running OUR llama-server binary whose parent is gone is an
|
||||
orphan of a previous router. Their VRAM must come back before
|
||||
the new router loads models next to the ghosts. External
|
||||
llama-servers (different binary path) never match.
|
||||
"""
|
||||
try:
|
||||
import psutil
|
||||
|
||||
exe = str(server_binary(self.install_dir))
|
||||
except Exception: # noqa: BLE001
|
||||
return
|
||||
for p in psutil.process_iter(["exe", "ppid"]):
|
||||
try:
|
||||
if p.info.get("exe") != exe:
|
||||
continue
|
||||
if self.proc is not None and p.pid == self.proc.pid:
|
||||
continue
|
||||
ppid = p.info.get("ppid") or 0
|
||||
if ppid and psutil.pid_exists(ppid):
|
||||
continue
|
||||
logger.warning("reaping orphaned llama-server child pid=%s", p.pid)
|
||||
p.kill()
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
continue
|
||||
|
||||
# ── model management (router endpoints) ──────────────────
|
||||
|
||||
def models(self) -> dict:
|
||||
"""{model_id: status_value} from GET /models."""
|
||||
data = self._request("/models")
|
||||
return {m["id"]: m.get("status", {}).get("value", "unknown")
|
||||
for m in data.get("data", [])}
|
||||
|
||||
def model_failures(self) -> dict:
|
||||
"""{model_id: exit_code} for children that died — surfaced to the
|
||||
UI, never auto-retried (design: router children are its problem)."""
|
||||
data = self._request("/models")
|
||||
out = {}
|
||||
for m in data.get("data", []):
|
||||
status = m.get("status", {})
|
||||
if status.get("value") == "failed" or status.get("exit_code"):
|
||||
out[m["id"]] = status.get("exit_code")
|
||||
return out
|
||||
|
||||
def load_model(self, model_id: str, timeout_s: int = 600) -> None:
|
||||
self._request("/models/load", {"model": model_id}, timeout_s=timeout_s)
|
||||
|
||||
def unload_model(self, model_id: str) -> None:
|
||||
"""Free the child's VRAM now. Route existence verified empirically
|
||||
on b10290 (POST /models/unload; bogus name -> 400 'model is not
|
||||
found'). Momentary action: never touches primary_model — the
|
||||
declaration is durable, an eject is not (residency design).
|
||||
|
||||
Settle before returning: for a few seconds after unload returns,
|
||||
the router still routes to the dying child and answers chat with
|
||||
500 'proxy error: Could not establish connection' (probed on
|
||||
b10362). Waiting for the model to report unloaded means the next
|
||||
message autoloads cleanly instead of racing the teardown.
|
||||
"""
|
||||
self._request("/models/unload", {"model": model_id}, timeout_s=120)
|
||||
deadline = time.monotonic() + 15
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if self.models().get(model_id) not in ("loaded", "ready", "unloading"):
|
||||
return
|
||||
except Exception: # noqa: BLE001
|
||||
return
|
||||
time.sleep(0.3)
|
||||
|
||||
# ── idle residency (non-primary models) ──────────────────
|
||||
|
||||
# A model that has gone quiet gets its VRAM back after this long. A
|
||||
# constant, not a knob: long enough that an active conversation never
|
||||
# trips it, short enough that a wandered-off session frees ~20 GiB
|
||||
# within the hour. No exemptions (residency v2): demand reloads
|
||||
# anything the user comes back to.
|
||||
IDLE_UNLOAD_S = 15 * 60
|
||||
|
||||
def sweep_idle(self, now: float | None = None) -> list[str]:
|
||||
"""Unload models idle past IDLE_UNLOAD_S. Returns the model ids
|
||||
unloaded. Idle means no busy slots and no queued work, tracked
|
||||
per model across calls; a model seen busy resets its clock."""
|
||||
now = time.monotonic() if now is None else now
|
||||
unloaded: list[str] = []
|
||||
try:
|
||||
statuses = self.models()
|
||||
except Exception: # noqa: BLE001
|
||||
return unloaded
|
||||
for model_id, status in statuses.items():
|
||||
if status not in ("loaded", "ready"):
|
||||
self._idle_since.pop(model_id, None)
|
||||
continue
|
||||
if not self.is_idle(model_id):
|
||||
self._idle_since.pop(model_id, None)
|
||||
continue
|
||||
first_idle = self._idle_since.setdefault(model_id, now)
|
||||
if now - first_idle >= self.IDLE_UNLOAD_S:
|
||||
try:
|
||||
self.unload_model(model_id)
|
||||
self._idle_since.pop(model_id, None)
|
||||
unloaded.append(model_id)
|
||||
logger.info("idle-unloaded %s (idle %ds)", model_id,
|
||||
int(now - first_idle))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("idle unload of %s failed: %s", model_id, exc)
|
||||
return unloaded
|
||||
|
||||
def touch_generate(self, model_id: str, timeout_s: int = 300) -> bool:
|
||||
"""The readiness proof. Generous budget + reasoning_content scan —
|
||||
small token budgets false-fail reasoning models, which spend their
|
||||
first tokens thinking."""
|
||||
try:
|
||||
resp = self._request("/v1/chat/completions", {
|
||||
"model": model_id,
|
||||
"messages": [{"role": "user", "content": TOUCH_PROMPT}],
|
||||
"max_tokens": 512, "temperature": 0,
|
||||
}, timeout_s=timeout_s)
|
||||
msg = resp["choices"][0]["message"]
|
||||
blob = (msg.get("content") or "") + " " + (msg.get("reasoning_content") or "")
|
||||
return TOUCH_EXPECT in blob.lower()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("touch generation failed for %s: %s", model_id, exc)
|
||||
return False
|
||||
|
||||
def ensure_model_ready(self, model_id: str, timeout_s: int = 600) -> bool:
|
||||
"""Load if needed, then prove readiness with a touch generation."""
|
||||
status = self.models().get(model_id)
|
||||
if status is None:
|
||||
raise KeyError(f"model {model_id} not present in models dir")
|
||||
if status not in ("loaded", "ready"):
|
||||
self.load_model(model_id, timeout_s=timeout_s)
|
||||
return self.touch_generate(model_id)
|
||||
|
||||
def actual_n_ctx(self, model_id: str) -> int | None:
|
||||
"""/props reconciliation: the granted window as the child reports
|
||||
it — the compressor's budget and the picker's 'running at 87K of
|
||||
262K' both read THIS value, never the request (design step 4)."""
|
||||
try:
|
||||
props = self._request(f"/props?model={model_id}")
|
||||
return props.get("default_generation_settings", {}).get("n_ctx")
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
def keep_primary_loaded(self) -> None:
|
||||
"""The router's LRU eviction has no pin, so after any other load
|
||||
re-touch the primary to keep it most-recently-used. Best-effort
|
||||
under bursty multi-model load — replaced when an upstream pin
|
||||
exists."""
|
||||
if self.primary_model and self.models().get(self.primary_model) in (
|
||||
"loaded", "ready"):
|
||||
self.touch_generate(self.primary_model, timeout_s=60)
|
||||
|
||||
# ── telemetry ────────────────────────────────────────────
|
||||
|
||||
def is_idle(self, model_id: str | None = None) -> bool:
|
||||
"""No processing requests and no busy slots. Router quirk: /slots
|
||||
and /metrics are per-child and require ?model= (bare calls 400),
|
||||
and no KV-usage metric exists. With ``model_id`` checks that one
|
||||
child; without, every loaded child."""
|
||||
try:
|
||||
if model_id is not None:
|
||||
loaded = [model_id]
|
||||
else:
|
||||
loaded = [m for m, status in self.models().items()
|
||||
if status in ("loaded", "ready")]
|
||||
for mid in loaded:
|
||||
slots = self._request(f"/slots?model={mid}")
|
||||
if any(s.get("is_processing") for s in slots):
|
||||
return False
|
||||
req = urllib.request.Request(
|
||||
self._url(f"/metrics?model={mid}"),
|
||||
headers={"Authorization": f"Bearer {self.api_key}"})
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
text = r.read().decode()
|
||||
for line in text.splitlines():
|
||||
if line.startswith("llamacpp:requests_processing"):
|
||||
if float(line.split()[-1]) != 0.0:
|
||||
return False
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
Reference in New Issue
Block a user