1427 lines
58 KiB
Python
1427 lines
58 KiB
Python
"""Local-models dashboard routes — the desktop's window into the managed
|
|
llama.cpp runtime.
|
|
|
|
Everything here is designed for a first-run user on an RTX laptop: every
|
|
payload carries plain-language, pre-formatted facts the UI can show verbatim
|
|
(what will this model do ON THIS MACHINE, how big is the download, what is
|
|
the runtime doing right now), never raw internals the renderer would have to
|
|
interpret.
|
|
|
|
Long jobs (runtime install, model download) follow the repo's job pattern:
|
|
start-POST -> {job_id} -> GET poll with byte progress. Downloads are
|
|
byte-size checked against the catalog (no hash verification by design);
|
|
a short download deletes the file and reports it plainly.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Optional
|
|
|
|
from fastapi import APIRouter, HTTPException
|
|
from pydantic import BaseModel
|
|
|
|
from hermes_cli.local_runtime.endpoint import _state_endpoint
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
router = APIRouter()
|
|
|
|
_GIB = 1 << 30
|
|
_JOBS: Dict[str, Dict[str, Any]] = {}
|
|
_JOBS_LOCK = threading.Lock()
|
|
|
|
|
|
def _human_gb(n: int | float) -> str:
|
|
return f"{n / _GIB:.1f} GB"
|
|
|
|
|
|
def _job(kind: str, target: str, model_id: str | None = None) -> Dict[str, Any]:
|
|
job = {
|
|
"job_id": uuid.uuid4().hex[:12],
|
|
"kind": kind, # "runtime-install" | "model-download"
|
|
"target": target,
|
|
"model_id": model_id, # catalog id for downloads; None otherwise
|
|
"status": "running", # running | done | error
|
|
"phase": "starting", # human-readable step name
|
|
"detail": "",
|
|
"total_bytes": None,
|
|
"done_bytes": 0,
|
|
"started_at": time.time(),
|
|
"error": None,
|
|
}
|
|
with _JOBS_LOCK:
|
|
_JOBS[job["job_id"]] = job
|
|
return job
|
|
|
|
|
|
# ── fast download: ranged parallel streams ───────────────────
|
|
|
|
# One TCP stream to a CDN rarely fills a fast line; 8 ranged connections
|
|
# writing into a preallocated file saturate consumer gigabit.
|
|
_DOWNLOAD_CONNECTIONS = 8
|
|
_CHUNK = 4 << 20
|
|
|
|
|
|
def _probe_range_support(url: str) -> int:
|
|
"""Total size when the server honors Range requests, else 0.
|
|
|
|
Auth-shaped failures raise with a plain-language message — a 401/403
|
|
from the CDN means the repo is gated or the catalog entry names a
|
|
wrong repo, and the user deserves better than a bare status code.
|
|
"""
|
|
req = urllib.request.Request(url, headers={"Range": "bytes=0-0"})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=60) as r:
|
|
if r.status == 206:
|
|
content_range = r.headers.get("Content-Range", "")
|
|
if "/" in content_range:
|
|
return int(content_range.rsplit("/", 1)[1])
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code in (401, 403):
|
|
raise RuntimeError(
|
|
"The model host refused the download (gated or moved). "
|
|
"This is a catalog problem, not yours — please report it.") from exc
|
|
raise
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
return 0
|
|
|
|
|
|
def _model_id_for(gguf: Path) -> str:
|
|
"""Variant model id for a staged file (strips split-part suffixes)."""
|
|
import re
|
|
|
|
return re.sub(r"-\d{5}-of-\d{5}$", "", gguf.stem)
|
|
|
|
|
|
def _variant_files_on_disk(model_id: str) -> "list[Path]":
|
|
"""Every local file belonging to a staged model: all split parts plus
|
|
its catalog-declared assets (mmproj/draft) when present."""
|
|
from hermes_cli.local_runtime.bootstrap import assets_dir
|
|
from hermes_cli.local_runtime.catalog import find_entry_for_model
|
|
|
|
mdir = _models_dir()
|
|
files = [p for p in mdir.glob("*.gguf") if _model_id_for(p) == model_id]
|
|
hit = find_entry_for_model(model_id)
|
|
if hit is not None:
|
|
entry, _variant = hit
|
|
for asset in (entry.mmproj, entry.draft):
|
|
if asset is not None:
|
|
p = assets_dir() / asset.local_name
|
|
if p.exists():
|
|
files.append(p)
|
|
return files
|
|
|
|
|
|
def download_file(url: str, dest: Path, job: Dict[str, Any],
|
|
*,
|
|
base_done: int = 0, keep_totals: bool = False) -> None:
|
|
"""Download url -> dest with byte progress on ``job``.
|
|
|
|
Ranged-parallel when the server supports it, single-stream fallback
|
|
otherwise. There is no integrity check against the CATALOG by
|
|
design: catalog sizes may lag an upstream re-upload, and a
|
|
newer file than we know about must download fine. Completeness is
|
|
checked only against what the SERVER declared for this transfer
|
|
(range-probe total / Content-Length) — self-consistent and always
|
|
current — so a dropped connection still errors instead of staging a
|
|
truncated file. Never leaves a .part behind.
|
|
|
|
Multi-file variants: ``base_done`` offsets the progress so this file's
|
|
bytes accumulate onto the files before it, and ``keep_totals=True``
|
|
stops the per-file size from overwriting the variant's total.
|
|
"""
|
|
import shutil
|
|
import threading as _threading
|
|
|
|
tmp = dest.with_suffix(".part")
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
file_done = [0]
|
|
progress_lock = _threading.Lock()
|
|
|
|
def bump(n: int) -> None:
|
|
with progress_lock:
|
|
file_done[0] += n
|
|
job["done_bytes"] = base_done + file_done[0]
|
|
|
|
try:
|
|
# The probe and the preallocation both take real seconds on a
|
|
# 20+ GB file — narrate them, or the pane shows a dead '— of X GB'
|
|
# until the first ranged byte lands.
|
|
job["detail"] = "Connecting"
|
|
total = _probe_range_support(url)
|
|
if total:
|
|
if not keep_totals:
|
|
job["total_bytes"] = total
|
|
# Preallocate so each worker writes at its own offset.
|
|
job["detail"] = f"Reserving {_human_gb(total)} of disk space"
|
|
with open(tmp, "wb") as f:
|
|
f.truncate(total)
|
|
job["detail"] = ""
|
|
errors: list[Exception] = []
|
|
bounds = [(i * total // _DOWNLOAD_CONNECTIONS,
|
|
(i + 1) * total // _DOWNLOAD_CONNECTIONS - 1)
|
|
for i in range(_DOWNLOAD_CONNECTIONS)]
|
|
|
|
def fetch_range(start: int, end: int) -> None:
|
|
try:
|
|
req = urllib.request.Request(
|
|
url, headers={"Range": f"bytes={start}-{end}"})
|
|
with urllib.request.urlopen(req, timeout=120) as r, \
|
|
open(tmp, "r+b") as f:
|
|
f.seek(start)
|
|
while True:
|
|
chunk = r.read(_CHUNK)
|
|
if not chunk:
|
|
break
|
|
f.write(chunk)
|
|
bump(len(chunk))
|
|
except Exception as exc: # noqa: BLE001
|
|
errors.append(exc)
|
|
|
|
threads = [_threading.Thread(target=fetch_range, args=b, daemon=True,
|
|
name=f"lm-dl-{i}")
|
|
for i, b in enumerate(bounds)]
|
|
for t in threads:
|
|
t.start()
|
|
for t in threads:
|
|
t.join()
|
|
if errors:
|
|
raise errors[0]
|
|
if file_done[0] != total:
|
|
raise RuntimeError(
|
|
f"download incomplete ({file_done[0]} of {total} bytes)")
|
|
else:
|
|
# No range support: single stream, large chunks. Completeness
|
|
# is judged by the server's own Content-Length when it sent
|
|
# one — never by the catalog, which may lag a re-upload.
|
|
with urllib.request.urlopen(url, timeout=120) as r, open(tmp, "wb") as f:
|
|
length = int(r.headers.get("Content-Length") or 0)
|
|
if length and not keep_totals:
|
|
job["total_bytes"] = length
|
|
while True:
|
|
chunk = r.read(_CHUNK)
|
|
if not chunk:
|
|
break
|
|
f.write(chunk)
|
|
bump(len(chunk))
|
|
if length and file_done[0] != length:
|
|
raise RuntimeError(
|
|
f"Download ended at {file_done[0]:,} bytes but the server "
|
|
f"said {length:,} — connection dropped? Removed; try again")
|
|
|
|
shutil.move(str(tmp), str(dest))
|
|
except Exception:
|
|
tmp.unlink(missing_ok=True)
|
|
raise
|
|
|
|
|
|
def _models_dir() -> Path:
|
|
from hermes_cli.local_runtime.bootstrap import models_dir
|
|
|
|
return models_dir()
|
|
|
|
|
|
def _engine_too_old(min_engine: str) -> bool:
|
|
"""True when the installed llama.cpp predates a model's requirement.
|
|
Tags are release numbers (b10362); no engine installed compares as
|
|
too old only when the model states a requirement."""
|
|
if not min_engine:
|
|
return False
|
|
try:
|
|
from hermes_cli.local_runtime.binaries import default_tag, installed_tags
|
|
|
|
tags = installed_tags() or [default_tag()]
|
|
newest = max(int(t.lstrip("b")) for t in tags if t.lstrip("b").isdigit())
|
|
return newest < int(min_engine.lstrip("b"))
|
|
except Exception: # noqa: BLE001
|
|
return False
|
|
|
|
|
|
def _load_config() -> dict:
|
|
from hermes_cli.config import load_config
|
|
|
|
try:
|
|
return load_config()
|
|
except Exception: # noqa: BLE001
|
|
return {}
|
|
|
|
|
|
def _runtime_section() -> dict:
|
|
return (_load_config() or {}).get("local_runtime") or {}
|
|
|
|
|
|
# ── status: the one call the pane opens with ─────────────────
|
|
|
|
|
|
@router.get("/api/local-models/status")
|
|
def local_models_status():
|
|
"""Cheap, immediate, never blocks on probes (responsiveness standard):
|
|
config state + installed runtime + staged models + supervisor state.
|
|
GPU facts come from /api/local-models/hardware (slower, polled).
|
|
|
|
Sync def on purpose: the body does blocking urlopen/scans, so it runs
|
|
in FastAPI's threadpool instead of stalling the event loop."""
|
|
from hermes_cli.local_runtime.binaries import (
|
|
default_tag,
|
|
installed_tags,
|
|
runtimes_root,
|
|
server_binary,
|
|
)
|
|
|
|
section = _runtime_section()
|
|
configured_tag = section.get("tag") or default_tag()
|
|
have = installed_tags()
|
|
|
|
# The tag actually serving (boot ladder: configured if installed, else
|
|
# newest installed). Present tense for the pane header.
|
|
tag = configured_tag if configured_tag in have else (have[0] if have else configured_tag)
|
|
|
|
# A pending engine update exists when the user runs the local engine
|
|
# (enabled + something installed) and the configured tag — pinned or
|
|
# the Hermes-release default — is newer than anything on disk. The
|
|
# download is a button click, never automatic.
|
|
update_available = bool(
|
|
section.get("enabled") and have and configured_tag not in have)
|
|
|
|
runtime_installed = False
|
|
runtime_backend = None
|
|
root = runtimes_root() / tag
|
|
if root.exists():
|
|
for backend_dir in sorted(p for p in root.iterdir() if p.is_dir()):
|
|
try:
|
|
server_binary(backend_dir)
|
|
runtime_installed = True
|
|
runtime_backend = backend_dir.name
|
|
break
|
|
except Exception: # noqa: BLE001
|
|
continue
|
|
|
|
staged = []
|
|
mdir = _models_dir()
|
|
if mdir.exists():
|
|
from hermes_cli.local_runtime.bootstrap import staged_models
|
|
|
|
# Split models: report the whole variant's bytes, not one part's.
|
|
from hermes_cli.local_runtime.catalog import find_entry_for_model
|
|
|
|
for gguf in staged_models():
|
|
model_id = _model_id_for(gguf)
|
|
size = gguf.stat().st_size
|
|
hit = find_entry_for_model(model_id)
|
|
if hit is not None:
|
|
size = hit[1].size_bytes
|
|
staged.append({
|
|
"id": model_id,
|
|
"size_bytes": size,
|
|
"size_label": _human_gb(size),
|
|
})
|
|
|
|
running = _state_endpoint()
|
|
|
|
# Which staged models are resident right now (loaded in VRAM). Read
|
|
# from the live router when it's up; {} when down. Feeds the pane's
|
|
# Loaded pills and eject buttons.
|
|
loaded: Dict[str, str] = {}
|
|
placement: Dict[str, Any] = {}
|
|
if running is not None:
|
|
try:
|
|
import urllib.request as _url
|
|
|
|
req = _url.Request(
|
|
running["base_url"].rsplit("/v1", 1)[0] + "/models",
|
|
headers={"Authorization": f"Bearer {running.get('api_key', '')}"})
|
|
with _url.urlopen(req, timeout=3) as r:
|
|
data = json.loads(r.read())
|
|
loaded = {
|
|
m["id"]: m.get("status", {}).get("value", "unknown")
|
|
for m in data.get("data", [])
|
|
# Everything resident or becoming resident: 'loading' renders
|
|
# as its own state in the pane (a 20-GB load in flight is the
|
|
# single most important thing the pane can show).
|
|
if m.get("status", {}).get("value") in ("loaded", "ready", "loading")
|
|
}
|
|
# How each loaded model is actually running: the granted window
|
|
# from the child itself, and the plan's spill facts from the
|
|
# preset decision. The pane shows this verbatim — placement is
|
|
# the difference between 'fast' and 'why is my CPU busy', so it
|
|
# must be inspectable, not inferred from Task Manager.
|
|
from hermes_cli.local_runtime.presets import read_preset_decisions
|
|
|
|
decisions = read_preset_decisions()
|
|
for model_id in loaded:
|
|
entry_facts: Dict[str, Any] = {}
|
|
plan = decisions.get(model_id)
|
|
if plan is not None:
|
|
entry_facts["window"] = plan.window
|
|
entry_facts["window_label"] = f"{plan.window // 1024}K"
|
|
entry_facts["spilled"] = plan.spilled
|
|
if loaded[model_id] in ("loaded", "ready"):
|
|
try:
|
|
preq = _url.Request(
|
|
running["base_url"].rsplit("/v1", 1)[0]
|
|
+ f"/props?model={model_id}",
|
|
headers={"Authorization":
|
|
f"Bearer {running.get('api_key', '')}"})
|
|
with _url.urlopen(preq, timeout=3) as pr:
|
|
props = json.loads(pr.read())
|
|
n_ctx = (props.get("default_generation_settings", {})
|
|
.get("n_ctx"))
|
|
if n_ctx:
|
|
entry_facts["granted_window"] = int(n_ctx)
|
|
entry_facts["granted_window_label"] = f"{int(n_ctx) // 1024}K"
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
if entry_facts:
|
|
placement[model_id] = entry_facts
|
|
except Exception as exc: # noqa: BLE001
|
|
# Never silent: an empty dict here renders as 'Not in memory'
|
|
# on a machine whose VRAM is visibly full.
|
|
logger.warning("loaded-models read failed: %r", exc)
|
|
loaded = {}
|
|
|
|
# The active main model, when it is one of ours (config authority: the
|
|
# same model.provider + model.default that /api/model/set writes).
|
|
active_model_id = None
|
|
try:
|
|
config = _load_config()
|
|
model_section = (config or {}).get("model") or {}
|
|
if str(model_section.get("provider", "")).strip().lower() in (
|
|
"llamacpp", "llama.cpp", "llama-cpp"):
|
|
active_model_id = str(
|
|
model_section.get("default") or model_section.get("name") or ""
|
|
).strip() or None
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
|
|
return {
|
|
"enabled": bool(section.get("enabled")),
|
|
"tag": tag,
|
|
"configured_tag": configured_tag,
|
|
"update_available": update_available,
|
|
"runtime_installed": runtime_installed,
|
|
"runtime_backend": runtime_backend,
|
|
"server_running": running is not None,
|
|
"server_base_url": (running or {}).get("base_url"),
|
|
"active_model_id": active_model_id,
|
|
"loaded_models": loaded,
|
|
# Live load progress per model (SSE-fed): {model_id: {stage, value,
|
|
# percent}}. The chat's loading bar and the picker rows poll this.
|
|
"loading": _loading_progress(),
|
|
"placement": placement,
|
|
"models": staged,
|
|
"models_dir": str(mdir),
|
|
}
|
|
|
|
|
|
def _loading_progress() -> Dict[str, Any]:
|
|
try:
|
|
from hermes_cli.local_runtime.load_progress import get_loading_progress
|
|
|
|
return get_loading_progress()
|
|
except Exception: # noqa: BLE001 — progress is garnish, never a 500
|
|
return {}
|
|
|
|
|
|
# ── hardware: what this machine can do ───────────────────────
|
|
|
|
|
|
@router.get("/api/local-models/hardware")
|
|
def local_models_hardware():
|
|
"""The budget as plain facts. Polled by the pane and the statusbar
|
|
resource item (throttled client-side). Sync def on purpose: the body
|
|
shells out to nvidia-smi and probes budgets — threadpool, not loop."""
|
|
from hermes_cli.local_runtime.hardware import probe_budget, _nvidia_vram, _ram_bytes
|
|
|
|
budget = probe_budget()
|
|
ram_total, ram_avail = _ram_bytes()
|
|
out = {
|
|
"uma": budget.uma,
|
|
"vram_total_bytes": budget.total_device_bytes,
|
|
"vram_usable_bytes": budget.usable_vram_bytes,
|
|
"ram_total_bytes": ram_total,
|
|
"ram_available_bytes": ram_avail,
|
|
"vram_label": _human_gb(budget.total_device_bytes),
|
|
"gpu_name": None,
|
|
"gpu_util_percent": None,
|
|
"vram_used_bytes": None,
|
|
}
|
|
# GPU identity + live utilization (NVIDIA; other vendors degrade to None
|
|
# and the UI hides those readouts).
|
|
try:
|
|
import subprocess
|
|
|
|
from hermes_cli.local_runtime.hardware import _nvidia_smi_path
|
|
|
|
smi_exe = _nvidia_smi_path()
|
|
smi = subprocess.run(
|
|
[smi_exe, "--query-gpu=name,utilization.gpu,memory.used",
|
|
"--format=csv,noheader,nounits"],
|
|
capture_output=True, text=True, timeout=5) if smi_exe else None
|
|
if smi and smi.returncode == 0 and smi.stdout.strip():
|
|
name, util, used_mib = (x.strip() for x in smi.stdout.strip().splitlines()[0].split(","))
|
|
out["gpu_name"] = name
|
|
out["gpu_util_percent"] = int(util)
|
|
out["vram_used_bytes"] = int(used_mib) << 20
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
return out
|
|
|
|
|
|
# ── catalog: priced for THIS machine before download ─────────
|
|
|
|
|
|
@router.get("/api/local-models/catalog")
|
|
def local_models_catalog():
|
|
"""Every entry answers the user's three questions up front: how big is
|
|
the download, will it fit, and what context/speed shape will I get —
|
|
computed from the catalog's measured numbers + this machine's
|
|
budget. Hardware-aware quant selection: the row advertises the BEST
|
|
build for this machine (highest quality that runs fully on the GPU at
|
|
the 64K floor; else the smallest that works, spilled and priced). No
|
|
entry is hidden; unaffordable models show WHY. Sync def on purpose:
|
|
probe_budget + catalog I/O block — threadpool, not loop."""
|
|
from hermes_cli.local_runtime.catalog import (
|
|
CATALOG,
|
|
recommended_entry,
|
|
refresh_catalog_soon,
|
|
select_variant,
|
|
)
|
|
from hermes_cli.local_runtime.context_policy import (
|
|
RUNTIME_OVERHEAD_BYTES,
|
|
initial_window,
|
|
ub_logits_bytes,
|
|
)
|
|
from hermes_cli.local_runtime.estimator import PhysicsRefusal
|
|
from hermes_cli.local_runtime.hardware import probe_budget
|
|
|
|
# This request serves the catalog already in memory; a TTL-gated
|
|
# background fetch from the repo lands new entries for the next one
|
|
# (day-0 models reach the pane without an app release).
|
|
refresh_catalog_soon()
|
|
|
|
# Planning budget: price against machine capacity, not live-free VRAM.
|
|
# A loaded model must not make the catalog call every row unaffordable.
|
|
budget = probe_budget(planning=True)
|
|
# The default pick for THIS machine: quality-ranked, fit- and
|
|
# speed-gated (recommended_entry). Engine-gated entries can't be
|
|
# activated today, so they can't be the recommendation either. The
|
|
# reason key ships with the row — the Recommended badge's tooltip is
|
|
# the branch that actually fired, not a re-derivation that can drift.
|
|
eligible = tuple(e for e in CATALOG if not _engine_too_old(e.min_engine))
|
|
picked = recommended_entry(budget, eligible)
|
|
recommended = picked[0].id if picked is not None else None
|
|
recommended_reason = picked[1] if picked is not None else None
|
|
# Completeness-checked staging (split parts all present) — the same
|
|
# answer the picker and the router see, so a mid-download model never
|
|
# reads as downloaded here.
|
|
from hermes_cli.local_runtime.bootstrap import staged_model_ids
|
|
|
|
staged_ids = set(staged_model_ids())
|
|
entries = []
|
|
for entry in CATALOG:
|
|
choice = select_variant(entry, budget)
|
|
# Any variant of this family already on disk counts as downloaded
|
|
# (split variants stage under their first part).
|
|
downloaded_variant = next(
|
|
(v for v in entry.variants if v.model_id in staged_ids), None)
|
|
row: Dict[str, Any] = {
|
|
"id": entry.id,
|
|
"display_name": entry.display_name,
|
|
"description": entry.description,
|
|
"native_context": entry.n_ctx_train,
|
|
"native_context_label": f"{entry.n_ctx_train // 1024}K",
|
|
"recommended": entry.id == recommended,
|
|
"recommended_reason": recommended_reason if entry.id == recommended else None,
|
|
"downloaded": downloaded_variant is not None,
|
|
"downloaded_model_id": downloaded_variant.model_id if downloaded_variant else None,
|
|
"downloaded_quant": downloaded_variant.quant if downloaded_variant else None,
|
|
"mtp": entry.mtp,
|
|
"vision": entry.mmproj is not None,
|
|
# Day-0 architectures need the llama.cpp release where their
|
|
# support landed. True gates download/activate in the pane
|
|
# until the engine updates; the row still renders (visible +
|
|
# explained beats hidden).
|
|
"needs_engine": _engine_too_old(entry.min_engine),
|
|
"min_engine": entry.min_engine or None,
|
|
}
|
|
if choice is None:
|
|
smallest = min(entry.variants, key=lambda v: v.size_bytes)
|
|
smallest_total = entry.download_bytes(smallest)
|
|
row.update({
|
|
"fits": False,
|
|
"size_bytes": smallest_total,
|
|
"size_label": _human_gb(smallest_total),
|
|
"fit_summary": "Needs more memory than this machine has",
|
|
"fit_detail": (f"even the most compact build ({smallest.quant}, "
|
|
f"{_human_gb(smallest_total)}) exceeds GPU + system memory"),
|
|
})
|
|
entries.append(row)
|
|
continue
|
|
|
|
variant = choice.variant
|
|
profile = entry.profile(variant)
|
|
# Same overhead the launch decision prices (runtime buffers +
|
|
# vision projector + the microbatch/MTP logits buffers): the row
|
|
# must advertise the window the model will actually get, not a
|
|
# paper number the server's own fit then shaves down.
|
|
overhead = (RUNTIME_OVERHEAD_BYTES
|
|
+ (entry.mmproj.size_bytes if entry.mmproj else 0)
|
|
+ ub_logits_bytes(entry.n_vocab, mtp_capable=entry.mtp))
|
|
decision = initial_window(profile, budget, overhead_bytes=overhead)
|
|
download_total = entry.download_bytes(variant)
|
|
row.update({
|
|
"fits": True,
|
|
"model_id": variant.model_id,
|
|
"quant": variant.quant,
|
|
"quant_validated": variant.validated,
|
|
"size_bytes": download_total,
|
|
"size_label": _human_gb(download_total),
|
|
"variant_count": len(entry.variants),
|
|
})
|
|
if choice.reason_key == "best-large-window":
|
|
row["quant_reason"] = (
|
|
f"Recommended build ({variant.quant}) — the quant class this "
|
|
"engine is optimized for; runs fully on your GPU with a "
|
|
"large context window")
|
|
elif choice.reason_key == "best-fits":
|
|
row["quant_reason"] = (
|
|
f"Recommended build ({variant.quant}) — the quant class this "
|
|
"engine is optimized for; runs fully on your GPU")
|
|
else:
|
|
row["quant_reason"] = (
|
|
f"Compact build sized for this machine ({variant.quant}) — "
|
|
"larger than GPU memory, runs slower")
|
|
if not isinstance(decision, PhysicsRefusal):
|
|
row["start_window"] = decision.window
|
|
row["start_window_label"] = f"{decision.window // 1024}K"
|
|
row["spilled"] = decision.spilled
|
|
if decision.window >= entry.n_ctx_train:
|
|
shape = f"runs at its full {row['native_context_label']} context"
|
|
else:
|
|
shape = (f"starts at {row['start_window_label']} and grows toward "
|
|
f"{row['native_context_label']} as you use it")
|
|
if decision.spilled:
|
|
shape += " (larger than your GPU memory — runs slower)"
|
|
row["fit_summary"] = shape
|
|
else:
|
|
row["fit_summary"] = row["quant_reason"]
|
|
entries.append(row)
|
|
return {"models": entries}
|
|
|
|
|
|
# ── runtime install (job) ────────────────────────────────────
|
|
|
|
|
|
class RuntimeInstallBody(BaseModel):
|
|
backend: Optional[str] = None # None/auto -> detect
|
|
|
|
|
|
def _runtime_progress_hook(job: Dict[str, Any]):
|
|
"""Adapter: ensure_runtime_installed's progress stream -> job fields.
|
|
|
|
Throttled to ~4 updates/s. The byte counters are CUMULATIVE across the
|
|
plan: a multi-asset engine (CUDA zip + cudart zip) reads as one growing
|
|
download, not a bar that restarts at zero per asset. The total grows as
|
|
each asset's size becomes known (sizes arrive with the response, not
|
|
the plan). Unpack/verify keep the download's counters in place — the
|
|
stage text says what's happening, and a bar that bounces back to zero
|
|
after the bytes finished reads as a failure."""
|
|
state = {"last": 0.0, "banked": 0, "asset": None, "asset_total": 0}
|
|
|
|
def hook(stage: str, done: int, total: int, label: str) -> None:
|
|
now = time.monotonic()
|
|
if now - state["last"] < 0.25 and done < total:
|
|
return
|
|
state["last"] = now
|
|
suffix = f" ({label})" if label else ""
|
|
if stage == "download":
|
|
if label != state["asset"]:
|
|
# Previous asset finished: bank its bytes so the counters
|
|
# keep climbing instead of restarting for the next asset.
|
|
state["banked"] += state["asset_total"]
|
|
state["asset"] = label
|
|
state["asset_total"] = total or done
|
|
plan_done = state["banked"] + done
|
|
plan_total = state["banked"] + (total or 0)
|
|
job["phase"] = "downloading-runtime"
|
|
if total:
|
|
job["detail"] = (f"Downloading the local engine{suffix} — "
|
|
f"{_human_gb(plan_done)} of {_human_gb(plan_total)}")
|
|
else:
|
|
job["detail"] = (f"Downloading the local engine{suffix} — "
|
|
f"{_human_gb(plan_done)}")
|
|
job["done_bytes"] = plan_done
|
|
job["total_bytes"] = plan_total or None
|
|
elif stage == "extract":
|
|
job["phase"] = "unpacking-runtime"
|
|
pct = f" — {min(100, round(done / total * 100))}%" if total else ""
|
|
job["detail"] = f"Unpacking the engine{suffix}{pct}"
|
|
else: # verify
|
|
job["phase"] = "verifying-runtime"
|
|
job["detail"] = f"Verifying the engine{suffix}"
|
|
|
|
return hook
|
|
|
|
|
|
@router.post("/api/local-models/runtime/install")
|
|
async def local_models_runtime_install(body: RuntimeInstallBody):
|
|
from hermes_cli.local_runtime.binaries import (
|
|
default_tag,
|
|
resolve_assets,
|
|
select_backend,
|
|
)
|
|
from hermes_cli.local_runtime.bootstrap import _detect_gpu_vendor
|
|
|
|
section = _runtime_section()
|
|
tag = section.get("tag") or default_tag()
|
|
backend = body.backend or section.get("backend", "auto")
|
|
if backend == "auto":
|
|
backend = select_backend(_detect_gpu_vendor())
|
|
# Resolve first so an impossible combination fails the POST, not the job.
|
|
try:
|
|
plan = resolve_assets(tag, backend)
|
|
except Exception as exc: # noqa: BLE001
|
|
raise HTTPException(status_code=400, detail=str(exc))
|
|
|
|
job = _job("runtime-install", f"llama.cpp {tag} ({backend})")
|
|
|
|
def _run():
|
|
try:
|
|
from hermes_cli.local_runtime.binaries import (
|
|
ensure_runtime_installed,
|
|
installed_tags,
|
|
prune_old_tags,
|
|
)
|
|
|
|
previous = installed_tags()
|
|
job["phase"] = "downloading"
|
|
job["detail"] = f"Fetching {len(plan.assets)} package(s) for {backend}"
|
|
ensure_runtime_installed(tag, backend,
|
|
progress=_runtime_progress_hook(job))
|
|
|
|
# Engine update path: a server already running on an older tag
|
|
# moves to the new one now — the click was the consent. Fresh
|
|
# installs (no server) skip this; Use/boot handles their start.
|
|
restarted = False
|
|
try:
|
|
from hermes_cli.local_runtime.bootstrap import (
|
|
ensure_local_runtime,
|
|
get_supervisor,
|
|
shutdown_local_runtime,
|
|
)
|
|
|
|
sup = get_supervisor()
|
|
if sup is not None and previous and tag not in previous:
|
|
job["phase"] = "restarting"
|
|
job["detail"] = "Switching the running server to the new build"
|
|
shutdown_local_runtime()
|
|
ensure_local_runtime(_load_config(), force=True)
|
|
restarted = True
|
|
except Exception as exc: # noqa: BLE001
|
|
# The new build is installed either way; the next boot serves
|
|
# it. Never fail the job on the restart nicety.
|
|
logger.warning("post-update restart skipped: %s", exc)
|
|
|
|
# N-1 retention, only after the new tag verified: keep it and the
|
|
# newest previous build as the rollback pin target.
|
|
try:
|
|
keep = [tag] + [t for t in previous if t != tag][:1]
|
|
prune_old_tags(keep)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("runtime prune skipped: %s", exc)
|
|
|
|
job["phase"] = "done"
|
|
job["status"] = "done"
|
|
job["detail"] = (f"llama.cpp {tag} ready ({backend})"
|
|
+ (" — server restarted on the new build" if restarted else ""))
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("runtime install failed: %s", exc)
|
|
job["status"] = "error"
|
|
job["error"] = str(exc)
|
|
|
|
threading.Thread(target=_run, daemon=True, name="lr-runtime-install").start()
|
|
return {"job_id": job["job_id"], "backend": backend, "tag": tag}
|
|
|
|
|
|
# ── model download (job with byte progress) ──────────────────
|
|
|
|
|
|
class ModelDownloadBody(BaseModel):
|
|
model_id: str
|
|
|
|
|
|
@router.post("/api/local-models/download")
|
|
async def local_models_download(body: ModelDownloadBody):
|
|
"""Accepts either a family id (downloads this machine's selected
|
|
variant) or an exact variant model_id."""
|
|
from hermes_cli.local_runtime.catalog import (
|
|
CATALOG,
|
|
catalog_by_id,
|
|
select_variant,
|
|
)
|
|
from hermes_cli.local_runtime.hardware import probe_budget
|
|
|
|
entry = catalog_by_id().get(body.model_id)
|
|
variant = None
|
|
if entry is not None:
|
|
if _engine_too_old(entry.min_engine):
|
|
raise HTTPException(
|
|
status_code=409,
|
|
detail=(f"{entry.display_name} needs llama.cpp {entry.min_engine} "
|
|
f"or newer — update the engine first"))
|
|
# Same planning budget as the catalog — the user downloads exactly
|
|
# the build the row advertised.
|
|
choice = select_variant(entry, probe_budget(planning=True))
|
|
if choice is None:
|
|
raise HTTPException(status_code=409,
|
|
detail=f"no variant of {entry.id} fits this machine")
|
|
variant = choice.variant
|
|
else:
|
|
for candidate in CATALOG:
|
|
for v in candidate.variants:
|
|
if v.model_id == body.model_id:
|
|
entry, variant = candidate, v
|
|
break
|
|
if variant:
|
|
break
|
|
if entry is None or variant is None:
|
|
raise HTTPException(status_code=404, detail=f"unknown model {body.model_id}")
|
|
|
|
from hermes_cli.local_runtime.bootstrap import assets_dir, staged_model_ids
|
|
|
|
if variant.model_id in staged_model_ids():
|
|
return {"job_id": None, "already_downloaded": True, "model_id": variant.model_id}
|
|
|
|
# Everything this variant needs: split parts + mmproj/draft assets.
|
|
plan = [] # (url, dest, bytes)
|
|
for asset in variant.files:
|
|
plan.append((f"https://huggingface.co/{entry.repo}/resolve/main/{asset.path}",
|
|
_models_dir() / asset.local_name, asset.size_bytes))
|
|
for asset in (entry.mmproj, entry.draft):
|
|
if asset is not None:
|
|
plan.append((f"https://huggingface.co/{entry.repo}/resolve/main/{asset.path}",
|
|
assets_dir() / asset.local_name, asset.size_bytes))
|
|
|
|
total = sum(p[2] for p in plan)
|
|
job = _job("model-download", f"{entry.display_name} ({variant.quant})",
|
|
model_id=entry.id)
|
|
job["total_bytes"] = total
|
|
|
|
def _run():
|
|
try:
|
|
job["phase"] = "downloading"
|
|
job["detail"] = f"{entry.display_name} — {_human_gb(total)}"
|
|
done_before = 0
|
|
for url, dest, size in plan:
|
|
if dest.exists():
|
|
done_before += size
|
|
job["done_bytes"] = done_before
|
|
continue
|
|
download_file(url, dest, job,
|
|
base_done=done_before, keep_totals=True)
|
|
job["phase"] = "downloading"
|
|
done_before += size
|
|
job["done_bytes"] = done_before
|
|
job["phase"] = "done"
|
|
job["status"] = "done"
|
|
job["detail"] = f"{entry.display_name} ready"
|
|
# A running router only scans models at spawn —
|
|
# bounce it so the new model is servable
|
|
# immediately instead of 400ing until the next app restart.
|
|
try:
|
|
from hermes_cli.local_runtime.bootstrap import refresh_local_runtime
|
|
|
|
refresh_local_runtime()
|
|
except Exception: # noqa: BLE001
|
|
logger.debug("post-download runtime refresh skipped", exc_info=True)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("model download failed: %s", exc)
|
|
job["status"] = "error"
|
|
job["error"] = str(exc)
|
|
|
|
threading.Thread(target=_run, daemon=True, name="lr-model-download").start()
|
|
return {"job_id": job["job_id"], "model_id": variant.model_id}
|
|
|
|
|
|
@router.delete("/api/local-models/models/{model_id}")
|
|
async def local_models_delete(model_id: str):
|
|
"""Remove a staged model: every split part plus its private assets.
|
|
A running router keeps serving from its spawn-time scan, so bounce it
|
|
off the request thread — deleting the active file mid-serve is the
|
|
kind of stale state the refresh exists for."""
|
|
files = _variant_files_on_disk(model_id)
|
|
if not files:
|
|
raise HTTPException(status_code=404, detail="model not found")
|
|
for path in files:
|
|
path.unlink(missing_ok=True)
|
|
# Growth state dies with the model: a re-download starts back at its
|
|
# zero-spill window instead of inheriting a stale grown one.
|
|
try:
|
|
from hermes_cli.local_runtime.growth import clear_window_override
|
|
|
|
clear_window_override(model_id)
|
|
except Exception: # noqa: BLE001
|
|
logger.debug("window-override clear skipped", exc_info=True)
|
|
|
|
def _refresh():
|
|
try:
|
|
from hermes_cli.local_runtime.bootstrap import refresh_local_runtime
|
|
|
|
refresh_local_runtime()
|
|
except Exception: # noqa: BLE001
|
|
logger.debug("post-delete runtime refresh skipped", exc_info=True)
|
|
|
|
threading.Thread(target=_refresh, daemon=True, name="lr-post-delete").start()
|
|
return {"ok": True}
|
|
|
|
|
|
# ── server lifecycle: turn the engine on/off ─────────────────
|
|
|
|
|
|
class ServerActionBody(BaseModel):
|
|
action: str # "stop" | "start"
|
|
|
|
|
|
# ── quickstart: one click from nothing to a working default ──
|
|
|
|
|
|
class QuickstartBody(BaseModel):
|
|
model_id: str | None = None # default: the catalog's recommended entry
|
|
|
|
|
|
# One quickstart at a time: the job sequences installs, downloads, a
|
|
# server bounce, and a config write — two racing runs would interleave
|
|
# all four. Held for the job's lifetime, released in the worker.
|
|
_QUICKSTART_LOCK = threading.Lock()
|
|
|
|
|
|
@router.post("/api/local-models/quickstart")
|
|
async def local_models_quickstart(body: QuickstartBody):
|
|
"""The dummy-proof path: one job that installs the runtime (if
|
|
missing), downloads this machine's build of the recommended model
|
|
(if missing), and makes it the default for new chats. Each leg is
|
|
the same code the individual routes run — this route only sequences
|
|
them, so 'Configure' (the existing pane) and quickstart can never
|
|
disagree about what gets installed.
|
|
|
|
Preflight rejects (no servable entry, engine too old) fail the POST
|
|
synchronously so the button can explain itself; everything slow runs
|
|
in the job with the usual phase/byte progress.
|
|
"""
|
|
from hermes_cli.local_runtime.binaries import (
|
|
default_tag,
|
|
installed_tags,
|
|
resolve_assets,
|
|
select_backend,
|
|
)
|
|
from hermes_cli.local_runtime.bootstrap import (
|
|
_detect_gpu_vendor,
|
|
assets_dir,
|
|
staged_model_ids,
|
|
)
|
|
from hermes_cli.local_runtime.catalog import (
|
|
CATALOG,
|
|
catalog_by_id,
|
|
recommended_entry,
|
|
select_variant,
|
|
)
|
|
from hermes_cli.local_runtime.hardware import probe_budget
|
|
|
|
# Resolve the target entry: explicit id, else this machine's
|
|
# recommendation (quality-ranked, fit- and speed-gated), else the
|
|
# first catalog entry this machine can serve.
|
|
budget = probe_budget(planning=True)
|
|
entry = None
|
|
if body.model_id:
|
|
entry = catalog_by_id().get(body.model_id)
|
|
if entry is None:
|
|
raise HTTPException(status_code=404,
|
|
detail=f"unknown model {body.model_id}")
|
|
candidates = [entry]
|
|
else:
|
|
eligible = tuple(e for e in CATALOG if not _engine_too_old(e.min_engine))
|
|
picked = recommended_entry(budget, eligible)
|
|
best = picked[0] if picked is not None else None
|
|
candidates = ([best] if best is not None else []) + [
|
|
e for e in CATALOG if best is None or e.id != best.id]
|
|
chosen = None
|
|
for candidate in candidates:
|
|
choice = select_variant(candidate, budget)
|
|
if choice is not None and not _engine_too_old(candidate.min_engine):
|
|
chosen = (candidate, choice.variant)
|
|
break
|
|
if chosen is None:
|
|
raise HTTPException(
|
|
status_code=409,
|
|
detail="no catalog model fits this machine — open Local Models "
|
|
"to browse for a smaller build")
|
|
entry, variant = chosen
|
|
|
|
section = _runtime_section()
|
|
tag = section.get("tag") or default_tag()
|
|
backend = section.get("backend", "auto")
|
|
if backend == "auto":
|
|
backend = select_backend(_detect_gpu_vendor())
|
|
need_runtime = not installed_tags()
|
|
if need_runtime:
|
|
# Same preflight as /runtime/install: impossible combos fail the POST.
|
|
try:
|
|
resolve_assets(tag, backend)
|
|
except Exception as exc: # noqa: BLE001
|
|
raise HTTPException(status_code=400, detail=str(exc))
|
|
|
|
need_download = variant.model_id not in staged_model_ids()
|
|
download_plan = [] # (url, dest, bytes)
|
|
if need_download:
|
|
for asset in variant.files:
|
|
download_plan.append(
|
|
(f"https://huggingface.co/{entry.repo}/resolve/main/{asset.path}",
|
|
_models_dir() / asset.local_name, asset.size_bytes))
|
|
for asset in (entry.mmproj, entry.draft):
|
|
if asset is not None:
|
|
download_plan.append(
|
|
(f"https://huggingface.co/{entry.repo}/resolve/main/{asset.path}",
|
|
assets_dir() / asset.local_name, asset.size_bytes))
|
|
|
|
if not _QUICKSTART_LOCK.acquire(blocking=False):
|
|
raise HTTPException(status_code=409,
|
|
detail="Setup is already running")
|
|
|
|
job = _job("quickstart", entry.display_name, model_id=entry.id)
|
|
job["total_bytes"] = sum(p[2] for p in download_plan) or None
|
|
|
|
def _run():
|
|
try:
|
|
if need_runtime:
|
|
from hermes_cli.local_runtime.binaries import ensure_runtime_installed
|
|
|
|
job["phase"] = "installing-runtime"
|
|
job["detail"] = "Installing the local engine"
|
|
ensure_runtime_installed(tag, backend,
|
|
progress=_runtime_progress_hook(job))
|
|
|
|
if need_download:
|
|
job["phase"] = "downloading"
|
|
total = sum(p[2] for p in download_plan)
|
|
# The runtime leg repurposed the byte counters for its own
|
|
# stages — reset them to the model plan before download.
|
|
job["done_bytes"] = 0
|
|
job["total_bytes"] = total
|
|
job["detail"] = f"{entry.display_name} — {_human_gb(total)}"
|
|
done_before = 0
|
|
for url, dest, size in download_plan:
|
|
if dest.exists():
|
|
done_before += size
|
|
job["done_bytes"] = done_before
|
|
continue
|
|
download_file(url, dest, job,
|
|
base_done=done_before, keep_totals=True)
|
|
job["phase"] = "downloading"
|
|
done_before += size
|
|
job["done_bytes"] = done_before
|
|
|
|
# Activate: same sequence as /activate's job body.
|
|
from hermes_cli.config import load_config, save_config
|
|
from hermes_cli.local_runtime.bootstrap import (
|
|
ensure_local_runtime,
|
|
refresh_local_runtime,
|
|
)
|
|
|
|
job["phase"] = "starting-server"
|
|
job["detail"] = "Starting the local server"
|
|
config = load_config()
|
|
config.setdefault("local_runtime", {})["enabled"] = True
|
|
save_config(config)
|
|
sup = ensure_local_runtime(config, force=True)
|
|
if sup is None and _state_endpoint() is None:
|
|
raise RuntimeError(
|
|
"The local server could not start — open Local Models for details")
|
|
if sup is not None:
|
|
try:
|
|
if variant.model_id not in sup.models():
|
|
job["detail"] = "Refreshing the local server"
|
|
refresh_local_runtime()
|
|
except Exception: # noqa: BLE001
|
|
logger.debug("quickstart rescan check skipped", exc_info=True)
|
|
|
|
job["phase"] = "setting-default"
|
|
job["detail"] = "Making it your default"
|
|
from hermes_cli.web_deps import late
|
|
|
|
late("_apply_model_assignment_sync")(
|
|
"main", "llamacpp", variant.model_id, "", "", "")
|
|
|
|
job["phase"] = "done"
|
|
job["status"] = "done"
|
|
job["detail"] = f"{entry.display_name} is ready — new chats use it"
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("quickstart failed: %s", exc)
|
|
job["status"] = "error"
|
|
job["error"] = str(exc)
|
|
finally:
|
|
_QUICKSTART_LOCK.release()
|
|
|
|
threading.Thread(target=_run, daemon=True, name="lr-quickstart").start()
|
|
return {
|
|
"job_id": job["job_id"],
|
|
"model_id": entry.id,
|
|
"display_name": entry.display_name,
|
|
"needs_runtime": need_runtime,
|
|
"needs_download": need_download,
|
|
"download_bytes": sum(p[2] for p in download_plan),
|
|
}
|
|
|
|
|
|
@router.post("/api/local-models/server")
|
|
async def local_models_server(body: ServerActionBody):
|
|
"""Turn the local engine off (stop the server, free ALL GPU memory,
|
|
and disable auto-start) or back on. The off switch is the whole-engine
|
|
counterpart of per-model eject — and unlike eject it IS durable: the
|
|
user said off, so boots stay off until they say on."""
|
|
import asyncio
|
|
|
|
from hermes_cli.config import load_config, save_config
|
|
|
|
action = (body.action or "").strip().lower()
|
|
if action not in ("stop", "start"):
|
|
raise HTTPException(status_code=400, detail="action must be 'stop' or 'start'")
|
|
|
|
def _stop():
|
|
from hermes_cli.local_runtime.bootstrap import (
|
|
get_supervisor,
|
|
shutdown_local_runtime,
|
|
)
|
|
|
|
sup = get_supervisor()
|
|
if sup is not None:
|
|
shutdown_local_runtime()
|
|
else:
|
|
# Server owned by another process (or an orphan): best-effort
|
|
# terminate via the state file's pid, then clear the state.
|
|
endpoint = _state_endpoint()
|
|
if endpoint is not None:
|
|
try:
|
|
import psutil # type: ignore
|
|
|
|
from hermes_cli.local_runtime.supervisor import state_path
|
|
|
|
state = json.loads(state_path().read_text(encoding="utf-8"))
|
|
pid = int(state.get("pid") or 0)
|
|
if pid > 0 and psutil.pid_exists(pid):
|
|
psutil.Process(pid).terminate()
|
|
state_path().unlink(missing_ok=True)
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
config = load_config()
|
|
config.setdefault("local_runtime", {})["enabled"] = False
|
|
save_config(config)
|
|
|
|
def _start():
|
|
from hermes_cli.local_runtime.bootstrap import ensure_local_runtime
|
|
|
|
config = load_config()
|
|
config.setdefault("local_runtime", {})["enabled"] = True
|
|
save_config(config)
|
|
sup = ensure_local_runtime(config, force=True)
|
|
if sup is None and _state_endpoint() is None:
|
|
raise RuntimeError("The local server could not start — check the "
|
|
"runtime is installed")
|
|
|
|
try:
|
|
await asyncio.to_thread(_stop if action == "stop" else _start)
|
|
except Exception as exc: # noqa: BLE001
|
|
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
|
return {"ok": True, "action": action}
|
|
|
|
|
|
# ── activate: make a downloaded model THE model ──────────────
|
|
|
|
|
|
class ModelEjectBody(BaseModel):
|
|
model_id: str
|
|
|
|
|
|
@router.post("/api/local-models/eject")
|
|
def local_models_eject(body: ModelEjectBody):
|
|
"""Free a loaded model's GPU memory now. Nothing reloads it except
|
|
demand — the next message to it (residency v2: no automatic loading
|
|
exists anywhere). Sync def on purpose: the fallback path blocks on a
|
|
urlopen with a 120s timeout — threadpool, never the event loop."""
|
|
from hermes_cli.local_runtime.bootstrap import get_supervisor
|
|
|
|
sup = get_supervisor()
|
|
if sup is not None:
|
|
try:
|
|
sup.unload_model(body.model_id)
|
|
return {"ok": True}
|
|
except Exception as exc: # noqa: BLE001
|
|
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
|
|
|
# Server owned by another process (or state-file only): drive the
|
|
# router directly with the persisted endpoint.
|
|
endpoint = _state_endpoint()
|
|
if endpoint is None:
|
|
raise HTTPException(status_code=409, detail="local server is not running")
|
|
try:
|
|
import urllib.request as _url
|
|
|
|
req = _url.Request(
|
|
endpoint["base_url"].rsplit("/v1", 1)[0] + "/models/unload",
|
|
data=json.dumps({"model": body.model_id}).encode(),
|
|
headers={"Content-Type": "application/json",
|
|
"Authorization": f"Bearer {endpoint.get('api_key', '')}"},
|
|
method="POST")
|
|
with _url.urlopen(req, timeout=120):
|
|
pass
|
|
return {"ok": True}
|
|
except Exception as exc: # noqa: BLE001
|
|
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
|
|
|
|
|
class ModelActivateBody(BaseModel):
|
|
model_id: str # exact variant id (a staged .gguf stem)
|
|
|
|
|
|
@router.post("/api/local-models/activate")
|
|
async def local_models_activate(body: ModelActivateBody):
|
|
"""Make a downloaded model the default for new chats. Pure selection
|
|
(residency v2): a config write through the same machinery as
|
|
/api/model/set, plus making sure the server is up. NO model loading —
|
|
models load on first inference, always; an empty router costs nothing.
|
|
Fast enough to be synchronous-feeling, but kept as a job for UI
|
|
continuity."""
|
|
# Split variants stage under their first part — resolve like the rest
|
|
# of the routes instead of assuming a single flat file.
|
|
from hermes_cli.local_runtime.bootstrap import staged_model_ids
|
|
|
|
if body.model_id not in staged_model_ids():
|
|
raise HTTPException(status_code=404, detail=f"{body.model_id} is not downloaded")
|
|
|
|
job = _job("model-activate", body.model_id, model_id=body.model_id)
|
|
|
|
def _run():
|
|
try:
|
|
from hermes_cli.config import load_config, save_config
|
|
from hermes_cli.local_runtime.bootstrap import (
|
|
ensure_local_runtime,
|
|
refresh_local_runtime,
|
|
)
|
|
|
|
job["phase"] = "starting-server"
|
|
job["detail"] = "Starting the local server"
|
|
config = load_config()
|
|
sup = ensure_local_runtime(config, force=True)
|
|
if sup is None:
|
|
if _state_endpoint() is None:
|
|
raise RuntimeError(
|
|
"The local server could not start — check the runtime is installed")
|
|
|
|
# Self-heal a stale router: the model list is spawn-only, so a
|
|
# server started before this model finished downloading can't
|
|
# serve it. If the router doesn't know the model, bounce it.
|
|
if sup is not None:
|
|
try:
|
|
if body.model_id not in sup.models():
|
|
job["detail"] = "Refreshing the local server"
|
|
refresh_local_runtime()
|
|
except Exception: # noqa: BLE001
|
|
logger.debug("activate rescan check skipped", exc_info=True)
|
|
|
|
job["phase"] = "setting-default"
|
|
job["detail"] = "Making it your default"
|
|
config = load_config()
|
|
config.setdefault("local_runtime", {})["enabled"] = True
|
|
save_config(config)
|
|
from hermes_cli.web_deps import late
|
|
|
|
late("_apply_model_assignment_sync")(
|
|
"main", "llamacpp", body.model_id, "", "", "")
|
|
|
|
job["phase"] = "done"
|
|
job["status"] = "done"
|
|
job["detail"] = f"{body.model_id} is the default for new chats"
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("model activate failed: %s", exc)
|
|
job["status"] = "error"
|
|
job["error"] = str(exc)
|
|
|
|
threading.Thread(target=_run, daemon=True, name="lr-model-activate").start()
|
|
return {"job_id": job["job_id"]}
|
|
|
|
|
|
# ── job polling ──────────────────────────────────────────────
|
|
|
|
|
|
@router.get("/api/local-models/jobs")
|
|
async def local_models_jobs():
|
|
"""All recent jobs, running first — the pane and the app-level poller
|
|
rediscover in-flight work here after a remount or app restart."""
|
|
with _JOBS_LOCK:
|
|
jobs = sorted(_JOBS.values(),
|
|
key=lambda j: (j["status"] != "running", -j["started_at"]))
|
|
out = []
|
|
for job in jobs[:20]:
|
|
entry = dict(job)
|
|
if entry["total_bytes"]:
|
|
entry["percent"] = min(100, round(entry["done_bytes"] / entry["total_bytes"] * 100))
|
|
out.append(entry)
|
|
return {"jobs": out}
|
|
|
|
|
|
@router.get("/api/local-models/jobs/{job_id}")
|
|
async def local_models_job(job_id: str):
|
|
with _JOBS_LOCK:
|
|
job = _JOBS.get(job_id)
|
|
if job is None:
|
|
raise HTTPException(status_code=404, detail="job not found")
|
|
out = dict(job)
|
|
if out["total_bytes"]:
|
|
out["percent"] = min(100, round(out["done_bytes"] / out["total_bytes"] * 100))
|
|
return out
|
|
|
|
|
|
# ── Hugging Face browser: search, repo files, arbitrary download ─
|
|
|
|
|
|
@router.get("/api/local-models/search")
|
|
async def local_models_search(q: str, limit: int = 20):
|
|
"""Full-text HF search over GGUF models — the firehose behind the
|
|
curated catalog. The pane's per-quant fit pills come from the
|
|
repo-files call once the user opens a hit."""
|
|
from starlette.concurrency import run_in_threadpool
|
|
|
|
from hermes_cli.local_runtime.hf_browse import search_models
|
|
|
|
if not q.strip():
|
|
return {"hits": []}
|
|
try:
|
|
hits = await run_in_threadpool(search_models, q, limit)
|
|
except Exception as exc: # noqa: BLE001
|
|
raise HTTPException(status_code=502,
|
|
detail=f"Hugging Face search unavailable: {exc}") from exc
|
|
return {"hits": [h.__dict__ for h in hits]}
|
|
|
|
|
|
@router.get("/api/local-models/search/files")
|
|
async def local_models_search_files(repo: str):
|
|
"""The servable GGUFs in one HF repo with a rough pre-download fit
|
|
verdict per quant (file size + conservative fill-ins — the GGUF
|
|
header refines it after download)."""
|
|
from starlette.concurrency import run_in_threadpool
|
|
|
|
from hermes_cli.local_runtime.hardware import probe_budget
|
|
from hermes_cli.local_runtime.hf_browse import priced_repo_files
|
|
|
|
try:
|
|
groups = await run_in_threadpool(
|
|
priced_repo_files, repo, probe_budget(planning=True))
|
|
except Exception as exc: # noqa: BLE001
|
|
raise HTTPException(status_code=502,
|
|
detail=f"Could not list {repo}: {exc}") from exc
|
|
return {"files": [dict(g.__dict__, paths=list(g.paths)) for g in groups]}
|
|
|
|
|
|
class BrowsedDownloadBody(BaseModel):
|
|
repo: str
|
|
paths: list[str] # one GGUF, or every part of a split, in order
|
|
|
|
|
|
@router.post("/api/local-models/download-browsed")
|
|
async def local_models_download_browsed(body: BrowsedDownloadBody):
|
|
"""Download an arbitrary HF GGUF (browsed or pasted) into the managed
|
|
models dir. From the moment it lands it is a normal staged model: the
|
|
post-download bounce regenerates presets from its real header and the
|
|
fit policy owns its launch. No catalog entry — it serves 'unverified',
|
|
capabilities answered from the live server only."""
|
|
import re as _re
|
|
|
|
from hermes_cli.local_runtime.bootstrap import staged_model_ids
|
|
|
|
paths = [p for p in (body.paths or []) if p.lower().endswith(".gguf")]
|
|
if not paths:
|
|
raise HTTPException(status_code=422, detail="no .gguf files given")
|
|
first = paths[0].rsplit("/", 1)[-1]
|
|
model_id = _re.sub(r"-\d{5}-of-\d{5}\.gguf$", "", first, flags=_re.IGNORECASE)
|
|
model_id = model_id[:-5] if model_id.lower().endswith(".gguf") else model_id
|
|
if model_id in staged_model_ids():
|
|
return {"job_id": None, "already_downloaded": True, "model_id": model_id}
|
|
|
|
job = _job("model-download", f"{model_id} (from {body.repo})",
|
|
model_id=model_id)
|
|
|
|
def _run():
|
|
try:
|
|
job["phase"] = "downloading"
|
|
for p in paths:
|
|
url = (f"https://huggingface.co/{body.repo}"
|
|
f"/resolve/main/{urllib.parse.quote(p)}")
|
|
dest = _models_dir() / p.rsplit("/", 1)[-1]
|
|
if dest.exists():
|
|
continue
|
|
download_file(url, dest, job,
|
|
base_done=int(job.get("done_bytes") or 0),
|
|
keep_totals=bool(job.get("total_bytes")))
|
|
job["phase"] = "downloading"
|
|
job["phase"] = "done"
|
|
job["status"] = "done"
|
|
job["detail"] = f"{model_id} ready"
|
|
try:
|
|
from hermes_cli.local_runtime.bootstrap import refresh_local_runtime
|
|
|
|
refresh_local_runtime()
|
|
except Exception: # noqa: BLE001
|
|
logger.debug("post-download runtime refresh skipped", exc_info=True)
|
|
except Exception as exc: # noqa: BLE001
|
|
job["status"] = "error"
|
|
job["error"] = str(exc)
|
|
|
|
threading.Thread(target=_run, daemon=True, name="lm-download-browsed").start()
|
|
return {"job_id": job["job_id"], "model_id": model_id}
|
|
|
|
|
|
class SideloadBody(BaseModel):
|
|
path: str # absolute path to a .gguf on this machine
|
|
|
|
|
|
@router.post("/api/local-models/sideload")
|
|
async def local_models_sideload(body: SideloadBody):
|
|
"""Register a GGUF that already exists on this machine: link it into
|
|
the managed models dir (copy only when linking is impossible) and
|
|
bounce the router so it serves immediately. The original stays where
|
|
it is; delete-from-Hermes removes only our link."""
|
|
import os
|
|
import shutil
|
|
|
|
from starlette.concurrency import run_in_threadpool
|
|
|
|
src = Path(body.path)
|
|
if not src.is_file() or src.suffix.lower() != ".gguf":
|
|
raise HTTPException(status_code=422, detail="Pick a .gguf model file")
|
|
dest = _models_dir() / src.name
|
|
if dest.exists():
|
|
return {"ok": True, "model_id": dest.stem, "already_present": True}
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
os.link(src, dest) # hardlink: instant, no extra disk
|
|
except OSError:
|
|
try:
|
|
os.symlink(src, dest) # cross-volume fallback
|
|
except OSError:
|
|
await run_in_threadpool(shutil.copyfile, src, dest)
|
|
try:
|
|
from hermes_cli.local_runtime.bootstrap import refresh_local_runtime
|
|
|
|
refresh_local_runtime()
|
|
except Exception: # noqa: BLE001
|
|
logger.debug("post-sideload runtime refresh skipped", exc_info=True)
|
|
return {"ok": True, "model_id": dest.stem}
|