Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

hermes_cli/local_runtime/presets.py

@@ -0,0 +1,265 @@
+"""Per-model preset generation (--models-preset INI) — the router-side
+carrier for context-policy launch decisions.
+
+The INI shape is what the router itself generates per child: a
+[model-id] section whose keys are long-form
+llama-server flag names without the leading dashes.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+
+from hermes_cli.local_runtime.context_policy import (
+    RUNTIME_OVERHEAD_BYTES,
+    WindowDecision,
+    initial_window,
+    launch_args,
+    ub_logits_bytes,
+)
+from hermes_cli.local_runtime.estimator import (
+    HardwareBudget,
+    PhysicsRefusal,
+    profile_from_gguf,
+)
+from hermes_cli.local_runtime.gguf import read_gguf_header
+
+logger = logging.getLogger(__name__)
+
+# args list -> INI keys. Flags the policy owns; everything else stays out
+# of the preset (recipe sampling defaults merge in a later pass).
+_FLAG_TO_KEY = {
+    "-c": "ctx-size",
+    "-b": "batch-size",
+    "-ub": "ubatch-size",
+    "-ctk": "cache-type-k",
+    "-ctv": "cache-type-v",
+    "-fa": "flash-attn",
+    "-ot": "override-tensor",
+    "--spec-type": "spec-type",
+    "--spec-draft-n-max": "spec-draft-n-max",
+}
+
+
+@dataclass
+class PresetEntry:
+    model_id: str
+    window: int
+    spilled: bool
+    refusal: str | None = None
+    keys: dict[str, str] | None = None
+
+
+def _args_to_keys(args: list[str]) -> dict[str, str]:
+    keys: dict[str, str] = {}
+    i = 0
+    while i < len(args):
+        flag = args[i]
+        key = _FLAG_TO_KEY.get(flag)
+        if key is None:
+            i += 1
+            continue
+        keys[key] = args[i + 1]
+        i += 2
+    return keys
+
+
+def generate_presets(models_dir: Path, budget: HardwareBudget,
+                     preset_path: Path,
+                     mtp_capable: set[str] | None = None) -> list[PresetEntry]:
+    """Walk the staged models, run the launch decision per model, and
+    write one INI. Refused models get no section (the router simply won't
+    have policy for them; the picker surfaces the refusal + smaller-quant
+    suggestion from the returned entries).
+
+    Catalog-declared companions merge in here: sampling defaults (policy
+    keys always win), the vision projector when present, and a spec-decode
+    draft model iff the decision spilled — the rule: speculative
+    decode is a spill amplifier, so a resident draft accelerates a spilled
+    main model; a zero-spill model doesn't pay the draft's memory."""
+    from hermes_cli.local_runtime.bootstrap import assets_dir
+    from hermes_cli.local_runtime.catalog import find_entry_for_model
+
+    entries: list[PresetEntry] = []
+    sections: list[str] = []
+    for gguf in _staged_in(models_dir):
+        model_id = _strip_part(gguf.stem)
+        try:
+            header = read_gguf_header(gguf)
+            profile = profile_from_gguf(header)
+        except (ValueError, OSError) as exc:
+            logger.warning("preset skip %s: %s", gguf.name, exc)
+            continue
+        # Overhead beyond weights+KV: runtime buffers, the vision projector
+        # when this model ships one, and the logits buffers of whichever
+        # microbatch/MTP posture launch_args will choose — flag and price
+        # decided together, from the same facts.
+        hit = find_entry_for_model(model_id)
+        entry = hit[0] if hit is not None else None
+        is_mtp = (entry.mtp if entry is not None
+                  else model_id in (mtp_capable or set()))
+        if is_mtp and profile.kv_scale == 1.0:
+            # Header-derived profiles don't know about MTP's draft
+            # context; apply the calibrated KV multiplier here so the
+            # launch fit prices what the server will actually allocate.
+            import dataclasses
+
+            profile = dataclasses.replace(profile, kv_scale=1.2)
+        mmproj_bytes = 0
+        if entry is not None and entry.mmproj is not None:
+            mmproj_path = assets_dir() / entry.mmproj.local_name
+            if mmproj_path.exists():
+                mmproj_bytes = entry.mmproj.size_bytes
+        # MTP posture ladder — window first, prefill second: price the
+        # launch under both postures and keep whichever grants the larger
+        # window (the stacked posture's bigger compute buffer buys ~3x
+        # short-prompt prefill but costs ~2 GiB that would otherwise be
+        # window; measured at 256K the ub512 posture still prefills at
+        # 2.7K tok/s, so window wins ties only one way: never trade
+        # context away for prefill). Same window -> stacked.
+        mtp_prefill = False
+        logits_bytes = ub_logits_bytes(profile.n_vocab, mtp_capable=is_mtp)
+        if is_mtp:
+            stacked_logits = ub_logits_bytes(profile.n_vocab, mtp_capable=True,
+                                             mtp_prefill=True)
+            stacked_probe = initial_window(
+                profile, budget,
+                overhead_bytes=(RUNTIME_OVERHEAD_BYTES + mmproj_bytes
+                                + stacked_logits))
+            plain_probe = initial_window(
+                profile, budget,
+                overhead_bytes=(RUNTIME_OVERHEAD_BYTES + mmproj_bytes
+                                + logits_bytes))
+            if (not isinstance(stacked_probe, PhysicsRefusal)
+                    and not stacked_probe.spilled
+                    and (isinstance(plain_probe, PhysicsRefusal)
+                         or stacked_probe.window >= plain_probe.window)):
+                mtp_prefill = True
+                logits_bytes = stacked_logits
+        decision = initial_window(
+            profile, budget,
+            overhead_bytes=RUNTIME_OVERHEAD_BYTES + mmproj_bytes + logits_bytes)
+        if isinstance(decision, PhysicsRefusal):
+            entries.append(PresetEntry(model_id=model_id, window=0,
+                                       spilled=False, refusal=decision.message))
+            continue
+
+        # Session growth (growth.py): a persisted override lifts the launch
+        # window to where the ladder last grew it — capped at native, and
+        # only when physics still clears the bigger window on THIS boot's
+        # budget (a smaller-VRAM day re-fits honestly back down).
+        try:
+            from hermes_cli.local_runtime.estimator import ctx_bytes
+            from hermes_cli.local_runtime.growth import load_window_overrides
+
+            override = load_window_overrides().get(model_id)
+            native = profile.n_ctx_train or decision.window
+            if override and override > decision.window:
+                target = min(int(override), native)
+                kv = ctx_bytes(profile, target)
+                need = (profile.weights_bytes + kv
+                        + RUNTIME_OVERHEAD_BYTES + mmproj_bytes + logits_bytes)
+                if need <= budget.usable_vram_bytes + budget.ram_available_bytes:
+                    spill = max(0, need - budget.usable_vram_bytes)
+                    decision = WindowDecision(
+                        window=target, spill_bytes=spill,
+                        kv_on_gpu=kv <= budget.usable_vram_bytes,
+                        reasons=[f"grown window restored ({target // 1024}K)"])
+        except Exception as exc:  # noqa: BLE001 — overrides are advisory
+            logger.debug("window override skipped for %s: %s", model_id, exc)
+
+        # (entry and is_mtp resolved above, where the overhead was priced —
+        # the launch flags below MUST match that pricing.)
+        args = launch_args(profile, decision, mtp_capable=is_mtp,
+                           mtp_draft_depth=(entry.mtp_draft_depth
+                                            if entry is not None else 3),
+                           uma=budget.uma, mtp_prefill=mtp_prefill)
+        keys = _args_to_keys(args)
+
+        if entry is not None and is_mtp:
+            # Integrated-MTP targets sample on the backend, and so does
+            # the draft (pairing validated against the vendor's published
+            # llama.cpp recipes for these models).
+            keys["backend-sampling"] = "on"
+            keys["spec-draft-backend-sampling"] = "on"
+
+        # Sampling deference ladder, under the policy keys (policy wins
+        # on clash). The GGUF's own general.sampling.* metadata is the
+        # publisher's recommendation — it arrives with the file, updates
+        # with every re-upload, and covers models the catalog has never
+        # heard of. Catalog sampling applies only where the file is
+        # silent; a model carrying neither runs llama.cpp defaults.
+        for k, v in header.sampling_defaults.items():
+            keys.setdefault(k, v)
+        if entry is not None:
+            for k, v in (entry.sampling or {}).items():
+                keys.setdefault(k, v)
+            if entry.mmproj is not None:
+                mmproj_path = assets_dir() / entry.mmproj.local_name
+                if mmproj_path.exists():
+                    keys["mmproj"] = str(mmproj_path)
+            if entry.draft is not None and decision.spilled:
+                draft_path = assets_dir() / entry.draft.local_name
+                if draft_path.exists():
+                    keys["model-draft"] = str(draft_path)
+                    keys["spec-type"] = "draft-dspark"
+                    # Unsloth's measured cliff: acceptance 83% at 2-3
+                    # drafts, collapses at 4.
+                    keys["spec-draft-n-max"] = "3"
+
+        entries.append(PresetEntry(model_id=model_id, window=decision.window,
+                                   spilled=decision.spilled, keys=keys))
+        body = "\n".join(f"{k} = {v}" for k, v in keys.items())
+        sections.append(f"[{model_id}]\n{body}\n")
+
+    preset_path.parent.mkdir(parents=True, exist_ok=True)
+    preset_path.write_text("\n".join(sections), encoding="utf-8")
+    logger.info("wrote %d preset sections to %s", len(sections), preset_path)
+    return entries
+
+
+def read_preset_decisions(preset_path: Path | None = None) -> dict[str, PresetEntry]:
+    """The launch decisions the running server was actually given, read
+    back from the preset INI (the INI is the record — it's what spawned
+    the children). Missing/unparseable file returns {}."""
+    import configparser
+
+    if preset_path is None:
+        from hermes_cli.local_runtime.binaries import runtimes_root
+
+        preset_path = runtimes_root() / "presets.ini"
+    out: dict[str, PresetEntry] = {}
+    try:
+        parser = configparser.ConfigParser()
+        parser.read(preset_path, encoding="utf-8")
+        for section in parser.sections():
+            window = parser.getint(section, "ctx-size", fallback=0)
+            spilled = parser.has_option(section, "override-tensor")
+            out[section] = PresetEntry(model_id=section, window=window,
+                                       spilled=spilled)
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("preset read-back failed: %s", exc)
+    return out
+
+
+def _strip_part(stem: str) -> str:
+    import re
+
+    return re.sub(r"-\d{5}-of-\d{5}$", "", stem)
+
+
+def _staged_in(models_dir: Path) -> "list[Path]":
+    """Servable models in an arbitrary directory (split first-parts only) —
+    the validation harness points at non-default dirs."""
+    import re
+
+    part = re.compile(r"-(\d{5})-of-\d{5}\.gguf$")
+    out = []
+    for p in sorted(models_dir.glob("*.gguf")):
+        m = part.search(p.name)
+        if m and m.group(1) != "00001":
+            continue
+        out.append(p)
+    return out