171 lines
7.3 KiB
Python
171 lines
7.3 KiB
Python
"""The recommendation decision table — the reviewable matrix.
|
|
|
|
The recommendation itself is DERIVED (catalog.recommended_entry: best
|
|
quality among resident entries clearing the pleasant speed floor, else
|
|
fastest resident, else least-painful spilled), so nobody hand-maintains
|
|
per-hardware-class picks. This table is the editorial control on that
|
|
derivation: it enumerates the real memory size classes x {discrete,
|
|
unified} and pins every cell. A catalog change (new model, quality
|
|
re-rank, quant swap) flips cells HERE, and the diff of this file in
|
|
review IS the sign-off on what each machine class gets.
|
|
|
|
These are decision pins, not change-detectors: each cell is a choice a
|
|
human approved, exactly like a golden file. When a cell flips on
|
|
purpose, update it in the same commit and say why. When one flips by
|
|
surprise, that is the test doing its job.
|
|
|
|
Budgets mirror hardware.probe_budget's planning-mode shapes (margins,
|
|
UMA headroom) so the cells match what a real machine of that class
|
|
resolves.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from hermes_cli.local_runtime.catalog import (
|
|
CATALOG,
|
|
PLEASANT_FLOOR_TOK_S,
|
|
predicted_decode_tok_s,
|
|
recommended_entry,
|
|
recommended_id,
|
|
select_variant,
|
|
)
|
|
from hermes_cli.local_runtime.estimator import HardwareBudget
|
|
|
|
_GIB = 1 << 30
|
|
|
|
|
|
def _discrete(size_gb: int) -> HardwareBudget:
|
|
total = size_gb * _GIB
|
|
margin = max(2 * _GIB, int(total * 0.09))
|
|
return HardwareBudget(usable_vram_bytes=max(0, total - margin),
|
|
total_device_bytes=total,
|
|
ram_available_bytes=64 * _GIB, uma=False)
|
|
|
|
|
|
def _unified(size_gb: int) -> HardwareBudget:
|
|
total = size_gb * _GIB
|
|
return HardwareBudget(usable_vram_bytes=int(total * 0.80),
|
|
total_device_bytes=total,
|
|
ram_available_bytes=0, uma=True)
|
|
|
|
|
|
# The decision table. Cells were generated by the resolver and then
|
|
# reviewed as editorial decisions:
|
|
#
|
|
# VRAM | discrete | unified
|
|
# -----+-------------------------+------------------------
|
|
# 8 | qwen3.6-35b-a3b spilled | (none fits)
|
|
# 16 | qwen3.6-35b-a3b spilled | (none fits)
|
|
# 24 | qwen3.8-27b | (none fits)
|
|
# 32 | qwen3.8-27b | qwen3.6-35b-a3b
|
|
# 48 | qwen3.8-27b | qwen3.6-35b-a3b
|
|
# 96 | qwen3.8-27b | qwen3.6-35b-a3b
|
|
# 128 | qwen3.8-flash-next | qwen3.6-35b-a3b
|
|
# 256 | qwen3.8-flash-next | qwen3.8-flash-next
|
|
# 512 | qwen3.8-flash-next | qwen3.8-flash-next
|
|
#
|
|
# Reading guide for reviewers:
|
|
# - Discrete <=16 GB: nothing runs resident; the 35B MoE is the least
|
|
# painful spill (active slice streams from host; a dense spill reads
|
|
# every weight over the bus).
|
|
# - Discrete 24-96 GB: the 27B is the flagship experience — dense reads
|
|
# at ~1 TB/s clear the floor easily, so quality decides.
|
|
# - Discrete/unified where Flash Next fits resident (128 GB discrete,
|
|
# 256+ GB unified): the frontier model is the pick — highest quality,
|
|
# and its sparse decode clears the floor even at UMA bandwidth
|
|
# (~24 tok/s predicted at 210 GB/s).
|
|
# - Unified 32-128 GB — the Spark class, the reason this resolver
|
|
# exists: the dense 27B predicts ~13 tok/s at UMA bandwidth (below
|
|
# the pleasant floor), so the 35B-A3B (~60 tok/s) wins.
|
|
# - Unified <=24 GB: no entry passes the physics check inside the UMA
|
|
# budget (spilling is impossible on UMA by construction — the pool IS
|
|
# the RAM). The pane's browse flow is the path for those machines
|
|
# until a small catalog entry lands (revisit when one does).
|
|
DECISION_TABLE = [
|
|
(8, "discrete", "qwen3.6-35b-a3b", "least-painful-spilled"),
|
|
(8, "unified", None, None),
|
|
(16, "discrete", "qwen3.6-35b-a3b", "least-painful-spilled"),
|
|
(16, "unified", None, None),
|
|
(24, "discrete", "qwen3.8-27b", "best-quality-resident"),
|
|
(24, "unified", None, None),
|
|
(32, "discrete", "qwen3.8-27b", "best-quality-resident"),
|
|
(32, "unified", "qwen3.6-35b-a3b", "speed-gated-quality"),
|
|
(48, "discrete", "qwen3.8-27b", "best-quality-resident"),
|
|
(48, "unified", "qwen3.6-35b-a3b", "speed-gated-quality"),
|
|
(96, "discrete", "qwen3.8-27b", "best-quality-resident"),
|
|
(96, "unified", "qwen3.6-35b-a3b", "speed-gated-quality"),
|
|
(128, "discrete", "qwen3.8-flash-next", "best-quality-resident"),
|
|
(128, "unified", "qwen3.6-35b-a3b", "speed-gated-quality"),
|
|
(256, "discrete", "qwen3.8-flash-next", "best-quality-resident"),
|
|
(256, "unified", "qwen3.8-flash-next", "best-quality-resident"),
|
|
(512, "discrete", "qwen3.8-flash-next", "best-quality-resident"),
|
|
(512, "unified", "qwen3.8-flash-next", "best-quality-resident"),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("size_gb", "kind", "expected", "expected_reason"),
|
|
DECISION_TABLE,
|
|
ids=[f"{s}GB-{k}" for s, k, _, _ in DECISION_TABLE])
|
|
def test_recommendation_decision_table(size_gb, kind, expected, expected_reason):
|
|
"""Pins the pick AND its reason per cell: the reason is user-facing
|
|
(the Recommended badge's tooltip), so a cell whose rationale flips
|
|
without the pick flipping is still a review-worthy change."""
|
|
budget = _discrete(size_gb) if kind == "discrete" else _unified(size_gb)
|
|
picked = recommended_entry(budget)
|
|
if expected is None:
|
|
assert picked is None
|
|
else:
|
|
assert picked is not None
|
|
assert (picked[0].id, picked[1]) == (expected, expected_reason)
|
|
|
|
|
|
# ── invariants behind the table (survive catalog changes) ──
|
|
|
|
|
|
def test_every_entry_carries_the_recommendation_axes():
|
|
"""quality and decode_fraction are authoring requirements: an entry
|
|
without them silently loses every quality comparison (quality=0) or
|
|
prices as dense (decode_fraction=1.0)."""
|
|
for entry in CATALOG:
|
|
assert entry.quality > 0, f"{entry.id} has no quality ordering"
|
|
assert 0.0 < entry.decode_fraction <= 1.0, entry.id
|
|
if not entry.moe:
|
|
assert entry.decode_fraction == 1.0, (
|
|
f"{entry.id} is dense — it reads every weight per token")
|
|
|
|
|
|
def test_unified_never_recommends_a_below_floor_dense_model():
|
|
"""The Spark rule, as an invariant: whatever the catalog holds, a
|
|
unified-memory machine must not be told to run a model whose
|
|
predicted decode is below the pleasant floor while a resident
|
|
alternative clears it."""
|
|
budget = _unified(128)
|
|
pick = recommended_id(budget)
|
|
assert pick is not None
|
|
entry = next(e for e in CATALOG if e.id == pick)
|
|
choice = select_variant(entry, budget)
|
|
assert choice is not None and choice.zero_spill
|
|
clears = [
|
|
e for e in CATALOG
|
|
if (c := select_variant(e, budget)) is not None and c.zero_spill
|
|
and predicted_decode_tok_s(e, c.variant, budget) >= PLEASANT_FLOOR_TOK_S
|
|
]
|
|
if clears:
|
|
assert predicted_decode_tok_s(entry, choice.variant, budget) >= PLEASANT_FLOOR_TOK_S
|
|
|
|
|
|
def test_quality_decides_where_speed_permits():
|
|
"""On big discrete hardware every resident entry clears the floor, so
|
|
the pick must be the highest-quality fitting entry — the axis that
|
|
justifies carrying an editorial field at all."""
|
|
budget = _discrete(512)
|
|
pick = recommended_id(budget)
|
|
resident = [
|
|
e for e in CATALOG
|
|
if (c := select_variant(e, budget)) is not None and c.zero_spill
|
|
]
|
|
assert pick == max(resident, key=lambda e: e.quality).id
|