Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/hermes_cli/test_local_recommendation.py

@@ -0,0 +1,170 @@
+"""The recommendation decision table — the reviewable matrix.
+
+The recommendation itself is DERIVED (catalog.recommended_entry: best
+quality among resident entries clearing the pleasant speed floor, else
+fastest resident, else least-painful spilled), so nobody hand-maintains
+per-hardware-class picks. This table is the editorial control on that
+derivation: it enumerates the real memory size classes x {discrete,
+unified} and pins every cell. A catalog change (new model, quality
+re-rank, quant swap) flips cells HERE, and the diff of this file in
+review IS the sign-off on what each machine class gets.
+
+These are decision pins, not change-detectors: each cell is a choice a
+human approved, exactly like a golden file. When a cell flips on
+purpose, update it in the same commit and say why. When one flips by
+surprise, that is the test doing its job.
+
+Budgets mirror hardware.probe_budget's planning-mode shapes (margins,
+UMA headroom) so the cells match what a real machine of that class
+resolves.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from hermes_cli.local_runtime.catalog import (
+    CATALOG,
+    PLEASANT_FLOOR_TOK_S,
+    predicted_decode_tok_s,
+    recommended_entry,
+    recommended_id,
+    select_variant,
+)
+from hermes_cli.local_runtime.estimator import HardwareBudget
+
+_GIB = 1 << 30
+
+
+def _discrete(size_gb: int) -> HardwareBudget:
+    total = size_gb * _GIB
+    margin = max(2 * _GIB, int(total * 0.09))
+    return HardwareBudget(usable_vram_bytes=max(0, total - margin),
+                          total_device_bytes=total,
+                          ram_available_bytes=64 * _GIB, uma=False)
+
+
+def _unified(size_gb: int) -> HardwareBudget:
+    total = size_gb * _GIB
+    return HardwareBudget(usable_vram_bytes=int(total * 0.80),
+                          total_device_bytes=total,
+                          ram_available_bytes=0, uma=True)
+
+
+# The decision table. Cells were generated by the resolver and then
+# reviewed as editorial decisions:
+#
+#   VRAM | discrete                | unified
+#   -----+-------------------------+------------------------
+#     8  | qwen3.6-35b-a3b spilled | (none fits)
+#    16  | qwen3.6-35b-a3b spilled | (none fits)
+#    24  | qwen3.8-27b             | (none fits)
+#    32  | qwen3.8-27b             | qwen3.6-35b-a3b
+#    48  | qwen3.8-27b             | qwen3.6-35b-a3b
+#    96  | qwen3.8-27b             | qwen3.6-35b-a3b
+#   128  | qwen3.8-flash-next      | qwen3.6-35b-a3b
+#   256  | qwen3.8-flash-next      | qwen3.8-flash-next
+#   512  | qwen3.8-flash-next      | qwen3.8-flash-next
+#
+# Reading guide for reviewers:
+# - Discrete <=16 GB: nothing runs resident; the 35B MoE is the least
+#   painful spill (active slice streams from host; a dense spill reads
+#   every weight over the bus).
+# - Discrete 24-96 GB: the 27B is the flagship experience — dense reads
+#   at ~1 TB/s clear the floor easily, so quality decides.
+# - Discrete/unified where Flash Next fits resident (128 GB discrete,
+#   256+ GB unified): the frontier model is the pick — highest quality,
+#   and its sparse decode clears the floor even at UMA bandwidth
+#   (~24 tok/s predicted at 210 GB/s).
+# - Unified 32-128 GB — the Spark class, the reason this resolver
+#   exists: the dense 27B predicts ~13 tok/s at UMA bandwidth (below
+#   the pleasant floor), so the 35B-A3B (~60 tok/s) wins.
+# - Unified <=24 GB: no entry passes the physics check inside the UMA
+#   budget (spilling is impossible on UMA by construction — the pool IS
+#   the RAM). The pane's browse flow is the path for those machines
+#   until a small catalog entry lands (revisit when one does).
+DECISION_TABLE = [
+    (8, "discrete", "qwen3.6-35b-a3b", "least-painful-spilled"),
+    (8, "unified", None, None),
+    (16, "discrete", "qwen3.6-35b-a3b", "least-painful-spilled"),
+    (16, "unified", None, None),
+    (24, "discrete", "qwen3.8-27b", "best-quality-resident"),
+    (24, "unified", None, None),
+    (32, "discrete", "qwen3.8-27b", "best-quality-resident"),
+    (32, "unified", "qwen3.6-35b-a3b", "speed-gated-quality"),
+    (48, "discrete", "qwen3.8-27b", "best-quality-resident"),
+    (48, "unified", "qwen3.6-35b-a3b", "speed-gated-quality"),
+    (96, "discrete", "qwen3.8-27b", "best-quality-resident"),
+    (96, "unified", "qwen3.6-35b-a3b", "speed-gated-quality"),
+    (128, "discrete", "qwen3.8-flash-next", "best-quality-resident"),
+    (128, "unified", "qwen3.6-35b-a3b", "speed-gated-quality"),
+    (256, "discrete", "qwen3.8-flash-next", "best-quality-resident"),
+    (256, "unified", "qwen3.8-flash-next", "best-quality-resident"),
+    (512, "discrete", "qwen3.8-flash-next", "best-quality-resident"),
+    (512, "unified", "qwen3.8-flash-next", "best-quality-resident"),
+]
+
+
+@pytest.mark.parametrize(
+    ("size_gb", "kind", "expected", "expected_reason"),
+    DECISION_TABLE,
+    ids=[f"{s}GB-{k}" for s, k, _, _ in DECISION_TABLE])
+def test_recommendation_decision_table(size_gb, kind, expected, expected_reason):
+    """Pins the pick AND its reason per cell: the reason is user-facing
+    (the Recommended badge's tooltip), so a cell whose rationale flips
+    without the pick flipping is still a review-worthy change."""
+    budget = _discrete(size_gb) if kind == "discrete" else _unified(size_gb)
+    picked = recommended_entry(budget)
+    if expected is None:
+        assert picked is None
+    else:
+        assert picked is not None
+        assert (picked[0].id, picked[1]) == (expected, expected_reason)
+
+
+# ── invariants behind the table (survive catalog changes) ──
+
+
+def test_every_entry_carries_the_recommendation_axes():
+    """quality and decode_fraction are authoring requirements: an entry
+    without them silently loses every quality comparison (quality=0) or
+    prices as dense (decode_fraction=1.0)."""
+    for entry in CATALOG:
+        assert entry.quality > 0, f"{entry.id} has no quality ordering"
+        assert 0.0 < entry.decode_fraction <= 1.0, entry.id
+        if not entry.moe:
+            assert entry.decode_fraction == 1.0, (
+                f"{entry.id} is dense — it reads every weight per token")
+
+
+def test_unified_never_recommends_a_below_floor_dense_model():
+    """The Spark rule, as an invariant: whatever the catalog holds, a
+    unified-memory machine must not be told to run a model whose
+    predicted decode is below the pleasant floor while a resident
+    alternative clears it."""
+    budget = _unified(128)
+    pick = recommended_id(budget)
+    assert pick is not None
+    entry = next(e for e in CATALOG if e.id == pick)
+    choice = select_variant(entry, budget)
+    assert choice is not None and choice.zero_spill
+    clears = [
+        e for e in CATALOG
+        if (c := select_variant(e, budget)) is not None and c.zero_spill
+        and predicted_decode_tok_s(e, c.variant, budget) >= PLEASANT_FLOOR_TOK_S
+    ]
+    if clears:
+        assert predicted_decode_tok_s(entry, choice.variant, budget) >= PLEASANT_FLOOR_TOK_S
+
+
+def test_quality_decides_where_speed_permits():
+    """On big discrete hardware every resident entry clears the floor, so
+    the pick must be the highest-quality fitting entry — the axis that
+    justifies carrying an editorial field at all."""
+    budget = _discrete(512)
+    pick = recommended_id(budget)
+    resident = [
+        e for e in CATALOG
+        if (c := select_variant(e, budget)) is not None and c.zero_spill
+    ]
+    assert pick == max(resident, key=lambda e: e.quality).id