Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/hermes_cli/test_openrouter_reasoning_metadata.py

@@ -0,0 +1,294 @@
+"""Tests for OpenRouter reasoning-capability metadata (prime-agent#1258 port).
+
+Covers:
+  - parse_openrouter_reasoning_capabilities: catalog-entry normalization
+  - clamp_reasoning_effort_to_supported: nearest-lower-level clamping
+  - openrouter_model_reasoning_capabilities: cache + tri-state contract
+  - AIAgent._supports_reasoning_extra_body: metadata-first, static fallback
+  - OpenRouterProfile._clamp_reasoning_to_catalog: emitted effort clamping
+"""
+
+import pytest
+
+from hermes_cli.models import (
+    clamp_reasoning_effort_to_supported,
+    parse_openrouter_reasoning_capabilities,
+)
+
+
+class TestParseReasoningCapabilities:
+    def test_reasoning_supported_with_efforts(self):
+        item = {
+            "id": "nvidia/nemotron-3-ultra",
+            "supported_parameters": ["temperature", "tools", "reasoning"],
+            "reasoning": {"mandatory": False, "supported_efforts": ["low", "medium", "high"]},
+        }
+        caps = parse_openrouter_reasoning_capabilities(item)
+        assert caps == {
+            "supports_reasoning": True,
+            "supported_efforts": ["low", "medium", "high"],
+            "mandatory": False,
+        }
+
+    def test_reasoning_supported_all_efforts_when_field_omitted(self):
+        item = {
+            "id": "deepseek/deepseek-chat",
+            "supported_parameters": ["reasoning", "tools"],
+            "reasoning": {},
+        }
+        caps = parse_openrouter_reasoning_capabilities(item)
+        assert caps["supports_reasoning"] is True
+        assert caps["supported_efforts"] is None  # None = every effort accepted
+        assert caps["mandatory"] is False
+
+    def test_reasoning_supported_without_reasoning_object(self):
+        # supported_parameters alone is authoritative for the on/off question.
+        item = {"supported_parameters": ["reasoning"]}
+        caps = parse_openrouter_reasoning_capabilities(item)
+        assert caps["supports_reasoning"] is True
+        assert caps["supported_efforts"] is None
+
+    def test_mandatory_flag(self):
+        item = {
+            "supported_parameters": ["reasoning"],
+            "reasoning": {"mandatory": True, "supported_efforts": None},
+        }
+        caps = parse_openrouter_reasoning_capabilities(item)
+        assert caps["mandatory"] is True
+
+    def test_reasoning_object_untrusted_without_supported_parameters_entry(self):
+        # Top-level reasoning object present but supported_parameters omits
+        # "reasoning" → the route rejects reasoning controls.
+        item = {
+            "supported_parameters": ["temperature", "tools"],
+            "reasoning": {"supported_efforts": ["high"]},
+        }
+        assert parse_openrouter_reasoning_capabilities(item) == {
+            "supports_reasoning": False
+        }
+
+    def test_unknown_when_supported_parameters_missing(self):
+        assert parse_openrouter_reasoning_capabilities({"id": "x"}) is None
+        assert parse_openrouter_reasoning_capabilities({"supported_parameters": "bad"}) is None
+        assert parse_openrouter_reasoning_capabilities("not-a-dict") is None
+
+    def test_effort_list_normalized_and_deduped(self):
+        item = {
+            "supported_parameters": ["reasoning"],
+            "reasoning": {"supported_efforts": [" High ", "high", "LOW", "", 3]},
+        }
+        caps = parse_openrouter_reasoning_capabilities(item)
+        assert caps["supported_efforts"] == ["high", "low", "3"]
+
+
+class TestClampReasoningEffort:
+    @pytest.mark.parametrize(
+        "effort,supported,expected",
+        [
+            # Supported as-is → unchanged.
+            ("high", ["low", "medium", "high"], "high"),
+            # Unknown supported list → pass through.
+            ("ultra", None, "ultra"),
+            ("ultra", [], "ultra"),
+            # Clamp DOWN to nearest lower supported level.
+            ("ultra", ["low", "medium", "high"], "high"),
+            ("xhigh", ["low", "medium", "high"], "high"),
+            ("max", ["minimal", "low"], "low"),
+            # No lower level exists → nearest (lowest) supported.
+            ("minimal", ["low", "medium"], "low"),
+            ("none", ["low", "high"], "low"),
+            # Unrecognized custom level → pass through untouched.
+            ("turbo-think", ["low", "high"], "turbo-think"),
+            # Supported list containing only unrecognized names → pass through.
+            ("high", ["banana"], "high"),
+            # Empty/None effort → pass through.
+            (None, ["low"], None),
+            ("", ["low"], ""),
+        ],
+    )
+    def test_clamping(self, effort, supported, expected):
+        assert clamp_reasoning_effort_to_supported(effort, supported) == expected
+
+    def test_never_escalates(self):
+        # A clamp must never pick a HIGHER level when a lower one exists.
+        assert clamp_reasoning_effort_to_supported("medium", ["low", "xhigh"]) == "low"
+
+
+class TestOpenRouterModelReasoningCapabilities:
+    def _prime_cache(self, monkeypatch, caps_by_id):
+        import hermes_cli.models as models_mod
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", caps_by_id)
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
+
+    def test_known_model(self, monkeypatch):
+        from hermes_cli.models import openrouter_model_reasoning_capabilities
+        self._prime_cache(monkeypatch, {
+            "nvidia/nemotron-3-ultra": {
+                "supports_reasoning": True,
+                "supported_efforts": ["low", "high"],
+                "mandatory": False,
+            },
+        })
+        caps = openrouter_model_reasoning_capabilities("nvidia/nemotron-3-ultra")
+        assert caps["supports_reasoning"] is True
+
+    def test_unlisted_model_returns_none(self, monkeypatch):
+        from hermes_cli.models import openrouter_model_reasoning_capabilities
+        self._prime_cache(monkeypatch, {"a/b": {"supports_reasoning": True}})
+        assert openrouter_model_reasoning_capabilities("private/custom") is None
+
+    def test_empty_model_returns_none(self, monkeypatch):
+        from hermes_cli.models import openrouter_model_reasoning_capabilities
+        self._prime_cache(monkeypatch, {"a/b": {"supports_reasoning": True}})
+        assert openrouter_model_reasoning_capabilities("") is None
+        assert openrouter_model_reasoning_capabilities(None) is None
+
+    def test_catalog_unreachable_returns_none_and_rate_limits(self, monkeypatch):
+        import hermes_cli.models as models_mod
+        from hermes_cli.models import openrouter_model_reasoning_capabilities
+
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", None)
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
+        calls = {"n": 0}
+
+        def _boom(req, *, timeout):
+            calls["n"] += 1
+            raise OSError("offline")
+
+        monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _boom)
+        assert openrouter_model_reasoning_capabilities("a/b", allow_fetch=True) is None
+        assert openrouter_model_reasoning_capabilities("a/b", allow_fetch=True) is None
+        # Second call inside the 60s failure TTL must not re-fetch.
+        assert calls["n"] == 1
+
+    def test_cache_only_by_default_never_fetches(self, monkeypatch):
+        import hermes_cli.models as models_mod
+        from hermes_cli.models import openrouter_model_reasoning_capabilities
+
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", None)
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
+
+        def _boom(req, *, timeout):
+            raise AssertionError("hot path must not fetch")
+
+        monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _boom)
+        # Default (allow_fetch=False) → cache-only, no HTTP even when cold.
+        assert openrouter_model_reasoning_capabilities("a/b") is None
+
+
+class TestSupportsReasoningExtraBodyMetadataGate:
+    """AIAgent._supports_reasoning_extra_body: metadata-first with fallback."""
+
+    def _make_agent(self, model):
+        from run_agent import AIAgent
+        agent = AIAgent(
+            api_key="test-key",
+            base_url="https://openrouter.ai/api/v1",
+            model=model,
+            quiet_mode=True,
+            skip_context_files=True,
+            skip_memory=True,
+        )
+        return agent
+
+    def test_metadata_positive_overrides_static_list(self, monkeypatch):
+        import hermes_cli.models as models_mod
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {
+            # nvidia/ is NOT in the static prefix allowlist (#75386) —
+            # metadata must make it work without a code change.
+            "nvidia/nemotron-3-ultra": {
+                "supports_reasoning": True,
+                "supported_efforts": ["low", "medium", "high"],
+                "mandatory": False,
+            },
+        })
+        agent = self._make_agent("nvidia/nemotron-3-ultra")
+        assert agent._supports_reasoning_extra_body() is True
+
+    def test_metadata_negative_overrides_static_list(self, monkeypatch):
+        import hermes_cli.models as models_mod
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {
+            # openai/ IS in the static prefix list, but the catalog says this
+            # route rejects reasoning controls — the definitive negative wins.
+            "openai/gpt-4o-mini": {"supports_reasoning": False},
+        })
+        agent = self._make_agent("openai/gpt-4o-mini")
+        assert agent._supports_reasoning_extra_body() is False
+
+    def test_unknown_falls_back_to_static_prefixes(self, monkeypatch):
+        import hermes_cli.models as models_mod
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {"a/b": None})
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
+        # deepseek/ is in the static list; unlisted in catalog → fallback True.
+        agent = self._make_agent("deepseek/deepseek-chat")
+        assert agent._supports_reasoning_extra_body() is True
+        # unknown vendor absent from both → False.
+        agent2 = self._make_agent("someveryunknown/model-x")
+        assert agent2._supports_reasoning_extra_body() is False
+
+
+class TestOpenRouterProfileClamp:
+    def test_clamp_applied_in_build_api_kwargs_extras(self, monkeypatch):
+        import hermes_cli.models as models_mod
+        from providers import get_provider_profile
+
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {
+            "qwen/qwen3-plus": {
+                "supports_reasoning": True,
+                "supported_efforts": ["low", "medium", "high"],
+                "mandatory": False,
+            },
+        })
+        profile = get_provider_profile("openrouter")
+        assert profile is not None
+        extra_body, _top = profile.build_api_kwargs_extras(
+            reasoning_config={"enabled": True, "effort": "ultra"},
+            supports_reasoning=True,
+            model="qwen/qwen3-plus",
+        )
+        assert extra_body["reasoning"]["effort"] == "high"
+
+    def test_no_clamp_when_catalog_unknown(self, monkeypatch):
+        import hermes_cli.models as models_mod
+        from providers import get_provider_profile
+
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {"a/b": None})
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
+        profile = get_provider_profile("openrouter")
+        assert profile is not None
+        extra_body, _top = profile.build_api_kwargs_extras(
+            reasoning_config={"enabled": True, "effort": "ultra"},
+            supports_reasoning=True,
+            model="unlisted/model",
+        )
+        # Unknown capability → passthrough unchanged (no silent downgrade).
+        assert extra_body["reasoning"]["effort"] == "ultra"
+
+    def test_disable_omitted_for_mandatory_route_kept_otherwise(self, monkeypatch):
+        """A reasoning-mandatory route 400s on ``{enabled: false}`` — omit it;
+        a route that can disable still gets the user's disable verbatim."""
+        import hermes_cli.models as models_mod
+        from providers import get_provider_profile
+
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_failed_at", None)
+        monkeypatch.setattr(models_mod, "_openrouter_reasoning_caps_cache", {
+            "z-ai/glm-5.3-flash": {
+                "supports_reasoning": True,
+                "supported_efforts": ["max", "high", "low"],
+                "mandatory": True,
+            },
+            "z-ai/glm-5.1": {"supports_reasoning": True, "supported_efforts": None, "mandatory": False},
+        })
+        profile = get_provider_profile("openrouter")
+        for disable in ({"enabled": False}, {"enabled": True, "effort": "none"}):
+            extra_body, _top = profile.build_api_kwargs_extras(
+                reasoning_config=disable, supports_reasoning=True, model="z-ai/glm-5.3-flash",
+            )
+            assert "reasoning" not in extra_body, disable
+        extra_body, _top = profile.build_api_kwargs_extras(
+            reasoning_config={"enabled": False}, supports_reasoning=True, model="z-ai/glm-5.1",
+        )
+        assert extra_body["reasoning"] == {"enabled": False}