Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,367 @@
|
||||
"""Unit tests for the CommandCode provider profiles.
|
||||
|
||||
CommandCode registers two profiles:
|
||||
|
||||
``commandcode``
|
||||
``api_mode=chat_completions`` — OpenAI-compatible. Defaults to
|
||||
``deepseek/deepseek-v4-pro``. 20+ models via a single base URL.
|
||||
|
||||
``commandcode-anthropic``
|
||||
``api_mode=anthropic_messages`` — Anthropic Messages API-compatible.
|
||||
Defaults to ``claude-sonnet-4-6``. Requires Bearer auth recognition
|
||||
in ``agent/anthropic_adapter.py``.
|
||||
|
||||
Both share ``COMMANDCODE_API_KEY`` and ``https://api.commandcode.ai/provider/v1``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ── Fixtures ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def commandcode_profile():
|
||||
"""Resolve the registered CommandCode (chat_completions) profile."""
|
||||
import model_tools # noqa: F401 — triggers discovery
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("commandcode")
|
||||
assert profile is not None, "commandcode provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def commandcode_anthropic_profile():
|
||||
"""Resolve the registered CommandCode Anthropic profile."""
|
||||
import model_tools # noqa: F401 — triggers discovery
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("commandcode-anthropic")
|
||||
assert profile is not None, "commandcode-anthropic profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
# ── Chat Completions profile ──────────────────────────────────────────────────
|
||||
|
||||
class TestCommandCodeProfileIdentity:
|
||||
"""Profile metadata matches the declared contract."""
|
||||
|
||||
def test_name(self, commandcode_profile):
|
||||
assert commandcode_profile.name == "commandcode"
|
||||
|
||||
def test_api_mode(self, commandcode_profile):
|
||||
assert commandcode_profile.api_mode == "chat_completions"
|
||||
|
||||
def test_aliases(self, commandcode_profile):
|
||||
assert "commandcode-chat" in commandcode_profile.aliases
|
||||
|
||||
def test_env_vars(self, commandcode_profile):
|
||||
assert "COMMANDCODE_API_KEY" in commandcode_profile.env_vars
|
||||
|
||||
def test_base_url(self, commandcode_profile):
|
||||
assert commandcode_profile.base_url == "https://api.commandcode.ai/provider/v1"
|
||||
|
||||
def test_display_name(self, commandcode_profile):
|
||||
assert "CommandCode" in commandcode_profile.display_name
|
||||
|
||||
def test_has_fallback_models(self, commandcode_profile):
|
||||
assert len(commandcode_profile.fallback_models) >= 5
|
||||
# Should include the major families
|
||||
names = " ".join(commandcode_profile.fallback_models)
|
||||
assert "deepseek" in names
|
||||
assert "Qwen" in names
|
||||
assert "Kimi" in names
|
||||
assert "gemini" in names
|
||||
|
||||
def test_default_aux_model(self, commandcode_profile):
|
||||
assert commandcode_profile.default_aux_model == "deepseek/deepseek-v4-flash"
|
||||
|
||||
def test_signup_url(self, commandcode_profile):
|
||||
assert "commandcode" in commandcode_profile.signup_url.lower()
|
||||
|
||||
def test_hostname_derived_from_base_url(self, commandcode_profile):
|
||||
assert commandcode_profile.get_hostname() == "api.commandcode.ai"
|
||||
|
||||
|
||||
class TestCommandCodeProfileNoThinkingInterference:
|
||||
"""Chat completions profile is a no-op for thinking config — it delegates
|
||||
to the underlying model's provider (DeepSeek, Qwen, etc.) for wire format.
|
||||
"""
|
||||
|
||||
def test_passthrough_no_reasoning_config(self, commandcode_profile):
|
||||
extra_body, top_level = commandcode_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None, model="deepseek/deepseek-v4-pro"
|
||||
)
|
||||
# Chat completions profile doesn't inject thinking params — that's
|
||||
# the DeepSeek provider's job when routed through DeepSeek's own profile.
|
||||
# When routed through CommandCode, the underlying model API handles it.
|
||||
assert isinstance(extra_body, dict)
|
||||
assert isinstance(top_level, dict)
|
||||
# Default ProviderProfile returns ({}, {}).
|
||||
|
||||
def test_passthrough_with_reasoning_config(self, commandcode_profile):
|
||||
extra_body, top_level = commandcode_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
model="deepseek/deepseek-v4-pro",
|
||||
)
|
||||
assert isinstance(extra_body, dict)
|
||||
assert isinstance(top_level, dict)
|
||||
|
||||
|
||||
# ── Anthropic Messages profile ────────────────────────────────────────────────
|
||||
|
||||
class TestCommandCodeAnthropicProfileIdentity:
|
||||
"""Anthropic-compatible profile metadata."""
|
||||
|
||||
def test_name(self, commandcode_anthropic_profile):
|
||||
assert commandcode_anthropic_profile.name == "commandcode-anthropic"
|
||||
|
||||
def test_api_mode(self, commandcode_anthropic_profile):
|
||||
assert commandcode_anthropic_profile.api_mode == "anthropic_messages"
|
||||
|
||||
def test_aliases(self, commandcode_anthropic_profile):
|
||||
assert "commandcode-claude" in commandcode_anthropic_profile.aliases
|
||||
|
||||
def test_env_vars(self, commandcode_anthropic_profile):
|
||||
assert "COMMANDCODE_API_KEY" in commandcode_anthropic_profile.env_vars
|
||||
|
||||
def test_base_url(self, commandcode_anthropic_profile):
|
||||
assert commandcode_anthropic_profile.base_url == "https://api.commandcode.ai/provider/v1"
|
||||
|
||||
def test_fallback_models_are_claude_family(self, commandcode_anthropic_profile):
|
||||
for model in commandcode_anthropic_profile.fallback_models:
|
||||
assert model.startswith("claude-"), (
|
||||
f"All anthropic fallback models should be claude-*: got {model}"
|
||||
)
|
||||
|
||||
def test_default_aux_model(self, commandcode_anthropic_profile):
|
||||
assert commandcode_anthropic_profile.default_aux_model == "claude-haiku-4-5-20251001"
|
||||
|
||||
def test_display_name_distinct_from_chat(self, commandcode_anthropic_profile):
|
||||
# The Anthropic profile should be distinguishable in /model picker
|
||||
assert "(Anthropic)" in commandcode_anthropic_profile.display_name
|
||||
|
||||
def test_hostname_derived_from_base_url(self, commandcode_anthropic_profile):
|
||||
assert commandcode_anthropic_profile.get_hostname() == "api.commandcode.ai"
|
||||
|
||||
|
||||
# ── Bearer Auth Recognition ───────────────────────────────────────────────────
|
||||
|
||||
class TestCommandCodeAnthropicBearerAuth:
|
||||
"""``agent/anthropic_adapter.py`` must recognize CommandCode as a
|
||||
Bearer-auth endpoint, or the chat_completions transport falls back to
|
||||
``x-api-key`` and gets a 401.
|
||||
"""
|
||||
|
||||
def test_requires_bearer_auth_recognizes_commandcode(self):
|
||||
from agent.anthropic_adapter import _requires_bearer_auth
|
||||
|
||||
assert _requires_bearer_auth("https://api.commandcode.ai/provider/v1") is True
|
||||
assert _requires_bearer_auth("https://api.commandcode.ai/provider/v1/models") is True
|
||||
assert _requires_bearer_auth("https://api.commandcode.ai/anthropic") is True
|
||||
|
||||
def test_bearer_auth_does_not_affect_unrelated(self):
|
||||
from agent.anthropic_adapter import _requires_bearer_auth
|
||||
|
||||
# Native Anthropic still uses x-api-key
|
||||
assert _requires_bearer_auth("https://api.anthropic.com") is False
|
||||
# OpenRouter still uses Bearer through its own transport path
|
||||
assert _requires_bearer_auth("https://openrouter.ai/api/v1") is False
|
||||
|
||||
def test_bearer_auth_case_insensitive(self):
|
||||
from agent.anthropic_adapter import _requires_bearer_auth
|
||||
|
||||
assert _requires_bearer_auth("https://API.COMMANDCODE.AI/provider/v1") is True
|
||||
|
||||
|
||||
# ── Registry integrity ───────────────────────────────────────────────────────
|
||||
|
||||
class TestCommandCodeRegistryIntegrity:
|
||||
"""Both profiles are discoverable and distinct."""
|
||||
|
||||
def test_both_profiles_registered(self):
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
chat = providers.get_provider_profile("commandcode")
|
||||
anth = providers.get_provider_profile("commandcode-anthropic")
|
||||
assert chat is not None
|
||||
assert anth is not None
|
||||
assert chat is not anth # distinct profile instances
|
||||
|
||||
def test_alias_lookup(self):
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
assert providers.get_provider_profile("commandcode-chat") is not None
|
||||
assert providers.get_provider_profile("commandcode-claude") is not None
|
||||
|
||||
def test_unknown_returns_none(self):
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
assert providers.get_provider_profile("commandcode-nonexistent") is None
|
||||
|
||||
|
||||
# ── Model list filtering ──────────────────────────────────────────────────────
|
||||
|
||||
class TestCommandCodeModelFiltering:
|
||||
"""``fetch_models`` filtering contracts."""
|
||||
|
||||
def test_anthropic_profile_filters_to_claude(self):
|
||||
"""If we mock a response with mixed models, anthropic profile
|
||||
should only return claude-* models.
|
||||
"""
|
||||
from plugins.model_providers.commandcode import CommandCodeAnthropicProfile
|
||||
|
||||
profile = CommandCodeAnthropicProfile(
|
||||
name="test-cc-anth",
|
||||
api_mode="anthropic_messages",
|
||||
env_vars=("COMMANDCODE_API_KEY",),
|
||||
base_url="https://api.commandcode.ai/provider/v1",
|
||||
)
|
||||
|
||||
# Don't actually hit the network — just test the filter logic.
|
||||
# The class has a fetch_models override that filters.
|
||||
# We verify the filter works by inspecting the method.
|
||||
import inspect
|
||||
|
||||
source = inspect.getsource(profile.fetch_models)
|
||||
assert "startswith(\"claude-\")" in source or '"claude-" in m' in source, (
|
||||
"CommandCodeAnthropicProfile.fetch_models should filter to claude-* models"
|
||||
)
|
||||
|
||||
|
||||
# ── Picker contract ──────────────────────────────────────────────────────────
|
||||
|
||||
class TestCommandCodeFetchModelsPickerContract:
|
||||
"""``fetch_models`` must accept the kwargs the model picker passes.
|
||||
|
||||
Regression: the generic live-fetch path in ``hermes_cli/models.py``
|
||||
(``provider_model_ids``) calls ``profile.fetch_models(api_key=...,
|
||||
base_url=...)``. The original CommandCode overrides only accepted
|
||||
``api_key``/``timeout``, so every picker open raised TypeError, which
|
||||
was swallowed, leaving the provider with zero models.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize("profile_name", ["commandcode", "commandcode-anthropic"])
|
||||
def test_accepts_base_url_kwarg(self, profile_name):
|
||||
import inspect
|
||||
|
||||
import model_tools # noqa: F401 — triggers discovery
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile(profile_name)
|
||||
assert profile is not None
|
||||
assert "base_url" in inspect.signature(profile.fetch_models).parameters
|
||||
|
||||
def test_resolve_provider_full(self):
|
||||
"""Both profiles must resolve through the model-switch path.
|
||||
|
||||
Regression: ``resolve_provider_full`` only knew models.dev + overlay
|
||||
providers, so plugin-only providers (commandcode) failed with
|
||||
"Unknown provider" on /model switches even though the picker listed
|
||||
them.
|
||||
"""
|
||||
from hermes_cli.providers import resolve_provider_full
|
||||
|
||||
chat = resolve_provider_full("commandcode", {}, [])
|
||||
assert chat is not None and chat.id == "commandcode"
|
||||
assert chat.transport == "openai_chat"
|
||||
assert "COMMANDCODE_API_KEY" in chat.api_key_env_vars
|
||||
|
||||
anth = resolve_provider_full("commandcode-anthropic", {}, [])
|
||||
assert anth is not None and anth.id == "commandcode-anthropic"
|
||||
assert anth.transport == "anthropic_messages"
|
||||
|
||||
|
||||
# ── base_url endpoint override ───────────────────────────────────────────────
|
||||
|
||||
class TestCommandCodeBaseUrlOverride:
|
||||
"""A custom base_url must redirect the catalog fetch; the default must not.
|
||||
|
||||
The picker passes ``base_url`` unconditionally (profile default when the
|
||||
user configured nothing), so only a value differing from the default
|
||||
``_COMMANDCODE_BASE`` counts as a customised endpoint.
|
||||
"""
|
||||
|
||||
def _serve(self, models):
|
||||
import json
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from threading import Thread
|
||||
|
||||
class H(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
body = json.dumps({"data": models}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
pass
|
||||
|
||||
server = HTTPServer(("127.0.0.1", 0), H)
|
||||
Thread(target=server.serve_forever, daemon=True).start()
|
||||
return server, server.server_address[1]
|
||||
|
||||
def test_custom_base_url_redirects_fetch(self, commandcode_profile):
|
||||
server, port = self._serve([{"id": "proxied/model-x"}])
|
||||
try:
|
||||
result = commandcode_profile.fetch_models(
|
||||
api_key="k", base_url=f"http://127.0.0.1:{port}"
|
||||
)
|
||||
assert result == ["proxied/model-x"]
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
def test_custom_base_url_redirects_anthropic_fetch(
|
||||
self, commandcode_anthropic_profile
|
||||
):
|
||||
server, port = self._serve(
|
||||
[{"id": "claude-sonnet-4-6"}, {"id": "deepseek/deepseek-v4-pro"}]
|
||||
)
|
||||
try:
|
||||
result = commandcode_anthropic_profile.fetch_models(
|
||||
api_key="k", base_url=f"http://127.0.0.1:{port}"
|
||||
)
|
||||
assert result == ["claude-sonnet-4-6"] # claude-* filter still applies
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
def test_default_base_url_hits_default_endpoint(self, commandcode_profile):
|
||||
"""Echoing the profile default back must NOT count as an override."""
|
||||
import sys
|
||||
from unittest.mock import patch as mock_patch
|
||||
|
||||
# The bundled plugin module is registered at discovery time under
|
||||
# ``plugins.model_providers.commandcode`` — resolve via the profile's
|
||||
# own __module__ so the test doesn't depend on discovery mechanics.
|
||||
cc_mod = sys.modules[type(commandcode_profile).__module__]
|
||||
|
||||
captured = {}
|
||||
|
||||
class _FakeResp:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return b'{"data": [{"id": "m1"}]}'
|
||||
|
||||
def fake_urlopen(req, timeout=0):
|
||||
captured["url"] = req.full_url
|
||||
return _FakeResp()
|
||||
|
||||
with mock_patch.object(
|
||||
cc_mod.urllib.request, "urlopen", side_effect=fake_urlopen
|
||||
):
|
||||
result = commandcode_profile.fetch_models(
|
||||
api_key="k", base_url=cc_mod._COMMANDCODE_BASE + "/"
|
||||
)
|
||||
assert result == ["m1"]
|
||||
assert captured["url"] == cc_mod._COMMANDCODE_MODELS_URL
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Unit tests for the Copilot provider profile's reasoning-effort wiring.
|
||||
|
||||
GitHub Copilot serves different models with different supported reasoning-effort
|
||||
sets (the live ``/models`` catalog reports them per model). The profile must
|
||||
forward the requested effort when the catalog lists it as supported, and only
|
||||
downgrade to the nearest weaker supported level when it does not, rather than
|
||||
unconditionally collapsing ``xhigh`` to ``high`` (which silently capped models
|
||||
that actually support the higher level).
|
||||
|
||||
These tests pin that contract without going live, by stubbing the catalog
|
||||
lookup ``github_model_reasoning_efforts``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def copilot_profile():
|
||||
"""Resolve the registered Copilot profile.
|
||||
|
||||
Importing ``model_tools`` triggers plugin discovery, which registers the
|
||||
Copilot profile. Going through ``get_provider_profile`` keeps the test
|
||||
honest: if the registered class is ever swapped for a plain
|
||||
``ProviderProfile`` the assertions below collapse.
|
||||
"""
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("copilot")
|
||||
assert profile is not None, "copilot provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
def _patch_efforts(monkeypatch, efforts):
|
||||
"""Stub the catalog lookup the profile calls for supported efforts."""
|
||||
import hermes_cli.models as models_mod
|
||||
monkeypatch.setattr(
|
||||
models_mod, "github_model_reasoning_efforts", lambda model: list(efforts)
|
||||
)
|
||||
|
||||
|
||||
class TestCopilotReasoningEffortClamp:
|
||||
def test_supported_effort_forwarded_verbatim(self, copilot_profile, monkeypatch):
|
||||
"""xhigh is forwarded unchanged when the catalog lists it."""
|
||||
_patch_efforts(monkeypatch, ["minimal", "low", "medium", "high", "xhigh"])
|
||||
extra_body, _ = copilot_profile.build_api_kwargs_extras(
|
||||
model="gpt-5.5",
|
||||
reasoning_config={"effort": "xhigh"},
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert extra_body["reasoning"] == {"effort": "xhigh"}
|
||||
|
||||
def test_xhigh_downgrades_to_high_when_unsupported(self, copilot_profile, monkeypatch):
|
||||
"""A model whose catalog lacks xhigh gets the nearest weaker level."""
|
||||
_patch_efforts(monkeypatch, ["low", "medium", "high"])
|
||||
extra_body, _ = copilot_profile.build_api_kwargs_extras(
|
||||
model="o-series-model",
|
||||
reasoning_config={"effort": "xhigh"},
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert extra_body["reasoning"] == {"effort": "high"}
|
||||
|
||||
def test_minimal_downgrades_to_low_when_unsupported(self, copilot_profile, monkeypatch):
|
||||
_patch_efforts(monkeypatch, ["low", "medium", "high"])
|
||||
extra_body, _ = copilot_profile.build_api_kwargs_extras(
|
||||
model="o-series-model",
|
||||
reasoning_config={"effort": "minimal"},
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert extra_body["reasoning"] == {"effort": "low"}
|
||||
|
||||
def test_unsupported_effort_falls_back_to_medium(self, copilot_profile, monkeypatch):
|
||||
"""An effort not in the set, with no specific rule, falls to medium."""
|
||||
_patch_efforts(monkeypatch, ["low", "medium", "high"])
|
||||
extra_body, _ = copilot_profile.build_api_kwargs_extras(
|
||||
model="some-model",
|
||||
reasoning_config={"effort": "garbage"},
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert extra_body["reasoning"] == {"effort": "medium"}
|
||||
|
||||
def test_falls_back_to_first_supported_when_no_medium(self, copilot_profile, monkeypatch):
|
||||
"""If medium isn't supported either, pick the first supported level."""
|
||||
_patch_efforts(monkeypatch, ["low", "high"])
|
||||
extra_body, _ = copilot_profile.build_api_kwargs_extras(
|
||||
model="weird-model",
|
||||
reasoning_config={"effort": "xhigh"},
|
||||
supports_reasoning=True,
|
||||
)
|
||||
# xhigh not supported, high IS supported → high wins via the xhigh rule.
|
||||
assert extra_body["reasoning"] == {"effort": "high"}
|
||||
|
||||
def test_first_supported_when_no_rule_matches(self, copilot_profile, monkeypatch):
|
||||
_patch_efforts(monkeypatch, ["low", "high"])
|
||||
extra_body, _ = copilot_profile.build_api_kwargs_extras(
|
||||
model="weird-model",
|
||||
reasoning_config={"effort": "garbage"},
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert extra_body["reasoning"] == {"effort": "low"}
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Unit tests for the custom provider profile's reasoning wiring.
|
||||
|
||||
``provider=custom`` covers any OpenAI-compatible endpoint the user points
|
||||
Hermes at — local Ollama, vLLM, llama.cpp, and hosted reasoning APIs like
|
||||
GLM-5.2 on Volcengine ARK. Before #57601's salvage, ``CustomProfile`` emitted
|
||||
nothing when reasoning was *enabled*, so a configured ``reasoning_effort``
|
||||
was silently dropped for every custom endpoint.
|
||||
|
||||
These tests pin the wire-shape contract:
|
||||
- disabled on Ollama → extra_body.think = False + reasoning_effort=none
|
||||
- disabled elsewhere → reasoning_effort=none, no think (strict APIs 422)
|
||||
- enabled + effort → top-level reasoning_effort (native OpenAI-compat
|
||||
format GLM/ARK expect), passed through verbatim
|
||||
including ``max``/``xhigh``
|
||||
- enabled + no effort → nothing emitted (endpoint's server default applies)
|
||||
- ollama_num_ctx → extra_body.options.num_ctx, orthogonal to reasoning
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def custom_profile():
|
||||
"""Resolve the registered custom profile via the global registry.
|
||||
|
||||
Importing ``model_tools`` triggers plugin discovery, which registers the
|
||||
``custom`` profile. Going through ``get_provider_profile`` keeps the test
|
||||
honest — if the registered class is ever downgraded to a plain
|
||||
``ProviderProfile``, the assertions below collapse.
|
||||
"""
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("custom")
|
||||
assert profile is not None, "custom provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
class TestCustomReasoningWireShape:
|
||||
"""``build_api_kwargs_extras`` produces the correct wire format."""
|
||||
|
||||
def test_no_reasoning_config_emits_nothing(self, custom_profile):
|
||||
"""Unset reasoning → omit everything so the endpoint's default applies."""
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None, model="glm-5.2"
|
||||
)
|
||||
assert eb == {}
|
||||
assert tl == {}
|
||||
|
||||
def test_disabled_sends_think_false(self, custom_profile):
|
||||
"""enabled=False on an Ollama URL → reasoning_effort='none' + think=False.
|
||||
|
||||
Both fields are required on Ollama: /v1/chat/completions silently
|
||||
ignores extra_body.think (only /api/chat honours it — ollama#14820)
|
||||
but respects top-level reasoning_effort (#25758). think=False stays
|
||||
for proxies and the native /api/chat path.
|
||||
"""
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
model="qwen3",
|
||||
base_url="http://127.0.0.1:11434/v1",
|
||||
)
|
||||
assert eb == {"think": False}
|
||||
assert tl == {"reasoning_effort": "none"}
|
||||
|
||||
def test_effort_none_sends_think_false(self, custom_profile):
|
||||
"""effort='none' is the disable alias → same dual emission on Ollama."""
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "none"},
|
||||
model="qwen3",
|
||||
base_url="http://localhost:11434/v1",
|
||||
)
|
||||
assert eb == {"think": False}
|
||||
assert tl == {"reasoning_effort": "none"}
|
||||
|
||||
def test_disabled_omits_think_on_mistral(self, custom_profile):
|
||||
"""Strict OpenAI-compat hosts forbid extra ``think`` (HTTP 422)."""
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "none"},
|
||||
model="mistral-small-latest",
|
||||
base_url="https://api.mistral.ai/v1",
|
||||
)
|
||||
assert "think" not in eb
|
||||
assert tl == {"reasoning_effort": "none"}
|
||||
|
||||
def test_disabled_omits_think_without_base_url(self, custom_profile):
|
||||
"""Unknown custom endpoint — do not send the Ollama-only flag."""
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False}, model="glm-5.2"
|
||||
)
|
||||
assert "think" not in eb
|
||||
assert tl == {"reasoning_effort": "none"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"http://127.0.0.1:8080/v1",
|
||||
"http://localhost:1234/v1",
|
||||
"https://api.groq.com/openai/v1",
|
||||
],
|
||||
)
|
||||
def test_disabled_omits_think_on_non_ollama_relays(self, custom_profile, base_url):
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"effort": "none"},
|
||||
model="llama3",
|
||||
base_url=base_url,
|
||||
)
|
||||
assert "think" not in eb
|
||||
assert tl == {"reasoning_effort": "none"}
|
||||
|
||||
def test_disabled_sends_think_false_on_ollama_cloud_host(self, custom_profile):
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
model="qwen3",
|
||||
base_url="https://ollama.com/v1",
|
||||
)
|
||||
assert eb == {"think": False}
|
||||
assert tl == {"reasoning_effort": "none"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"http://myhost:99999/v1", # out-of-range port: OpenAI client accepts it
|
||||
"http://localhost:80a/v1", # non-integer port
|
||||
"http://localhost:11434./v1", # trailing-dot port
|
||||
],
|
||||
)
|
||||
def test_malformed_port_does_not_raise(self, custom_profile, base_url):
|
||||
"""Malformed ports must not raise — urlparse's ``port`` is ValueError-happy.
|
||||
|
||||
The OpenAI client accepts ``http://myhost:99999/v1`` at construction
|
||||
(only httpx fails later), so these URLs reach ``build_api_kwargs_extras``
|
||||
in production. The heuristic must treat them as non-Ollama rather than
|
||||
killing the kwargs build.
|
||||
"""
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
model="qwen3",
|
||||
base_url=base_url,
|
||||
)
|
||||
assert "think" not in eb
|
||||
assert tl == {"reasoning_effort": "none"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"effort", ["minimal", "low", "medium", "high", "xhigh", "max"]
|
||||
)
|
||||
def test_enabled_effort_goes_top_level(self, custom_profile, effort):
|
||||
"""enabled + effort → TOP-LEVEL reasoning_effort, passed through verbatim.
|
||||
|
||||
GLM-5.2/ARK and OpenAI-compatible reasoning APIs read reasoning_effort
|
||||
as a top-level string, not nested in extra_body. ``max`` is GLM's
|
||||
native deep-reasoning level and must survive.
|
||||
"""
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort}, model="glm-5.2"
|
||||
)
|
||||
assert tl == {"reasoning_effort": effort}
|
||||
assert "reasoning_effort" not in eb
|
||||
assert "think" not in eb
|
||||
|
||||
|
||||
def test_does_not_force_think_true_on_enable(self, custom_profile):
|
||||
"""We must never send think=True on enable — it's Ollama-only and
|
||||
would 400 on GLM/vLLM endpoints that don't recognize it."""
|
||||
eb, _ = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"}, model="glm-5.2"
|
||||
)
|
||||
assert eb.get("think") is not True
|
||||
|
||||
|
||||
class TestCustomReasoningWithNumCtx:
|
||||
"""Ollama num_ctx and reasoning are independent and compose."""
|
||||
|
||||
def test_num_ctx_alone(self, custom_profile):
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None, ollama_num_ctx=8192, model="qwen3"
|
||||
)
|
||||
assert eb == {"options": {"num_ctx": 8192}}
|
||||
assert tl == {}
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
"""Unit tests for the DeepSeek provider profile's thinking-mode wiring.
|
||||
|
||||
DeepSeek V4 expects every request to carry an explicit ``extra_body.thinking``
|
||||
parameter. Omitting it makes the server default to thinking-mode ON, which
|
||||
then enforces the ``reasoning_content``-must-be-echoed-back contract on
|
||||
subsequent turns and breaks the conversation with HTTP 400 (#15700, #17212,
|
||||
#17825).
|
||||
|
||||
These tests pin the profile's wire-shape contract so DeepSeek requests stay
|
||||
correctly shaped without going live.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def deepseek_profile():
|
||||
"""Resolve the registered DeepSeek profile.
|
||||
|
||||
Going through ``providers.get_provider_profile`` keeps the test honest —
|
||||
if someone later replaces the registered class with a plain
|
||||
``ProviderProfile``, every assertion below collapses.
|
||||
"""
|
||||
# ``model_tools`` triggers plugin discovery on import, which is what
|
||||
# registers the DeepSeek profile in the global provider registry.
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("deepseek")
|
||||
assert profile is not None, "deepseek provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
class TestDeepSeekThinkingWireShape:
|
||||
"""``build_api_kwargs_extras`` produces DeepSeek's exact wire format."""
|
||||
|
||||
def test_v4_pro_default_enables_thinking_without_effort(self, deepseek_profile):
|
||||
"""No reasoning_config → thinking enabled, server picks default effort."""
|
||||
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None, model="deepseek-v4-pro"
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
|
||||
def test_standard_efforts_pass_through(self, deepseek_profile, effort):
|
||||
_, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert top_level == {"reasoning_effort": effort}
|
||||
|
||||
@pytest.mark.parametrize("effort", ["xhigh", "max", "MAX", " Max "])
|
||||
def test_xhigh_and_max_normalize_to_max(self, deepseek_profile, effort):
|
||||
_, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert top_level == {"reasoning_effort": "max"}
|
||||
|
||||
def test_explicitly_disabled_sends_disabled_marker(self, deepseek_profile):
|
||||
"""``reasoning_config.enabled=False`` → ``thinking.type=disabled``.
|
||||
|
||||
The crucial bit is that the parameter is *sent* at all — DeepSeek
|
||||
defaults to thinking-on when ``thinking`` is absent.
|
||||
"""
|
||||
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False}, model="deepseek-v4-pro"
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "disabled"}}
|
||||
# No effort when disabled — DeepSeek rejects it.
|
||||
assert top_level == {}
|
||||
|
||||
def test_disabled_ignores_effort_field(self, deepseek_profile):
|
||||
"""Effort silently dropped when thinking is off."""
|
||||
_, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False, "effort": "high"},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
def test_unknown_effort_omits_top_level(self, deepseek_profile):
|
||||
"""Garbage effort → omit reasoning_effort so DeepSeek applies its default."""
|
||||
_, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "garbage"},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
def test_empty_effort_omits_top_level(self, deepseek_profile):
|
||||
_, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": ""},
|
||||
model="deepseek-v4-pro",
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestDeepSeekModelGating:
|
||||
"""V4 family gets thinking; V3 / unknown stay untouched."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"deepseek-v4-pro",
|
||||
"deepseek-v4-flash",
|
||||
"deepseek-v4-future-variant",
|
||||
"DEEPSEEK-V4-PRO", # case-insensitive
|
||||
],
|
||||
)
|
||||
def test_thinking_capable_models_emit_thinking(self, deepseek_profile, model):
|
||||
extra_body, _ = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None, model=model
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"deepseek-v3-0324", # explicit V3
|
||||
"deepseek-v3.1", # V3 minor revisions
|
||||
"", # bare/unknown
|
||||
None, # missing
|
||||
"deepseek-unknown", # unrecognized
|
||||
],
|
||||
)
|
||||
def test_non_thinking_models_emit_nothing(self, deepseek_profile, model):
|
||||
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"}, model=model
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestDeepSeekFullKwargsIntegration:
|
||||
"""End-to-end: the transport's full kwargs match DeepSeek's live wire format.
|
||||
|
||||
The live test harness in ``tests/run_agent/test_deepseek_v4_thinking_live.py``
|
||||
sends ``{"reasoning_effort": "high", "extra_body": {"thinking": {"type":
|
||||
"enabled"}}}``. Confirm the transport produces that exact shape when wired
|
||||
through the registered DeepSeek profile.
|
||||
"""
|
||||
|
||||
def test_full_kwargs_match_live_wire_shape(self, deepseek_profile):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
kwargs = ChatCompletionsTransport().build_kwargs(
|
||||
model="deepseek-v4-pro",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=deepseek_profile,
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
base_url="https://api.deepseek.com/v1",
|
||||
provider_name="deepseek",
|
||||
)
|
||||
assert kwargs["model"] == "deepseek-v4-pro"
|
||||
assert kwargs["reasoning_effort"] == "high"
|
||||
assert kwargs["extra_body"] == {"thinking": {"type": "enabled"}}
|
||||
|
||||
def test_v3_full_kwargs_omit_thinking(self, deepseek_profile):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
kwargs = ChatCompletionsTransport().build_kwargs(
|
||||
model="deepseek-v3-0324",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=deepseek_profile,
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
base_url="https://api.deepseek.com/v1",
|
||||
provider_name="deepseek",
|
||||
)
|
||||
assert "reasoning_effort" not in kwargs
|
||||
assert "extra_body" not in kwargs or "thinking" not in kwargs.get("extra_body", {})
|
||||
|
||||
|
||||
class TestDeepSeekAuxModel:
|
||||
"""DeepSeek aux model is set on the profile so users stop seeing the
|
||||
bogus 'No auxiliary LLM provider configured' warning (#26924).
|
||||
|
||||
Pinned at the profile layer rather than the legacy
|
||||
`_API_KEY_PROVIDER_AUX_MODELS_FALLBACK` dict — new providers are
|
||||
expected to set `default_aux_model` on `ProviderProfile`, and the
|
||||
fallback dict only exists for providers that predate the profiles
|
||||
system.
|
||||
"""
|
||||
|
||||
def test_profile_advertises_deepseek_v4_flash(self, deepseek_profile):
|
||||
assert deepseek_profile.default_aux_model == "deepseek-v4-flash"
|
||||
|
||||
def test_fallback_models_are_v4_only(self, deepseek_profile):
|
||||
assert deepseek_profile.fallback_models == (
|
||||
"deepseek-v4-pro",
|
||||
"deepseek-v4-flash",
|
||||
)
|
||||
|
||||
def test_consumer_api_returns_deepseek_v4_flash(self):
|
||||
from agent.auxiliary_client import _get_aux_model_for_provider
|
||||
assert _get_aux_model_for_provider("deepseek") == "deepseek-v4-flash"
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Unit tests for the Fireworks AI provider profile.
|
||||
|
||||
Pins the profile's contract without going live: identity, alias registration,
|
||||
and the pay-as-you-go model defaults (direct catalog ``/models/``
|
||||
IDs, not the router-only tier).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fireworks_profile():
|
||||
"""Resolve the registered Fireworks profile through the real discovery path."""
|
||||
# Importing model_tools triggers plugin discovery, registering the profile.
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("fireworks")
|
||||
assert profile is not None, "fireworks provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
class TestFireworksIdentity:
|
||||
def test_core_fields(self, fireworks_profile):
|
||||
p = fireworks_profile
|
||||
assert p.name == "fireworks"
|
||||
assert p.auth_type == "api_key"
|
||||
assert p.base_url == "https://api.fireworks.ai/inference/v1"
|
||||
assert "FIREWORKS_API_KEY" in p.env_vars
|
||||
assert "FIREWORKS_BASE_URL" not in p.env_vars
|
||||
|
||||
def test_display_metadata_present(self, fireworks_profile):
|
||||
# Prominence copy is surfaced in the picker; keep it non-empty rather
|
||||
# than pinning exact marketing wording (that's expected to change).
|
||||
assert fireworks_profile.display_name
|
||||
assert fireworks_profile.description
|
||||
assert fireworks_profile.signup_url.startswith("https://")
|
||||
|
||||
|
||||
class TestFireworksHeaders:
|
||||
def test_attribution_matches_canonical_hermes_values(self, fireworks_profile):
|
||||
"""Fireworks requests carry the same attribution identity Hermes sends
|
||||
everywhere else.
|
||||
|
||||
Asserted against the shared constant rather than the literals so a
|
||||
rebrand can't leave one provider on a stale referer/title.
|
||||
"""
|
||||
from agent.auxiliary_client import _OR_HEADERS_BASE
|
||||
|
||||
headers = fireworks_profile.default_headers
|
||||
assert headers["HTTP-Referer"] == _OR_HEADERS_BASE["HTTP-Referer"]
|
||||
assert headers["X-Title"] == _OR_HEADERS_BASE["X-Title"]
|
||||
|
||||
def test_user_agent_identifies_hermes(self, fireworks_profile):
|
||||
# Prefix, not the full string — the version moves every release.
|
||||
assert fireworks_profile.default_headers["User-Agent"].startswith("HermesAgent/")
|
||||
|
||||
|
||||
class TestFireworksAliases:
|
||||
@pytest.mark.parametrize("alias", ["fireworks-ai", "fw"])
|
||||
def test_alias_resolves_via_registry(self, fireworks_profile, alias):
|
||||
import providers
|
||||
|
||||
resolved = providers.get_provider_profile(alias)
|
||||
assert resolved is not None
|
||||
assert resolved.name == "fireworks"
|
||||
|
||||
def test_aliases_declared_on_profile(self, fireworks_profile):
|
||||
assert "fireworks-ai" in fireworks_profile.aliases
|
||||
assert "fw" in fireworks_profile.aliases
|
||||
|
||||
|
||||
class TestFireworksModelDefaults:
|
||||
"""Defaults must be usable with a standard pay-as-you-go key.
|
||||
|
||||
PAYG keys address ``accounts/fireworks/models/...`` directly; the bundled
|
||||
defaults target that (the BYOK motion) so a fresh key works out of the box,
|
||||
and use the standard tier rather than turbo as the out-of-box default.
|
||||
"""
|
||||
|
||||
def test_aux_model_is_payg_model_not_router(self, fireworks_profile):
|
||||
aux = fireworks_profile.default_aux_model
|
||||
assert aux.startswith("accounts/fireworks/models/"), aux
|
||||
assert "/routers/" not in aux
|
||||
assert "turbo" not in aux.lower()
|
||||
|
||||
def test_fallback_models_are_payg_models_not_routers(self, fireworks_profile):
|
||||
assert fireworks_profile.fallback_models, "expected curated fallbacks"
|
||||
for model in fireworks_profile.fallback_models:
|
||||
assert model.startswith("accounts/fireworks/models/"), model
|
||||
assert "/routers/" not in model
|
||||
assert "turbo" not in model.lower(), model
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Contract tests for the native Google Gemini provider profile."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gemini_profile():
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("gemini")
|
||||
assert profile is not None, "gemini provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
def test_native_gemini_auxiliary_default_is_in_curated_catalog(gemini_profile):
|
||||
"""The profile's default_aux_model must stay in lockstep with the curated
|
||||
model picker catalog — whatever model the default points at has to be
|
||||
one the picker can actually offer. Deliberately durable against future
|
||||
model-generation bumps: it does not pin either side to a frozen
|
||||
model-name string, only to the invariant that they never drift apart.
|
||||
"""
|
||||
from hermes_cli.models import _PROVIDER_MODELS
|
||||
|
||||
assert gemini_profile.default_aux_model in _PROVIDER_MODELS["gemini"]
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Unit tests for the Kimi/Moonshot provider profile's reasoning wiring.
|
||||
|
||||
Moonshot's OpenAI-compat endpoint (``api.moonshot.ai/v1``) treats
|
||||
``extra_body.thinking`` and a top-level ``reasoning_effort`` as mutually
|
||||
exclusive. The profile must send at most one of them — never both — so a
|
||||
request can't trip "cannot specify both 'thinking' and 'reasoning_effort'".
|
||||
|
||||
This mirrors the kimi-k2 handling already shipped for the opencode-go relay
|
||||
(see ``tests/plugins/model_providers/test_opencode_go_profile.py``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def kimi_profile():
|
||||
"""Resolve the registered Kimi profile via the provider registry.
|
||||
|
||||
Importing ``model_tools`` triggers plugin discovery, which registers the
|
||||
Kimi profile. Going through ``get_provider_profile`` keeps the test honest:
|
||||
if the registered class is ever swapped for a plain ``ProviderProfile`` the
|
||||
assertions below collapse.
|
||||
"""
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("kimi-coding")
|
||||
assert profile is not None, "kimi-coding provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
class TestKimiReasoningWireShape:
|
||||
"""``build_api_kwargs_extras`` never emits thinking + reasoning_effort together."""
|
||||
|
||||
def test_no_config_enables_thinking_without_effort(self, kimi_profile):
|
||||
"""No reasoning_config → thinking on, server picks the depth.
|
||||
|
||||
Regression guard: this path previously also sent
|
||||
``reasoning_effort="medium"``, pairing thinking + effort on every
|
||||
default call.
|
||||
"""
|
||||
extra_body, top_level = kimi_profile.build_api_kwargs_extras(reasoning_config=None)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"effort,expected",
|
||||
[
|
||||
("low", "low"),
|
||||
("minimal", "low"),
|
||||
("medium", "high"),
|
||||
("high", "high"),
|
||||
("xhigh", "max"),
|
||||
("max", "max"),
|
||||
("ultra", "max"),
|
||||
],
|
||||
)
|
||||
def test_effort_mapped_to_k3_vocabulary(self, kimi_profile, effort, expected):
|
||||
"""Hermes' wider effort vocabulary is mapped onto K3's low/high/max."""
|
||||
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort}
|
||||
)
|
||||
assert top_level == {"reasoning_effort": expected}
|
||||
assert "thinking" not in extra_body
|
||||
|
||||
def test_enabled_without_effort_falls_back_to_thinking(self, kimi_profile):
|
||||
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True}
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
@pytest.mark.parametrize("effort", ["", "garbage"])
|
||||
def test_unrecognized_effort_falls_back_to_thinking(self, kimi_profile, effort):
|
||||
"""Unknown efforts drop to the thinking toggle rather than sending
|
||||
an invalid effort."""
|
||||
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort}
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
def test_disabled_sends_thinking_disabled_only(self, kimi_profile):
|
||||
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False}
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "disabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reasoning_config",
|
||||
[
|
||||
None,
|
||||
{"enabled": True},
|
||||
{"enabled": True, "effort": "high"},
|
||||
{"enabled": True, "effort": "garbage"},
|
||||
{"enabled": False},
|
||||
{"enabled": False, "effort": "low"},
|
||||
],
|
||||
)
|
||||
def test_never_emits_both(self, kimi_profile, reasoning_config):
|
||||
"""The core invariant: thinking and reasoning_effort are never both set."""
|
||||
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
|
||||
reasoning_config=reasoning_config
|
||||
)
|
||||
assert not ("thinking" in extra_body and "reasoning_effort" in top_level)
|
||||
|
||||
|
||||
class TestKimiModelDiscovery:
|
||||
def test_malformed_base_url_is_unconfirmed_and_filters_k3(self, kimi_profile):
|
||||
"""Malformed user URLs must fall through safely, never authorize K3."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from providers.base import ProviderProfile
|
||||
|
||||
with patch.object(
|
||||
ProviderProfile,
|
||||
"fetch_models",
|
||||
return_value=["k3", "kimi-k2.6"],
|
||||
):
|
||||
models = kimi_profile.fetch_models(
|
||||
api_key="test-key",
|
||||
base_url="https://[api.kimi.com/coding",
|
||||
)
|
||||
|
||||
assert models == ["kimi-k2.6"]
|
||||
|
||||
|
||||
class TestKimiFullKwargsIntegration:
|
||||
"""The transport's full kwargs carry at most one reasoning knob."""
|
||||
|
||||
def _build(self, kimi_profile, reasoning_config):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
return ChatCompletionsTransport().build_kwargs(
|
||||
model="kimi-k2-turbo-preview",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=kimi_profile,
|
||||
reasoning_config=reasoning_config,
|
||||
base_url="https://api.moonshot.ai/v1",
|
||||
provider_name="kimi-coding",
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Unit tests for the MiniMax provider profile.
|
||||
|
||||
Three MiniMax provider profiles (`minimax` direct API, `minimax-cn` China direct
|
||||
API, `minimax-oauth` browser OAuth) all advertise a `default_aux_model` on
|
||||
their `ProviderProfile`. The previous M2.7 / M2.7-highspeed values were
|
||||
stale relative to the current frontier model (M3, released 2026-06-01) and
|
||||
inconsistent with the `_PROVIDER_MODELS["minimax"]` catalog top entry in
|
||||
`hermes_cli/models.py`.
|
||||
|
||||
This file pins the new defaults so the choice is reviewable and any future
|
||||
revert shows up in a failing test rather than silent behavior drift.
|
||||
|
||||
Refs:
|
||||
- Issue #36196: M3 support request
|
||||
- PR #36205 (closed unmerged): Csrayz's M3 + 1M context work
|
||||
- PR #36212 (open): adds M3 to `_PROVIDER_MODELS["minimax"]` catalog
|
||||
- PR #6082: M2.7-highspeed → M2.7 for aux model (half-price fix)
|
||||
- Commit 773a0faca: same profile-layer fix pattern for `deepseek`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(params=["minimax", "minimax-cn", "minimax-oauth"])
|
||||
def minimax_profile(request):
|
||||
"""Resolve each registered MiniMax profile.
|
||||
|
||||
Going through ``providers.get_provider_profile`` keeps the test honest —
|
||||
if someone later replaces the registered class with a plain
|
||||
``ProviderProfile``, every assertion below collapses.
|
||||
"""
|
||||
import model_tools # noqa: F401 -- triggers plugin discovery
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile(request.param)
|
||||
assert profile is not None, f"{request.param} provider profile must be registered"
|
||||
return profile, request.param
|
||||
|
||||
|
||||
class TestMinimaxAuxModelM3:
|
||||
"""MiniMax profile aux model is the new frontier M3, not the stale M2.7.
|
||||
|
||||
The catalog top entry is ``MiniMax-M3`` in
|
||||
``hermes_cli.models._PROVIDER_MODELS['minimax']`` and the
|
||||
user-facing ``model.default`` for a Token-Plan install is M3,
|
||||
so pinning the aux default to the same model keeps the runtime
|
||||
consistent (same auth, same billing pool, same rate limits, no
|
||||
surprise 2x-cost highspeed variant). M3 was released 2026-06-01
|
||||
— picking it as the aux default matches the forward-looking
|
||||
catalog order rather than the pre-M3 era.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider_id,expected",
|
||||
[
|
||||
("minimax", "MiniMax-M3"),
|
||||
("minimax-cn", "MiniMax-M3"),
|
||||
# minimax-oauth sticks with M2.7: the OAuth / Coding Plan
|
||||
# tier historically used -highspeed (PR #6082 collapsed that
|
||||
# to plain M2.7 to avoid the 2x TPS surcharge). M3 is not on
|
||||
# the OAuth/Coding Plan tier per platform docs as of this PR,
|
||||
# so the safe choice is the cheapest generally-available
|
||||
# M2.7 — matching PR #6082's intent.
|
||||
("minimax-oauth", "MiniMax-M2.7"),
|
||||
],
|
||||
)
|
||||
def test_profile_advertises_expected_aux_model(
|
||||
self, provider_id, expected
|
||||
):
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile(provider_id)
|
||||
assert profile is not None
|
||||
assert profile.default_aux_model == expected, (
|
||||
f"{provider_id} default_aux_model drifted to "
|
||||
f"{profile.default_aux_model!r}, expected {expected!r}"
|
||||
)
|
||||
|
||||
def test_consumer_api_returns_non_empty_for_each_provider(self, minimax_profile):
|
||||
from agent.auxiliary_client import _get_aux_model_for_provider
|
||||
|
||||
profile, provider_id = minimax_profile
|
||||
resolved = _get_aux_model_for_provider(provider_id)
|
||||
assert resolved != "", (
|
||||
f"_get_aux_model_for_provider({provider_id!r}) returned empty — "
|
||||
"the 'No auxiliary LLM provider configured' warning will fire on "
|
||||
f"every {provider_id} session even though the profile advertises "
|
||||
f"default_aux_model={profile.default_aux_model!r}"
|
||||
)
|
||||
assert resolved == profile.default_aux_model, (
|
||||
f"_get_aux_model_for_provider({provider_id!r}) returned "
|
||||
f"{resolved!r} but profile advertises {profile.default_aux_model!r} "
|
||||
"— the consumer API and the profile have drifted out of sync"
|
||||
)
|
||||
|
||||
|
||||
class TestMinimaxAuxModelNotHighspeed:
|
||||
"""Regression guard against re-introducing the M2.7-highspeed aux default.
|
||||
|
||||
PR #6082 collapsed the highspeed aux choice to plain M2.7 because the
|
||||
highspeed variant costs 2x with no real benefit for compression / vision /
|
||||
session-search aux tasks. None of the three MiniMax profiles should
|
||||
silently re-introduce that 2x-cost path.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize("provider_id", ["minimax", "minimax-cn", "minimax-oauth"])
|
||||
def test_default_aux_model_is_not_highspeed(self, provider_id):
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile(provider_id)
|
||||
assert profile is not None
|
||||
assert "highspeed" not in profile.default_aux_model.lower(), (
|
||||
f"{provider_id} default_aux_model={profile.default_aux_model!r} "
|
||||
"is a -highspeed variant — that costs 2x for the same model and "
|
||||
"broke #4082 the first time. Revert to plain M2.7 or M3."
|
||||
)
|
||||
|
||||
|
||||
class TestMinimaxM3OpenAIReasoningWireShape:
|
||||
"""MiniMax-M3 on api.minimax.io/v1 gets MiniMax's OpenAI-compatible knobs."""
|
||||
|
||||
def test_m3_openai_route_requests_reasoning_split_by_default(self):
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("minimax")
|
||||
assert profile is not None
|
||||
extra_body, top_level = profile.build_api_kwargs_extras(
|
||||
reasoning_config=None,
|
||||
model="MiniMax-M3",
|
||||
base_url="https://api.minimax.io/v1",
|
||||
)
|
||||
assert extra_body == {"reasoning_split": True}
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,base_url",
|
||||
[
|
||||
("MiniMax-M2.7", "https://api.minimax.io/v1"),
|
||||
("MiniMax-M3", "https://api.minimax.io/anthropic"),
|
||||
("MiniMax-M3", "https://api.minimaxi.com/v1"),
|
||||
],
|
||||
)
|
||||
def test_non_m3_or_non_global_openai_routes_emit_no_openai_reasoning_knobs(
|
||||
self, model, base_url
|
||||
):
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("minimax")
|
||||
assert profile is not None
|
||||
extra_body, top_level = profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
def test_transport_threads_base_url_to_profile(self):
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
profile = providers.get_provider_profile("minimax")
|
||||
assert profile is not None
|
||||
kwargs = ChatCompletionsTransport().build_kwargs(
|
||||
model="MiniMax-M3",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=profile,
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
base_url="https://api.minimax.io/v1",
|
||||
)
|
||||
assert kwargs["extra_body"] == {
|
||||
"reasoning_split": True,
|
||||
"thinking": {"type": "adaptive"},
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Unit tests for the Nous Portal profile's reasoning wiring.
|
||||
|
||||
The Portal honors ``reasoning: {enabled: false}`` — it is the only wire shape
|
||||
that does, and ``extra_body.thinking`` is not forwarded upstream. The profile
|
||||
used to drop a disable for every model, which on a thinking-first route like
|
||||
``deepseek/deepseek-v4-pro`` (catalog: ``default_effort: high``) meant the
|
||||
upstream default applied and "thinking off" burned reasoning tokens anyway.
|
||||
|
||||
A disable is still dropped for reasoning-mandatory routes, which answer
|
||||
``reasoning: {enabled: false}`` with HTTP 400, and for models the catalog
|
||||
can't speak to — an unknown model errs toward the old behavior rather than
|
||||
risking a 400 on a cold first turn.
|
||||
|
||||
These tests pin that contract without going live.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def nous_profile():
|
||||
"""Resolve the registered Nous profile through the real discovery path."""
|
||||
# ``model_tools`` triggers plugin discovery on import, which is what
|
||||
# registers the Nous profile in the global provider registry.
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("nous")
|
||||
assert profile is not None, "nous provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def portal_catalog(monkeypatch):
|
||||
"""Prime the Portal reasoning-capability cache with known entries."""
|
||||
import hermes_cli.models as models_mod
|
||||
|
||||
monkeypatch.setattr(models_mod, "_nous_reasoning_caps_failed_at", None)
|
||||
monkeypatch.setattr(models_mod, "_nous_reasoning_caps_cache", {
|
||||
"deepseek/deepseek-v4-pro": {
|
||||
"supports_reasoning": True,
|
||||
"supported_efforts": ["xhigh", "high"],
|
||||
"mandatory": False,
|
||||
},
|
||||
"arcee-ai/trinity-large-thinking": {
|
||||
"supports_reasoning": True,
|
||||
"supported_efforts": None,
|
||||
"mandatory": True,
|
||||
},
|
||||
# Catalogued, and it takes no reasoning parameter at all.
|
||||
"moonshotai/kimi-k3-instruct": {"supports_reasoning": False},
|
||||
})
|
||||
|
||||
|
||||
class TestNousReasoningWireShape:
|
||||
"""``build_api_kwargs_extras`` produces the Portal's wire format."""
|
||||
|
||||
def test_disable_reaches_optional_reasoning_model(self, nous_profile, portal_catalog):
|
||||
"""The knob the user set is the knob the Portal receives."""
|
||||
extra_body, top_level = nous_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
supports_reasoning=True,
|
||||
model="deepseek/deepseek-v4-pro",
|
||||
)
|
||||
assert extra_body == {"reasoning": {"enabled": False}}
|
||||
assert top_level == {}
|
||||
|
||||
def test_disable_dropped_for_mandatory_reasoning_model(self, nous_profile, portal_catalog):
|
||||
"""Mandatory routes 400 on a disable — send nothing instead."""
|
||||
extra_body, _ = nous_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
supports_reasoning=True,
|
||||
model="arcee-ai/trinity-large-thinking",
|
||||
)
|
||||
assert "reasoning" not in extra_body
|
||||
|
||||
def test_disable_dropped_for_unknown_model(self, nous_profile, portal_catalog):
|
||||
"""Unlisted / cold catalog → fail safe, never risk the 400."""
|
||||
extra_body, _ = nous_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
supports_reasoning=True,
|
||||
model="private/unlisted-route",
|
||||
)
|
||||
assert "reasoning" not in extra_body
|
||||
|
||||
def test_disable_dropped_for_non_reasoning_route(self, nous_profile, portal_catalog):
|
||||
"""A route the catalog says takes no reasoning parameter gets none.
|
||||
|
||||
Hermes' own ``supports_reasoning`` can disagree with the Portal about a
|
||||
given route; when it does, the catalog of the service actually serving
|
||||
the model wins, and we don't send it a parameter it doesn't accept.
|
||||
"""
|
||||
extra_body, _ = nous_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
supports_reasoning=True,
|
||||
model="moonshotai/kimi-k3-instruct",
|
||||
)
|
||||
assert "reasoning" not in extra_body
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["deepseek/deepseek-v4-pro", "arcee-ai/trinity-large-thinking", "private/unlisted-route"],
|
||||
)
|
||||
def test_enabled_config_always_forwarded(self, nous_profile, portal_catalog, model):
|
||||
"""Mandatory-ness only gates the disable; an enable always ships."""
|
||||
extra_body, _ = nous_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
supports_reasoning=True,
|
||||
model=model,
|
||||
)
|
||||
assert extra_body["reasoning"] == {"enabled": True, "effort": "high"}
|
||||
|
||||
def test_no_config_defaults_to_medium(self, nous_profile, portal_catalog):
|
||||
extra_body, _ = nous_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None,
|
||||
supports_reasoning=True,
|
||||
model="deepseek/deepseek-v4-pro",
|
||||
)
|
||||
assert extra_body["reasoning"] == {"enabled": True, "effort": "medium"}
|
||||
|
||||
def test_nothing_emitted_without_reasoning_support(self, nous_profile, portal_catalog):
|
||||
extra_body, top_level = nous_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
supports_reasoning=False,
|
||||
model="deepseek/deepseek-v4-pro",
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
def test_caller_config_not_mutated(self, nous_profile, portal_catalog):
|
||||
cfg = {"enabled": False}
|
||||
nous_profile.build_api_kwargs_extras(
|
||||
reasoning_config=cfg,
|
||||
supports_reasoning=True,
|
||||
model="deepseek/deepseek-v4-pro",
|
||||
)
|
||||
assert cfg == {"enabled": False}
|
||||
@@ -0,0 +1,234 @@
|
||||
"""Unit tests for the Ollama Cloud provider profile's reasoning-effort wiring.
|
||||
|
||||
Ollama Cloud's ``/v1/chat/completions`` endpoint supports top-level
|
||||
``reasoning_effort`` with values ``none``, ``low``, ``medium``, ``high``,
|
||||
and (undocumented but empirically confirmed) ``max``. The profile maps
|
||||
Hermes's ``xhigh`` → ``max`` to unlock DeepSeek V4's "Max thinking" tier
|
||||
and passes the standard levels through unchanged.
|
||||
|
||||
These tests pin the profile's wire-shape contract so Ollama Cloud
|
||||
requests carry the correct ``reasoning_effort`` field.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ollama_cloud_profile():
|
||||
"""Resolve the registered Ollama Cloud profile.
|
||||
|
||||
Going through ``providers.get_provider_profile`` keeps the test
|
||||
honest — if someone replaces the registered class with a plain
|
||||
``ProviderProfile``, every assertion below collapses.
|
||||
"""
|
||||
# ``model_tools`` triggers plugin discovery on import, which is what
|
||||
# registers the Ollama Cloud profile in the global provider registry.
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("ollama-cloud")
|
||||
assert profile is not None, "ollama-cloud provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
class TestOllamaCloudReasoningEffort:
|
||||
"""``build_api_kwargs_extras`` emits correct top-level ``reasoning_effort``."""
|
||||
|
||||
# ── xhigh / max → max ──────────────────────────────────────────
|
||||
|
||||
@pytest.mark.parametrize("effort", ["xhigh", "max", "MAX", " Max "])
|
||||
def test_xhigh_and_max_normalize_to_max(self, ollama_cloud_profile, effort):
|
||||
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {"reasoning_effort": "max"}
|
||||
|
||||
# ── low / medium / high pass through ───────────────────────────
|
||||
|
||||
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
|
||||
def test_standard_efforts_pass_through(self, ollama_cloud_profile, effort):
|
||||
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
)
|
||||
assert top_level == {"reasoning_effort": effort}
|
||||
|
||||
# ── disabled → reasoning_effort:"none" (the only working off switch) ──
|
||||
|
||||
def test_explicitly_disabled_sends_none(self, ollama_cloud_profile):
|
||||
"""Ollama Cloud defaults to thinking ON and ignores extra_body.thinking,
|
||||
so disabling requires top-level reasoning_effort:"none" (verified live);
|
||||
omitting the field would leave thinking on."""
|
||||
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": False},
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {"reasoning_effort": "none"}
|
||||
|
||||
def test_disabled_ignores_effort_field(self, ollama_cloud_profile):
|
||||
"""Effort is overridden by the disable off switch when thinking is off."""
|
||||
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": False, "effort": "high"},
|
||||
)
|
||||
assert top_level == {"reasoning_effort": "none"}
|
||||
|
||||
# ── none effort → reasoning_effort:"none" ──────────────────────
|
||||
|
||||
def test_none_effort_sends_none(self, ollama_cloud_profile):
|
||||
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": "none"},
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {"reasoning_effort": "none"}
|
||||
|
||||
# ── missing / empty effort → let model default ─────────────────
|
||||
|
||||
def test_no_reasoning_config_emits_nothing(self, ollama_cloud_profile):
|
||||
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config=None,
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
def test_empty_effort_emits_nothing(self, ollama_cloud_profile):
|
||||
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": ""},
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
# ── unknown / minimal effort → omitted (server default) ────────
|
||||
|
||||
def test_unknown_effort_omitted(self, ollama_cloud_profile):
|
||||
"""Unrecognized effort is omitted, not forwarded verbatim, so the
|
||||
model applies its own default. Matches the sibling deepseek profile,
|
||||
which targets the same backend."""
|
||||
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": "future-tier"},
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
def test_minimal_effort_clamps_to_low(self, ollama_cloud_profile):
|
||||
"""``minimal`` is a real Hermes effort level but is rejected by
|
||||
Ollama Cloud's /v1/chat/completions. The shared clamp degrades it to
|
||||
``low`` — the nearest supported level — instead of silently dropping
|
||||
the user's ask (old behavior left the server default, i.e. MORE
|
||||
thinking than requested: a ladder inversion)."""
|
||||
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": "minimal"},
|
||||
)
|
||||
assert top_level == {"reasoning_effort": "low"}
|
||||
|
||||
|
||||
class TestOllamaCloudFullKwargsIntegration:
|
||||
"""End-to-end: the transport's full kwargs include reasoning_effort."""
|
||||
|
||||
def test_full_kwargs_with_xhigh(self, ollama_cloud_profile):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
kwargs = ChatCompletionsTransport().build_kwargs(
|
||||
model="deepseek-v4-pro:cloud",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=ollama_cloud_profile,
|
||||
reasoning_config={"enabled": True, "effort": "xhigh"},
|
||||
base_url="https://ollama.com/v1",
|
||||
provider_name="ollama-cloud",
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert kwargs["model"] == "deepseek-v4-pro:cloud"
|
||||
assert kwargs["reasoning_effort"] == "max"
|
||||
# No extra_body — Ollama Cloud uses top-level reasoning_effort
|
||||
assert "extra_body" not in kwargs or "reasoning" not in kwargs.get("extra_body", {})
|
||||
|
||||
|
||||
class TestOllamaCloudCapabilityGating:
|
||||
"""reasoning_effort is gated on the model's thinking capability."""
|
||||
|
||||
def test_non_thinking_model_emits_nothing(self, ollama_cloud_profile):
|
||||
"""A model that doesn't support thinking (supports_reasoning=False)
|
||||
gets no reasoning_effort, even when an effort is requested — Ollama
|
||||
resolves thinking capability from /api/show, and we don't send a
|
||||
meaningless field to e.g. gemma3 / qwen3-coder."""
|
||||
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "xhigh"},
|
||||
supports_reasoning=False,
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestOllamaModelSupportsThinking:
|
||||
"""The /api/show capability probe used to resolve supports_reasoning."""
|
||||
|
||||
def _patch_show(self, monkeypatch, *, status=200, capabilities=None, raise_exc=None):
|
||||
import httpx
|
||||
|
||||
class _Resp:
|
||||
status_code = status
|
||||
|
||||
def json(self):
|
||||
return {"capabilities": capabilities} if capabilities is not None else {}
|
||||
|
||||
class _Client:
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def post(self, *a, **k):
|
||||
if raise_exc:
|
||||
raise raise_exc
|
||||
return _Resp()
|
||||
|
||||
monkeypatch.setattr(httpx, "Client", _Client)
|
||||
|
||||
def test_thinking_capability_true(self, monkeypatch):
|
||||
from hermes_cli.models import ollama_model_supports_thinking
|
||||
|
||||
self._patch_show(monkeypatch, capabilities=["completion", "tools", "thinking"])
|
||||
assert (
|
||||
ollama_model_supports_thinking(
|
||||
"deepseek-v4-pro", "https://ollama.com/v1", "key"
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def test_probe_failure_returns_none(self, monkeypatch):
|
||||
from hermes_cli.models import ollama_model_supports_thinking
|
||||
|
||||
self._patch_show(monkeypatch, status=404)
|
||||
assert (
|
||||
ollama_model_supports_thinking("x", "https://ollama.com/v1", "key") is None
|
||||
)
|
||||
|
||||
def test_exception_returns_none(self, monkeypatch):
|
||||
from hermes_cli.models import ollama_model_supports_thinking
|
||||
|
||||
self._patch_show(monkeypatch, raise_exc=RuntimeError("boom"))
|
||||
assert (
|
||||
ollama_model_supports_thinking("x", "https://ollama.com/v1", "key") is None
|
||||
)
|
||||
|
||||
|
||||
class TestOllamaCloudAuxModel:
|
||||
"""Ollama Cloud aux model is set on the profile."""
|
||||
|
||||
def test_profile_advertises_aux_model(self, ollama_cloud_profile):
|
||||
assert ollama_cloud_profile.default_aux_model == "nemotron-3-nano:30b"
|
||||
@@ -0,0 +1,255 @@
|
||||
"""Unit tests for OpenCode Go reasoning-control wiring."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def opencode_go_profile():
|
||||
"""Resolve the registered OpenCode Go provider profile."""
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("opencode-go")
|
||||
assert profile is not None, "opencode-go provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def opencode_zen_profile():
|
||||
"""Resolve the registered OpenCode Zen provider profile."""
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("opencode-zen")
|
||||
assert profile is not None, "opencode-zen provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
class TestOpenCodeZenOxReasoning:
|
||||
"""Ox Alpha Free uses OpenCode Zen's native reasoning_effort control."""
|
||||
|
||||
def test_max_effort_is_emitted(self, opencode_zen_profile):
|
||||
extra_body, top_level = opencode_zen_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "max"},
|
||||
model="x-preview-f-free",
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {"reasoning_effort": "max"}
|
||||
|
||||
@pytest.mark.parametrize("reasoning_config", [None, {"enabled": False}])
|
||||
def test_unset_or_disabled_preserves_server_default(
|
||||
self, opencode_zen_profile, reasoning_config
|
||||
):
|
||||
extra_body, top_level = opencode_zen_profile.build_api_kwargs_extras(
|
||||
reasoning_config=reasoning_config,
|
||||
model="x-preview-f-free",
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
def test_other_zen_models_are_untouched(self, opencode_zen_profile):
|
||||
extra_body, top_level = opencode_zen_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "max"},
|
||||
model="gemini-3-flash",
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
def test_max_reaches_chat_completions_request(self, opencode_zen_profile):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
kwargs = ChatCompletionsTransport().build_kwargs(
|
||||
model="x-preview-f-free",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=opencode_zen_profile,
|
||||
reasoning_config={"enabled": True, "effort": "max"},
|
||||
base_url="https://opencode.ai/zen/v1",
|
||||
)
|
||||
assert "extra_body" not in kwargs
|
||||
assert kwargs["reasoning_effort"] == "max"
|
||||
|
||||
def test_unsupported_efforts_clamp_to_wire_vocabulary(self, opencode_zen_profile):
|
||||
"""medium/xhigh are not on Ox Alpha's wire (400 raw); they must clamp
|
||||
to the nearest supported level, never pass through."""
|
||||
for requested, expected in (("medium", "low"), ("xhigh", "max")):
|
||||
_, top_level = opencode_zen_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": requested},
|
||||
model="x-preview-f-free",
|
||||
)
|
||||
assert top_level == {"reasoning_effort": expected}, requested
|
||||
|
||||
def test_opencode_free_profile_shares_the_translation(self):
|
||||
"""Ox Alpha is reachable via the keyless opencode-free provider too;
|
||||
its profile must emit the identical clamped reasoning_effort."""
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
from providers.base import ProviderProfile
|
||||
|
||||
profile = providers.get_provider_profile("opencode-free")
|
||||
assert profile is not None
|
||||
assert (
|
||||
type(profile).build_api_kwargs_extras
|
||||
is not ProviderProfile.build_api_kwargs_extras
|
||||
), "opencode-free must override build_api_kwargs_extras (aux gate)"
|
||||
_, top_level = profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
model="x-preview-f-free",
|
||||
)
|
||||
assert top_level == {"reasoning_effort": "low"}
|
||||
_, other = profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "max"},
|
||||
model="big-pickle",
|
||||
)
|
||||
assert other == {}
|
||||
|
||||
|
||||
class TestOpenCodeGoKimiReasoning:
|
||||
"""Kimi K2 models use Moonshot's thinking + reasoning_effort shape on OpenCode Go."""
|
||||
|
||||
def test_high_effort_emits_thinking_and_effort(self, opencode_go_profile):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
model="kimi-k2.6",
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {"reasoning_effort": "high"}
|
||||
|
||||
def test_disabled_emits_thinking_disabled_without_effort(self, opencode_go_profile):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
model="kimi-k2.6",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "disabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
def test_minimal_effort_clamps_to_low(self, opencode_go_profile):
|
||||
# "minimal" is below Moonshot's floor — the shared clamp degrades it
|
||||
# to "low" (nearest supported) instead of dropping the ask and
|
||||
# leaving the server default (which was MORE thinking than asked).
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "minimal"},
|
||||
model="kimi-k2.6",
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {"reasoning_effort": "low"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"effort",
|
||||
[
|
||||
"xhigh",
|
||||
"max",
|
||||
],
|
||||
)
|
||||
def test_strong_efforts_clamp_to_high(self, opencode_go_profile, effort):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="moonshotai/kimi-k2.6",
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {"reasoning_effort": "high"}
|
||||
|
||||
def test_low_and_medium_pass_through(self, opencode_go_profile):
|
||||
for effort in ("low", "medium"):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="kimi-k2.5",
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {"reasoning_effort": effort}
|
||||
|
||||
def test_no_config_preserves_server_default(self, opencode_go_profile):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None,
|
||||
model="kimi-k2.6",
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestOpenCodeGoDeepSeekThinking:
|
||||
"""DeepSeek V4 models use DeepSeek-style thinking controls on OpenCode Go."""
|
||||
|
||||
|
||||
def test_xhigh_and_max_normalize_to_max(self, opencode_go_profile):
|
||||
for effort in ("xhigh", "max"):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="deepseek/deepseek-v4-pro",
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {"reasoning_effort": "max"}
|
||||
|
||||
|
||||
class TestOpenCodeGoGLM52Reasoning:
|
||||
"""GLM-5.2 uses its native high/max reasoning_effort knob on OpenCode Go."""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["glm-5-2", "glm-5p2"])
|
||||
def test_alias_spellings_recognized(self, opencode_go_profile, model):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "max"},
|
||||
model=model,
|
||||
)
|
||||
assert top_level == {"reasoning_effort": "max"}
|
||||
|
||||
|
||||
class TestOpenCodeGoModelGating:
|
||||
"""Other OpenCode Go models must not receive Kimi/DeepSeek/GLM controls."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"glm-5.1",
|
||||
"glm-5",
|
||||
"qwen3.6-plus",
|
||||
"minimax-m2.7",
|
||||
"deepseek-v3.1",
|
||||
"deepseek-chat",
|
||||
"",
|
||||
None,
|
||||
],
|
||||
)
|
||||
def test_non_target_models_emit_nothing(self, opencode_go_profile, model):
|
||||
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
model=model,
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestOpenCodeGoFullKwargsIntegration:
|
||||
"""End-to-end transport kwargs include the profile-provided controls."""
|
||||
|
||||
def test_kimi_reasoning_reaches_extra_body_and_top_level(self, opencode_go_profile):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
kwargs = ChatCompletionsTransport().build_kwargs(
|
||||
model="kimi-k2.6",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=opencode_go_profile,
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
base_url="https://opencode.ai/zen/go/v1",
|
||||
)
|
||||
assert "extra_body" not in kwargs
|
||||
assert kwargs["reasoning_effort"] == "high"
|
||||
|
||||
def test_deepseek_thinking_reaches_extra_body_and_top_level(
|
||||
self, opencode_go_profile
|
||||
):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
kwargs = ChatCompletionsTransport().build_kwargs(
|
||||
model="deepseek-v4-pro",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=opencode_go_profile,
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
base_url="https://opencode.ai/zen/go/v1",
|
||||
)
|
||||
assert "extra_body" not in kwargs
|
||||
assert kwargs["reasoning_effort"] == "high"
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Unit tests for the Upstage Solar provider profile.
|
||||
|
||||
Upstage Solar is a plain OpenAI-compatible api-key provider, so this verifies
|
||||
the profile is registered correctly and wires the expected identity, endpoint,
|
||||
auth, and catalog fields — the contract every downstream layer (auth, models,
|
||||
doctor, runtime_provider, transport) reads from.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def upstage_profile():
|
||||
"""Resolve the registered Upstage profile via the provider registry.
|
||||
|
||||
Importing ``model_tools`` triggers plugin discovery, which registers the
|
||||
Upstage profile. Going through ``get_provider_profile`` keeps the test
|
||||
honest about the actual registration path (name + alias resolution).
|
||||
"""
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("upstage")
|
||||
assert profile is not None, "upstage provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
class TestUpstageProfile:
|
||||
def test_identity_and_endpoint(self, upstage_profile):
|
||||
assert upstage_profile.name == "upstage"
|
||||
assert upstage_profile.api_mode == "chat_completions"
|
||||
assert upstage_profile.auth_type == "api_key"
|
||||
assert upstage_profile.base_url == "https://api.upstage.ai/v1"
|
||||
assert upstage_profile.get_hostname() == "api.upstage.ai"
|
||||
|
||||
def test_solar_alias_resolves(self):
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
assert providers.get_provider_profile("solar") is upstage_profile_singleton()
|
||||
|
||||
def test_env_vars(self, upstage_profile):
|
||||
# API key first, optional base-url override second (priority order).
|
||||
assert upstage_profile.env_vars == ("UPSTAGE_API_KEY", "UPSTAGE_BASE_URL")
|
||||
|
||||
def test_fallback_models_are_agentic_pro_only(self, upstage_profile):
|
||||
# Only the agentic, tool-calling Solar Pro models belong in the offline
|
||||
# catalog — Mini is capable but not agentic, so it's never promoted as a
|
||||
# default. Live /v1/models still surfaces everything when a key is set.
|
||||
# Behavior contract (not a frozen list): non-empty, no denied families.
|
||||
assert upstage_profile.fallback_models
|
||||
for denied in ("solar-mini", "syn-pro"):
|
||||
assert not any(
|
||||
denied in m for m in upstage_profile.fallback_models
|
||||
), f"non-agentic family {denied!r} must not be a fallback default"
|
||||
|
||||
def test_default_model_is_solar_pro3(self, upstage_profile):
|
||||
# Entry [0] is the setup default (get_default_model_for_provider).
|
||||
assert upstage_profile.fallback_models[0] == "solar-pro3"
|
||||
|
||||
def test_aux_model_left_empty(self, upstage_profile):
|
||||
# Unset → auxiliary side tasks fall back to the user's main model.
|
||||
assert upstage_profile.default_aux_model == ""
|
||||
|
||||
|
||||
class TestUpstageReasoning:
|
||||
"""``build_api_kwargs_extras`` wires Solar's top-level ``reasoning_effort``.
|
||||
|
||||
Solar Pro accepts ``reasoning_effort`` (minimal|low|medium|high, default
|
||||
minimal=off) and never requires echoing ``reasoning_content`` back, so only
|
||||
the request field is emitted — always top-level, never in extra_body.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
|
||||
def test_pro_explicit_effort_passes_through(self, upstage_profile, effort):
|
||||
extra_body, top_level = upstage_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort}, model="solar-pro3"
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {"reasoning_effort": effort}
|
||||
|
||||
|
||||
def test_unknown_future_effort_collapses_to_high(self, upstage_profile):
|
||||
# Guard against the #62650 recurrence: a future effort level Hermes
|
||||
# adds above "high" must collapse to Solar's strongest, not silently
|
||||
# downgrade to the medium default.
|
||||
_, top_level = upstage_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "hyperthink"},
|
||||
model="solar-pro3",
|
||||
)
|
||||
assert top_level == {"reasoning_effort": "high"}
|
||||
|
||||
|
||||
def test_disabled_omits_field(self, upstage_profile):
|
||||
# `/reasoning none` → enabled False → explicitly off.
|
||||
_, top_level = upstage_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False, "effort": "high"}, model="solar-pro3"
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
@pytest.mark.parametrize("model", ["solar-pro3", "solar-pro", "solar-open2"])
|
||||
def test_no_config_defaults_reasoning_on(self, upstage_profile, model):
|
||||
# Unset reasoning_config → default ON at medium (matches the /reasoning
|
||||
# "medium (default)" label), not Solar's server default of minimal/off.
|
||||
_, top_level = upstage_profile.build_api_kwargs_extras(model=model)
|
||||
assert top_level == {"reasoning_effort": "medium"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["solar-mini", "solar-mini-202610", "syn-pro"])
|
||||
def test_deny_listed_models_never_send_reasoning(self, upstage_profile, model):
|
||||
# solar-mini / syn-pro ignore reasoning_effort, so never send it —
|
||||
# even when the user explicitly enables reasoning.
|
||||
extra_body, top_level = upstage_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"}, model=model
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
def test_none_model_defaults_to_reasoning(self, upstage_profile):
|
||||
# No model in context → treated as reasoning-capable, consistent with
|
||||
# the provider default (fallback_models[0] == "solar-pro3").
|
||||
_, top_level = upstage_profile.build_api_kwargs_extras(model=None)
|
||||
assert top_level == {"reasoning_effort": "medium"}
|
||||
|
||||
|
||||
def upstage_profile_singleton():
|
||||
import providers
|
||||
|
||||
return providers.get_provider_profile("upstage")
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Unit tests for the Z.AI / GLM provider profile's reasoning wiring.
|
||||
|
||||
Z.AI's GLM-4.5-and-later chat models default to thinking-mode ON when the
|
||||
request omits ``thinking``. Before the profile emitted the parameter,
|
||||
``reasoning_config = {"enabled": False}`` was a silent no-op on the direct
|
||||
Z.AI route — users who turned thinking off kept burning thinking tokens on
|
||||
every turn (the desktop "thinking reverts to medium" report).
|
||||
|
||||
GLM-5.2 additionally exposes a native ``reasoning_effort`` knob with two
|
||||
enabled levels (high / max) on the OpenAI-compatible ``/api/paas/v4``
|
||||
endpoint; the Hermes effort scale is collapsed onto those.
|
||||
|
||||
These tests pin the profile's wire-shape contract so Z.AI requests stay
|
||||
correctly shaped without going live.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def zai_profile():
|
||||
"""Resolve the registered Z.AI profile through the real discovery path."""
|
||||
# ``model_tools`` triggers plugin discovery on import, which is what
|
||||
# registers the Z.AI profile in the global provider registry.
|
||||
import model_tools # noqa: F401
|
||||
import providers
|
||||
|
||||
profile = providers.get_provider_profile("zai")
|
||||
assert profile is not None, "zai provider profile must be registered"
|
||||
return profile
|
||||
|
||||
|
||||
class TestZaiThinkingWireShape:
|
||||
"""``build_api_kwargs_extras`` produces Z.AI's exact wire format."""
|
||||
|
||||
def test_no_preference_omits_thinking(self, zai_profile):
|
||||
"""No reasoning_config → omit ``thinking`` so the server default
|
||||
applies (matches prior behavior for users with no preference)."""
|
||||
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None, model="glm-5"
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
def test_enabled_sends_enabled_marker(self, zai_profile):
|
||||
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "medium"}, model="glm-5"
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
def test_explicitly_disabled_sends_disabled_marker(self, zai_profile):
|
||||
"""``reasoning_config.enabled=False`` → ``thinking.type=disabled``.
|
||||
|
||||
The crucial bit is that the parameter is *sent* at all — GLM defaults
|
||||
to thinking-on when ``thinking`` is absent, so an unsent disable
|
||||
burns thinking tokens forever.
|
||||
"""
|
||||
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False}, model="glm-5"
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "disabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestZaiGLM52ReasoningEffort:
|
||||
"""GLM-5.2's native ``reasoning_effort`` knob (two enabled levels)."""
|
||||
|
||||
def test_high_maps_to_high(self, zai_profile):
|
||||
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
model="glm-5.2",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {"reasoning_effort": "high"}
|
||||
|
||||
@pytest.mark.parametrize("effort", ["low", "medium", "minimal"])
|
||||
def test_lower_efforts_clamp_up_to_high(self, zai_profile, effort):
|
||||
"""GLM-5.2's minimum thinking level is high — lower Hermes levels
|
||||
clamp onto it."""
|
||||
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="glm-5.2",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {"reasoning_effort": "high"}
|
||||
|
||||
@pytest.mark.parametrize("effort", ["xhigh", "max"])
|
||||
def test_strong_efforts_map_to_max(self, zai_profile, effort):
|
||||
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="glm-5.2",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {"reasoning_effort": "max"}
|
||||
|
||||
def test_disabled_sends_no_effort(self, zai_profile):
|
||||
"""Disabled reasoning still sends the thinking-off marker but never
|
||||
an effort level."""
|
||||
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False, "effort": "high"},
|
||||
model="glm-5.2",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "disabled"}}
|
||||
assert top_level == {}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"z-ai/glm-5.2",
|
||||
"glm-5-2",
|
||||
"glm-5p2",
|
||||
"accounts/fireworks/models/glm-5p2",
|
||||
"zai-org-glm-5-2",
|
||||
],
|
||||
)
|
||||
def test_alias_spellings_recognized(self, zai_profile, model):
|
||||
_, top_level = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "max"},
|
||||
model=model,
|
||||
)
|
||||
assert top_level == {"reasoning_effort": "max"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["glm-5.1", "glm-5", "glm-4.7", "glm-4-9b", "", None],
|
||||
)
|
||||
def test_non_glm_5_2_models_get_no_effort(self, zai_profile, model):
|
||||
_, top_level = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
model=model,
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestZaiGLM53ReasoningEffort:
|
||||
"""GLM-5.3's graded low/medium/high/max effort scale (issue #91789).
|
||||
|
||||
Verified live on api.z.ai/api/coding/paas/v4: all four levels accepted
|
||||
with monotonic reasoning-token scaling. Unlike 5.2, low and medium must
|
||||
reach the wire instead of clamping up to high.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("effort", "expected"),
|
||||
[
|
||||
("low", "low"),
|
||||
("medium", "medium"),
|
||||
("high", "high"),
|
||||
("max", "max"),
|
||||
("xhigh", "max"),
|
||||
("minimal", "low"),
|
||||
],
|
||||
)
|
||||
def test_graded_efforts_pass_through(self, zai_profile, effort, expected):
|
||||
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
model="glm-5.3",
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "enabled"}}
|
||||
assert top_level == {"reasoning_effort": expected}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["z-ai/glm-5.3", "glm-5-3", "glm-5p3", "zai-org-glm-5-3"],
|
||||
)
|
||||
def test_alias_spellings_get_graded_scale(self, zai_profile, model):
|
||||
_, top_level = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "low"},
|
||||
model=model,
|
||||
)
|
||||
assert top_level == {"reasoning_effort": "low"}
|
||||
|
||||
def test_glm_5_2_still_clamps_low_to_high(self, zai_profile):
|
||||
"""The 5.3 widening must not leak into 5.2's two-level wire."""
|
||||
_, top_level = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "low"},
|
||||
model="glm-5.2",
|
||||
)
|
||||
assert top_level == {"reasoning_effort": "high"}
|
||||
|
||||
|
||||
class TestZaiModelGating:
|
||||
"""GLM 4.5+ get thinking; earlier GLM models are left untouched."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"glm-4.5",
|
||||
"glm-4.5-air",
|
||||
"glm-4.5-flash",
|
||||
"glm-4.6",
|
||||
"glm-5",
|
||||
"glm-5.2",
|
||||
"GLM-5", # case-insensitive
|
||||
],
|
||||
)
|
||||
def test_thinking_capable_models_emit_thinking(self, zai_profile, model):
|
||||
extra_body, _ = zai_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False}, model=model
|
||||
)
|
||||
assert extra_body == {"thinking": {"type": "disabled"}}
|
||||
|
||||
|
||||
class TestZaiFullKwargsIntegration:
|
||||
"""End-to-end: the transport's full kwargs carry the reasoning wiring."""
|
||||
|
||||
|
||||
def test_glm_5_2_effort_reaches_top_level(self, zai_profile):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
kwargs = ChatCompletionsTransport().build_kwargs(
|
||||
model="glm-5.2",
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
tools=None,
|
||||
provider_profile=zai_profile,
|
||||
reasoning_config={"enabled": True, "effort": "max"},
|
||||
base_url="https://api.z.ai/api/paas/v4",
|
||||
provider_name="zai",
|
||||
)
|
||||
assert kwargs["reasoning_effort"] == "max"
|
||||
assert kwargs["extra_body"]["thinking"] == {"type": "enabled"}
|
||||
Reference in New Issue
Block a user