Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

This commit is contained in:
2026-09-05 13:26:46 +03:00
commit 03634b1ca3
11340 changed files with 3442369 additions and 0 deletions
@@ -0,0 +1,367 @@
"""Unit tests for the CommandCode provider profiles.
CommandCode registers two profiles:
``commandcode``
``api_mode=chat_completions`` — OpenAI-compatible. Defaults to
``deepseek/deepseek-v4-pro``. 20+ models via a single base URL.
``commandcode-anthropic``
``api_mode=anthropic_messages`` — Anthropic Messages API-compatible.
Defaults to ``claude-sonnet-4-6``. Requires Bearer auth recognition
in ``agent/anthropic_adapter.py``.
Both share ``COMMANDCODE_API_KEY`` and ``https://api.commandcode.ai/provider/v1``.
"""
from __future__ import annotations
import pytest
# ── Fixtures ──────────────────────────────────────────────────────────────────
@pytest.fixture
def commandcode_profile():
"""Resolve the registered CommandCode (chat_completions) profile."""
import model_tools # noqa: F401 — triggers discovery
import providers
profile = providers.get_provider_profile("commandcode")
assert profile is not None, "commandcode provider profile must be registered"
return profile
@pytest.fixture
def commandcode_anthropic_profile():
"""Resolve the registered CommandCode Anthropic profile."""
import model_tools # noqa: F401 — triggers discovery
import providers
profile = providers.get_provider_profile("commandcode-anthropic")
assert profile is not None, "commandcode-anthropic profile must be registered"
return profile
# ── Chat Completions profile ──────────────────────────────────────────────────
class TestCommandCodeProfileIdentity:
"""Profile metadata matches the declared contract."""
def test_name(self, commandcode_profile):
assert commandcode_profile.name == "commandcode"
def test_api_mode(self, commandcode_profile):
assert commandcode_profile.api_mode == "chat_completions"
def test_aliases(self, commandcode_profile):
assert "commandcode-chat" in commandcode_profile.aliases
def test_env_vars(self, commandcode_profile):
assert "COMMANDCODE_API_KEY" in commandcode_profile.env_vars
def test_base_url(self, commandcode_profile):
assert commandcode_profile.base_url == "https://api.commandcode.ai/provider/v1"
def test_display_name(self, commandcode_profile):
assert "CommandCode" in commandcode_profile.display_name
def test_has_fallback_models(self, commandcode_profile):
assert len(commandcode_profile.fallback_models) >= 5
# Should include the major families
names = " ".join(commandcode_profile.fallback_models)
assert "deepseek" in names
assert "Qwen" in names
assert "Kimi" in names
assert "gemini" in names
def test_default_aux_model(self, commandcode_profile):
assert commandcode_profile.default_aux_model == "deepseek/deepseek-v4-flash"
def test_signup_url(self, commandcode_profile):
assert "commandcode" in commandcode_profile.signup_url.lower()
def test_hostname_derived_from_base_url(self, commandcode_profile):
assert commandcode_profile.get_hostname() == "api.commandcode.ai"
class TestCommandCodeProfileNoThinkingInterference:
"""Chat completions profile is a no-op for thinking config — it delegates
to the underlying model's provider (DeepSeek, Qwen, etc.) for wire format.
"""
def test_passthrough_no_reasoning_config(self, commandcode_profile):
extra_body, top_level = commandcode_profile.build_api_kwargs_extras(
reasoning_config=None, model="deepseek/deepseek-v4-pro"
)
# Chat completions profile doesn't inject thinking params — that's
# the DeepSeek provider's job when routed through DeepSeek's own profile.
# When routed through CommandCode, the underlying model API handles it.
assert isinstance(extra_body, dict)
assert isinstance(top_level, dict)
# Default ProviderProfile returns ({}, {}).
def test_passthrough_with_reasoning_config(self, commandcode_profile):
extra_body, top_level = commandcode_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model="deepseek/deepseek-v4-pro",
)
assert isinstance(extra_body, dict)
assert isinstance(top_level, dict)
# ── Anthropic Messages profile ────────────────────────────────────────────────
class TestCommandCodeAnthropicProfileIdentity:
"""Anthropic-compatible profile metadata."""
def test_name(self, commandcode_anthropic_profile):
assert commandcode_anthropic_profile.name == "commandcode-anthropic"
def test_api_mode(self, commandcode_anthropic_profile):
assert commandcode_anthropic_profile.api_mode == "anthropic_messages"
def test_aliases(self, commandcode_anthropic_profile):
assert "commandcode-claude" in commandcode_anthropic_profile.aliases
def test_env_vars(self, commandcode_anthropic_profile):
assert "COMMANDCODE_API_KEY" in commandcode_anthropic_profile.env_vars
def test_base_url(self, commandcode_anthropic_profile):
assert commandcode_anthropic_profile.base_url == "https://api.commandcode.ai/provider/v1"
def test_fallback_models_are_claude_family(self, commandcode_anthropic_profile):
for model in commandcode_anthropic_profile.fallback_models:
assert model.startswith("claude-"), (
f"All anthropic fallback models should be claude-*: got {model}"
)
def test_default_aux_model(self, commandcode_anthropic_profile):
assert commandcode_anthropic_profile.default_aux_model == "claude-haiku-4-5-20251001"
def test_display_name_distinct_from_chat(self, commandcode_anthropic_profile):
# The Anthropic profile should be distinguishable in /model picker
assert "(Anthropic)" in commandcode_anthropic_profile.display_name
def test_hostname_derived_from_base_url(self, commandcode_anthropic_profile):
assert commandcode_anthropic_profile.get_hostname() == "api.commandcode.ai"
# ── Bearer Auth Recognition ───────────────────────────────────────────────────
class TestCommandCodeAnthropicBearerAuth:
"""``agent/anthropic_adapter.py`` must recognize CommandCode as a
Bearer-auth endpoint, or the chat_completions transport falls back to
``x-api-key`` and gets a 401.
"""
def test_requires_bearer_auth_recognizes_commandcode(self):
from agent.anthropic_adapter import _requires_bearer_auth
assert _requires_bearer_auth("https://api.commandcode.ai/provider/v1") is True
assert _requires_bearer_auth("https://api.commandcode.ai/provider/v1/models") is True
assert _requires_bearer_auth("https://api.commandcode.ai/anthropic") is True
def test_bearer_auth_does_not_affect_unrelated(self):
from agent.anthropic_adapter import _requires_bearer_auth
# Native Anthropic still uses x-api-key
assert _requires_bearer_auth("https://api.anthropic.com") is False
# OpenRouter still uses Bearer through its own transport path
assert _requires_bearer_auth("https://openrouter.ai/api/v1") is False
def test_bearer_auth_case_insensitive(self):
from agent.anthropic_adapter import _requires_bearer_auth
assert _requires_bearer_auth("https://API.COMMANDCODE.AI/provider/v1") is True
# ── Registry integrity ───────────────────────────────────────────────────────
class TestCommandCodeRegistryIntegrity:
"""Both profiles are discoverable and distinct."""
def test_both_profiles_registered(self):
import model_tools # noqa: F401
import providers
chat = providers.get_provider_profile("commandcode")
anth = providers.get_provider_profile("commandcode-anthropic")
assert chat is not None
assert anth is not None
assert chat is not anth # distinct profile instances
def test_alias_lookup(self):
import model_tools # noqa: F401
import providers
assert providers.get_provider_profile("commandcode-chat") is not None
assert providers.get_provider_profile("commandcode-claude") is not None
def test_unknown_returns_none(self):
import model_tools # noqa: F401
import providers
assert providers.get_provider_profile("commandcode-nonexistent") is None
# ── Model list filtering ──────────────────────────────────────────────────────
class TestCommandCodeModelFiltering:
"""``fetch_models`` filtering contracts."""
def test_anthropic_profile_filters_to_claude(self):
"""If we mock a response with mixed models, anthropic profile
should only return claude-* models.
"""
from plugins.model_providers.commandcode import CommandCodeAnthropicProfile
profile = CommandCodeAnthropicProfile(
name="test-cc-anth",
api_mode="anthropic_messages",
env_vars=("COMMANDCODE_API_KEY",),
base_url="https://api.commandcode.ai/provider/v1",
)
# Don't actually hit the network — just test the filter logic.
# The class has a fetch_models override that filters.
# We verify the filter works by inspecting the method.
import inspect
source = inspect.getsource(profile.fetch_models)
assert "startswith(\"claude-\")" in source or '"claude-" in m' in source, (
"CommandCodeAnthropicProfile.fetch_models should filter to claude-* models"
)
# ── Picker contract ──────────────────────────────────────────────────────────
class TestCommandCodeFetchModelsPickerContract:
"""``fetch_models`` must accept the kwargs the model picker passes.
Regression: the generic live-fetch path in ``hermes_cli/models.py``
(``provider_model_ids``) calls ``profile.fetch_models(api_key=...,
base_url=...)``. The original CommandCode overrides only accepted
``api_key``/``timeout``, so every picker open raised TypeError, which
was swallowed, leaving the provider with zero models.
"""
@pytest.mark.parametrize("profile_name", ["commandcode", "commandcode-anthropic"])
def test_accepts_base_url_kwarg(self, profile_name):
import inspect
import model_tools # noqa: F401 — triggers discovery
import providers
profile = providers.get_provider_profile(profile_name)
assert profile is not None
assert "base_url" in inspect.signature(profile.fetch_models).parameters
def test_resolve_provider_full(self):
"""Both profiles must resolve through the model-switch path.
Regression: ``resolve_provider_full`` only knew models.dev + overlay
providers, so plugin-only providers (commandcode) failed with
"Unknown provider" on /model switches even though the picker listed
them.
"""
from hermes_cli.providers import resolve_provider_full
chat = resolve_provider_full("commandcode", {}, [])
assert chat is not None and chat.id == "commandcode"
assert chat.transport == "openai_chat"
assert "COMMANDCODE_API_KEY" in chat.api_key_env_vars
anth = resolve_provider_full("commandcode-anthropic", {}, [])
assert anth is not None and anth.id == "commandcode-anthropic"
assert anth.transport == "anthropic_messages"
# ── base_url endpoint override ───────────────────────────────────────────────
class TestCommandCodeBaseUrlOverride:
"""A custom base_url must redirect the catalog fetch; the default must not.
The picker passes ``base_url`` unconditionally (profile default when the
user configured nothing), so only a value differing from the default
``_COMMANDCODE_BASE`` counts as a customised endpoint.
"""
def _serve(self, models):
import json
from http.server import BaseHTTPRequestHandler, HTTPServer
from threading import Thread
class H(BaseHTTPRequestHandler):
def do_GET(self):
body = json.dumps({"data": models}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(body)
def log_message(self, fmt, *args):
pass
server = HTTPServer(("127.0.0.1", 0), H)
Thread(target=server.serve_forever, daemon=True).start()
return server, server.server_address[1]
def test_custom_base_url_redirects_fetch(self, commandcode_profile):
server, port = self._serve([{"id": "proxied/model-x"}])
try:
result = commandcode_profile.fetch_models(
api_key="k", base_url=f"http://127.0.0.1:{port}"
)
assert result == ["proxied/model-x"]
finally:
server.shutdown()
def test_custom_base_url_redirects_anthropic_fetch(
self, commandcode_anthropic_profile
):
server, port = self._serve(
[{"id": "claude-sonnet-4-6"}, {"id": "deepseek/deepseek-v4-pro"}]
)
try:
result = commandcode_anthropic_profile.fetch_models(
api_key="k", base_url=f"http://127.0.0.1:{port}"
)
assert result == ["claude-sonnet-4-6"] # claude-* filter still applies
finally:
server.shutdown()
def test_default_base_url_hits_default_endpoint(self, commandcode_profile):
"""Echoing the profile default back must NOT count as an override."""
import sys
from unittest.mock import patch as mock_patch
# The bundled plugin module is registered at discovery time under
# ``plugins.model_providers.commandcode`` — resolve via the profile's
# own __module__ so the test doesn't depend on discovery mechanics.
cc_mod = sys.modules[type(commandcode_profile).__module__]
captured = {}
class _FakeResp:
def __enter__(self):
return self
def __exit__(self, *a):
return False
def read(self):
return b'{"data": [{"id": "m1"}]}'
def fake_urlopen(req, timeout=0):
captured["url"] = req.full_url
return _FakeResp()
with mock_patch.object(
cc_mod.urllib.request, "urlopen", side_effect=fake_urlopen
):
result = commandcode_profile.fetch_models(
api_key="k", base_url=cc_mod._COMMANDCODE_BASE + "/"
)
assert result == ["m1"]
assert captured["url"] == cc_mod._COMMANDCODE_MODELS_URL
@@ -0,0 +1,102 @@
"""Unit tests for the Copilot provider profile's reasoning-effort wiring.
GitHub Copilot serves different models with different supported reasoning-effort
sets (the live ``/models`` catalog reports them per model). The profile must
forward the requested effort when the catalog lists it as supported, and only
downgrade to the nearest weaker supported level when it does not, rather than
unconditionally collapsing ``xhigh`` to ``high`` (which silently capped models
that actually support the higher level).
These tests pin that contract without going live, by stubbing the catalog
lookup ``github_model_reasoning_efforts``.
"""
from __future__ import annotations
import pytest
@pytest.fixture
def copilot_profile():
"""Resolve the registered Copilot profile.
Importing ``model_tools`` triggers plugin discovery, which registers the
Copilot profile. Going through ``get_provider_profile`` keeps the test
honest: if the registered class is ever swapped for a plain
``ProviderProfile`` the assertions below collapse.
"""
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("copilot")
assert profile is not None, "copilot provider profile must be registered"
return profile
def _patch_efforts(monkeypatch, efforts):
"""Stub the catalog lookup the profile calls for supported efforts."""
import hermes_cli.models as models_mod
monkeypatch.setattr(
models_mod, "github_model_reasoning_efforts", lambda model: list(efforts)
)
class TestCopilotReasoningEffortClamp:
def test_supported_effort_forwarded_verbatim(self, copilot_profile, monkeypatch):
"""xhigh is forwarded unchanged when the catalog lists it."""
_patch_efforts(monkeypatch, ["minimal", "low", "medium", "high", "xhigh"])
extra_body, _ = copilot_profile.build_api_kwargs_extras(
model="gpt-5.5",
reasoning_config={"effort": "xhigh"},
supports_reasoning=True,
)
assert extra_body["reasoning"] == {"effort": "xhigh"}
def test_xhigh_downgrades_to_high_when_unsupported(self, copilot_profile, monkeypatch):
"""A model whose catalog lacks xhigh gets the nearest weaker level."""
_patch_efforts(monkeypatch, ["low", "medium", "high"])
extra_body, _ = copilot_profile.build_api_kwargs_extras(
model="o-series-model",
reasoning_config={"effort": "xhigh"},
supports_reasoning=True,
)
assert extra_body["reasoning"] == {"effort": "high"}
def test_minimal_downgrades_to_low_when_unsupported(self, copilot_profile, monkeypatch):
_patch_efforts(monkeypatch, ["low", "medium", "high"])
extra_body, _ = copilot_profile.build_api_kwargs_extras(
model="o-series-model",
reasoning_config={"effort": "minimal"},
supports_reasoning=True,
)
assert extra_body["reasoning"] == {"effort": "low"}
def test_unsupported_effort_falls_back_to_medium(self, copilot_profile, monkeypatch):
"""An effort not in the set, with no specific rule, falls to medium."""
_patch_efforts(monkeypatch, ["low", "medium", "high"])
extra_body, _ = copilot_profile.build_api_kwargs_extras(
model="some-model",
reasoning_config={"effort": "garbage"},
supports_reasoning=True,
)
assert extra_body["reasoning"] == {"effort": "medium"}
def test_falls_back_to_first_supported_when_no_medium(self, copilot_profile, monkeypatch):
"""If medium isn't supported either, pick the first supported level."""
_patch_efforts(monkeypatch, ["low", "high"])
extra_body, _ = copilot_profile.build_api_kwargs_extras(
model="weird-model",
reasoning_config={"effort": "xhigh"},
supports_reasoning=True,
)
# xhigh not supported, high IS supported → high wins via the xhigh rule.
assert extra_body["reasoning"] == {"effort": "high"}
def test_first_supported_when_no_rule_matches(self, copilot_profile, monkeypatch):
_patch_efforts(monkeypatch, ["low", "high"])
extra_body, _ = copilot_profile.build_api_kwargs_extras(
model="weird-model",
reasoning_config={"effort": "garbage"},
supports_reasoning=True,
)
assert extra_body["reasoning"] == {"effort": "low"}
@@ -0,0 +1,182 @@
"""Unit tests for the custom provider profile's reasoning wiring.
``provider=custom`` covers any OpenAI-compatible endpoint the user points
Hermes at — local Ollama, vLLM, llama.cpp, and hosted reasoning APIs like
GLM-5.2 on Volcengine ARK. Before #57601's salvage, ``CustomProfile`` emitted
nothing when reasoning was *enabled*, so a configured ``reasoning_effort``
was silently dropped for every custom endpoint.
These tests pin the wire-shape contract:
- disabled on Ollama → extra_body.think = False + reasoning_effort=none
- disabled elsewhere → reasoning_effort=none, no think (strict APIs 422)
- enabled + effort → top-level reasoning_effort (native OpenAI-compat
format GLM/ARK expect), passed through verbatim
including ``max``/``xhigh``
- enabled + no effort → nothing emitted (endpoint's server default applies)
- ollama_num_ctx → extra_body.options.num_ctx, orthogonal to reasoning
"""
from __future__ import annotations
import pytest
@pytest.fixture
def custom_profile():
"""Resolve the registered custom profile via the global registry.
Importing ``model_tools`` triggers plugin discovery, which registers the
``custom`` profile. Going through ``get_provider_profile`` keeps the test
honest — if the registered class is ever downgraded to a plain
``ProviderProfile``, the assertions below collapse.
"""
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("custom")
assert profile is not None, "custom provider profile must be registered"
return profile
class TestCustomReasoningWireShape:
"""``build_api_kwargs_extras`` produces the correct wire format."""
def test_no_reasoning_config_emits_nothing(self, custom_profile):
"""Unset reasoning → omit everything so the endpoint's default applies."""
eb, tl = custom_profile.build_api_kwargs_extras(
reasoning_config=None, model="glm-5.2"
)
assert eb == {}
assert tl == {}
def test_disabled_sends_think_false(self, custom_profile):
"""enabled=False on an Ollama URL → reasoning_effort='none' + think=False.
Both fields are required on Ollama: /v1/chat/completions silently
ignores extra_body.think (only /api/chat honours it — ollama#14820)
but respects top-level reasoning_effort (#25758). think=False stays
for proxies and the native /api/chat path.
"""
eb, tl = custom_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False},
model="qwen3",
base_url="http://127.0.0.1:11434/v1",
)
assert eb == {"think": False}
assert tl == {"reasoning_effort": "none"}
def test_effort_none_sends_think_false(self, custom_profile):
"""effort='none' is the disable alias → same dual emission on Ollama."""
eb, tl = custom_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "none"},
model="qwen3",
base_url="http://localhost:11434/v1",
)
assert eb == {"think": False}
assert tl == {"reasoning_effort": "none"}
def test_disabled_omits_think_on_mistral(self, custom_profile):
"""Strict OpenAI-compat hosts forbid extra ``think`` (HTTP 422)."""
eb, tl = custom_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "none"},
model="mistral-small-latest",
base_url="https://api.mistral.ai/v1",
)
assert "think" not in eb
assert tl == {"reasoning_effort": "none"}
def test_disabled_omits_think_without_base_url(self, custom_profile):
"""Unknown custom endpoint — do not send the Ollama-only flag."""
eb, tl = custom_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False}, model="glm-5.2"
)
assert "think" not in eb
assert tl == {"reasoning_effort": "none"}
@pytest.mark.parametrize(
"base_url",
[
"http://127.0.0.1:8080/v1",
"http://localhost:1234/v1",
"https://api.groq.com/openai/v1",
],
)
def test_disabled_omits_think_on_non_ollama_relays(self, custom_profile, base_url):
eb, tl = custom_profile.build_api_kwargs_extras(
reasoning_config={"effort": "none"},
model="llama3",
base_url=base_url,
)
assert "think" not in eb
assert tl == {"reasoning_effort": "none"}
def test_disabled_sends_think_false_on_ollama_cloud_host(self, custom_profile):
eb, tl = custom_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False},
model="qwen3",
base_url="https://ollama.com/v1",
)
assert eb == {"think": False}
assert tl == {"reasoning_effort": "none"}
@pytest.mark.parametrize(
"base_url",
[
"http://myhost:99999/v1", # out-of-range port: OpenAI client accepts it
"http://localhost:80a/v1", # non-integer port
"http://localhost:11434./v1", # trailing-dot port
],
)
def test_malformed_port_does_not_raise(self, custom_profile, base_url):
"""Malformed ports must not raise — urlparse's ``port`` is ValueError-happy.
The OpenAI client accepts ``http://myhost:99999/v1`` at construction
(only httpx fails later), so these URLs reach ``build_api_kwargs_extras``
in production. The heuristic must treat them as non-Ollama rather than
killing the kwargs build.
"""
eb, tl = custom_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False},
model="qwen3",
base_url=base_url,
)
assert "think" not in eb
assert tl == {"reasoning_effort": "none"}
@pytest.mark.parametrize(
"effort", ["minimal", "low", "medium", "high", "xhigh", "max"]
)
def test_enabled_effort_goes_top_level(self, custom_profile, effort):
"""enabled + effort → TOP-LEVEL reasoning_effort, passed through verbatim.
GLM-5.2/ARK and OpenAI-compatible reasoning APIs read reasoning_effort
as a top-level string, not nested in extra_body. ``max`` is GLM's
native deep-reasoning level and must survive.
"""
eb, tl = custom_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort}, model="glm-5.2"
)
assert tl == {"reasoning_effort": effort}
assert "reasoning_effort" not in eb
assert "think" not in eb
def test_does_not_force_think_true_on_enable(self, custom_profile):
"""We must never send think=True on enable — it's Ollama-only and
would 400 on GLM/vLLM endpoints that don't recognize it."""
eb, _ = custom_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"}, model="glm-5.2"
)
assert eb.get("think") is not True
class TestCustomReasoningWithNumCtx:
"""Ollama num_ctx and reasoning are independent and compose."""
def test_num_ctx_alone(self, custom_profile):
eb, tl = custom_profile.build_api_kwargs_extras(
reasoning_config=None, ollama_num_ctx=8192, model="qwen3"
)
assert eb == {"options": {"num_ctx": 8192}}
assert tl == {}
@@ -0,0 +1,201 @@
"""Unit tests for the DeepSeek provider profile's thinking-mode wiring.
DeepSeek V4 expects every request to carry an explicit ``extra_body.thinking``
parameter. Omitting it makes the server default to thinking-mode ON, which
then enforces the ``reasoning_content``-must-be-echoed-back contract on
subsequent turns and breaks the conversation with HTTP 400 (#15700, #17212,
#17825).
These tests pin the profile's wire-shape contract so DeepSeek requests stay
correctly shaped without going live.
"""
from __future__ import annotations
import pytest
@pytest.fixture
def deepseek_profile():
"""Resolve the registered DeepSeek profile.
Going through ``providers.get_provider_profile`` keeps the test honest —
if someone later replaces the registered class with a plain
``ProviderProfile``, every assertion below collapses.
"""
# ``model_tools`` triggers plugin discovery on import, which is what
# registers the DeepSeek profile in the global provider registry.
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("deepseek")
assert profile is not None, "deepseek provider profile must be registered"
return profile
class TestDeepSeekThinkingWireShape:
"""``build_api_kwargs_extras`` produces DeepSeek's exact wire format."""
def test_v4_pro_default_enables_thinking_without_effort(self, deepseek_profile):
"""No reasoning_config → thinking enabled, server picks default effort."""
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config=None, model="deepseek-v4-pro"
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
def test_standard_efforts_pass_through(self, deepseek_profile, effort):
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="deepseek-v4-pro",
)
assert top_level == {"reasoning_effort": effort}
@pytest.mark.parametrize("effort", ["xhigh", "max", "MAX", " Max "])
def test_xhigh_and_max_normalize_to_max(self, deepseek_profile, effort):
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="deepseek-v4-pro",
)
assert top_level == {"reasoning_effort": "max"}
def test_explicitly_disabled_sends_disabled_marker(self, deepseek_profile):
"""``reasoning_config.enabled=False`` → ``thinking.type=disabled``.
The crucial bit is that the parameter is *sent* at all — DeepSeek
defaults to thinking-on when ``thinking`` is absent.
"""
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False}, model="deepseek-v4-pro"
)
assert extra_body == {"thinking": {"type": "disabled"}}
# No effort when disabled — DeepSeek rejects it.
assert top_level == {}
def test_disabled_ignores_effort_field(self, deepseek_profile):
"""Effort silently dropped when thinking is off."""
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False, "effort": "high"},
model="deepseek-v4-pro",
)
assert top_level == {}
def test_unknown_effort_omits_top_level(self, deepseek_profile):
"""Garbage effort → omit reasoning_effort so DeepSeek applies its default."""
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "garbage"},
model="deepseek-v4-pro",
)
assert top_level == {}
def test_empty_effort_omits_top_level(self, deepseek_profile):
_, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": ""},
model="deepseek-v4-pro",
)
assert top_level == {}
class TestDeepSeekModelGating:
"""V4 family gets thinking; V3 / unknown stay untouched."""
@pytest.mark.parametrize(
"model",
[
"deepseek-v4-pro",
"deepseek-v4-flash",
"deepseek-v4-future-variant",
"DEEPSEEK-V4-PRO", # case-insensitive
],
)
def test_thinking_capable_models_emit_thinking(self, deepseek_profile, model):
extra_body, _ = deepseek_profile.build_api_kwargs_extras(
reasoning_config=None, model=model
)
assert extra_body == {"thinking": {"type": "enabled"}}
@pytest.mark.parametrize(
"model",
[
"deepseek-v3-0324", # explicit V3
"deepseek-v3.1", # V3 minor revisions
"", # bare/unknown
None, # missing
"deepseek-unknown", # unrecognized
],
)
def test_non_thinking_models_emit_nothing(self, deepseek_profile, model):
extra_body, top_level = deepseek_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"}, model=model
)
assert extra_body == {}
assert top_level == {}
class TestDeepSeekFullKwargsIntegration:
"""End-to-end: the transport's full kwargs match DeepSeek's live wire format.
The live test harness in ``tests/run_agent/test_deepseek_v4_thinking_live.py``
sends ``{"reasoning_effort": "high", "extra_body": {"thinking": {"type":
"enabled"}}}``. Confirm the transport produces that exact shape when wired
through the registered DeepSeek profile.
"""
def test_full_kwargs_match_live_wire_shape(self, deepseek_profile):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="deepseek-v4-pro",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=deepseek_profile,
reasoning_config={"enabled": True, "effort": "high"},
base_url="https://api.deepseek.com/v1",
provider_name="deepseek",
)
assert kwargs["model"] == "deepseek-v4-pro"
assert kwargs["reasoning_effort"] == "high"
assert kwargs["extra_body"] == {"thinking": {"type": "enabled"}}
def test_v3_full_kwargs_omit_thinking(self, deepseek_profile):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="deepseek-v3-0324",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=deepseek_profile,
reasoning_config={"enabled": True, "effort": "high"},
base_url="https://api.deepseek.com/v1",
provider_name="deepseek",
)
assert "reasoning_effort" not in kwargs
assert "extra_body" not in kwargs or "thinking" not in kwargs.get("extra_body", {})
class TestDeepSeekAuxModel:
"""DeepSeek aux model is set on the profile so users stop seeing the
bogus 'No auxiliary LLM provider configured' warning (#26924).
Pinned at the profile layer rather than the legacy
`_API_KEY_PROVIDER_AUX_MODELS_FALLBACK` dict — new providers are
expected to set `default_aux_model` on `ProviderProfile`, and the
fallback dict only exists for providers that predate the profiles
system.
"""
def test_profile_advertises_deepseek_v4_flash(self, deepseek_profile):
assert deepseek_profile.default_aux_model == "deepseek-v4-flash"
def test_fallback_models_are_v4_only(self, deepseek_profile):
assert deepseek_profile.fallback_models == (
"deepseek-v4-pro",
"deepseek-v4-flash",
)
def test_consumer_api_returns_deepseek_v4_flash(self):
from agent.auxiliary_client import _get_aux_model_for_provider
assert _get_aux_model_for_provider("deepseek") == "deepseek-v4-flash"
@@ -0,0 +1,94 @@
"""Unit tests for the Fireworks AI provider profile.
Pins the profile's contract without going live: identity, alias registration,
and the pay-as-you-go model defaults (direct catalog ``/models/``
IDs, not the router-only tier).
"""
from __future__ import annotations
import pytest
@pytest.fixture
def fireworks_profile():
"""Resolve the registered Fireworks profile through the real discovery path."""
# Importing model_tools triggers plugin discovery, registering the profile.
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("fireworks")
assert profile is not None, "fireworks provider profile must be registered"
return profile
class TestFireworksIdentity:
def test_core_fields(self, fireworks_profile):
p = fireworks_profile
assert p.name == "fireworks"
assert p.auth_type == "api_key"
assert p.base_url == "https://api.fireworks.ai/inference/v1"
assert "FIREWORKS_API_KEY" in p.env_vars
assert "FIREWORKS_BASE_URL" not in p.env_vars
def test_display_metadata_present(self, fireworks_profile):
# Prominence copy is surfaced in the picker; keep it non-empty rather
# than pinning exact marketing wording (that's expected to change).
assert fireworks_profile.display_name
assert fireworks_profile.description
assert fireworks_profile.signup_url.startswith("https://")
class TestFireworksHeaders:
def test_attribution_matches_canonical_hermes_values(self, fireworks_profile):
"""Fireworks requests carry the same attribution identity Hermes sends
everywhere else.
Asserted against the shared constant rather than the literals so a
rebrand can't leave one provider on a stale referer/title.
"""
from agent.auxiliary_client import _OR_HEADERS_BASE
headers = fireworks_profile.default_headers
assert headers["HTTP-Referer"] == _OR_HEADERS_BASE["HTTP-Referer"]
assert headers["X-Title"] == _OR_HEADERS_BASE["X-Title"]
def test_user_agent_identifies_hermes(self, fireworks_profile):
# Prefix, not the full string — the version moves every release.
assert fireworks_profile.default_headers["User-Agent"].startswith("HermesAgent/")
class TestFireworksAliases:
@pytest.mark.parametrize("alias", ["fireworks-ai", "fw"])
def test_alias_resolves_via_registry(self, fireworks_profile, alias):
import providers
resolved = providers.get_provider_profile(alias)
assert resolved is not None
assert resolved.name == "fireworks"
def test_aliases_declared_on_profile(self, fireworks_profile):
assert "fireworks-ai" in fireworks_profile.aliases
assert "fw" in fireworks_profile.aliases
class TestFireworksModelDefaults:
"""Defaults must be usable with a standard pay-as-you-go key.
PAYG keys address ``accounts/fireworks/models/...`` directly; the bundled
defaults target that (the BYOK motion) so a fresh key works out of the box,
and use the standard tier rather than turbo as the out-of-box default.
"""
def test_aux_model_is_payg_model_not_router(self, fireworks_profile):
aux = fireworks_profile.default_aux_model
assert aux.startswith("accounts/fireworks/models/"), aux
assert "/routers/" not in aux
assert "turbo" not in aux.lower()
def test_fallback_models_are_payg_models_not_routers(self, fireworks_profile):
assert fireworks_profile.fallback_models, "expected curated fallbacks"
for model in fireworks_profile.fallback_models:
assert model.startswith("accounts/fireworks/models/"), model
assert "/routers/" not in model
assert "turbo" not in model.lower(), model
@@ -0,0 +1,27 @@
"""Contract tests for the native Google Gemini provider profile."""
from __future__ import annotations
import pytest
@pytest.fixture
def gemini_profile():
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("gemini")
assert profile is not None, "gemini provider profile must be registered"
return profile
def test_native_gemini_auxiliary_default_is_in_curated_catalog(gemini_profile):
"""The profile's default_aux_model must stay in lockstep with the curated
model picker catalog — whatever model the default points at has to be
one the picker can actually offer. Deliberately durable against future
model-generation bumps: it does not pin either side to a frozen
model-name string, only to the invariant that they never drift apart.
"""
from hermes_cli.models import _PROVIDER_MODELS
assert gemini_profile.default_aux_model in _PROVIDER_MODELS["gemini"]
@@ -0,0 +1,148 @@
"""Unit tests for the Kimi/Moonshot provider profile's reasoning wiring.
Moonshot's OpenAI-compat endpoint (``api.moonshot.ai/v1``) treats
``extra_body.thinking`` and a top-level ``reasoning_effort`` as mutually
exclusive. The profile must send at most one of them — never both — so a
request can't trip "cannot specify both 'thinking' and 'reasoning_effort'".
This mirrors the kimi-k2 handling already shipped for the opencode-go relay
(see ``tests/plugins/model_providers/test_opencode_go_profile.py``).
"""
from __future__ import annotations
import pytest
@pytest.fixture
def kimi_profile():
"""Resolve the registered Kimi profile via the provider registry.
Importing ``model_tools`` triggers plugin discovery, which registers the
Kimi profile. Going through ``get_provider_profile`` keeps the test honest:
if the registered class is ever swapped for a plain ``ProviderProfile`` the
assertions below collapse.
"""
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("kimi-coding")
assert profile is not None, "kimi-coding provider profile must be registered"
return profile
class TestKimiReasoningWireShape:
"""``build_api_kwargs_extras`` never emits thinking + reasoning_effort together."""
def test_no_config_enables_thinking_without_effort(self, kimi_profile):
"""No reasoning_config → thinking on, server picks the depth.
Regression guard: this path previously also sent
``reasoning_effort="medium"``, pairing thinking + effort on every
default call.
"""
extra_body, top_level = kimi_profile.build_api_kwargs_extras(reasoning_config=None)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
@pytest.mark.parametrize(
"effort,expected",
[
("low", "low"),
("minimal", "low"),
("medium", "high"),
("high", "high"),
("xhigh", "max"),
("max", "max"),
("ultra", "max"),
],
)
def test_effort_mapped_to_k3_vocabulary(self, kimi_profile, effort, expected):
"""Hermes' wider effort vocabulary is mapped onto K3's low/high/max."""
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort}
)
assert top_level == {"reasoning_effort": expected}
assert "thinking" not in extra_body
def test_enabled_without_effort_falls_back_to_thinking(self, kimi_profile):
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True}
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
@pytest.mark.parametrize("effort", ["", "garbage"])
def test_unrecognized_effort_falls_back_to_thinking(self, kimi_profile, effort):
"""Unknown efforts drop to the thinking toggle rather than sending
an invalid effort."""
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort}
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
def test_disabled_sends_thinking_disabled_only(self, kimi_profile):
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False}
)
assert extra_body == {"thinking": {"type": "disabled"}}
assert top_level == {}
@pytest.mark.parametrize(
"reasoning_config",
[
None,
{"enabled": True},
{"enabled": True, "effort": "high"},
{"enabled": True, "effort": "garbage"},
{"enabled": False},
{"enabled": False, "effort": "low"},
],
)
def test_never_emits_both(self, kimi_profile, reasoning_config):
"""The core invariant: thinking and reasoning_effort are never both set."""
extra_body, top_level = kimi_profile.build_api_kwargs_extras(
reasoning_config=reasoning_config
)
assert not ("thinking" in extra_body and "reasoning_effort" in top_level)
class TestKimiModelDiscovery:
def test_malformed_base_url_is_unconfirmed_and_filters_k3(self, kimi_profile):
"""Malformed user URLs must fall through safely, never authorize K3."""
from unittest.mock import patch
from providers.base import ProviderProfile
with patch.object(
ProviderProfile,
"fetch_models",
return_value=["k3", "kimi-k2.6"],
):
models = kimi_profile.fetch_models(
api_key="test-key",
base_url="https://[api.kimi.com/coding",
)
assert models == ["kimi-k2.6"]
class TestKimiFullKwargsIntegration:
"""The transport's full kwargs carry at most one reasoning knob."""
def _build(self, kimi_profile, reasoning_config):
from agent.transports.chat_completions import ChatCompletionsTransport
return ChatCompletionsTransport().build_kwargs(
model="kimi-k2-turbo-preview",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=kimi_profile,
reasoning_config=reasoning_config,
base_url="https://api.moonshot.ai/v1",
provider_name="kimi-coding",
)
@@ -0,0 +1,183 @@
"""Unit tests for the MiniMax provider profile.
Three MiniMax provider profiles (`minimax` direct API, `minimax-cn` China direct
API, `minimax-oauth` browser OAuth) all advertise a `default_aux_model` on
their `ProviderProfile`. The previous M2.7 / M2.7-highspeed values were
stale relative to the current frontier model (M3, released 2026-06-01) and
inconsistent with the `_PROVIDER_MODELS["minimax"]` catalog top entry in
`hermes_cli/models.py`.
This file pins the new defaults so the choice is reviewable and any future
revert shows up in a failing test rather than silent behavior drift.
Refs:
- Issue #36196: M3 support request
- PR #36205 (closed unmerged): Csrayz's M3 + 1M context work
- PR #36212 (open): adds M3 to `_PROVIDER_MODELS["minimax"]` catalog
- PR #6082: M2.7-highspeed → M2.7 for aux model (half-price fix)
- Commit 773a0faca: same profile-layer fix pattern for `deepseek`
"""
from __future__ import annotations
import pytest
@pytest.fixture(params=["minimax", "minimax-cn", "minimax-oauth"])
def minimax_profile(request):
"""Resolve each registered MiniMax profile.
Going through ``providers.get_provider_profile`` keeps the test honest —
if someone later replaces the registered class with a plain
``ProviderProfile``, every assertion below collapses.
"""
import model_tools # noqa: F401 -- triggers plugin discovery
import providers
profile = providers.get_provider_profile(request.param)
assert profile is not None, f"{request.param} provider profile must be registered"
return profile, request.param
class TestMinimaxAuxModelM3:
"""MiniMax profile aux model is the new frontier M3, not the stale M2.7.
The catalog top entry is ``MiniMax-M3`` in
``hermes_cli.models._PROVIDER_MODELS['minimax']`` and the
user-facing ``model.default`` for a Token-Plan install is M3,
so pinning the aux default to the same model keeps the runtime
consistent (same auth, same billing pool, same rate limits, no
surprise 2x-cost highspeed variant). M3 was released 2026-06-01
— picking it as the aux default matches the forward-looking
catalog order rather than the pre-M3 era.
"""
@pytest.mark.parametrize(
"provider_id,expected",
[
("minimax", "MiniMax-M3"),
("minimax-cn", "MiniMax-M3"),
# minimax-oauth sticks with M2.7: the OAuth / Coding Plan
# tier historically used -highspeed (PR #6082 collapsed that
# to plain M2.7 to avoid the 2x TPS surcharge). M3 is not on
# the OAuth/Coding Plan tier per platform docs as of this PR,
# so the safe choice is the cheapest generally-available
# M2.7 — matching PR #6082's intent.
("minimax-oauth", "MiniMax-M2.7"),
],
)
def test_profile_advertises_expected_aux_model(
self, provider_id, expected
):
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile(provider_id)
assert profile is not None
assert profile.default_aux_model == expected, (
f"{provider_id} default_aux_model drifted to "
f"{profile.default_aux_model!r}, expected {expected!r}"
)
def test_consumer_api_returns_non_empty_for_each_provider(self, minimax_profile):
from agent.auxiliary_client import _get_aux_model_for_provider
profile, provider_id = minimax_profile
resolved = _get_aux_model_for_provider(provider_id)
assert resolved != "", (
f"_get_aux_model_for_provider({provider_id!r}) returned empty — "
"the 'No auxiliary LLM provider configured' warning will fire on "
f"every {provider_id} session even though the profile advertises "
f"default_aux_model={profile.default_aux_model!r}"
)
assert resolved == profile.default_aux_model, (
f"_get_aux_model_for_provider({provider_id!r}) returned "
f"{resolved!r} but profile advertises {profile.default_aux_model!r} "
"— the consumer API and the profile have drifted out of sync"
)
class TestMinimaxAuxModelNotHighspeed:
"""Regression guard against re-introducing the M2.7-highspeed aux default.
PR #6082 collapsed the highspeed aux choice to plain M2.7 because the
highspeed variant costs 2x with no real benefit for compression / vision /
session-search aux tasks. None of the three MiniMax profiles should
silently re-introduce that 2x-cost path.
"""
@pytest.mark.parametrize("provider_id", ["minimax", "minimax-cn", "minimax-oauth"])
def test_default_aux_model_is_not_highspeed(self, provider_id):
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile(provider_id)
assert profile is not None
assert "highspeed" not in profile.default_aux_model.lower(), (
f"{provider_id} default_aux_model={profile.default_aux_model!r} "
"is a -highspeed variant — that costs 2x for the same model and "
"broke #4082 the first time. Revert to plain M2.7 or M3."
)
class TestMinimaxM3OpenAIReasoningWireShape:
"""MiniMax-M3 on api.minimax.io/v1 gets MiniMax's OpenAI-compatible knobs."""
def test_m3_openai_route_requests_reasoning_split_by_default(self):
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("minimax")
assert profile is not None
extra_body, top_level = profile.build_api_kwargs_extras(
reasoning_config=None,
model="MiniMax-M3",
base_url="https://api.minimax.io/v1",
)
assert extra_body == {"reasoning_split": True}
assert top_level == {}
@pytest.mark.parametrize(
"model,base_url",
[
("MiniMax-M2.7", "https://api.minimax.io/v1"),
("MiniMax-M3", "https://api.minimax.io/anthropic"),
("MiniMax-M3", "https://api.minimaxi.com/v1"),
],
)
def test_non_m3_or_non_global_openai_routes_emit_no_openai_reasoning_knobs(
self, model, base_url
):
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("minimax")
assert profile is not None
extra_body, top_level = profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model=model,
base_url=base_url,
)
assert extra_body == {}
assert top_level == {}
def test_transport_threads_base_url_to_profile(self):
import model_tools # noqa: F401
import providers
from agent.transports.chat_completions import ChatCompletionsTransport
profile = providers.get_provider_profile("minimax")
assert profile is not None
kwargs = ChatCompletionsTransport().build_kwargs(
model="MiniMax-M3",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=profile,
reasoning_config={"enabled": True, "effort": "medium"},
base_url="https://api.minimax.io/v1",
)
assert kwargs["extra_body"] == {
"reasoning_split": True,
"thinking": {"type": "adaptive"},
}
@@ -0,0 +1,139 @@
"""Unit tests for the Nous Portal profile's reasoning wiring.
The Portal honors ``reasoning: {enabled: false}`` — it is the only wire shape
that does, and ``extra_body.thinking`` is not forwarded upstream. The profile
used to drop a disable for every model, which on a thinking-first route like
``deepseek/deepseek-v4-pro`` (catalog: ``default_effort: high``) meant the
upstream default applied and "thinking off" burned reasoning tokens anyway.
A disable is still dropped for reasoning-mandatory routes, which answer
``reasoning: {enabled: false}`` with HTTP 400, and for models the catalog
can't speak to — an unknown model errs toward the old behavior rather than
risking a 400 on a cold first turn.
These tests pin that contract without going live.
"""
from __future__ import annotations
import pytest
@pytest.fixture
def nous_profile():
"""Resolve the registered Nous profile through the real discovery path."""
# ``model_tools`` triggers plugin discovery on import, which is what
# registers the Nous profile in the global provider registry.
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("nous")
assert profile is not None, "nous provider profile must be registered"
return profile
@pytest.fixture
def portal_catalog(monkeypatch):
"""Prime the Portal reasoning-capability cache with known entries."""
import hermes_cli.models as models_mod
monkeypatch.setattr(models_mod, "_nous_reasoning_caps_failed_at", None)
monkeypatch.setattr(models_mod, "_nous_reasoning_caps_cache", {
"deepseek/deepseek-v4-pro": {
"supports_reasoning": True,
"supported_efforts": ["xhigh", "high"],
"mandatory": False,
},
"arcee-ai/trinity-large-thinking": {
"supports_reasoning": True,
"supported_efforts": None,
"mandatory": True,
},
# Catalogued, and it takes no reasoning parameter at all.
"moonshotai/kimi-k3-instruct": {"supports_reasoning": False},
})
class TestNousReasoningWireShape:
"""``build_api_kwargs_extras`` produces the Portal's wire format."""
def test_disable_reaches_optional_reasoning_model(self, nous_profile, portal_catalog):
"""The knob the user set is the knob the Portal receives."""
extra_body, top_level = nous_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False},
supports_reasoning=True,
model="deepseek/deepseek-v4-pro",
)
assert extra_body == {"reasoning": {"enabled": False}}
assert top_level == {}
def test_disable_dropped_for_mandatory_reasoning_model(self, nous_profile, portal_catalog):
"""Mandatory routes 400 on a disable — send nothing instead."""
extra_body, _ = nous_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False},
supports_reasoning=True,
model="arcee-ai/trinity-large-thinking",
)
assert "reasoning" not in extra_body
def test_disable_dropped_for_unknown_model(self, nous_profile, portal_catalog):
"""Unlisted / cold catalog → fail safe, never risk the 400."""
extra_body, _ = nous_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False},
supports_reasoning=True,
model="private/unlisted-route",
)
assert "reasoning" not in extra_body
def test_disable_dropped_for_non_reasoning_route(self, nous_profile, portal_catalog):
"""A route the catalog says takes no reasoning parameter gets none.
Hermes' own ``supports_reasoning`` can disagree with the Portal about a
given route; when it does, the catalog of the service actually serving
the model wins, and we don't send it a parameter it doesn't accept.
"""
extra_body, _ = nous_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False},
supports_reasoning=True,
model="moonshotai/kimi-k3-instruct",
)
assert "reasoning" not in extra_body
@pytest.mark.parametrize(
"model",
["deepseek/deepseek-v4-pro", "arcee-ai/trinity-large-thinking", "private/unlisted-route"],
)
def test_enabled_config_always_forwarded(self, nous_profile, portal_catalog, model):
"""Mandatory-ness only gates the disable; an enable always ships."""
extra_body, _ = nous_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
supports_reasoning=True,
model=model,
)
assert extra_body["reasoning"] == {"enabled": True, "effort": "high"}
def test_no_config_defaults_to_medium(self, nous_profile, portal_catalog):
extra_body, _ = nous_profile.build_api_kwargs_extras(
reasoning_config=None,
supports_reasoning=True,
model="deepseek/deepseek-v4-pro",
)
assert extra_body["reasoning"] == {"enabled": True, "effort": "medium"}
def test_nothing_emitted_without_reasoning_support(self, nous_profile, portal_catalog):
extra_body, top_level = nous_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False},
supports_reasoning=False,
model="deepseek/deepseek-v4-pro",
)
assert extra_body == {}
assert top_level == {}
def test_caller_config_not_mutated(self, nous_profile, portal_catalog):
cfg = {"enabled": False}
nous_profile.build_api_kwargs_extras(
reasoning_config=cfg,
supports_reasoning=True,
model="deepseek/deepseek-v4-pro",
)
assert cfg == {"enabled": False}
@@ -0,0 +1,234 @@
"""Unit tests for the Ollama Cloud provider profile's reasoning-effort wiring.
Ollama Cloud's ``/v1/chat/completions`` endpoint supports top-level
``reasoning_effort`` with values ``none``, ``low``, ``medium``, ``high``,
and (undocumented but empirically confirmed) ``max``. The profile maps
Hermes's ``xhigh`` → ``max`` to unlock DeepSeek V4's "Max thinking" tier
and passes the standard levels through unchanged.
These tests pin the profile's wire-shape contract so Ollama Cloud
requests carry the correct ``reasoning_effort`` field.
"""
from __future__ import annotations
import pytest
@pytest.fixture
def ollama_cloud_profile():
"""Resolve the registered Ollama Cloud profile.
Going through ``providers.get_provider_profile`` keeps the test
honest — if someone replaces the registered class with a plain
``ProviderProfile``, every assertion below collapses.
"""
# ``model_tools`` triggers plugin discovery on import, which is what
# registers the Ollama Cloud profile in the global provider registry.
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("ollama-cloud")
assert profile is not None, "ollama-cloud provider profile must be registered"
return profile
class TestOllamaCloudReasoningEffort:
"""``build_api_kwargs_extras`` emits correct top-level ``reasoning_effort``."""
# ── xhigh / max → max ──────────────────────────────────────────
@pytest.mark.parametrize("effort", ["xhigh", "max", "MAX", " Max "])
def test_xhigh_and_max_normalize_to_max(self, ollama_cloud_profile, effort):
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": effort},
)
assert extra_body == {}
assert top_level == {"reasoning_effort": "max"}
# ── low / medium / high pass through ───────────────────────────
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
def test_standard_efforts_pass_through(self, ollama_cloud_profile, effort):
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": effort},
)
assert top_level == {"reasoning_effort": effort}
# ── disabled → reasoning_effort:"none" (the only working off switch) ──
def test_explicitly_disabled_sends_none(self, ollama_cloud_profile):
"""Ollama Cloud defaults to thinking ON and ignores extra_body.thinking,
so disabling requires top-level reasoning_effort:"none" (verified live);
omitting the field would leave thinking on."""
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": False},
)
assert extra_body == {}
assert top_level == {"reasoning_effort": "none"}
def test_disabled_ignores_effort_field(self, ollama_cloud_profile):
"""Effort is overridden by the disable off switch when thinking is off."""
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": False, "effort": "high"},
)
assert top_level == {"reasoning_effort": "none"}
# ── none effort → reasoning_effort:"none" ──────────────────────
def test_none_effort_sends_none(self, ollama_cloud_profile):
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": "none"},
)
assert extra_body == {}
assert top_level == {"reasoning_effort": "none"}
# ── missing / empty effort → let model default ─────────────────
def test_no_reasoning_config_emits_nothing(self, ollama_cloud_profile):
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config=None,
)
assert extra_body == {}
assert top_level == {}
def test_empty_effort_emits_nothing(self, ollama_cloud_profile):
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": ""},
)
assert top_level == {}
# ── unknown / minimal effort → omitted (server default) ────────
def test_unknown_effort_omitted(self, ollama_cloud_profile):
"""Unrecognized effort is omitted, not forwarded verbatim, so the
model applies its own default. Matches the sibling deepseek profile,
which targets the same backend."""
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": "future-tier"},
)
assert top_level == {}
def test_minimal_effort_clamps_to_low(self, ollama_cloud_profile):
"""``minimal`` is a real Hermes effort level but is rejected by
Ollama Cloud's /v1/chat/completions. The shared clamp degrades it to
``low`` — the nearest supported level — instead of silently dropping
the user's ask (old behavior left the server default, i.e. MORE
thinking than requested: a ladder inversion)."""
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": "minimal"},
)
assert top_level == {"reasoning_effort": "low"}
class TestOllamaCloudFullKwargsIntegration:
"""End-to-end: the transport's full kwargs include reasoning_effort."""
def test_full_kwargs_with_xhigh(self, ollama_cloud_profile):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="deepseek-v4-pro:cloud",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=ollama_cloud_profile,
reasoning_config={"enabled": True, "effort": "xhigh"},
base_url="https://ollama.com/v1",
provider_name="ollama-cloud",
supports_reasoning=True,
)
assert kwargs["model"] == "deepseek-v4-pro:cloud"
assert kwargs["reasoning_effort"] == "max"
# No extra_body — Ollama Cloud uses top-level reasoning_effort
assert "extra_body" not in kwargs or "reasoning" not in kwargs.get("extra_body", {})
class TestOllamaCloudCapabilityGating:
"""reasoning_effort is gated on the model's thinking capability."""
def test_non_thinking_model_emits_nothing(self, ollama_cloud_profile):
"""A model that doesn't support thinking (supports_reasoning=False)
gets no reasoning_effort, even when an effort is requested — Ollama
resolves thinking capability from /api/show, and we don't send a
meaningless field to e.g. gemma3 / qwen3-coder."""
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "xhigh"},
supports_reasoning=False,
)
assert extra_body == {}
assert top_level == {}
class TestOllamaModelSupportsThinking:
"""The /api/show capability probe used to resolve supports_reasoning."""
def _patch_show(self, monkeypatch, *, status=200, capabilities=None, raise_exc=None):
import httpx
class _Resp:
status_code = status
def json(self):
return {"capabilities": capabilities} if capabilities is not None else {}
class _Client:
def __init__(self, *a, **k):
pass
def __enter__(self):
return self
def __exit__(self, *a):
return False
def post(self, *a, **k):
if raise_exc:
raise raise_exc
return _Resp()
monkeypatch.setattr(httpx, "Client", _Client)
def test_thinking_capability_true(self, monkeypatch):
from hermes_cli.models import ollama_model_supports_thinking
self._patch_show(monkeypatch, capabilities=["completion", "tools", "thinking"])
assert (
ollama_model_supports_thinking(
"deepseek-v4-pro", "https://ollama.com/v1", "key"
)
is True
)
def test_probe_failure_returns_none(self, monkeypatch):
from hermes_cli.models import ollama_model_supports_thinking
self._patch_show(monkeypatch, status=404)
assert (
ollama_model_supports_thinking("x", "https://ollama.com/v1", "key") is None
)
def test_exception_returns_none(self, monkeypatch):
from hermes_cli.models import ollama_model_supports_thinking
self._patch_show(monkeypatch, raise_exc=RuntimeError("boom"))
assert (
ollama_model_supports_thinking("x", "https://ollama.com/v1", "key") is None
)
class TestOllamaCloudAuxModel:
"""Ollama Cloud aux model is set on the profile."""
def test_profile_advertises_aux_model(self, ollama_cloud_profile):
assert ollama_cloud_profile.default_aux_model == "nemotron-3-nano:30b"
@@ -0,0 +1,255 @@
"""Unit tests for OpenCode Go reasoning-control wiring."""
from __future__ import annotations
import pytest
@pytest.fixture
def opencode_go_profile():
"""Resolve the registered OpenCode Go provider profile."""
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("opencode-go")
assert profile is not None, "opencode-go provider profile must be registered"
return profile
@pytest.fixture
def opencode_zen_profile():
"""Resolve the registered OpenCode Zen provider profile."""
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("opencode-zen")
assert profile is not None, "opencode-zen provider profile must be registered"
return profile
class TestOpenCodeZenOxReasoning:
"""Ox Alpha Free uses OpenCode Zen's native reasoning_effort control."""
def test_max_effort_is_emitted(self, opencode_zen_profile):
extra_body, top_level = opencode_zen_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "max"},
model="x-preview-f-free",
)
assert extra_body == {}
assert top_level == {"reasoning_effort": "max"}
@pytest.mark.parametrize("reasoning_config", [None, {"enabled": False}])
def test_unset_or_disabled_preserves_server_default(
self, opencode_zen_profile, reasoning_config
):
extra_body, top_level = opencode_zen_profile.build_api_kwargs_extras(
reasoning_config=reasoning_config,
model="x-preview-f-free",
)
assert extra_body == {}
assert top_level == {}
def test_other_zen_models_are_untouched(self, opencode_zen_profile):
extra_body, top_level = opencode_zen_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "max"},
model="gemini-3-flash",
)
assert extra_body == {}
assert top_level == {}
def test_max_reaches_chat_completions_request(self, opencode_zen_profile):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="x-preview-f-free",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=opencode_zen_profile,
reasoning_config={"enabled": True, "effort": "max"},
base_url="https://opencode.ai/zen/v1",
)
assert "extra_body" not in kwargs
assert kwargs["reasoning_effort"] == "max"
def test_unsupported_efforts_clamp_to_wire_vocabulary(self, opencode_zen_profile):
"""medium/xhigh are not on Ox Alpha's wire (400 raw); they must clamp
to the nearest supported level, never pass through."""
for requested, expected in (("medium", "low"), ("xhigh", "max")):
_, top_level = opencode_zen_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": requested},
model="x-preview-f-free",
)
assert top_level == {"reasoning_effort": expected}, requested
def test_opencode_free_profile_shares_the_translation(self):
"""Ox Alpha is reachable via the keyless opencode-free provider too;
its profile must emit the identical clamped reasoning_effort."""
import model_tools # noqa: F401
import providers
from providers.base import ProviderProfile
profile = providers.get_provider_profile("opencode-free")
assert profile is not None
assert (
type(profile).build_api_kwargs_extras
is not ProviderProfile.build_api_kwargs_extras
), "opencode-free must override build_api_kwargs_extras (aux gate)"
_, top_level = profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "medium"},
model="x-preview-f-free",
)
assert top_level == {"reasoning_effort": "low"}
_, other = profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "max"},
model="big-pickle",
)
assert other == {}
class TestOpenCodeGoKimiReasoning:
"""Kimi K2 models use Moonshot's thinking + reasoning_effort shape on OpenCode Go."""
def test_high_effort_emits_thinking_and_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model="kimi-k2.6",
)
assert extra_body == {}
assert top_level == {"reasoning_effort": "high"}
def test_disabled_emits_thinking_disabled_without_effort(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False},
model="kimi-k2.6",
)
assert extra_body == {"thinking": {"type": "disabled"}}
assert top_level == {}
def test_minimal_effort_clamps_to_low(self, opencode_go_profile):
# "minimal" is below Moonshot's floor — the shared clamp degrades it
# to "low" (nearest supported) instead of dropping the ask and
# leaving the server default (which was MORE thinking than asked).
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "minimal"},
model="kimi-k2.6",
)
assert extra_body == {}
assert top_level == {"reasoning_effort": "low"}
@pytest.mark.parametrize(
"effort",
[
"xhigh",
"max",
],
)
def test_strong_efforts_clamp_to_high(self, opencode_go_profile, effort):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="moonshotai/kimi-k2.6",
)
assert extra_body == {}
assert top_level == {"reasoning_effort": "high"}
def test_low_and_medium_pass_through(self, opencode_go_profile):
for effort in ("low", "medium"):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="kimi-k2.5",
)
assert extra_body == {}
assert top_level == {"reasoning_effort": effort}
def test_no_config_preserves_server_default(self, opencode_go_profile):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config=None,
model="kimi-k2.6",
)
assert extra_body == {}
assert top_level == {}
class TestOpenCodeGoDeepSeekThinking:
"""DeepSeek V4 models use DeepSeek-style thinking controls on OpenCode Go."""
def test_xhigh_and_max_normalize_to_max(self, opencode_go_profile):
for effort in ("xhigh", "max"):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="deepseek/deepseek-v4-pro",
)
assert extra_body == {}
assert top_level == {"reasoning_effort": "max"}
class TestOpenCodeGoGLM52Reasoning:
"""GLM-5.2 uses its native high/max reasoning_effort knob on OpenCode Go."""
@pytest.mark.parametrize("model", ["glm-5-2", "glm-5p2"])
def test_alias_spellings_recognized(self, opencode_go_profile, model):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "max"},
model=model,
)
assert top_level == {"reasoning_effort": "max"}
class TestOpenCodeGoModelGating:
"""Other OpenCode Go models must not receive Kimi/DeepSeek/GLM controls."""
@pytest.mark.parametrize(
"model",
[
"glm-5.1",
"glm-5",
"qwen3.6-plus",
"minimax-m2.7",
"deepseek-v3.1",
"deepseek-chat",
"",
None,
],
)
def test_non_target_models_emit_nothing(self, opencode_go_profile, model):
extra_body, top_level = opencode_go_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model=model,
)
assert extra_body == {}
assert top_level == {}
class TestOpenCodeGoFullKwargsIntegration:
"""End-to-end transport kwargs include the profile-provided controls."""
def test_kimi_reasoning_reaches_extra_body_and_top_level(self, opencode_go_profile):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="kimi-k2.6",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=opencode_go_profile,
reasoning_config={"enabled": True, "effort": "high"},
base_url="https://opencode.ai/zen/go/v1",
)
assert "extra_body" not in kwargs
assert kwargs["reasoning_effort"] == "high"
def test_deepseek_thinking_reaches_extra_body_and_top_level(
self, opencode_go_profile
):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="deepseek-v4-pro",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=opencode_go_profile,
reasoning_config={"enabled": True, "effort": "high"},
base_url="https://opencode.ai/zen/go/v1",
)
assert "extra_body" not in kwargs
assert kwargs["reasoning_effort"] == "high"
@@ -0,0 +1,132 @@
"""Unit tests for the Upstage Solar provider profile.
Upstage Solar is a plain OpenAI-compatible api-key provider, so this verifies
the profile is registered correctly and wires the expected identity, endpoint,
auth, and catalog fields — the contract every downstream layer (auth, models,
doctor, runtime_provider, transport) reads from.
"""
from __future__ import annotations
import pytest
@pytest.fixture
def upstage_profile():
"""Resolve the registered Upstage profile via the provider registry.
Importing ``model_tools`` triggers plugin discovery, which registers the
Upstage profile. Going through ``get_provider_profile`` keeps the test
honest about the actual registration path (name + alias resolution).
"""
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("upstage")
assert profile is not None, "upstage provider profile must be registered"
return profile
class TestUpstageProfile:
def test_identity_and_endpoint(self, upstage_profile):
assert upstage_profile.name == "upstage"
assert upstage_profile.api_mode == "chat_completions"
assert upstage_profile.auth_type == "api_key"
assert upstage_profile.base_url == "https://api.upstage.ai/v1"
assert upstage_profile.get_hostname() == "api.upstage.ai"
def test_solar_alias_resolves(self):
import model_tools # noqa: F401
import providers
assert providers.get_provider_profile("solar") is upstage_profile_singleton()
def test_env_vars(self, upstage_profile):
# API key first, optional base-url override second (priority order).
assert upstage_profile.env_vars == ("UPSTAGE_API_KEY", "UPSTAGE_BASE_URL")
def test_fallback_models_are_agentic_pro_only(self, upstage_profile):
# Only the agentic, tool-calling Solar Pro models belong in the offline
# catalog — Mini is capable but not agentic, so it's never promoted as a
# default. Live /v1/models still surfaces everything when a key is set.
# Behavior contract (not a frozen list): non-empty, no denied families.
assert upstage_profile.fallback_models
for denied in ("solar-mini", "syn-pro"):
assert not any(
denied in m for m in upstage_profile.fallback_models
), f"non-agentic family {denied!r} must not be a fallback default"
def test_default_model_is_solar_pro3(self, upstage_profile):
# Entry [0] is the setup default (get_default_model_for_provider).
assert upstage_profile.fallback_models[0] == "solar-pro3"
def test_aux_model_left_empty(self, upstage_profile):
# Unset → auxiliary side tasks fall back to the user's main model.
assert upstage_profile.default_aux_model == ""
class TestUpstageReasoning:
"""``build_api_kwargs_extras`` wires Solar's top-level ``reasoning_effort``.
Solar Pro accepts ``reasoning_effort`` (minimal|low|medium|high, default
minimal=off) and never requires echoing ``reasoning_content`` back, so only
the request field is emitted — always top-level, never in extra_body.
"""
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
def test_pro_explicit_effort_passes_through(self, upstage_profile, effort):
extra_body, top_level = upstage_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort}, model="solar-pro3"
)
assert extra_body == {}
assert top_level == {"reasoning_effort": effort}
def test_unknown_future_effort_collapses_to_high(self, upstage_profile):
# Guard against the #62650 recurrence: a future effort level Hermes
# adds above "high" must collapse to Solar's strongest, not silently
# downgrade to the medium default.
_, top_level = upstage_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "hyperthink"},
model="solar-pro3",
)
assert top_level == {"reasoning_effort": "high"}
def test_disabled_omits_field(self, upstage_profile):
# `/reasoning none` → enabled False → explicitly off.
_, top_level = upstage_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False, "effort": "high"}, model="solar-pro3"
)
assert top_level == {}
@pytest.mark.parametrize("model", ["solar-pro3", "solar-pro", "solar-open2"])
def test_no_config_defaults_reasoning_on(self, upstage_profile, model):
# Unset reasoning_config → default ON at medium (matches the /reasoning
# "medium (default)" label), not Solar's server default of minimal/off.
_, top_level = upstage_profile.build_api_kwargs_extras(model=model)
assert top_level == {"reasoning_effort": "medium"}
@pytest.mark.parametrize("model", ["solar-mini", "solar-mini-202610", "syn-pro"])
def test_deny_listed_models_never_send_reasoning(self, upstage_profile, model):
# solar-mini / syn-pro ignore reasoning_effort, so never send it —
# even when the user explicitly enables reasoning.
extra_body, top_level = upstage_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"}, model=model
)
assert extra_body == {}
assert top_level == {}
def test_none_model_defaults_to_reasoning(self, upstage_profile):
# No model in context → treated as reasoning-capable, consistent with
# the provider default (fallback_models[0] == "solar-pro3").
_, top_level = upstage_profile.build_api_kwargs_extras(model=None)
assert top_level == {"reasoning_effort": "medium"}
def upstage_profile_singleton():
import providers
return providers.get_provider_profile("upstage")
@@ -0,0 +1,224 @@
"""Unit tests for the Z.AI / GLM provider profile's reasoning wiring.
Z.AI's GLM-4.5-and-later chat models default to thinking-mode ON when the
request omits ``thinking``. Before the profile emitted the parameter,
``reasoning_config = {"enabled": False}`` was a silent no-op on the direct
Z.AI route — users who turned thinking off kept burning thinking tokens on
every turn (the desktop "thinking reverts to medium" report).
GLM-5.2 additionally exposes a native ``reasoning_effort`` knob with two
enabled levels (high / max) on the OpenAI-compatible ``/api/paas/v4``
endpoint; the Hermes effort scale is collapsed onto those.
These tests pin the profile's wire-shape contract so Z.AI requests stay
correctly shaped without going live.
"""
from __future__ import annotations
import pytest
@pytest.fixture
def zai_profile():
"""Resolve the registered Z.AI profile through the real discovery path."""
# ``model_tools`` triggers plugin discovery on import, which is what
# registers the Z.AI profile in the global provider registry.
import model_tools # noqa: F401
import providers
profile = providers.get_provider_profile("zai")
assert profile is not None, "zai provider profile must be registered"
return profile
class TestZaiThinkingWireShape:
"""``build_api_kwargs_extras`` produces Z.AI's exact wire format."""
def test_no_preference_omits_thinking(self, zai_profile):
"""No reasoning_config → omit ``thinking`` so the server default
applies (matches prior behavior for users with no preference)."""
extra_body, top_level = zai_profile.build_api_kwargs_extras(
reasoning_config=None, model="glm-5"
)
assert extra_body == {}
assert top_level == {}
def test_enabled_sends_enabled_marker(self, zai_profile):
extra_body, top_level = zai_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "medium"}, model="glm-5"
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {}
def test_explicitly_disabled_sends_disabled_marker(self, zai_profile):
"""``reasoning_config.enabled=False`` → ``thinking.type=disabled``.
The crucial bit is that the parameter is *sent* at all — GLM defaults
to thinking-on when ``thinking`` is absent, so an unsent disable
burns thinking tokens forever.
"""
extra_body, top_level = zai_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False}, model="glm-5"
)
assert extra_body == {"thinking": {"type": "disabled"}}
assert top_level == {}
class TestZaiGLM52ReasoningEffort:
"""GLM-5.2's native ``reasoning_effort`` knob (two enabled levels)."""
def test_high_maps_to_high(self, zai_profile):
extra_body, top_level = zai_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model="glm-5.2",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {"reasoning_effort": "high"}
@pytest.mark.parametrize("effort", ["low", "medium", "minimal"])
def test_lower_efforts_clamp_up_to_high(self, zai_profile, effort):
"""GLM-5.2's minimum thinking level is high — lower Hermes levels
clamp onto it."""
extra_body, top_level = zai_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="glm-5.2",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {"reasoning_effort": "high"}
@pytest.mark.parametrize("effort", ["xhigh", "max"])
def test_strong_efforts_map_to_max(self, zai_profile, effort):
extra_body, top_level = zai_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="glm-5.2",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {"reasoning_effort": "max"}
def test_disabled_sends_no_effort(self, zai_profile):
"""Disabled reasoning still sends the thinking-off marker but never
an effort level."""
extra_body, top_level = zai_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False, "effort": "high"},
model="glm-5.2",
)
assert extra_body == {"thinking": {"type": "disabled"}}
assert top_level == {}
@pytest.mark.parametrize(
"model",
[
"z-ai/glm-5.2",
"glm-5-2",
"glm-5p2",
"accounts/fireworks/models/glm-5p2",
"zai-org-glm-5-2",
],
)
def test_alias_spellings_recognized(self, zai_profile, model):
_, top_level = zai_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "max"},
model=model,
)
assert top_level == {"reasoning_effort": "max"}
@pytest.mark.parametrize(
"model",
["glm-5.1", "glm-5", "glm-4.7", "glm-4-9b", "", None],
)
def test_non_glm_5_2_models_get_no_effort(self, zai_profile, model):
_, top_level = zai_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "high"},
model=model,
)
assert top_level == {}
class TestZaiGLM53ReasoningEffort:
"""GLM-5.3's graded low/medium/high/max effort scale (issue #91789).
Verified live on api.z.ai/api/coding/paas/v4: all four levels accepted
with monotonic reasoning-token scaling. Unlike 5.2, low and medium must
reach the wire instead of clamping up to high.
"""
@pytest.mark.parametrize(
("effort", "expected"),
[
("low", "low"),
("medium", "medium"),
("high", "high"),
("max", "max"),
("xhigh", "max"),
("minimal", "low"),
],
)
def test_graded_efforts_pass_through(self, zai_profile, effort, expected):
extra_body, top_level = zai_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": effort},
model="glm-5.3",
)
assert extra_body == {"thinking": {"type": "enabled"}}
assert top_level == {"reasoning_effort": expected}
@pytest.mark.parametrize(
"model",
["z-ai/glm-5.3", "glm-5-3", "glm-5p3", "zai-org-glm-5-3"],
)
def test_alias_spellings_get_graded_scale(self, zai_profile, model):
_, top_level = zai_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "low"},
model=model,
)
assert top_level == {"reasoning_effort": "low"}
def test_glm_5_2_still_clamps_low_to_high(self, zai_profile):
"""The 5.3 widening must not leak into 5.2's two-level wire."""
_, top_level = zai_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "low"},
model="glm-5.2",
)
assert top_level == {"reasoning_effort": "high"}
class TestZaiModelGating:
"""GLM 4.5+ get thinking; earlier GLM models are left untouched."""
@pytest.mark.parametrize(
"model",
[
"glm-4.5",
"glm-4.5-air",
"glm-4.5-flash",
"glm-4.6",
"glm-5",
"glm-5.2",
"GLM-5", # case-insensitive
],
)
def test_thinking_capable_models_emit_thinking(self, zai_profile, model):
extra_body, _ = zai_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False}, model=model
)
assert extra_body == {"thinking": {"type": "disabled"}}
class TestZaiFullKwargsIntegration:
"""End-to-end: the transport's full kwargs carry the reasoning wiring."""
def test_glm_5_2_effort_reaches_top_level(self, zai_profile):
from agent.transports.chat_completions import ChatCompletionsTransport
kwargs = ChatCompletionsTransport().build_kwargs(
model="glm-5.2",
messages=[{"role": "user", "content": "ping"}],
tools=None,
provider_profile=zai_profile,
reasoning_config={"enabled": True, "effort": "max"},
base_url="https://api.z.ai/api/paas/v4",
provider_name="zai",
)
assert kwargs["reasoning_effort"] == "max"
assert kwargs["extra_body"]["thinking"] == {"type": "enabled"}