Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

This commit is contained in:
2026-09-05 13:26:46 +03:00
commit 03634b1ca3
11340 changed files with 3442369 additions and 0 deletions
View File
@@ -0,0 +1,146 @@
"""Tests for the BedrockTransport."""
import pytest
from types import SimpleNamespace
from agent.transports import get_transport
from agent.transports.types import NormalizedResponse
@pytest.fixture
def transport():
import agent.transports.bedrock # noqa: F401
return get_transport("bedrock_converse")
class TestBedrockBasic:
def test_api_mode(self, transport):
assert transport.api_mode == "bedrock_converse"
def test_registered(self, transport):
assert transport is not None
class TestBedrockBuildKwargs:
def test_basic_kwargs(self, transport):
msgs = [{"role": "user", "content": "Hello"}]
kw = transport.build_kwargs(model="anthropic.claude-3-5-sonnet-20241022-v2:0", messages=msgs)
assert kw["modelId"] == "anthropic.claude-3-5-sonnet-20241022-v2:0"
assert kw["__bedrock_converse__"] is True
assert kw["__bedrock_region__"] == "us-east-1"
assert "messages" in kw
def test_custom_region(self, transport):
msgs = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="anthropic.claude-3-5-sonnet-20241022-v2:0",
messages=msgs,
region="eu-west-1",
)
assert kw["__bedrock_region__"] == "eu-west-1"
def test_max_tokens(self, transport):
msgs = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="anthropic.claude-3-5-sonnet-20241022-v2:0",
messages=msgs,
max_tokens=8192,
)
assert kw["inferenceConfig"]["maxTokens"] == 8192
class TestBedrockConvertTools:
def test_convert_tools(self, transport):
tools = [{
"type": "function",
"function": {
"name": "terminal",
"description": "Run commands",
"parameters": {"type": "object", "properties": {"command": {"type": "string"}}},
}
}]
result = transport.convert_tools(tools)
assert len(result) == 1
assert result[0]["toolSpec"]["name"] == "terminal"
class TestBedrockValidate:
def test_none(self, transport):
assert transport.validate_response(None) is False
def test_normalized_valid(self, transport):
r = SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content="hi"))])
assert transport.validate_response(r) is True
class TestBedrockMapFinishReason:
def test_end_turn(self, transport):
assert transport.map_finish_reason("end_turn") == "stop"
class TestBedrockNormalize:
def _make_bedrock_response(self, text="Hello", tool_calls=None, stop_reason="end_turn"):
"""Build a raw Bedrock converse response dict."""
content = []
if text:
content.append({"text": text})
if tool_calls:
for tc in tool_calls:
content.append({
"toolUse": {
"toolUseId": tc["id"],
"name": tc["name"],
"input": tc["input"],
}
})
return {
"output": {"message": {"role": "assistant", "content": content}},
"stopReason": stop_reason,
"usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15},
}
def test_tool_call_response(self, transport):
raw = self._make_bedrock_response(
text=None,
tool_calls=[{"id": "tool_1", "name": "terminal", "input": {"command": "ls"}}],
stop_reason="tool_use",
)
nr = transport.normalize_response(raw)
assert nr.finish_reason == "tool_calls"
assert len(nr.tool_calls) == 1
assert nr.tool_calls[0].name == "terminal"
def test_already_normalized_response(self, transport):
"""Test normalize_response handles already-normalized SimpleNamespace (from dispatch site)."""
pre_normalized = SimpleNamespace(
choices=[SimpleNamespace(
message=SimpleNamespace(
content="Hello from Bedrock",
tool_calls=None,
reasoning=None,
reasoning_content=None,
),
finish_reason="stop",
)],
usage=SimpleNamespace(prompt_tokens=10, completion_tokens=5, total_tokens=15),
)
nr = transport.normalize_response(pre_normalized)
assert isinstance(nr, NormalizedResponse)
assert nr.content == "Hello from Bedrock"
assert nr.finish_reason == "stop"
assert nr.usage is not None
assert nr.usage.prompt_tokens == 10
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,129 @@
"""Tests for empty / null ``tool_calls`` stripping in ChatCompletionsTransport.
Strict OpenAI-compatible providers (onerouter / Qwen, DeepSeek v4) reject an
assistant message carrying ``tool_calls: []`` (or ``null``) with HTTP 400
"Empty tool_calls is not supported in message." The pre-API sanitizer in
``agent_runtime_helpers.sanitize_api_messages`` already drops these on the
conversation_loop path, but the transport layer must also normalize them so
auxiliary / custom-provider routes that bypass that sanitizer cannot reach the
wire with an invalid array. See #58755 (follow-up).
"""
import pytest
from agent.transports import get_transport
@pytest.fixture
def transport():
import agent.transports.chat_completions # noqa: F401
return get_transport("chat_completions")
class TestEmptyToolCallsStripping:
"""Assistant messages with empty/invalid tool_calls must be normalized."""
def test_assistant_empty_list_dropped(self, transport):
msgs = [{"role": "assistant", "content": "ok", "tool_calls": []}]
out = transport.convert_messages(msgs)
assert "tool_calls" not in out[0]
assert out[0]["content"] == "ok"
def test_assistant_null_dropped(self, transport):
msgs = [{"role": "assistant", "content": "ok", "tool_calls": None}]
out = transport.convert_messages(msgs)
assert "tool_calls" not in out[0]
def test_assistant_real_calls_preserved(self, transport):
real_tc = [{
"id": "call_abc",
"type": "function",
"function": {"name": "read_file", "arguments": "{}"},
}]
msgs = [{"role": "assistant", "content": "c", "tool_calls": real_tc}]
out = transport.convert_messages(msgs)
assert out[0]["tool_calls"] == real_tc
def test_only_empty_assistant_stripped_in_mixed_batch(self, transport):
msgs = [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "thinking", "tool_calls": []},
{
"role": "assistant",
"content": "acting",
"tool_calls": [{
"id": "call_x",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}],
},
]
out = transport.convert_messages(msgs)
assert "tool_calls" not in out[1]
assert out[1]["content"] == "thinking"
assert out[2]["tool_calls"] and out[2]["tool_calls"][0]["id"] == "call_x"
def test_user_role_empty_tool_calls_untouched(self, transport):
# User messages should not carry tool_calls at all, but if a stray
# empty array is present we must NOT strip it (it's not the invalid
# assistant shape, and mutating unrelated roles risks breaking the
# schema assumptions elsewhere). The transport only normalizes
# assistant messages.
msgs = [{"role": "user", "content": "hi", "tool_calls": []}]
out = transport.convert_messages(msgs)
assert "tool_calls" in out[0]
assert out[0]["tool_calls"] == []
def test_nonempty_array_codex_fields_stripped(self, transport):
# A non-empty tool_calls array carrying codex scaffolding markers
# (call_id, response_item_id) must have those fields stripped while
# the call itself is preserved.
msgs = [{
"role": "assistant",
"content": "ok",
"tool_calls": [{
"id": "fc_1",
"call_id": "call_1",
"response_item_id": "fc_1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}],
}]
out = transport.convert_messages(msgs, model="gpt-4o")
# Non-empty array: codex fields stripped, call preserved.
assert out[0]["tool_calls"]
tc = out[0]["tool_calls"][0]
assert "call_id" not in tc
assert "response_item_id" not in tc
def test_clean_list_is_identity(self, transport):
msgs = [
{"role": "user", "content": "hi"},
{
"role": "assistant",
"content": "c",
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}],
},
]
assert transport.convert_messages(msgs) is msgs
def test_empty_array_triggers_copy_on_write(self, transport):
msgs = [{"role": "assistant", "content": "ok", "tool_calls": []}]
out = transport.convert_messages(msgs)
# Original list/message must not be mutated in place.
assert msgs[0]["tool_calls"] == []
assert "tool_calls" not in out[0]
assert out is not msgs
@pytest.mark.parametrize(
"model",
["qwen/qwen3.8-max-preview:free", "deepseek/deepseek-v4-flash", "gpt-4o"],
)
def test_empty_array_stripped_across_providers(self, transport, model):
msgs = [{"role": "assistant", "content": "ok", "tool_calls": []}]
out = transport.convert_messages(msgs, model=model)
assert "tool_calls" not in out[0]
@@ -0,0 +1,141 @@
"""Tests for the xAI ``tool_search`` reserved-name alias (#95003).
xAI's chat-completions API reserves the function name ``tool_search`` for
its native server-side tool and rejects the whole request when the client
Tool Search bridge declares it (HTTP 400 "The function name tool_search is
reserved for the tool_search tool"). The fix mirrors the web_search treatment
in ``transports/codex.py``: rename the bridge's wire declaration to
``hermes_tool_search`` for xAI targets and map the alias back to
``tool_search`` in ``normalize_response`` so dispatch is unchanged.
The reverse map is request-local: ``normalize_response`` only rewrites
aliases the paired request actually emitted (stashed on the transport as
``_last_wire_aliases``), so a real user/plugin/MCP tool that happens to be
named ``hermes_tool_search`` is never silently dispatched as the bridge.
"""
from types import SimpleNamespace
import pytest
from agent.transports import get_transport
from agent.transports.chat_completions import (
_XAI_TOOL_SEARCH_ALIAS,
_rename_tool_search_bridge_for_xai,
)
@pytest.fixture
def transport():
import agent.transports.chat_completions # noqa: F401
return get_transport("chat_completions")
class TestRenameToolSearchBridgeForXai:
def test_tool_search_renamed_alias_value(self):
tools = [{
"type": "function",
"function": {
"name": "tool_search",
"description": "Search deferred tools",
"parameters": {"type": "object", "properties": {}},
},
}]
out, alias_map = _rename_tool_search_bridge_for_xai(tools)
assert out[0]["function"]["name"] == _XAI_TOOL_SEARCH_ALIAS
assert out[0]["function"]["name"] == "hermes_tool_search"
assert alias_map == {"hermes_tool_search": "tool_search"}
def test_schema_and_description_untouched(self):
fn = {
"name": "tool_search",
"description": "Search the deferred tool catalog",
"parameters": {"type": "object", "properties": {"query": {"type": "string"}}},
}
out, _ = _rename_tool_search_bridge_for_xai([{"type": "function", "function": fn}])
assert out[0]["function"]["description"] == fn["description"]
assert out[0]["function"]["parameters"] == fn["parameters"]
def test_sibling_bridge_names_not_reserved(self):
# xAI's error names only tool_search; tool_describe / tool_call stay
# on the wire unchanged so the model keeps calling them directly.
tools = [
{"type": "function", "function": {"name": "tool_describe"}},
{"type": "function", "function": {"name": "tool_call"}},
]
out, alias_map = _rename_tool_search_bridge_for_xai(tools)
assert [t["function"]["name"] for t in out] == ["tool_describe", "tool_call"]
assert alias_map == {}
def test_ordinary_tools_untouched(self):
tools = [{"type": "function", "function": {"name": "web_search"}}]
out, alias_map = _rename_tool_search_bridge_for_xai(tools)
assert out[0]["function"]["name"] == "web_search"
assert alias_map == {}
def test_input_not_mutated(self):
# The helper feeds a deep-copied list on the helper-layer path, but
# pin copy semantics anyway: the shared per-agent tool registry must
# never see the alias (#27907 lesson).
tools = [{"type": "function", "function": {"name": "tool_search"}}]
_rename_tool_search_bridge_for_xai(tools)
assert tools[0]["function"]["name"] == "tool_search"
def test_collision_with_real_hermes_tool_search_takes_suffix(self):
# A legitimate tool already using the alias name must NOT be
# shadowed and no duplicate wire names may be produced: the bridge
# takes hermes_tool_search_2 instead.
tools = [
{"type": "function", "function": {"name": "hermes_tool_search"}},
{"type": "function", "function": {"name": "tool_search"}},
]
out, alias_map = _rename_tool_search_bridge_for_xai(tools)
names = [t["function"]["name"] for t in out]
assert names == ["hermes_tool_search", "hermes_tool_search_2"]
assert len(names) == len(set(names))
assert alias_map == {"hermes_tool_search_2": "tool_search"}
def _fake_response(tool_name):
tc = SimpleNamespace(
id="call_1",
function=SimpleNamespace(name=tool_name, arguments='{"query": "x"}'),
)
msg = SimpleNamespace(tool_calls=[tc], reasoning=None, reasoning_content=None)
choice = SimpleNamespace(message=msg, finish_reason="tool_calls")
return SimpleNamespace(choices=[choice], usage=None)
class TestNormalizeResponseMapsAliasBack:
def test_alias_call_maps_back_to_bridge_name(self, transport):
transport._last_wire_aliases = {"hermes_tool_search": "tool_search"}
resp = transport.normalize_response(_fake_response(_XAI_TOOL_SEARCH_ALIAS))
assert resp.tool_calls[0].name == "tool_search"
def test_ordinary_call_name_preserved(self, transport):
transport._last_wire_aliases = {"hermes_tool_search": "tool_search"}
resp = transport.normalize_response(_fake_response("tool_describe"))
assert resp.tool_calls[0].name == "tool_describe"
def test_no_alias_emitted_means_no_reverse_rewrite(self, transport):
# Provenance contract: if THIS request emitted no aliases, a tool
# call named hermes_tool_search is a REAL tool (user/plugin/MCP)
# and must dispatch under its own name.
transport._last_wire_aliases = {}
resp = transport.normalize_response(_fake_response("hermes_tool_search"))
assert resp.tool_calls[0].name == "hermes_tool_search"
def test_suffixed_alias_maps_back(self, transport):
transport._last_wire_aliases = {"hermes_tool_search_2": "tool_search"}
resp = transport.normalize_response(_fake_response("hermes_tool_search_2"))
assert resp.tool_calls[0].name == "tool_search"
# And the real tool occupying the plain alias name is untouched.
resp2 = transport.normalize_response(_fake_response("hermes_tool_search"))
assert resp2.tool_calls[0].name == "hermes_tool_search"
def test_legacy_fallback_without_provenance(self, transport):
# Normalize-only call sites (no request built on this instance)
# keep the historical unconditional mapping.
transport._last_wire_aliases = None
resp = transport.normalize_response(_fake_response(_XAI_TOOL_SEARCH_ALIAS))
assert resp.tool_calls[0].name == "tool_search"
@@ -0,0 +1,342 @@
"""Tests for the optional codex app-server runtime gate.
These are unit tests for the api_mode rewriter and the wire-level transport
module. They do NOT require the `codex` CLI to be installed — that's
covered by a separate live test gated on `codex --version`.
"""
from __future__ import annotations
import pytest
from hermes_cli.runtime_provider import (
_VALID_API_MODES,
_maybe_apply_codex_app_server_runtime,
)
class TestApiModeRegistration:
"""The new api_mode must be registered or downstream parsing rejects it."""
def test_codex_app_server_is_a_valid_api_mode(self) -> None:
assert "codex_app_server" in _VALID_API_MODES
def test_existing_api_modes_still_present(self) -> None:
# Regression guard: don't accidentally delete other api_modes when
# touching this set.
for mode in (
"chat_completions",
"codex_responses",
"anthropic_messages",
"bedrock_converse",
):
assert mode in _VALID_API_MODES
class TestMaybeApplyCodexAppServerRuntime:
"""The opt-in helper that rewrites api_mode → codex_app_server."""
@pytest.mark.parametrize(
"model_cfg",
[
None,
{},
{"openai_runtime": ""},
{"openai_runtime": "auto"},
{"openai_runtime": "AUTO"},
{"other_key": "codex_app_server"}, # wrong key
],
)
def test_default_off_for_openai(self, model_cfg) -> None:
"""Default behavior is preserved when the flag is unset/auto."""
got = _maybe_apply_codex_app_server_runtime(
provider="openai", api_mode="chat_completions", model_cfg=model_cfg
)
assert got == "chat_completions"
def test_opt_in_rewrites_openai(self) -> None:
got = _maybe_apply_codex_app_server_runtime(
provider="openai",
api_mode="chat_completions",
model_cfg={"openai_runtime": "codex_app_server"},
)
assert got == "codex_app_server"
@pytest.mark.parametrize(
"provider",
[
"anthropic",
"openrouter",
"xai",
"qwen-oauth",
"opencode-zen",
"bedrock",
"",
],
)
def test_other_providers_never_rerouted(self, provider) -> None:
"""Non-OpenAI providers MUST NOT be rerouted even with the flag set —
codex's app-server can only run OpenAI/Codex auth flows."""
got = _maybe_apply_codex_app_server_runtime(
provider=provider,
api_mode="anthropic_messages",
model_cfg={"openai_runtime": "codex_app_server"},
)
assert got == "anthropic_messages", (
f"provider={provider!r} should not be rerouted to codex_app_server"
)
class TestCodexAppServerModule:
"""Module-surface tests for the JSON-RPC speaker. Don't require codex CLI."""
def test_check_binary_handles_missing_executable(self) -> None:
from agent.transports.codex_app_server import check_codex_binary
ok, msg = check_codex_binary(codex_bin="/nonexistent/codex/binary/path")
assert ok is False
assert "not found" in msg.lower() or "no such" in msg.lower()
def test_codex_error_class_is_runtimeerror(self) -> None:
from agent.transports.codex_app_server import CodexAppServerError
err = CodexAppServerError(code=-32600, message="boom")
assert isinstance(err, RuntimeError)
assert "boom" in str(err)
assert "-32600" in str(err)
class TestSpawnEnvIsolation:
"""The codex spawn must NOT rewrite HOME — codex's shell tool spawns
subprocesses (gh, git, npm, aws, gcloud, ...) that need to find their
config in the real user $HOME. CODEX_HOME isolates codex's own state,
HOME stays unchanged.
OpenClaw hit this footgun (openclaw/openclaw#81562) — they were
rewriting HOME to a synthetic per-agent dir alongside CODEX_HOME,
and then `gh auth status` / git config / etc. all broke inside codex
shell calls. We avoid the same bug by only overlaying CODEX_HOME and
RUST_LOG on top of os.environ.copy().
"""
def test_spawn_env_preserves_HOME(self, monkeypatch):
"""The spawn env must contain the parent process's HOME unchanged.
Verifies via a subprocess-monkey-patch."""
import subprocess
from agent.transports import codex_app_server as cas
captured = {}
class FakePopen:
def __init__(self, cmd, *args, **kwargs):
captured["env"] = kwargs.get("env", {}).copy()
# Provide minimal Popen surface so __init__ doesn't crash
# on attribute access during construction.
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = 1
self.returncode = None
def poll(self):
return None
def terminate(self):
pass
def wait(self, timeout=None):
return 0
def kill(self):
pass
monkeypatch.setattr(subprocess, "Popen", FakePopen)
monkeypatch.setenv("HOME", "/users/alice")
client = cas.CodexAppServerClient(codex_bin="codex")
client._closed = True # so close() is a no-op
# The spawn env must have HOME=/users/alice unchanged
assert captured["env"].get("HOME") == "/users/alice", (
f"HOME got rewritten in codex spawn env: "
f"{captured['env'].get('HOME')!r}. Codex's shell tool's "
"subprocesses (gh, git, aws, npm) need the user's real HOME."
)
def test_spawn_env_sets_CODEX_HOME_when_provided(self, monkeypatch):
"""CODEX_HOME isolation must still work — that's the whole point
of the codex_home arg."""
import subprocess
from agent.transports import codex_app_server as cas
captured = {}
class FakePopen:
def __init__(self, cmd, *args, **kwargs):
captured["env"] = kwargs.get("env", {}).copy()
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = 1
self.returncode = None
def poll(self):
return None
def terminate(self):
pass
def wait(self, timeout=None):
return 0
def kill(self):
pass
monkeypatch.setattr(subprocess, "Popen", FakePopen)
monkeypatch.setenv("HOME", "/users/alice")
client = cas.CodexAppServerClient(
codex_bin="codex", codex_home="/tmp/profile/codex"
)
client._closed = True
assert captured["env"].get("CODEX_HOME") == "/tmp/profile/codex"
# And HOME still passes through unchanged
assert captured["env"].get("HOME") == "/users/alice"
def test_kanban_worker_adds_only_kanban_writable_root(self, monkeypatch):
"""Codex-runtime Kanban workers need to write board state outside
their scratch/worktree workspace, but should not fall back to
danger-full-access. Hermes passes a narrow app-server config override
for the Kanban root only.
"""
import subprocess
from agent.transports import codex_app_server as cas
captured = {}
class FakePopen:
def __init__(self, cmd, *args, **kwargs):
captured["cmd"] = list(cmd)
captured["env"] = kwargs.get("env", {}).copy()
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = 1
self.returncode = None
def poll(self):
return None
def terminate(self):
pass
def wait(self, timeout=None):
return 0
def kill(self):
pass
monkeypatch.setattr(subprocess, "Popen", FakePopen)
monkeypatch.setenv("HOME", "/users/alice")
monkeypatch.setenv("HERMES_HOME", "/users/alice/.hermes/profiles/backend-worker")
monkeypatch.setenv("HERMES_KANBAN_TASK", "t_smoke")
monkeypatch.setenv(
"HERMES_KANBAN_DB",
"/users/alice/.hermes/kanban/boards/smoke/kanban.db",
)
client = cas.CodexAppServerClient(codex_bin="codex")
client._closed = True
cmd = captured["cmd"]
assert cmd[:2] == ["codex", "app-server"]
assert 'sandbox_mode="workspace-write"' in cmd
assert (
'sandbox_workspace_write.writable_roots=["/users/alice/.hermes/kanban/boards/smoke"]'
in cmd
)
assert "sandbox_workspace_write.network_access=false" in cmd
assert all("danger" not in part for part in cmd)
class TestSpawnEnvSecretStripping:
"""codex app-server routes its spawn env through hermes_subprocess_env(
inherit_credentials=True) instead of a raw os.environ.copy().
codex is a model-driving CLI executor: it legitimately needs LLM provider
credentials to authenticate, but it must NOT inherit Tier-1 Hermes secrets
(gateway bot tokens, GitHub/infra auth, dashboard session token) or the
dynamic-internal secrets (AUXILIARY_*_API_KEY / _BASE_URL side-LLM keys,
GATEWAY_RELAY_* relay-auth) — a coding subprocess has no use for those and
a model-controlled action could exfiltrate them. This closes the #29157
sibling spawn-site gap (copilot_acp_client already routes through the
helper; codex app-server predated it).
"""
@staticmethod
def _capture_spawn_env(monkeypatch):
import subprocess
from agent.transports import codex_app_server as cas
captured = {}
class FakePopen:
def __init__(self, cmd, *args, **kwargs):
captured["env"] = kwargs.get("env", {}).copy()
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = 1
self.returncode = None
def poll(self):
return None
def terminate(self):
pass
def wait(self, timeout=None):
return 0
def kill(self):
pass
monkeypatch.setattr(subprocess, "Popen", FakePopen)
client = cas.CodexAppServerClient(codex_bin="codex")
client._closed = True
return captured["env"]
def test_tier1_and_internal_secrets_stripped_from_spawn_env(self, monkeypatch):
for var, val in {
"GH_TOKEN": "ghp-secret",
"TELEGRAM_BOT_TOKEN": "bot-secret",
"MODAL_TOKEN_SECRET": "modal-secret",
"HERMES_DASHBOARD_SESSION_TOKEN": "dash-secret",
"AUXILIARY_VISION_API_KEY": "aux-secret",
"GATEWAY_RELAY_SECRET": "relay-secret",
"GATEWAY_RELAY_ID": "relay-id",
"GATEWAY_RELAY_DELIVERY_KEY": "relay-delivery",
}.items():
monkeypatch.setenv(var, val)
env = self._capture_spawn_env(monkeypatch)
for var in (
"GH_TOKEN", "TELEGRAM_BOT_TOKEN", "MODAL_TOKEN_SECRET",
"HERMES_DASHBOARD_SESSION_TOKEN", "AUXILIARY_VISION_API_KEY",
"GATEWAY_RELAY_SECRET", "GATEWAY_RELAY_ID", "GATEWAY_RELAY_DELIVERY_KEY",
):
assert var not in env, f"{var} leaked into codex app-server spawn env"
def test_provider_credentials_still_reach_codex(self, monkeypatch):
"""codex authenticates against the model endpoint — provider keys must
still flow through (inherit_credentials=True)."""
monkeypatch.setenv("OPENAI_API_KEY", "sk-codex-needs-this")
env = self._capture_spawn_env(monkeypatch)
assert env.get("OPENAI_API_KEY") == "sk-codex-needs-this"
@@ -0,0 +1,898 @@
"""Tests for CodexAppServerSession — drive turns through a mock client.
The session adapter has the most complex behavior of the three new modules:
notification draining, server-request handling (approvals), interrupt,
deadline timeouts. These tests pin all of that without spawning real codex.
"""
from __future__ import annotations
import time
from unittest.mock import patch
from typing import Any, Optional
import pytest
import agent.transports.codex_app_server_session as session_mod
from agent.transports.codex_app_server_session import (
CodexAppServerSession,
_ServerRequestRouting,
_approval_choice_to_codex_decision,
_coerce_turn_input_text,
)
class FakeClient:
"""Stand-in for CodexAppServerClient that records calls and lets the test
drive the notification / server-request streams synchronously."""
def __init__(self, *, codex_bin: str = "codex", codex_home=None) -> None:
self.codex_bin = codex_bin
self.codex_home = codex_home
self.requests: list[tuple[str, dict]] = []
self.notifications_responses: list[dict] = []
self.responses: list[tuple[Any, dict]] = []
self.error_responses: list[tuple[Any, int, str]] = []
self._initialized = False
self._closed = False
self._notifications: list[dict] = []
self._server_requests: list[dict] = []
self._request_handler = None # Optional[Callable[[str, dict], dict]]
# API matching CodexAppServerClient
def initialize(self, **kwargs):
self._initialized = True
return {"userAgent": "fake/0.0.0", "codexHome": "/tmp",
"platformOs": "linux", "platformFamily": "unix"}
def request(self, method: str, params: Optional[dict] = None, timeout: float = 30.0):
self.requests.append((method, params or {}))
if self._request_handler is not None:
return self._request_handler(method, params or {})
# Sensible defaults for protocol methods used by the session
if method == "thread/start":
return {"thread": {"id": "thread-fake-001"},
"activePermissionProfile": {"id": "workspace-write"}}
if method == "turn/start":
return {"turn": {"id": "turn-fake-001"}}
if method == "turn/interrupt":
return {}
if method == "turn/steer":
return {"turnId": (params or {}).get("expectedTurnId")}
return {}
def notify(self, method: str, params=None):
pass
def respond(self, request_id, result):
self.responses.append((request_id, result))
def respond_error(self, request_id, code, message, data=None):
self.error_responses.append((request_id, code, message))
def take_notification(self, timeout: float = 0.0):
if self._notifications:
return self._notifications.pop(0)
# Honor a tiny sleep so the loop doesn't hot-spin; the real client
# blocks on a queue. For tests we want determinism.
if timeout > 0:
time.sleep(min(timeout, 0.001))
return None
def take_server_request(self, timeout: float = 0.0):
if self._server_requests:
return self._server_requests.pop(0)
return None
def close(self):
self._closed = True
def is_alive(self) -> bool:
# Fake is "alive" until close() is called; tests that want a dead
# subprocess can patch this attribute or call close() directly.
return not self._closed
def stderr_tail(self, n: int = 20):
return list(getattr(self, "_stderr_tail", []))[-n:]
# Test helpers
def queue_notification(self, method: str, **params):
# Keep legacy fixture shorthand aligned with the IDs returned by the
# fake thread/start and turn/start responses.
if params.get("threadId") in {"t", "th"}:
params["threadId"] = "thread-fake-001"
if params.get("turnId") == "tu1":
params["turnId"] = "turn-fake-001"
turn = params.get("turn")
if isinstance(turn, dict) and turn.get("id") == "tu1":
turn = dict(turn)
turn["id"] = "turn-fake-001"
params["turn"] = turn
self._notifications.append({"method": method, "params": params})
def queue_server_request(self, method: str, request_id: Any = "srv-1", **params):
self._server_requests.append({"id": request_id, "method": method, "params": params})
def set_stderr_tail(self, lines):
"""Test helper: seed stderr_tail() output for OAuth-refresh classifier tests."""
self._stderr_tail = list(lines)
def make_session(client: FakeClient, **kwargs) -> CodexAppServerSession:
return CodexAppServerSession(
cwd="/tmp",
client_factory=lambda **kw: client,
**kwargs,
)
# ---- choice mapping ----
class TestApprovalChoiceMapping:
@pytest.mark.parametrize("choice,expected", [
("once", "accept"),
("session", "acceptForSession"),
("always", "acceptForSession"),
("deny", "decline"),
("anything-else", "decline"),
])
def test_mapping(self, choice, expected):
assert _approval_choice_to_codex_decision(choice) == expected
class TestTurnInputCoercion:
def test_list_content_keeps_text_and_marks_images(self):
text = _coerce_turn_input_text([
{"type": "text", "text": "caption"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}},
])
assert text == "caption\n\n[image attached]"
# ---- lifecycle ----
class TestLifecycle:
def test_ensure_started_is_idempotent(self):
client = FakeClient()
s = make_session(client)
tid_a = s.ensure_started()
tid_b = s.ensure_started()
assert tid_a == tid_b == "thread-fake-001"
# thread/start should be called exactly once
method_calls = [m for (m, _) in client.requests if m == "thread/start"]
assert len(method_calls) == 1
def test_thread_start_passes_cwd_only(self):
"""thread/start carries cwd. We intentionally do NOT pass `permissions`
on this codex version (experimentalApi-gated + requires matching
config.toml [permissions] table). Letting codex use its default
(read-only unless user configures otherwise) is the documented path."""
client = FakeClient()
s = make_session(client, permission_profile="workspace-write")
s.ensure_started()
method, params = next(r for r in client.requests if r[0] == "thread/start")
assert params["cwd"] == "/tmp"
assert "permissions" not in params # see session.ensure_started() comment
def test_close_idempotent(self):
client = FakeClient()
s = make_session(client)
s.ensure_started()
s.close()
s.close()
assert client._closed is True
# ---- turn loop ----
class TestRunTurn:
def test_simple_text_turn_returns_final_message(self):
client = FakeClient()
client.queue_notification("turn/started", threadId="t", turn={"id": "tu1"})
client.queue_notification(
"item/completed",
item={"type": "agentMessage", "id": "m1", "text": "hello world"},
threadId="t", turnId="tu1",
)
client.queue_notification(
"turn/completed",
threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
s = make_session(client)
r = s.run_turn("hi", turn_timeout=2.0)
assert r.final_text == "hello world"
assert r.interrupted is False
assert r.error is None
assert any(m["role"] == "assistant" and m.get("content") == "hello world"
for m in r.projected_messages)
# turn_id propagated for downstream session-DB linkage
assert r.turn_id == "turn-fake-001"
def test_foreign_completion_in_server_request_drain_is_ignored(self):
"""Approval draining must not project a child result into the parent."""
client = FakeClient()
client.queue_server_request(
"item/commandExecution/requestApproval",
request_id="approval-1",
command="pwd",
cwd="/tmp",
)
client.queue_notification(
"item/completed",
threadId="thread-child-001",
turnId="turn-child-001",
item={
"type": "agentMessage",
"id": "child-message",
"text": "child drain summary",
},
)
client.queue_notification(
"turn/completed",
threadId="thread-child-001",
turn={
"id": "turn-child-001",
"status": "completed",
"error": None,
},
)
original_respond = client.respond
def respond_and_release_parent(request_id, response):
original_respond(request_id, response)
client.queue_notification(
"item/completed",
threadId="thread-fake-001",
turnId="turn-fake-001",
item={
"type": "agentMessage",
"id": "parent-message",
"text": "parent after approval",
},
)
client.queue_notification(
"turn/completed",
threadId="thread-fake-001",
turn={
"id": "turn-fake-001",
"status": "completed",
"error": None,
},
)
client.respond = respond_and_release_parent
session = make_session(
client,
request_routing=_ServerRequestRouting(auto_approve_exec=True),
)
result = session.run_turn("delegate then continue", turn_timeout=2.0)
assert client.responses == [("approval-1", {"decision": "accept"})]
assert result.final_text == "parent after approval"
assert result.projected_messages == [
{"role": "assistant", "content": "parent after approval"}
]
def test_tool_iteration_counter_ticks(self):
client = FakeClient()
# Two completed exec items + one final agent message
for i, item_id in enumerate(("ex1", "ex2"), start=1):
client.queue_notification(
"item/completed",
item={
"type": "commandExecution", "id": item_id,
"command": f"cmd{i}", "cwd": "/tmp",
"status": "completed", "aggregatedOutput": "ok",
"exitCode": 0, "commandActions": [],
},
threadId="t", turnId="tu1",
)
client.queue_notification(
"item/completed",
item={"type": "agentMessage", "id": "m1", "text": "done"},
threadId="t", turnId="tu1",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
s = make_session(client)
r = s.run_turn("do stuff", turn_timeout=2.0)
assert r.tool_iterations == 2
# Each tool item produces (assistant, tool) — 2*2 + final assistant = 5 msgs
assert len(r.projected_messages) == 5
def test_turn_start_failure_attaches_redacted_stderr_tail(self):
"""When codex stderr has content (non-OAuth), the tail gets attached
to the user-facing error so config/provider problems are debuggable
instead of just 'Internal error'. Credential-shaped values in stderr
are redacted via agent.redact(force=True); web-URL query params pass
through (see fix(redact): pass web URLs through unchanged)."""
client = FakeClient()
client.set_stderr_tail([
"ERROR: provider auth failed",
"Authorization: Bearer sk-live-deadbeefdeadbeef",
"url=https://api.example.com/v1?token=querysecret12345",
])
from agent.transports.codex_app_server import CodexAppServerError
def boom(method, params):
if method == "turn/start":
raise CodexAppServerError(code=-32603, message="Internal error")
return {"thread": {"id": "t"}, "activePermissionProfile": {"id": "x"}}
client._request_handler = boom
s = make_session(client)
r = s.run_turn("hi", turn_timeout=2.0)
assert r.error is not None
assert "turn/start failed" in r.error
assert "Internal error" in r.error
# Stderr tail attached
assert "codex stderr" in r.error
assert "provider auth failed" in r.error
# Credential-shaped values still redacted (sk- prefix + Bearer header)
assert "sk-live-deadbeefdeadbeef" not in r.error
# Non-OAuth → should NOT retire (subprocess JSON-RPC is still healthy).
assert r.should_retire is False
def test_turn_start_timeout_attaches_redacted_stderr_tail(self):
"""A non-OAuth TimeoutError on turn/start surfaces with codex stderr
context attached and marks the session for retirement."""
client = FakeClient()
client.set_stderr_tail([
"WARN: provider request stalled",
"Authorization: Bearer sk-stalled-secret-abc123",
])
def stall(method, params):
if method == "turn/start":
raise TimeoutError("codex method 'turn/start' timed out after 10s")
return {"thread": {"id": "t"}, "activePermissionProfile": {"id": "x"}}
client._request_handler = stall
s = make_session(client)
r = s.run_turn("hi", turn_timeout=2.0)
assert r.error is not None
assert "turn/start timed out" in r.error
assert "provider request stalled" in r.error
assert "sk-stalled-secret-abc123" not in r.error
assert r.should_retire is True
def test_steer_appends_input_to_active_turn(self):
client = FakeClient()
s = make_session(client)
s.ensure_started()
with s._active_turn_lock:
s._active_turn_id = "turn-live-123"
assert s.request_steer("Use Postgres instead") is True
method, params = client.requests[-1]
assert method == "turn/steer"
assert params == {
"threadId": "thread-fake-001",
"input": [{"type": "text", "text": "Use Postgres instead"}],
"expectedTurnId": "turn-live-123",
}
class TestCompactThread:
def test_compact_thread_sends_rpc_and_waits_for_completion(self):
client = FakeClient()
client.queue_notification(
"turn/started",
threadId="thread-fake-001",
turn={"id": "compact-turn-1"},
)
client.queue_notification(
"item/completed",
threadId="thread-fake-001",
turnId="compact-turn-1",
item={"type": "contextCompaction", "id": "compact-item-1"},
)
client.queue_notification(
"item/completed",
threadId="thread-fake-001",
turnId="compact-turn-1",
item={"type": "agentMessage", "id": "m1", "text": "compacted"},
)
client.queue_notification(
"thread/tokenUsage/updated",
threadId="thread-fake-001",
turnId="compact-turn-1",
tokenUsage={
"last": {"inputTokens": 10, "outputTokens": 2, "totalTokens": 12},
"total": {"inputTokens": 100, "outputTokens": 20, "totalTokens": 120},
"modelContextWindow": 200000,
},
)
client.queue_notification(
"turn/completed",
threadId="thread-fake-001",
turn={"id": "compact-turn-1", "status": "completed", "error": None},
)
r = make_session(client).compact_thread(turn_timeout=2.0)
assert ("thread/compact/start", {"threadId": "thread-fake-001"}) in client.requests
assert r.error is None
assert r.thread_id == "thread-fake-001"
assert r.turn_id == "compact-turn-1"
assert r.compacted is True
assert r.final_text == "compacted"
assert r.token_usage_last["totalTokens"] == 12
assert r.model_context_window == 200000
def test_compact_thread_ignores_foreign_child_completion(self):
client = FakeClient()
client.queue_notification(
"turn/started",
threadId="thread-child-001",
turn={"id": "child-compact-turn"},
)
client.queue_notification(
"item/completed",
threadId="thread-child-001",
turnId="child-compact-turn",
item={
"type": "agentMessage",
"id": "child-compact-message",
"text": "child compact summary",
},
)
client.queue_notification(
"turn/completed",
threadId="thread-child-001",
turn={
"id": "child-compact-turn",
"status": "completed",
"error": None,
},
)
client.queue_notification(
"turn/started",
threadId="thread-fake-001",
turn={"id": "compact-turn-1"},
)
client.queue_notification(
"item/completed",
threadId="thread-fake-001",
turnId="compact-turn-1",
item={
"type": "agentMessage",
"id": "parent-compact-message",
"text": "parent compacted",
},
)
client.queue_notification(
"turn/completed",
threadId="thread-fake-001",
turn={
"id": "compact-turn-1",
"status": "completed",
"error": None,
},
)
result = make_session(client).compact_thread(turn_timeout=2.0)
assert result.error is None
assert result.turn_id == "compact-turn-1"
assert result.final_text == "parent compacted"
assert result.projected_messages == [
{"role": "assistant", "content": "parent compacted"}
]
# ---- approval bridge ----
class TestServerRequestRouting:
def test_unknown_server_request_replied_with_error(self):
client = FakeClient()
client.queue_server_request("totally/unknown", request_id="req-3")
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
s = make_session(client)
s.run_turn("hi", turn_timeout=1.0)
assert any(
rid == "req-3" and code == -32601
for (rid, code, _msg) in client.error_responses
)
def test_on_event_fires_during_approval_drain(self):
"""When a server-initiated approval request arrives, the session
drains up to 8 pending notifications first so per-turn state
(e.g. _pending_file_changes for fileChange approvals) is current.
Those drained notifications must also reach the on_event display
hook — otherwise tool bubbles around approvals silently disappear.
Regression for the issue where item/started events that landed
in the queue alongside (or just before) an approval request got
projected into messages but never displayed.
"""
client = FakeClient()
# An item/started notification is queued first, then a server
# request — the session sees both during a single drain loop.
client.queue_notification(
"item/started",
item={
"type": "commandExecution",
"id": "exec-1",
"command": "echo drained",
"cwd": "/tmp",
},
)
client.queue_server_request(
"item/commandExecution/requestApproval", request_id="req-d",
command="echo drained",
cwd="/tmp",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
events: list[dict] = []
def cb(command, description, *, allow_permanent=True):
return "once"
s = make_session(
client,
approval_callback=cb,
on_event=events.append,
)
s.run_turn("hi", turn_timeout=1.0)
# The on_event hook must have seen the item/started even though
# it was drained as part of the approval roundtrip — not just
# events that arrive on the main notification path.
item_started_events = [
e for e in events
if e.get("method") == "item/started"
]
assert item_started_events, (
"item/started drained alongside the approval was not "
"forwarded to on_event — display will miss tool bubbles "
"around approvals"
)
def test_routing_auto_approve_bypass(self):
client = FakeClient()
client.queue_server_request("item/commandExecution/requestApproval", request_id="r1",
command="ls", cwd="/")
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
# No callback, but routing says auto-approve. Should approve.
s = make_session(client, request_routing=_ServerRequestRouting(
auto_approve_exec=True))
s.run_turn("hi", turn_timeout=1.0)
assert ("r1", {"decision": "accept"}) in client.responses
# ---- enriched approval prompts ----
class TestApprovalPromptEnrichment:
"""Quirk #4: apply_patch prompt should show what's changing.
Quirk #10: exec prompt should never show empty cwd."""
def test_exec_falls_back_to_session_cwd(self):
"""When codex omits cwd from the approval params, the prompt shows
the session cwd, not an empty string."""
client = FakeClient()
client.queue_server_request(
"item/commandExecution/requestApproval", request_id="r1",
command="ls", # no cwd
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
captured = {}
def cb(command, description, *, allow_permanent=True):
captured["description"] = description
return "once"
s = make_session(client, approval_callback=cb)
s.run_turn("hi", turn_timeout=1.0)
# Session cwd is /tmp by default in make_session()
assert "/tmp" in captured["description"]
assert "Codex requests exec in <unknown>" not in captured["description"]
def test_apply_patch_prompt_summarizes_pending_changes(self):
"""When the projector has cached the fileChange item from item/started,
the approval prompt surfaces the change summary."""
client = FakeClient()
# item/started fires first (carries the changes), then approval request
client.queue_notification(
"item/started",
item={"type": "fileChange", "id": "fc-1",
"changes": [
{"kind": {"type": "add"}, "path": "/tmp/new.py"},
{"kind": {"type": "update"}, "path": "/tmp/old.py"},
]},
threadId="t", turnId="tu1",
)
client.queue_server_request(
"item/fileChange/requestApproval", request_id="req-2",
itemId="fc-1", turnId="tu1", threadId="t",
startedAtMs=1234567890,
reason="add and update files",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
captured = {}
def cb(command, description, *, allow_permanent=True):
captured["command"] = command
captured["description"] = description
return "once"
s = make_session(client, approval_callback=cb)
s.run_turn("hi", turn_timeout=1.0)
# Both add and update kinds should be in the summary
assert "1 add" in captured["command"] or "1 add" in captured["description"]
assert "1 update" in captured["command"] or "1 update" in captured["description"]
# And at least one of the paths
joined = captured["command"] + " " + captured["description"]
assert "/tmp/new.py" in joined or "/tmp/old.py" in joined
def test_apply_patch_prompt_works_without_cached_summary(self):
"""When approval arrives before item/started (or without changes
info), prompt falls back to whatever codex provided."""
client = FakeClient()
client.queue_server_request(
"item/fileChange/requestApproval", request_id="req-2",
itemId="fc-orphan", turnId="tu1", threadId="t",
startedAtMs=1234567890,
reason="apply some changes",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
captured = {}
def cb(command, description, *, allow_permanent=True):
captured["command"] = command
return "once"
s = make_session(client, approval_callback=cb)
s.run_turn("hi", turn_timeout=1.0)
# Falls back to the reason
assert "apply some changes" in captured["command"]
# ---- openclaw beta.8 parity: retire/wedge/oauth/abort marker ----
class TestSessionRetirement:
"""Mirrors openclaw beta.8's resilience fixes:
- retire timed-out app-server clients (should_retire on deadline)
- post-tool completion watchdog (don't burn the full deadline after a
tool result if codex goes silent)
- <turn_aborted> raw marker as terminal (don't wait for turn/completed
that never comes)
- OAuth refresh failure classification (suggest `codex login` instead
of raw RPC error strings)
- dead subprocess detection between iterations
"""
def test_final_agent_message_without_turn_completed_is_recovered(self):
"""A completed assistant item is still a usable terminal response when
codex omits turn/completed and then goes quiet.
"""
client = FakeClient()
client.queue_notification(
"item/completed",
item={"type": "agentMessage", "id": "m1", "text": "done"},
threadId="t",
turnId="tu1",
)
s = make_session(client)
r = s.run_turn(
"hi",
turn_timeout=0.05,
notification_poll_timeout=0.01,
)
assert r.final_text == "done"
assert r.interrupted is False
assert r.error is None
assert r.should_retire is False
assert any(
msg["role"] == "assistant" and msg.get("content") == "done"
for msg in r.projected_messages
)
assert not any(method == "turn/interrupt" for method, _ in client.requests)
def test_post_tool_watchdog_uses_monotonic_clock(self):
client = FakeClient()
client.queue_notification(
"item/completed",
item={
"type": "commandExecution", "id": "ex1",
"command": "echo hi", "cwd": "/tmp",
"status": "completed", "aggregatedOutput": "hi",
"exitCode": 0, "commandActions": [],
},
threadId="t", turnId="tu1",
)
s = make_session(client)
monotonic_values = iter([1000.0, 999.0, 999.0, 999.0, 1000.2])
with patch.object(
session_mod.time,
"monotonic",
side_effect=lambda: next(monotonic_values),
):
r = s.run_turn(
"tool then silence",
turn_timeout=5.0,
notification_poll_timeout=0.0,
post_tool_quiet_timeout=0.15,
)
assert r.interrupted is True
assert r.should_retire is True
assert r.error and "silent" in r.error
def test_post_tool_watchdog_resets_on_further_activity(self):
"""A tool completion followed by an agent message should NOT trip
the watchdog — further activity = codex still alive."""
client = FakeClient()
client.queue_notification(
"item/completed",
item={
"type": "commandExecution", "id": "ex1",
"command": "echo hi", "cwd": "/tmp",
"status": "completed", "aggregatedOutput": "hi",
"exitCode": 0, "commandActions": [],
},
threadId="t", turnId="tu1",
)
# Non-tool activity immediately after — resets watchdog.
client.queue_notification(
"item/completed",
item={"type": "agentMessage", "id": "m1", "text": "tool finished"},
threadId="t", turnId="tu1",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
s = make_session(client)
r = s.run_turn(
"tool then talk", turn_timeout=2.0,
notification_poll_timeout=0.01,
post_tool_quiet_timeout=0.05,
)
# Tool ran, then text reset the watchdog, then turn/completed.
# Should NOT be a retirement case.
assert r.tool_iterations == 1
assert r.final_text == "tool finished"
assert r.should_retire is False
assert r.interrupted is False
def test_dead_subprocess_detected_between_iterations(self):
"""If codex dies (segfault, OOM, killed by its auth refresh
thread), the inter-iteration is_alive check breaks the loop
instead of waiting on a queue that will never fill."""
client = FakeClient()
s = make_session(client)
s.ensure_started()
# Simulate subprocess death by setting _closed (FakeClient's
# is_alive returns False when closed).
client._closed = True
client.set_stderr_tail([
"thread 'tokio-runtime-worker' panicked at 'oauth: invalid_grant'",
])
r = s.run_turn("x", turn_timeout=2.0,
notification_poll_timeout=0.01)
assert r.should_retire is True
# Stderr-derived auth hint takes precedence over generic message
assert r.error and "codex login" in r.error
# ---- thread/start cross-fill ----
class TestThreadStartCrossFill:
"""Mirrors openclaw beta.8's tolerance for thread.id/sessionId aliasing."""
def test_thread_id_under_thread_key(self):
client = FakeClient()
s = make_session(client)
tid = s.ensure_started()
assert tid == "thread-fake-001"
def test_missing_thread_id_raises(self):
from agent.transports.codex_app_server import CodexAppServerError
client = FakeClient()
client._request_handler = lambda method, params: (
{"thread": {}, "activePermissionProfile": {"id": "x"}}
if method == "thread/start" else
{"turn": {"id": "tu1"}}
)
s = make_session(client)
with pytest.raises(CodexAppServerError, match="no thread id"):
s.ensure_started()
class TestHasTurnAbortedMarker:
"""Unit coverage for the marker matcher itself."""
def test_empty_string(self):
from agent.transports.codex_app_server_session import (
_has_turn_aborted_marker,
)
assert _has_turn_aborted_marker("") is False
assert _has_turn_aborted_marker(None) is False # type: ignore[arg-type]
def test_plain_text_no_marker(self):
from agent.transports.codex_app_server_session import (
_has_turn_aborted_marker,
)
assert _has_turn_aborted_marker("normal response with no markers") is False
def test_open_marker(self):
from agent.transports.codex_app_server_session import (
_has_turn_aborted_marker,
)
assert _has_turn_aborted_marker("blah <turn_aborted> blah") is True
class TestClassifyOAuthFailure:
"""Unit coverage for the OAuth classifier; conservative on purpose."""
def test_401_classified(self):
from agent.transports.codex_app_server_session import (
_classify_oauth_failure,
)
hint = _classify_oauth_failure("HTTP 401 Unauthorized")
assert hint is not None
def test_empty_inputs(self):
from agent.transports.codex_app_server_session import (
_classify_oauth_failure,
)
assert _classify_oauth_failure() is None
assert _classify_oauth_failure("") is None
assert _classify_oauth_failure("", None) is None # type: ignore[arg-type]
@@ -0,0 +1,279 @@
"""Tests for CodexEventProjector — codex item/* events → Hermes messages list.
Drives projection against fixture notifications captured from codex 0.130.0
plus synthetic ones for item types we couldn't auth-test live."""
from __future__ import annotations
import json
import pytest
from agent.transports.codex_event_projector import (
CodexEventProjector,
_deterministic_call_id,
_format_tool_args,
)
# --- Fixture: real `commandExecution` notification captured from codex 0.130.0
COMMAND_EXEC_COMPLETED = {
"method": "item/completed",
"params": {
"item": {
"type": "commandExecution",
"id": "f8a75c66-a89e-4fd7-8bcf-2d58e664fa9e",
"command": "/bin/bash -lc 'echo hello && ls /tmp | head -3'",
"cwd": "/tmp",
"processId": None,
"source": "userShell",
"status": "completed",
"commandActions": [
{"type": "listFiles", "command": "ls /tmp", "path": "tmp"}
],
"aggregatedOutput": "hello\naa_lang.json\n",
"exitCode": 0,
"durationMs": 10,
},
"threadId": "019e1a94-352b-71e1-b214-e5c67c9ec190",
"turnId": "019e1a94-3553-7940-8af3-4ca57142deb7",
"completedAtMs": 1778562381151,
},
}
class TestProjectionInvariants:
"""Universal invariants that must hold across all projection paths."""
def test_streaming_deltas_dont_materialize(self) -> None:
p = CodexEventProjector()
for delta_method in (
"item/commandExecution/outputDelta",
"item/agentMessage/delta",
"item/reasoning/delta",
):
r = p.project({"method": delta_method, "params": {"delta": "x"}})
assert r.messages == [], (
f"{delta_method} should NOT produce messages — only "
f"item/completed materializes"
)
assert r.is_tool_iteration is False
assert r.final_text is None
def test_turn_started_and_completed_are_silent(self) -> None:
p = CodexEventProjector()
for method in ("turn/started", "turn/completed", "thread/started"):
r = p.project({"method": method, "params": {}})
assert r.messages == []
def test_unknown_method_silent(self) -> None:
p = CodexEventProjector()
r = p.project({"method": "totally/unknown", "params": {}})
assert r.messages == []
class TestCommandExecutionProjection:
"""Real captured notification → assistant tool_call + tool result."""
def test_first_message_is_assistant_tool_call(self) -> None:
p = CodexEventProjector()
msgs = p.project(COMMAND_EXEC_COMPLETED).messages
assistant = msgs[0]
assert assistant["role"] == "assistant"
assert assistant["content"] is None
assert len(assistant["tool_calls"]) == 1
tc = assistant["tool_calls"][0]
assert tc["type"] == "function"
assert tc["function"]["name"] == "exec_command"
args = json.loads(tc["function"]["arguments"])
assert "echo hello" in args["command"]
assert args["cwd"] == "/tmp"
def test_second_message_is_tool_result_correlating_by_id(self) -> None:
p = CodexEventProjector()
msgs = p.project(COMMAND_EXEC_COMPLETED).messages
assistant, tool = msgs
assert tool["role"] == "tool"
assert tool["tool_call_id"] == assistant["tool_calls"][0]["id"]
assert "hello" in tool["content"]
class TestAgentMessageProjection:
"""assistant text → final_text + assistant message."""
def test_agent_message_projects_to_assistant(self) -> None:
p = CodexEventProjector()
r = p.project({
"method": "item/completed",
"params": {"item": {"type": "agentMessage", "id": "x",
"text": "hi there"}},
})
assert r.final_text == "hi there"
assert r.messages == [{"role": "assistant", "content": "hi there"}]
assert r.is_tool_iteration is False
def test_pending_reasoning_attaches_to_next_assistant_message(self) -> None:
p = CodexEventProjector()
# First a reasoning item lands
r1 = p.project({
"method": "item/completed",
"params": {"item": {"type": "reasoning", "id": "r1",
"summary": ["thinking..."],
"content": ["step 1", "step 2"]}},
})
assert r1.messages == [] # reasoning alone produces no message
# Then the assistant message
r2 = p.project({
"method": "item/completed",
"params": {"item": {"type": "agentMessage", "id": "a1",
"text": "ok"}},
})
assistant = r2.messages[0]
assert "reasoning" in assistant
assert "thinking" in assistant["reasoning"]
assert "step 1" in assistant["reasoning"]
def test_reasoning_consumed_after_attaching(self) -> None:
p = CodexEventProjector()
p.project({"method": "item/completed", "params": {"item": {
"type": "reasoning", "id": "r1", "summary": ["once"], "content": []}}})
first = p.project({"method": "item/completed", "params": {"item": {
"type": "agentMessage", "id": "a", "text": "first"}}}).messages[0]
second = p.project({"method": "item/completed", "params": {"item": {
"type": "agentMessage", "id": "b", "text": "second"}}}).messages[0]
assert "reasoning" in first
assert "reasoning" not in second
class TestFileChangeProjection:
def test_file_change_summary_no_inlined_content(self) -> None:
item = {
"type": "fileChange",
"id": "fc1",
"status": "applied",
"changes": [
{"kind": {"type": "add"}, "path": "/tmp/new.py"},
{"kind": {"type": "update"}, "path": "/tmp/old.py"},
],
}
p = CodexEventProjector()
msgs = p.project({"method": "item/completed",
"params": {"item": item}}).messages
assert len(msgs) == 2
tc = msgs[0]["tool_calls"][0]
assert tc["function"]["name"] == "apply_patch"
args = json.loads(tc["function"]["arguments"])
assert len(args["changes"]) == 2
assert all("kind" in c and "path" in c for c in args["changes"])
assert "applied" in msgs[1]["content"]
class TestMcpToolCallProjection:
def test_mcp_tool_call_namespaced(self) -> None:
item = {
"type": "mcpToolCall",
"id": "m1",
"server": "obsidian",
"tool": "search_notes",
"status": "completed",
"arguments": {"query": "hermes"},
"result": {"content": [{"text": "found"}]},
"error": None,
}
msgs = CodexEventProjector().project(
{"method": "item/completed", "params": {"item": item}}
).messages
assert msgs[0]["tool_calls"][0]["function"]["name"] == "mcp.obsidian.search_notes"
assert "found" in msgs[1]["content"]
def test_mcp_error_surfaced(self) -> None:
item = {
"type": "mcpToolCall", "id": "m2",
"server": "x", "tool": "y", "status": "failed",
"arguments": {}, "result": None,
"error": {"code": -1, "message": "no"},
}
msgs = CodexEventProjector().project(
{"method": "item/completed", "params": {"item": item}}
).messages
assert "error" in msgs[1]["content"]
class TestUserAndOpaqueProjection:
def test_user_message_text_fragments_only(self) -> None:
item = {
"type": "userMessage", "id": "u1",
"content": [
{"type": "text", "text": "hello"},
{"type": "image", "url": "http://x/y"},
{"type": "text", "text": "world"},
],
}
msgs = CodexEventProjector().project(
{"method": "item/completed", "params": {"item": item}}
).messages
assert msgs[0]["role"] == "user"
assert "hello" in msgs[0]["content"]
assert "world" in msgs[0]["content"]
def test_opaque_item_recorded_without_fabricated_tool_calls(self) -> None:
item = {"type": "plan", "id": "p1", "text": "do the thing"}
msgs = CodexEventProjector().project(
{"method": "item/completed", "params": {"item": item}}
).messages
assert len(msgs) == 1
assert msgs[0]["role"] == "assistant"
assert "plan" in msgs[0]["content"].lower()
assert "tool_calls" not in msgs[0]
class TestHelpers:
def test_deterministic_call_id_stable(self) -> None:
assert _deterministic_call_id("exec", "abc") == _deterministic_call_id("exec", "abc")
assert _deterministic_call_id("exec", "abc") != _deterministic_call_id("exec", "xyz")
def test_deterministic_call_id_handles_missing_id(self) -> None:
# Should not raise, should be stable for same item type
a = _deterministic_call_id("exec", "")
b = _deterministic_call_id("exec", "")
assert a == b
assert "exec" in a
def test_format_tool_args_sorted_keys(self) -> None:
# Sorted keys = deterministic across replays = prefix cache stays valid
a = _format_tool_args({"b": 1, "a": 2})
b = _format_tool_args({"a": 2, "b": 1})
assert a == b
class TestRoleAlternationInvariant:
"""The project must never emit two assistant messages back-to-back from
one item — that breaks Hermes' message alternation invariant."""
@pytest.mark.parametrize(
"item",
[
{"type": "commandExecution", "id": "c1", "command": "x",
"cwd": "/", "status": "completed", "aggregatedOutput": "",
"exitCode": 0, "commandActions": []},
{"type": "fileChange", "id": "f1", "status": "applied",
"changes": []},
{"type": "mcpToolCall", "id": "m1", "server": "s", "tool": "t",
"status": "completed", "arguments": {}, "result": None,
"error": None},
{"type": "dynamicToolCall", "id": "d1", "tool": "x",
"arguments": {}, "status": "completed",
"contentItems": [], "success": True},
],
)
def test_tool_items_emit_assistant_then_tool(self, item) -> None:
msgs = CodexEventProjector().project(
{"method": "item/completed", "params": {"item": item}}
).messages
assert len(msgs) == 2
assert msgs[0]["role"] == "assistant"
assert msgs[1]["role"] == "tool"
assert msgs[1]["tool_call_id"] == msgs[0]["tool_calls"][0]["id"]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,148 @@
"""Tests for the hermes-tools-as-MCP server module surface.
We don't run a live MCP session in unit tests — that requires the codex
subprocess + client + an event loop. These tests pin the static
contract: the module imports, the EXPOSED_TOOLS list is sane, and the
build helper assembles a server when the SDK is present.
"""
from __future__ import annotations
import inspect
from typing import get_args
from agent.transports.hermes_tools_mcp_server import (
_signature_from_schema,
)
class TestSignatureFromSchema:
"""Test the JSON Schema -> Python signature conversion."""
def test_simple_required_string_param(self):
"""A required string param becomes str with no default."""
schema = {
"type": "object",
"properties": {"query": {"type": "string"}},
"required": ["query"],
}
sig, annots = _signature_from_schema(schema)
assert len(sig.parameters) == 1
param = sig.parameters["query"]
assert param.name == "query"
assert param.kind == inspect.Parameter.KEYWORD_ONLY
assert annots["query"] == str
assert param.default is inspect.Parameter.empty
def test_skip_private_params(self):
"""Params starting with '_' are excluded from the signature."""
schema = {
"type": "object",
"properties": {
"query": {"type": "string"},
"_internal": {"type": "string"},
},
"required": ["query", "_internal"],
}
sig, annots = _signature_from_schema(schema)
assert "_internal" not in sig.parameters
assert "_internal" not in annots
assert "query" in sig.parameters
def test_all_json_types(self):
"""All JSON schema types map to correct Python types."""
schema = {
"type": "object",
"properties": {
"s": {"type": "string"},
"i": {"type": "integer"},
"n": {"type": "number"},
"b": {"type": "boolean"},
"a": {"type": "array"},
"o": {"type": "object"},
},
"required": ["s", "i", "n", "b", "a", "o"],
}
sig, annots = _signature_from_schema(schema)
assert annots["s"] == str
assert annots["i"] == int
assert annots["n"] == float
assert annots["b"] == bool
assert annots["a"] == list
assert annots["o"] == dict
class TestModuleSurface:
def test_module_imports_clean(self):
from agent.transports import hermes_tools_mcp_server as m
assert callable(m.main)
assert callable(m._build_server)
assert isinstance(m.EXPOSED_TOOLS, tuple)
assert len(m.EXPOSED_TOOLS) > 0
def test_exposed_tools_are_safe_subset(self):
"""We MUST NOT expose tools codex already has, because codex'
own builtins are better-integrated with its sandbox + approvals.
Specifically: no terminal/shell, no read_file/write_file, no
patch — those are codex's built-in tools."""
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
forbidden = {
"terminal", "shell", "read_file", "write_file", "patch",
"search_files", "process",
}
leaked = forbidden & set(EXPOSED_TOOLS)
assert not leaked, (
f"these tools must NOT be exposed via the codex callback "
f"because codex has built-in equivalents: {leaked}"
)
class TestMain:
def test_main_returns_2_when_mcp_unavailable(self, monkeypatch):
"""When the mcp package isn't installed, main() should exit
cleanly with code 2 and an install hint, not crash."""
import agent.transports.hermes_tools_mcp_server as m
def boom_build(*a, **kw):
raise ImportError("mcp not installed")
monkeypatch.setattr(m, "_build_server", boom_build)
rc = m.main(["--verbose"])
assert rc == 2
def test_main_handles_keyboard_interrupt(self, monkeypatch):
import agent.transports.hermes_tools_mcp_server as m
class FakeServer:
def run(self):
raise KeyboardInterrupt()
monkeypatch.setattr(m, "_build_server", lambda: FakeServer())
rc = m.main([])
assert rc == 0
def test_main_returns_1_on_runtime_error(self, monkeypatch):
import agent.transports.hermes_tools_mcp_server as m
class CrashingServer:
def run(self):
raise RuntimeError("boom")
monkeypatch.setattr(m, "_build_server", lambda: CrashingServer())
rc = m.main([])
assert rc == 1
@@ -0,0 +1,114 @@
"""Tests for Meta api.meta.ai prompt_cache_retention and transport plumbing."""
import json
from types import SimpleNamespace
import pytest
from agent.transports import get_transport
from agent.transports.codex import _default_prompt_cache_retention_for_request
@pytest.fixture
def transport():
import agent.transports.codex # noqa: F401
return get_transport("codex_responses")
class TestMetaRetention:
def test_meta_retention_24h(self, transport):
kw = transport.build_kwargs(
model="muse-spark-1.2",
messages=[{"role": "user", "content": "Hi"}],
tools=[],
base_url="https://api.meta.ai/v1",
session_id="test-session",
)
assert kw.get("prompt_cache_retention") == "24h"
def test_meta_retention_also_for_generic_model_name(self, transport):
for model in ["muse-spark", "meta/muse-spark-1.2-2026-04-01", "gpt-5.4", ""]:
kw = transport.build_kwargs(
model=model,
messages=[{"role": "user", "content": "Hi"}],
tools=[],
base_url="https://api.meta.ai/v1",
session_id="sid",
)
assert kw.get("prompt_cache_retention") == "24h", f"model={model!r}"
def test_meta_retention_helper_direct(self):
assert _default_prompt_cache_retention_for_request("muse-spark-1.2", "https://api.meta.ai/v1") == "24h"
assert _default_prompt_cache_retention_for_request("muse-spark-1.2", "https://API.META.AI/v1") == "24h"
assert _default_prompt_cache_retention_for_request("muse-spark-1.2", "https://api.meta.ai:443/v1") == "24h"
def test_meta_retention_override_wins(self, transport):
kw = transport.build_kwargs(
model="muse-spark-1.2",
messages=[{"role": "user", "content": "Hi"}],
tools=[],
base_url="https://api.meta.ai/v1",
session_id="sid",
request_overrides={"prompt_cache_retention": "in_memory"},
)
assert kw.get("prompt_cache_retention") == "in_memory"
def test_non_meta_no_retention(self, transport):
kw = transport.build_kwargs(
model="muse-spark-1.2",
messages=[{"role": "user", "content": "Hi"}],
tools=[],
base_url="https://generic.example.com/v1",
session_id="sid",
)
assert "prompt_cache_retention" not in kw
def test_non_meta_no_retention_helper(self):
assert _default_prompt_cache_retention_for_request("muse-spark-1.2", "https://generic.example.com/v1") is None
def test_meta_prompt_cache_key_is_content_addressed(self, transport):
messages = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="muse-spark-1.2",
messages=messages,
tools=[],
base_url="https://api.meta.ai/v1",
session_id="cron_job_xxx_20260624_143000",
)
pck = kw.get("prompt_cache_key", "")
assert pck.startswith("pck_")
# stable across different cron fire timestamps (same scope)
kw2 = transport.build_kwargs(
model="muse-spark-1.2",
messages=messages,
tools=[],
base_url="https://api.meta.ai/v1",
session_id="cron_job_xxx_20260624_143500",
)
assert kw["prompt_cache_key"] == kw2["prompt_cache_key"]
def test_meta_reasoning_effort_passthrough(self, transport):
kw = transport.build_kwargs(
model="muse-spark-1.2",
messages=[{"role": "user", "content": "Hi"}],
tools=[],
base_url="https://api.meta.ai/v1",
session_id="sid",
reasoning_config={"effort": "high", "enabled": True},
)
assert kw.get("reasoning") == {"effort": "high", "summary": "auto"}
def test_meta_request_hits_responses_plan(self, transport):
# Transport api_mode must be codex_responses so conversation_loop hits /v1/responses
assert transport.api_mode == "codex_responses"
kw = transport.build_kwargs(
model="muse-spark-1.2",
messages=[{"role": "user", "content": "Hi"}],
tools=[{"type": "function", "function": {"name": "terminal", "description": "x", "parameters": {"type": "object", "properties": {"command": {"type": "string"}}}}}],
base_url="https://api.meta.ai/v1",
session_id="sid",
)
# Ensure instructions/input/tools and retention present - i.e., responses shape
assert "instructions" in kw or "input" in kw
assert kw.get("prompt_cache_retention") == "24h"
assert "prompt_cache_key" in kw
@@ -0,0 +1,130 @@
"""Sibling-site coverage for the reasoning-effort wire-vocabulary class (#89503).
The chat-completions chokepoint fix (ultra → max for every model) is covered
by tests/agent/test_reasoning_effort_wire_translation.py. These tests pin the
sibling sites fixed in the same sweep:
- Kimi/Moonshot top-level ``reasoning_effort``: K3 accepts low/high/max only
(docs: default high); K2-era models accept low/medium/high. Previously the
transport forwarded only {low,medium,high} and silently dropped everything
else to "medium", so K3 400'd on "medium" requests and ultra resolved
WEAKER than an explicit high (ladder inversion).
- Tencent TokenHub: accepts low/medium/high; upper-ladder levels previously
dropped to the "high" default (accidentally right) but "minimal" also
dropped to high — asked for the least, got the most.
- Codex/Responses transport: ultra → max for EVERY model, not just gpt-5.6.
"""
from agent.transports import get_transport
import agent.transports.chat_completions # noqa: F401
import agent.transports.codex # noqa: F401
def _cc():
return get_transport("chat_completions")
def _kimi_kwargs(model, reasoning_config):
return _cc().build_kwargs(
model=model,
messages=[{"role": "user", "content": "hi"}],
is_kimi=True,
reasoning_config=reasoning_config,
)
class TestKimiEffortVocabulary:
def test_k3_maps_full_hermes_ladder(self):
expected = {
"minimal": "low",
"low": "low",
"medium": "high",
"high": "high",
"xhigh": "max",
"max": "max",
"ultra": "max",
}
for hermes_level, wire_level in expected.items():
kw = _kimi_kwargs(
"kimi-k3", {"enabled": True, "effort": hermes_level}
)
assert kw["reasoning_effort"] == wire_level, hermes_level
def test_k3_default_is_high(self):
kw = _kimi_kwargs("kimi-k3", None)
assert kw["reasoning_effort"] == "high"
def test_k2_upper_ladder_caps_at_high_not_medium(self):
"""Pre-fix, ultra/max/xhigh on K2-era models silently dropped to the
'medium' default — the strongest ask resolved weaker than an explicit
high (ladder inversion, same class as #74295)."""
for level in ("xhigh", "max", "ultra"):
kw = _kimi_kwargs(
"moonshotai/kimi-k2.6", {"enabled": True, "effort": level}
)
assert kw["reasoning_effort"] == "high", level
def test_k2_native_levels_pass_through(self):
for level in ("low", "medium", "high"):
kw = _kimi_kwargs(
"moonshotai/kimi-k2.6", {"enabled": True, "effort": level}
)
assert kw["reasoning_effort"] == level
def test_k2_minimal_maps_to_low(self):
kw = _kimi_kwargs(
"moonshotai/kimi-k2.6", {"enabled": True, "effort": "minimal"}
)
assert kw["reasoning_effort"] == "low"
def test_disabled_omits_effort(self):
kw = _kimi_kwargs("kimi-k3", {"enabled": False})
assert "reasoning_effort" not in kw
class TestTokenHubEffortVocabulary:
def _kwargs(self, reasoning_config):
return _cc().build_kwargs(
model="hunyuan-t2",
messages=[{"role": "user", "content": "hi"}],
is_tokenhub=True,
reasoning_config=reasoning_config,
)
def test_upper_ladder_caps_at_high(self):
for level in ("xhigh", "max", "ultra"):
kw = self._kwargs({"enabled": True, "effort": level})
assert kw["reasoning_effort"] == "high", level
def test_minimal_maps_to_low_not_high(self):
"""Pre-fix, 'minimal' fell through to the 'high' default — asked for
the least reasoning, got the most."""
kw = self._kwargs({"enabled": True, "effort": "minimal"})
assert kw["reasoning_effort"] == "low"
def test_native_levels_pass_through(self):
for level in ("low", "medium", "high"):
kw = self._kwargs({"enabled": True, "effort": level})
assert kw["reasoning_effort"] == level
class TestCodexUltraForEveryModel:
def test_ultra_never_leaks_and_respects_per_model_ceiling(self):
"""'ultra' must never reach the Codex wire. Live-verified vocabulary
(#68365): gpt-5.6 accepts max (its ceiling), gpt-5.5/o5 do not —
their ceiling is xhigh."""
transport = get_transport("codex_responses")
expected = {
"gpt-5.6-codex": "max",
"o5-pro": "xhigh",
"gpt-5.5": "xhigh",
"some-responses-model": "xhigh",
}
for model, wire in expected.items():
kw = transport.build_kwargs(
model=model,
messages=[{"role": "user", "content": "hi"}],
tools=[],
reasoning_config={"enabled": True, "effort": "ultra"},
)
assert kw["reasoning"]["effort"] == wire, model
@@ -0,0 +1,234 @@
"""Router catalog-declared reasoning-effort clamping on the codex transport.
Ramp Router (api.router.com) validates ``reasoning.effort`` against each
model's published vocabulary — HTTP 400 ``invalid-argument`` on an
unsupported level, and 400 ``unsupported_parameter`` when a non-reasoning
model receives any reasoning field (both verified live, Aug 2026). The
router profile declares each model's vocabulary from its cached catalog via
``ProviderProfile.supported_reasoning_efforts``; these tests pin how the
codex transport consumes that declaration.
All tests seed the plugin's in-memory cache directly — no network.
"""
import sys
import pytest
from agent.transports import get_transport
def _router_plugin_module():
from providers import get_provider_profile
profile = get_provider_profile("router")
assert profile is not None, "router profile must be registered"
return profile, sys.modules[type(profile).__module__]
@pytest.fixture
def transport():
import agent.transports.codex # noqa: F401
return get_transport("codex_responses")
@pytest.fixture
def seeded_catalog(monkeypatch):
"""Seed the router efforts cache with catalog-shaped verdicts."""
profile, mod = _router_plugin_module()
monkeypatch.setattr(mod, "_efforts_cache", {
# grok via Router: no "none", no "max" (live catalog shape)
"grok-4.6": ["minimal", "low", "medium", "high", "xhigh"],
# non-reasoning model: any reasoning field 400s
"gpt-4.1-mini": [],
# full ladder including max
"accounts/fireworks/models/kimi-k3": [
"minimal", "low", "medium", "high", "xhigh", "max",
],
})
monkeypatch.setattr(mod, "_disk_checked", True)
return profile
class TestProfileContract:
def test_declared_vocabulary(self, seeded_catalog):
assert seeded_catalog.supported_reasoning_efforts("grok-4.6") == (
"minimal", "low", "medium", "high", "xhigh",
)
def test_non_reasoning_model_is_definitive_empty(self, seeded_catalog):
assert seeded_catalog.supported_reasoning_efforts("gpt-4.1-mini") == ()
def test_unknown_model_is_none(self, seeded_catalog):
assert seeded_catalog.supported_reasoning_efforts("some-byok-route") is None
def test_cold_cache_is_none_and_never_blocks(self, monkeypatch):
profile, mod = _router_plugin_module()
monkeypatch.setattr(mod, "_efforts_cache", None)
monkeypatch.setattr(mod, "_disk_checked", True)
monkeypatch.setattr(mod, "_warm_efforts_async", lambda: None)
assert profile.supported_reasoning_efforts("grok-4.6") is None
def test_parse_efforts_catalog_shapes(self):
_, mod = _router_plugin_module()
parsed = mod._parse_efforts([
{
"id": "grok-4.6",
"router": {"capabilities": {"reasoning": {
"supported": True,
"efforts": [{"value": "low"}, {"value": "high"}],
}}},
},
{
"id": "gpt-4.1",
"router": {"capabilities": {"reasoning": {"supported": False, "efforts": []}}},
},
# reasoning supported but vocabulary unpublished -> omitted (unknown)
{
"id": "mystery-model",
"router": {"capabilities": {"reasoning": {"supported": True, "efforts": []}}},
},
# no router metadata at all -> omitted
{"id": "bare-model"},
])
assert parsed == {"grok-4.6": ["low", "high"], "gpt-4.1": []}
class TestTransportClamp:
def _kwargs(self, transport, model, reasoning_config=None):
return transport.build_kwargs(
model=model,
messages=[{"role": "user", "content": "Hi"}],
tools=[],
base_url="https://api.router.com/v1",
session_id="sid",
provider="router",
reasoning_config=reasoning_config,
)
def test_clamps_to_catalog_vocabulary(self, transport, seeded_catalog):
# grok-4.6 via Router has no "max" — nearest weaker supported is xhigh.
kw = self._kwargs(transport, "grok-4.6", {"effort": "max"})
assert kw["reasoning"]["effort"] == "xhigh"
def test_supported_effort_passes_through(self, transport, seeded_catalog):
kw = self._kwargs(
transport, "accounts/fireworks/models/kimi-k3", {"effort": "max"}
)
assert kw["reasoning"]["effort"] == "max"
def test_non_reasoning_model_suppresses_reasoning(self, transport, seeded_catalog):
# Default reasoning_config is enabled — the () verdict must strip the
# reasoning field entirely (Router 400s rather than ignoring it).
kw = self._kwargs(transport, "gpt-4.1-mini")
assert "reasoning" not in kw
assert kw.get("include") == []
def test_unknown_model_falls_back_to_codex_default(self, transport, seeded_catalog):
# Not in the catalog -> default codex vocabulary applies (legacy has
# xhigh but no max: max clamps to xhigh, medium is untouched).
kw = self._kwargs(transport, "some-byok-route", {"effort": "max"})
assert kw["reasoning"]["effort"] == "xhigh"
kw = self._kwargs(transport, "some-byok-route", {"effort": "medium"})
assert kw["reasoning"]["effort"] == "medium"
def test_cold_cache_keeps_default_behavior(self, transport, monkeypatch):
_, mod = _router_plugin_module()
monkeypatch.setattr(mod, "_efforts_cache", None)
monkeypatch.setattr(mod, "_disk_checked", True)
monkeypatch.setattr(mod, "_warm_efforts_async", lambda: None)
kw = self._kwargs(transport, "grok-4.6", {"effort": "xhigh"})
# Cold cache -> no declaration -> default codex vocabulary (xhigh ok).
assert kw["reasoning"]["effort"] == "xhigh"
def test_other_providers_unaffected(self, transport, seeded_catalog):
kw = transport.build_kwargs(
model="gpt-4.1-mini",
messages=[{"role": "user", "content": "Hi"}],
tools=[],
base_url="https://generic.example.com/v1",
session_id="sid",
provider="some-other-provider",
reasoning_config={"effort": "medium"},
)
# The router catalog's () verdict for gpt-4.1-mini must not leak
# into other providers' requests.
assert kw["reasoning"]["effort"] == "medium"
class TestHostResolvedProfile:
def test_named_custom_provider_at_router_host_gets_the_clamp(
self, transport, seeded_catalog
):
# A providers.my-proxy entry pointed at api.router.com rides the same
# host mandate onto this transport; the vocabulary must follow the
# host, not the config-entry name.
kw = transport.build_kwargs(
model="grok-4.6",
messages=[{"role": "user", "content": "Hi"}],
tools=[],
base_url="https://api.router.com/v1",
session_id="sid",
provider="my-proxy",
reasoning_config={"effort": "max"},
)
assert kw["reasoning"]["effort"] == "xhigh"
def test_foreign_host_does_not_borrow_the_router_vocabulary(
self, transport, seeded_catalog
):
kw = transport.build_kwargs(
model="grok-4.6",
messages=[{"role": "user", "content": "Hi"}],
tools=[],
base_url="https://generic.example.com/v1",
session_id="sid",
provider="my-proxy",
reasoning_config={"effort": "max"},
)
# Default codex vocabulary applies (legacy: no max -> xhigh).
assert kw["reasoning"]["effort"] == "xhigh"
class TestCatalogIngestValidation:
def test_unrecognized_effort_levels_are_dropped_at_ingest(self):
_, mod = _router_plugin_module()
parsed = mod._parse_efforts([
{
"id": "future-model",
"router": {"capabilities": {"reasoning": {
"supported": True,
"efforts": [
{"value": "low"},
{"value": "hyperthink"}, # a new vendor tier
{"value": "high"},
],
}}},
},
{
# every level unknown -> omitted entirely (unknown model), so
# the transport keeps its defaults instead of suppressing or
# passing garbage through.
"id": "alien-model",
"router": {"capabilities": {"reasoning": {
"supported": True,
"efforts": [{"value": "hyperthink"}, {"value": "galaxy"}],
}}},
},
])
assert parsed == {"future-model": ["low", "high"]}
def test_fetch_models_dedupes_while_preserving_catalog_order(self, monkeypatch):
profile, mod = _router_plugin_module()
monkeypatch.setattr(mod, "_disk_path", lambda: None)
monkeypatch.setattr(
mod,
"_fetch_catalog_items",
lambda **_kwargs: [
{"id": "b"},
{"id": "a"},
{"id": "b"},
{"id": "c"},
],
)
assert profile.fetch_models() == ["b", "a", "c"]
+170
View File
@@ -0,0 +1,170 @@
"""Tests for the transport ABC, registry, and AnthropicTransport."""
import pytest
from types import SimpleNamespace
from agent.transports.base import ProviderTransport
from agent.transports.types import NormalizedResponse
from agent.transports import get_transport, register_transport, _REGISTRY
# ── ABC contract tests ──────────────────────────────────────────────────
class TestProviderTransportABC:
"""Verify the ABC contract is enforceable."""
def test_cannot_instantiate_abc(self):
with pytest.raises(TypeError):
ProviderTransport()
def test_concrete_must_implement_all_abstract(self):
class Incomplete(ProviderTransport):
@property
def api_mode(self):
return "test"
with pytest.raises(TypeError):
Incomplete()
def test_minimal_concrete(self):
class Minimal(ProviderTransport):
@property
def api_mode(self):
return "test_minimal"
def convert_messages(self, messages, **kw):
return messages
def convert_tools(self, tools):
return tools
def build_kwargs(self, model, messages, tools=None, **params):
return {"model": model, "messages": messages}
def normalize_response(self, response, **kw):
return NormalizedResponse(content="ok", tool_calls=None, finish_reason="stop")
t = Minimal()
assert t.api_mode == "test_minimal"
assert t.validate_response(None) is True # default
assert t.extract_cache_stats(None) is None # default
assert t.map_finish_reason("end_turn") == "end_turn" # default passthrough
# ── Registry tests ───────────────────────────────────────────────────────
class TestTransportRegistry:
def test_get_unregistered_returns_none(self):
assert get_transport("nonexistent_mode") is None
def test_register_and_get(self):
class DummyTransport(ProviderTransport):
@property
def api_mode(self):
return "dummy_test"
def convert_messages(self, messages, **kw):
return messages
def convert_tools(self, tools):
return tools
def build_kwargs(self, model, messages, tools=None, **params):
return {}
def normalize_response(self, response, **kw):
return NormalizedResponse(content=None, tool_calls=None, finish_reason="stop")
register_transport("dummy_test", DummyTransport)
t = get_transport("dummy_test")
assert t.api_mode == "dummy_test"
# Cleanup
_REGISTRY.pop("dummy_test", None)
# ── AnthropicTransport tests ────────────────────────────────────────────
class TestAnthropicTransport:
@pytest.fixture
def transport(self):
import agent.transports.anthropic # noqa: F401
return get_transport("anthropic_messages")
def test_convert_tools_simple(self, transport):
tools = [{
"type": "function",
"function": {
"name": "test_tool",
"description": "A test",
"parameters": {"type": "object", "properties": {}},
}
}]
result = transport.convert_tools(tools)
assert len(result) == 1
assert result[0]["name"] == "test_tool"
assert "input_schema" in result[0]
def test_map_finish_reason(self, transport):
assert transport.map_finish_reason("end_turn") == "stop"
assert transport.map_finish_reason("tool_use") == "tool_calls"
assert transport.map_finish_reason("max_tokens") == "length"
assert transport.map_finish_reason("stop_sequence") == "stop"
assert transport.map_finish_reason("refusal") == "content_filter"
assert transport.map_finish_reason("model_context_window_exceeded") == "length"
assert transport.map_finish_reason("unknown") == "stop"
def test_normalize_response_text(self, transport):
"""Test normalization of a simple text response."""
r = SimpleNamespace(
content=[SimpleNamespace(type="text", text="Hello world")],
stop_reason="end_turn",
usage=SimpleNamespace(input_tokens=10, output_tokens=5),
model="claude-sonnet-4-6",
)
nr = transport.normalize_response(r)
assert isinstance(nr, NormalizedResponse)
assert nr.content == "Hello world"
assert nr.tool_calls is None or nr.tool_calls == []
assert nr.finish_reason == "stop"
def test_normalize_response_tool_calls(self, transport):
"""Test normalization of a tool-use response."""
r = SimpleNamespace(
content=[
SimpleNamespace(
type="tool_use",
id="toolu_123",
name="terminal",
input={"command": "ls"},
),
],
stop_reason="tool_use",
usage=SimpleNamespace(input_tokens=10, output_tokens=20),
model="claude-sonnet-4-6",
)
nr = transport.normalize_response(r)
assert nr.finish_reason == "tool_calls"
assert len(nr.tool_calls) == 1
tc = nr.tool_calls[0]
assert tc.name == "terminal"
assert tc.id == "toolu_123"
assert '"command"' in tc.arguments
def test_convert_messages_extracts_system(self, transport):
"""Test convert_messages separates system from messages."""
messages = [
{"role": "system", "content": "You are helpful."},
{"role": "user", "content": "Hi"},
]
system, msgs = transport.convert_messages(messages)
# System should be extracted
assert system is not None
# Messages should only have user
assert len(msgs) >= 1
+200
View File
@@ -0,0 +1,200 @@
"""Tests for agent/transports/types.py — dataclass construction + helpers."""
import json
from agent.transports.types import (
NormalizedResponse,
ToolCall,
Usage,
build_tool_call,
map_finish_reason,
)
# ---------------------------------------------------------------------------
# ToolCall
# ---------------------------------------------------------------------------
class TestToolCall:
def test_basic_construction(self):
tc = ToolCall(id="call_abc", name="terminal", arguments='{"cmd": "ls"}')
assert tc.id == "call_abc"
assert tc.name == "terminal"
assert tc.arguments == '{"cmd": "ls"}'
assert tc.provider_data is None
def test_none_id(self):
tc = ToolCall(id=None, name="read_file", arguments="{}")
assert tc.id is None
def test_provider_data(self):
tc = ToolCall(
id="call_x",
name="t",
arguments="{}",
provider_data={"call_id": "call_x", "response_item_id": "fc_x"},
)
assert tc.provider_data["call_id"] == "call_x"
assert tc.provider_data["response_item_id"] == "fc_x"
# ---------------------------------------------------------------------------
# Usage
# ---------------------------------------------------------------------------
class TestUsage:
def test_defaults(self):
u = Usage()
assert u.prompt_tokens == 0
assert u.completion_tokens == 0
assert u.total_tokens == 0
assert u.cached_tokens == 0
def test_explicit(self):
u = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150, cached_tokens=80)
assert u.total_tokens == 150
# ---------------------------------------------------------------------------
# NormalizedResponse
# ---------------------------------------------------------------------------
class TestNormalizedResponse:
def test_text_only(self):
r = NormalizedResponse(content="hello", tool_calls=None, finish_reason="stop")
assert r.content == "hello"
assert r.tool_calls is None
assert r.finish_reason == "stop"
assert r.reasoning is None
assert r.usage is None
assert r.provider_data is None
def test_with_tool_calls(self):
tcs = [ToolCall(id="call_1", name="terminal", arguments='{"cmd":"pwd"}')]
r = NormalizedResponse(content=None, tool_calls=tcs, finish_reason="tool_calls")
assert r.finish_reason == "tool_calls"
assert len(r.tool_calls) == 1
assert r.tool_calls[0].name == "terminal"
# ---------------------------------------------------------------------------
# build_tool_call
# ---------------------------------------------------------------------------
class TestBuildToolCall:
def test_dict_arguments_serialized(self):
tc = build_tool_call(id="call_1", name="terminal", arguments={"cmd": "ls"})
assert tc.arguments == json.dumps({"cmd": "ls"})
assert tc.provider_data is None
def test_none_id(self):
tc = build_tool_call(id=None, name="t", arguments="{}")
assert tc.id is None
# ---------------------------------------------------------------------------
# map_finish_reason
# ---------------------------------------------------------------------------
class TestMapFinishReason:
ANTHROPIC_MAP = {
"end_turn": "stop",
"tool_use": "tool_calls",
"max_tokens": "length",
"stop_sequence": "stop",
"refusal": "content_filter",
}
def test_known_reason(self):
assert map_finish_reason("end_turn", self.ANTHROPIC_MAP) == "stop"
assert map_finish_reason("tool_use", self.ANTHROPIC_MAP) == "tool_calls"
assert map_finish_reason("max_tokens", self.ANTHROPIC_MAP) == "length"
assert map_finish_reason("refusal", self.ANTHROPIC_MAP) == "content_filter"
def test_unknown_reason_defaults_to_stop(self):
assert map_finish_reason("something_new", self.ANTHROPIC_MAP) == "stop"
def test_none_reason(self):
assert map_finish_reason(None, self.ANTHROPIC_MAP) == "stop"
# ---------------------------------------------------------------------------
# Backward-compat property tests
# ---------------------------------------------------------------------------
class TestToolCallBackwardCompat:
"""Test duck-typing properties that let ToolCall pass through code expecting
the old SimpleNamespace(id, type, function=SimpleNamespace(name, arguments)) shape."""
def test_function_name_matches(self):
tc = ToolCall(id="1", name="search", arguments='{"q":"test"}')
assert tc.function.name == "search"
assert tc.function.name == tc.name
def test_function_arguments_matches(self):
tc = ToolCall(id="1", name="search", arguments='{"q":"test"}')
assert tc.function.arguments == '{"q":"test"}'
assert tc.function.arguments == tc.arguments
def test_getattr_pattern_matches_agent_loop(self):
"""run_agent.py uses getattr(tool_call, 'call_id', None) — verify it works."""
tc = ToolCall(id="1", name="fn", arguments="{}", provider_data={"call_id": "c1"})
assert getattr(tc, "call_id", None) == "c1"
tc_no_pd = ToolCall(id="1", name="fn", arguments="{}")
assert getattr(tc_no_pd, "call_id", None) is None
def test_extra_content_getattr_pattern(self):
"""_build_assistant_message uses getattr(tc, 'extra_content', None).
This is the exact pattern that was broken before the extra_content
property was added — ToolCall lacked the property so getattr always
returned None, silently dropping the Gemini thought_signature and
causing HTTP 400 on subsequent turns (issue #14488).
"""
ec = {"google": {"thought_signature": "SIG_ABC123"}}
tc = ToolCall(id="1", name="fn", arguments="{}", provider_data={"extra_content": ec})
assert getattr(tc, "extra_content", None) == ec
tc_no_extra = ToolCall(id="1", name="fn", arguments="{}")
assert getattr(tc_no_extra, "extra_content", None) is None
class TestNormalizedResponseBackwardCompat:
"""Test properties that replaced _nr_to_assistant_message() shim."""
def test_reasoning_content_from_provider_data(self):
nr = NormalizedResponse(
content="hi", tool_calls=None, finish_reason="stop",
provider_data={"reasoning_content": "thought process"},
)
assert nr.reasoning_content == "thought process"
def test_reasoning_content_none_when_absent(self):
nr = NormalizedResponse(content="hi", tool_calls=None, finish_reason="stop")
assert nr.reasoning_content is None
def test_reasoning_details_from_provider_data(self):
details = [{"type": "thinking", "thinking": "hmm"}]
nr = NormalizedResponse(
content="hi", tool_calls=None, finish_reason="stop",
provider_data={"reasoning_details": details},
)
assert nr.reasoning_details == details