246 lines
7.5 KiB
Python
246 lines
7.5 KiB
Python
"""Regression coverage for OpenRouter preset references (issue #31739)."""
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from hermes_cli.models import validate_requested_model
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"model_name",
|
|
["@preset/email-copywriter", "@preset/Foo_bar.~9"],
|
|
)
|
|
def test_direct_openrouter_preset_reference_skips_model_listing(model_name):
|
|
"""An account-scoped direct preset has no public model row to probe."""
|
|
with patch(
|
|
"hermes_cli.models.fetch_api_models",
|
|
side_effect=AssertionError("direct preset references must not probe /models"),
|
|
):
|
|
result = validate_requested_model(
|
|
model_name,
|
|
"openrouter",
|
|
api_key="key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
|
|
assert result == {
|
|
"accepted": True,
|
|
"persist": True,
|
|
"recognized": False,
|
|
"message": None,
|
|
}
|
|
|
|
|
|
def test_combined_openrouter_preset_reference_validates_base_model():
|
|
"""Combined references validate the base model, not the preset-decorated ID."""
|
|
with patch(
|
|
"hermes_cli.models.fetch_api_models",
|
|
return_value=["openai/gpt-5.4"],
|
|
) as mock_fetch:
|
|
result = validate_requested_model(
|
|
"openai/gpt-5.4@preset/email-copywriter",
|
|
"openrouter",
|
|
api_key="key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
|
|
mock_fetch.assert_called_once_with("key", "https://openrouter.ai/api/v1")
|
|
assert result == {
|
|
"accepted": True,
|
|
"persist": True,
|
|
"recognized": True,
|
|
"message": None,
|
|
}
|
|
|
|
|
|
def test_combined_openrouter_preset_reference_rejects_unknown_base_model():
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=["openai/gpt-5.4"]):
|
|
result = validate_requested_model(
|
|
"openai/gpt-5.4-preview@preset/email-copywriter",
|
|
"openrouter",
|
|
api_key="key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
|
|
assert result["accepted"] is False
|
|
assert result["persist"] is False
|
|
assert result["recognized"] is False
|
|
assert "Similar models" in result["message"]
|
|
assert "openai/gpt-5.4" in result["message"]
|
|
|
|
|
|
def test_combined_openrouter_preset_reference_preserves_suffix_on_autocorrect():
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=["openai/gpt-5.4"]):
|
|
result = validate_requested_model(
|
|
"openai/gpt-5.44@preset/email-copywriter",
|
|
"openrouter",
|
|
api_key="key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
|
|
corrected = "openai/gpt-5.4@preset/email-copywriter"
|
|
assert result["accepted"] is True
|
|
assert result["corrected_model"] == corrected
|
|
assert corrected in result["message"]
|
|
|
|
|
|
def test_combined_preset_preserves_suffix_on_catalog_autocorrect():
|
|
with (
|
|
patch("hermes_cli.models.fetch_api_models", return_value=None),
|
|
patch(
|
|
"hermes_cli.models.provider_model_ids",
|
|
return_value=["openai/gpt-5.4"],
|
|
),
|
|
):
|
|
result = validate_requested_model(
|
|
"openai/gpt-5.44@preset/email-copywriter",
|
|
"openrouter",
|
|
api_key="key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
|
|
corrected = "openai/gpt-5.4@preset/email-copywriter"
|
|
assert result["accepted"] is True
|
|
assert result["corrected_model"] == corrected
|
|
assert corrected in result["message"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"model_name",
|
|
[
|
|
"@preset/",
|
|
"openai/gpt-5.4@preset/",
|
|
"@preset/foo/bar",
|
|
"@preset/foo?bar",
|
|
"@preset/☃",
|
|
"@preset/foo@preset/bar",
|
|
"openai/gpt-5.4@preset/foo/bar",
|
|
],
|
|
)
|
|
def test_openrouter_preset_reference_requires_a_url_safe_slug(model_name):
|
|
"""Malformed preset references must fail before model-list probing."""
|
|
with patch(
|
|
"hermes_cli.models.fetch_api_models",
|
|
side_effect=AssertionError("malformed presets must not probe /models"),
|
|
):
|
|
result = validate_requested_model(
|
|
model_name,
|
|
"openrouter",
|
|
api_key="key",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
|
|
assert result["accepted"] is False
|
|
assert result["persist"] is False
|
|
assert result["recognized"] is False
|
|
assert "URL-safe" in result["message"]
|
|
|
|
|
|
def test_preset_reference_does_not_bypass_other_provider_validation():
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=["gpt-5.4"]):
|
|
result = validate_requested_model(
|
|
"@preset/email-copywriter",
|
|
"openai",
|
|
api_key="key",
|
|
base_url="https://api.openai.com/v1",
|
|
)
|
|
|
|
assert result["accepted"] is False
|
|
assert result["persist"] is False
|
|
|
|
|
|
def test_preset_reference_does_not_bypass_custom_endpoint_validation():
|
|
probe = {
|
|
"models": ["local-model"],
|
|
"probed_url": "https://proxy.example/v1/models",
|
|
"resolved_base_url": "https://proxy.example/v1",
|
|
"suggested_base_url": None,
|
|
"used_fallback": False,
|
|
}
|
|
with patch("hermes_cli.models.probe_api_models", return_value=probe) as mock_probe:
|
|
result = validate_requested_model(
|
|
"@preset/email-copywriter",
|
|
"openrouter",
|
|
api_key="key",
|
|
base_url="https://proxy.example/v1",
|
|
)
|
|
|
|
mock_probe.assert_called_once()
|
|
assert result["accepted"] is True
|
|
assert result["recognized"] is False
|
|
assert "custom endpoint's model listing" in result["message"]
|
|
|
|
|
|
def test_configured_alias_switches_preset_through_real_resolution_chain(tmp_path):
|
|
"""Exercise config loading, alias resolution, runtime resolution, and validation."""
|
|
(tmp_path / "config.yaml").write_text(
|
|
"""
|
|
model:
|
|
default: openai/gpt-5.4
|
|
provider: openrouter
|
|
base_url: https://openrouter.ai/api/v1
|
|
model_aliases:
|
|
email-copywriter:
|
|
model: '@preset/email-copywriter'
|
|
provider: openrouter
|
|
""".lstrip(),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
script = r"""
|
|
import json
|
|
import hermes_cli.model_switch as model_switch
|
|
import hermes_cli.models as models
|
|
|
|
|
|
def fail_model_probe(*args, **kwargs):
|
|
raise AssertionError("preset references must not probe /models")
|
|
|
|
|
|
models.fetch_api_models = fail_model_probe
|
|
model_switch.get_model_capabilities = lambda *args, **kwargs: None
|
|
model_switch.get_model_info = lambda *args, **kwargs: None
|
|
result = model_switch.switch_model(
|
|
"email-copywriter",
|
|
current_provider="openrouter",
|
|
current_model="openai/gpt-5.4",
|
|
current_base_url="https://openrouter.ai/api/v1",
|
|
current_api_key="key",
|
|
)
|
|
print(json.dumps(vars(result)))
|
|
"""
|
|
env = {
|
|
"HOME": str(tmp_path),
|
|
"HERMES_HOME": str(tmp_path),
|
|
"LANG": "C.UTF-8",
|
|
"OPENROUTER_API_KEY": "key",
|
|
"PATH": os.environ.get("PATH", ""),
|
|
"PYTHONIOENCODING": "utf-8",
|
|
}
|
|
completed = subprocess.run(
|
|
[sys.executable, "-c", script],
|
|
cwd=Path(__file__).resolve().parents[2],
|
|
env=env,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert completed.returncode == 0, (
|
|
f"subprocess failed with exit {completed.returncode}\n"
|
|
f"stdout:\n{completed.stdout}\n"
|
|
f"stderr:\n{completed.stderr}"
|
|
)
|
|
result = json.loads(completed.stdout.splitlines()[-1])
|
|
|
|
assert result["success"] is True
|
|
assert result["new_model"] == "@preset/email-copywriter"
|
|
assert result["target_provider"] == "openrouter"
|
|
assert result["resolved_via_alias"] == "email-copywriter"
|
|
assert result["warning_message"] == ""
|