Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/hermes_cli/test_openrouter_preset_validation.py

@@ -0,0 +1,245 @@
+"""Regression coverage for OpenRouter preset references (issue #31739)."""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from hermes_cli.models import validate_requested_model
+
+
+@pytest.mark.parametrize(
+    "model_name",
+    ["@preset/email-copywriter", "@preset/Foo_bar.~9"],
+)
+def test_direct_openrouter_preset_reference_skips_model_listing(model_name):
+    """An account-scoped direct preset has no public model row to probe."""
+    with patch(
+        "hermes_cli.models.fetch_api_models",
+        side_effect=AssertionError("direct preset references must not probe /models"),
+    ):
+        result = validate_requested_model(
+            model_name,
+            "openrouter",
+            api_key="key",
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    assert result == {
+        "accepted": True,
+        "persist": True,
+        "recognized": False,
+        "message": None,
+    }
+
+
+def test_combined_openrouter_preset_reference_validates_base_model():
+    """Combined references validate the base model, not the preset-decorated ID."""
+    with patch(
+        "hermes_cli.models.fetch_api_models",
+        return_value=["openai/gpt-5.4"],
+    ) as mock_fetch:
+        result = validate_requested_model(
+            "openai/gpt-5.4@preset/email-copywriter",
+            "openrouter",
+            api_key="key",
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    mock_fetch.assert_called_once_with("key", "https://openrouter.ai/api/v1")
+    assert result == {
+        "accepted": True,
+        "persist": True,
+        "recognized": True,
+        "message": None,
+    }
+
+
+def test_combined_openrouter_preset_reference_rejects_unknown_base_model():
+    with patch("hermes_cli.models.fetch_api_models", return_value=["openai/gpt-5.4"]):
+        result = validate_requested_model(
+            "openai/gpt-5.4-preview@preset/email-copywriter",
+            "openrouter",
+            api_key="key",
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    assert result["accepted"] is False
+    assert result["persist"] is False
+    assert result["recognized"] is False
+    assert "Similar models" in result["message"]
+    assert "openai/gpt-5.4" in result["message"]
+
+
+def test_combined_openrouter_preset_reference_preserves_suffix_on_autocorrect():
+    with patch("hermes_cli.models.fetch_api_models", return_value=["openai/gpt-5.4"]):
+        result = validate_requested_model(
+            "openai/gpt-5.44@preset/email-copywriter",
+            "openrouter",
+            api_key="key",
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    corrected = "openai/gpt-5.4@preset/email-copywriter"
+    assert result["accepted"] is True
+    assert result["corrected_model"] == corrected
+    assert corrected in result["message"]
+
+
+def test_combined_preset_preserves_suffix_on_catalog_autocorrect():
+    with (
+        patch("hermes_cli.models.fetch_api_models", return_value=None),
+        patch(
+            "hermes_cli.models.provider_model_ids",
+            return_value=["openai/gpt-5.4"],
+        ),
+    ):
+        result = validate_requested_model(
+            "openai/gpt-5.44@preset/email-copywriter",
+            "openrouter",
+            api_key="key",
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    corrected = "openai/gpt-5.4@preset/email-copywriter"
+    assert result["accepted"] is True
+    assert result["corrected_model"] == corrected
+    assert corrected in result["message"]
+
+
+@pytest.mark.parametrize(
+    "model_name",
+    [
+        "@preset/",
+        "openai/gpt-5.4@preset/",
+        "@preset/foo/bar",
+        "@preset/foo?bar",
+        "@preset/☃",
+        "@preset/foo@preset/bar",
+        "openai/gpt-5.4@preset/foo/bar",
+    ],
+)
+def test_openrouter_preset_reference_requires_a_url_safe_slug(model_name):
+    """Malformed preset references must fail before model-list probing."""
+    with patch(
+        "hermes_cli.models.fetch_api_models",
+        side_effect=AssertionError("malformed presets must not probe /models"),
+    ):
+        result = validate_requested_model(
+            model_name,
+            "openrouter",
+            api_key="key",
+            base_url="https://openrouter.ai/api/v1",
+        )
+
+    assert result["accepted"] is False
+    assert result["persist"] is False
+    assert result["recognized"] is False
+    assert "URL-safe" in result["message"]
+
+
+def test_preset_reference_does_not_bypass_other_provider_validation():
+    with patch("hermes_cli.models.fetch_api_models", return_value=["gpt-5.4"]):
+        result = validate_requested_model(
+            "@preset/email-copywriter",
+            "openai",
+            api_key="key",
+            base_url="https://api.openai.com/v1",
+        )
+
+    assert result["accepted"] is False
+    assert result["persist"] is False
+
+
+def test_preset_reference_does_not_bypass_custom_endpoint_validation():
+    probe = {
+        "models": ["local-model"],
+        "probed_url": "https://proxy.example/v1/models",
+        "resolved_base_url": "https://proxy.example/v1",
+        "suggested_base_url": None,
+        "used_fallback": False,
+    }
+    with patch("hermes_cli.models.probe_api_models", return_value=probe) as mock_probe:
+        result = validate_requested_model(
+            "@preset/email-copywriter",
+            "openrouter",
+            api_key="key",
+            base_url="https://proxy.example/v1",
+        )
+
+    mock_probe.assert_called_once()
+    assert result["accepted"] is True
+    assert result["recognized"] is False
+    assert "custom endpoint's model listing" in result["message"]
+
+
+def test_configured_alias_switches_preset_through_real_resolution_chain(tmp_path):
+    """Exercise config loading, alias resolution, runtime resolution, and validation."""
+    (tmp_path / "config.yaml").write_text(
+        """
+model:
+  default: openai/gpt-5.4
+  provider: openrouter
+  base_url: https://openrouter.ai/api/v1
+model_aliases:
+  email-copywriter:
+    model: '@preset/email-copywriter'
+    provider: openrouter
+""".lstrip(),
+        encoding="utf-8",
+    )
+
+    script = r"""
+import json
+import hermes_cli.model_switch as model_switch
+import hermes_cli.models as models
+
+
+def fail_model_probe(*args, **kwargs):
+    raise AssertionError("preset references must not probe /models")
+
+
+models.fetch_api_models = fail_model_probe
+model_switch.get_model_capabilities = lambda *args, **kwargs: None
+model_switch.get_model_info = lambda *args, **kwargs: None
+result = model_switch.switch_model(
+    "email-copywriter",
+    current_provider="openrouter",
+    current_model="openai/gpt-5.4",
+    current_base_url="https://openrouter.ai/api/v1",
+    current_api_key="key",
+)
+print(json.dumps(vars(result)))
+"""
+    env = {
+        "HOME": str(tmp_path),
+        "HERMES_HOME": str(tmp_path),
+        "LANG": "C.UTF-8",
+        "OPENROUTER_API_KEY": "key",
+        "PATH": os.environ.get("PATH", ""),
+        "PYTHONIOENCODING": "utf-8",
+    }
+    completed = subprocess.run(
+        [sys.executable, "-c", script],
+        cwd=Path(__file__).resolve().parents[2],
+        env=env,
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    assert completed.returncode == 0, (
+        f"subprocess failed with exit {completed.returncode}\n"
+        f"stdout:\n{completed.stdout}\n"
+        f"stderr:\n{completed.stderr}"
+    )
+    result = json.loads(completed.stdout.splitlines()[-1])
+
+    assert result["success"] is True
+    assert result["new_model"] == "@preset/email-copywriter"
+    assert result["target_provider"] == "openrouter"
+    assert result["resolved_via_alias"] == "email-copywriter"
+    assert result["warning_message"] == ""