Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/test_output_cap_parsing.py

@@ -0,0 +1,193 @@
+import pytest
+from agent.model_metadata import (
+    is_output_cap_error,
+    parse_available_output_tokens_from_error,
+)
+
+
+class TestParseOpenRouterOutputCap:
+    """OpenRouter/Nous phrase the output-cap error as a context breakdown."""
+
+    def test_openrouter_breakdown_format(self):
+        msg = ("This endpoint's maximum context length is 200000 tokens. "
+               "However, you requested about 195000 tokens "
+               "(150000 of text input, 40000 of tool input, 5000 in the output).")
+        # available output = 200000 - 150000 - 40000 = 10000
+        assert parse_available_output_tokens_from_error(msg) == 10000
+
+
+
+
+
+class TestParseCharBasedOutputCap:
+    """LM Studio / llama.cpp report context in tokens but prompt in characters.
+
+    These servers send a hard 400 even on a trivial prompt when the default
+    output cap equals the context window (#42741): the request asks for the
+    whole window as output, leaving zero room for input.
+    """
+
+    def test_char_based_output_cap_format(self):
+        msg = ("This model's maximum context length is 65536 tokens. However, "
+               "you requested 65536 output tokens and your prompt contains "
+               "77409 characters (more than 0 characters, which is the upper "
+               "bound for 0 input tokens). Please reduce the length of the "
+               "input prompt or the number of requested output tokens.")
+        # est input = ceil(77409 / 3) = 25803; available = 65536 - 25803 = 39733
+        assert parse_available_output_tokens_from_error(msg) == 39733
+
+    def test_char_based_leaves_room_for_input(self):
+        # The whole point: the retried output cap + the estimated input must
+        # fit inside the reported context window.
+        ctx = 65536
+        chars = 77409
+        available = parse_available_output_tokens_from_error(
+            f"maximum context length is {ctx} tokens. However, you requested "
+            f"{ctx} output tokens and your prompt contains {chars} characters."
+        )
+        assert available is not None
+        assert available + (chars + 2) // 3 <= ctx
+
+
+
+class TestParseDashScopeOutputCap:
+    """DashScope / Alibaba Cloud (Qwen) reject an over-cap output request with
+    a bounded range whose upper bound is the real max-output cap (#55546)."""
+
+    def test_dashscope_range_format(self):
+        msg = ("HTTP 400: InternalError.Algo.InvalidParameter: "
+               "Range of max_tokens should be [1, 65536]")
+        assert parse_available_output_tokens_from_error(msg) == 65536
+
+    def test_dashscope_range_arbitrary_bound(self):
+        msg = "Range of max_tokens should be [1, 8192]"
+        assert parse_available_output_tokens_from_error(msg) == 8192
+
+    def test_dashscope_range_with_spaces(self):
+        msg = "range of max_tokens should be [ 1 , 32768 ]"
+        assert parse_available_output_tokens_from_error(msg) == 32768
+
+
+class TestParseMaximumOutputTokensCap:
+    """Some OpenAI-compatible relays report the model's separate output cap."""
+
+    def test_parenthesized_max_output_cap(self):
+        msg = (
+            "API call failed after 3 retries: [400]: max_tokens (98304) "
+            "exceeds model's maximum output tokens (65536)"
+        )
+        assert parse_available_output_tokens_from_error(msg) == 65536
+
+    def test_parenthesized_max_output_cap_is_output_cap(self):
+        assert is_output_cap_error(
+            "max_tokens (98304) exceeds model's maximum output tokens (65536)"
+        ) is True
+
+
+class TestIsOutputCapError:
+    """`is_output_cap_error` is the broader yes/no gate that keeps an
+    output-cap 400 out of the compression death-loop even when we can't parse
+    a number from the provider's wording (#55546)."""
+
+    def test_dashscope_is_output_cap(self):
+        assert is_output_cap_error(
+            "Range of max_tokens should be [1, 65536]"
+        ) is True
+
+
+    def test_anthropic_available_tokens_is_output_cap(self):
+        assert is_output_cap_error(
+            "max_tokens: 32768 > context_window: 200000 - "
+            "input_tokens: 190000 = available_tokens: 10000"
+        ) is True
+
+    def test_real_input_overflow_is_not_output_cap(self):
+        # Mentions max_tokens but the INPUT is the problem -> compression path.
+        assert is_output_cap_error(
+            "prompt is too long: 250000 tokens > 200000 max_tokens window"
+        ) is False
+
+    def test_gpt5_unsupported_param_is_not_output_cap(self):
+        # format_error caught earlier; must NOT be treated as an output cap.
+        assert is_output_cap_error(
+            "Unsupported parameter: 'max_tokens' is not supported with this "
+            "model. Use 'max_completion_tokens' instead."
+        ) is False
+
+    def test_unrelated_error_is_not_output_cap(self):
+        assert is_output_cap_error("some unrelated 400 error") is False
+
+
+class TestParseVllmTokenBasedOutputCap:
+    """vLLM reports both the window and the prompt in TOKENS.
+
+    Until this format was parsed, the recovery path misclassified it as
+    prompt-too-long and looped through compression (which frees little) while
+    retrying with the same oversized max_tokens — terminating in "cannot
+    compress further" even though simply lowering the output cap would have
+    succeeded.
+    """
+
+    # Verbatim vLLM 0.22 / OpenAI-compatible server response (max_tokens set).
+    _VLLM_MSG = (
+        "This model's maximum context length is 131072 tokens. However, you "
+        "requested 65536 output tokens and your prompt contains at least "
+        "65537 input tokens, for a total of at least 131073 tokens. Please "
+        "reduce the length of the input prompt or the number of requested "
+        "output tokens."
+    )
+
+    # Verbatim vLLM response where the input is MEASURED, not back-computed:
+    # window - input != requested - 1, so the reported figure is real.
+    _VLLM_MSG_REAL_INPUT = (
+        "This model's maximum context length is 131072 tokens. However, you "
+        "requested 65536 output tokens and your prompt contains 100000 "
+        "input tokens, for a total of 165536 tokens. Please reduce the length "
+        "of the input prompt or the number of requested output tokens."
+    )
+
+    def test_vllm_token_based_format(self):
+        # The reported input is a LOWER BOUND that vLLM back-computes from the
+        # constraint (65537 == 131072 + 1 - 65536), so window - input is just
+        # requested - 1 and carries no information about the real prompt.
+        # Halve the requested cap instead so the retry actually converges.
+        assert parse_available_output_tokens_from_error(self._VLLM_MSG) == 32768
+
+    def test_vllm_measured_input_is_trusted(self):
+        # When the input is measured rather than derived, use it as-is.
+        # available output = 131072 - 100000 = 31072
+        assert parse_available_output_tokens_from_error(
+            self._VLLM_MSG_REAL_INPUT
+        ) == 31072
+
+    def test_vllm_retry_fits_inside_window(self):
+        # The retried cap plus the reported input must fit in the window.
+        available = parse_available_output_tokens_from_error(self._VLLM_MSG)
+        assert available is not None
+        assert available + 65537 <= 131072
+
+    def test_vllm_retry_converges(self):
+        """The retry sequence must reach a working cap in a few attempts.
+
+        Regression test for the 65-tokens-per-retry crawl: with a 102400
+        window and a real prompt of ~37000 tokens, retrying from a 65536 cap
+        used to produce 65471 -> 65406 -> 65341 and exhaust the compression
+        budget without ever fitting.
+        """
+        window, real_input, cap = 102400, 37000, 65536
+        for _ in range(5):
+            if real_input + cap <= window:
+                break
+            # vLLM's message when max_tokens is the binding constraint.
+            msg = (
+                f"This model's maximum context length is {window} tokens. "
+                f"However, you requested {cap} output tokens and your prompt "
+                f"contains at least {window + 1 - cap} input tokens, for a "
+                f"total of at least {window + 1} tokens."
+            )
+            available = parse_available_output_tokens_from_error(msg)
+            assert available is not None
+            assert available < cap, "each retry must lower the cap"
+            cap = available
+        assert real_input + cap <= window, f"did not converge: cap={cap}"
+