Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/run_agent/test_image_rejection_fallback.py

@@ -0,0 +1,299 @@
+"""Tests for the image-rejection fallback in run_agent.
+
+When a server rejects image content (e.g. text-only endpoints), the agent
+strips image parts from message history and retries text-only.  These tests
+verify that stripping preserves the role-alternation invariants providers
+require, and that the phrase detector fires on the expected error bodies.
+"""
+
+from run_agent import _looks_like_image_content_rejection, _strip_images_from_messages
+
+
+class TestStripImagesPreservesAlternation:
+    """_strip_images_from_messages must not break message role alternation."""
+
+    def test_noop_when_no_images(self):
+        msgs = [
+            {"role": "user", "content": "hello"},
+            {"role": "assistant", "content": "hi"},
+        ]
+        changed = _strip_images_from_messages(msgs)
+        assert changed is False
+        assert msgs == [
+            {"role": "user", "content": "hello"},
+            {"role": "assistant", "content": "hi"},
+        ]
+
+
+
+
+    def test_tool_message_with_all_images_replaced_not_deleted(self):
+        """CRITICAL: tool messages must NEVER be deleted — their tool_call_id
+        pairs with an assistant tool_call and providers reject unmatched IDs.
+        """
+        msgs = [
+            {"role": "user", "content": "take a screenshot"},
+            {
+                "role": "assistant",
+                "content": None,
+                "tool_calls": [{
+                    "id": "call_abc",
+                    "type": "function",
+                    "function": {"name": "computer_use", "arguments": "{}"},
+                }],
+            },
+            {
+                "role": "tool",
+                "tool_call_id": "call_abc",
+                "content": [
+                    {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}},
+                ],
+            },
+        ]
+        changed = _strip_images_from_messages(msgs)
+        assert changed is True
+        # Length preserved — tool message NOT deleted
+        assert len(msgs) == 3
+        # tool_call_id still present
+        assert msgs[2]["tool_call_id"] == "call_abc"
+        # Content replaced with text placeholder (now a string, not a list)
+        assert isinstance(msgs[2]["content"], str)
+        assert "image content removed" in msgs[2]["content"].lower()
+
+    def test_tool_message_with_mixed_content_keeps_text_parts(self):
+        msgs = [
+            {"role": "user", "content": "screenshot plz"},
+            {
+                "role": "assistant",
+                "content": None,
+                "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "x", "arguments": "{}"}}],
+            },
+            {
+                "role": "tool",
+                "tool_call_id": "call_1",
+                "content": [
+                    {"type": "text", "text": "Captured 1024x768"},
+                    {"type": "image_url", "image_url": {"url": "data:..."}},
+                ],
+            },
+        ]
+        changed = _strip_images_from_messages(msgs)
+        assert changed is True
+        assert len(msgs) == 3
+        assert msgs[2]["content"] == [{"type": "text", "text": "Captured 1024x768"}]
+        assert msgs[2]["tool_call_id"] == "call_1"
+
+    def test_assistant_with_tool_calls_and_image_only_content_preserved(self):
+        """Assistant messages carrying tool_calls must NEVER be deleted —
+        dropping them would orphan the paired tool responses, which providers
+        reject with unmatched tool_call_id errors.
+        """
+        msgs = [
+            {"role": "user", "content": "annotate this screenshot"},
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}},
+                ],
+                "tool_calls": [{
+                    "id": "call_xyz",
+                    "type": "function",
+                    "function": {"name": "annotate", "arguments": "{}"},
+                }],
+            },
+            {"role": "tool", "tool_call_id": "call_xyz", "content": "done"},
+        ]
+        changed = _strip_images_from_messages(msgs)
+        assert changed is True
+        # Length preserved — assistant message with tool_calls NOT deleted
+        assert len(msgs) == 3
+        assert msgs[1]["tool_calls"][0]["id"] == "call_xyz"
+        # Content replaced with text placeholder (now a string, not a list)
+        assert isinstance(msgs[1]["content"], str)
+        assert "image content removed" in msgs[1]["content"].lower()
+        # Paired tool response still matches
+        assert msgs[2]["tool_call_id"] == "call_xyz"
+
+    def test_image_only_user_message_dropped(self):
+        """Synthetic image-only user messages (gateway injection pattern) are
+        safe to drop — no tool_call_id linkage to preserve."""
+        msgs = [
+            {"role": "user", "content": "what's in this?"},
+            {"role": "assistant", "content": "I'll check."},
+            {
+                "role": "user",
+                "content": [{"type": "image_url", "image_url": {"url": "data:..."}}],
+            },
+        ]
+        changed = _strip_images_from_messages(msgs)
+        assert changed is True
+        # Synthetic image-only user message dropped
+        assert len(msgs) == 2
+        assert msgs[-1]["role"] == "assistant"
+
+    def test_multiple_tool_messages_all_preserved(self):
+        """Parallel tool calls: each tool_call_id must retain a paired message."""
+        msgs = [
+            {
+                "role": "assistant",
+                "content": None,
+                "tool_calls": [
+                    {"id": "c1", "type": "function", "function": {"name": "x", "arguments": "{}"}},
+                    {"id": "c2", "type": "function", "function": {"name": "x", "arguments": "{}"}},
+                ],
+            },
+            {
+                "role": "tool",
+                "tool_call_id": "c1",
+                "content": [{"type": "image_url", "image_url": {}}],
+            },
+            {
+                "role": "tool",
+                "tool_call_id": "c2",
+                "content": [{"type": "image_url", "image_url": {}}],
+            },
+        ]
+        changed = _strip_images_from_messages(msgs)
+        assert changed is True
+        tool_msgs = [m for m in msgs if m.get("role") == "tool"]
+        assert len(tool_msgs) == 2
+        assert {m["tool_call_id"] for m in tool_msgs} == {"c1", "c2"}
+
+
+
+
+class TestImageRejectionPhraseIsolation:
+    """The image-rejection phrase list must NOT false-match on other
+    image-related error categories (size-too-large, format errors, etc.)
+    so they route to the correct recovery handler (e.g. _try_shrink_image_parts).
+    """
+
+    def _matches(self, body: str) -> bool:
+        return _looks_like_image_content_rejection(body)
+
+    def test_kimi_truncated_image_trips_recovery(self):
+        # Kimi/Moonshot reject truncated image bytes with this 400; the
+        # bad bytes are in immutable history so stripping must fire.
+        body = ("HTTP 400: Invalid request: prepare image failed error, "
+                "status code: 400, message: failed to decode image: invalid "
+                "or unsupported image format")
+        assert self._matches(body) is True
+
+    def test_anthropic_image_too_large_does_not_trip(self):
+        # From agent/error_classifier.py _IMAGE_TOO_LARGE_PATTERNS —
+        # these must route to image_too_large / _try_shrink_image_parts_in_messages,
+        # NOT to our vision-unsupported fallback.
+        bodies = [
+            "messages.0.content.1.image.source.base64: image exceeds 5 MB maximum",
+            "image too large: 6291456 bytes > 5242880 limit",
+            "image_too_large",
+            "image size exceeds per-request limit",
+        ]
+        for body in bodies:
+            assert self._matches(body) is False, f"false positive on: {body}"
+
+
+
+    def test_real_image_rejection_bodies_trip(self):
+        """Positive cases — real-world error wordings that should trigger."""
+        bodies = [
+            "Only 'text' content type is supported.",
+            "Bad request: multimodal is not supported by this model",
+            "This model does not support images",
+            "vision is not supported on this endpoint",
+            "model does not support image input",
+            # ChatGPT-account Codex backend (issue #23570) — rejects
+            # data:image/...base64 URLs in input_image fields. Without this
+            # match the agent cascaded into compression / context-too-large
+            # recovery instead of just stripping the images.
+            "Invalid 'input[56].content[1].image_url'. Expected a valid URL, but got a value with an invalid format.",
+            # OpenRouter 404 when no upstream endpoint for the model accepts
+            # image input — issue #21160. The exact wording from the report.
+            "HTTP 404: No endpoints found that support image input",
+            # Alibaba/OpenAI-compatible endpoints can reject image-bearing
+            # messages without naming image_url explicitly. The first failed
+            # turn should still switch to text-only/aux-vision mode (#57948).
+            "The provided messages input is invalid. The error info is [Unexpected item type in content].",
+            "The image data you provided does not represent a valid image. Please check your input and try again.",
+        ]
+        for body in bodies:
+            assert self._matches(body) is True, f"false negative on: {body}"
+
+
+class TestStripImagesDropsStaleApiContent:
+    """The strip runs on the persistent history, not just the per-call copy.
+
+    ``api_content`` is the byte-stability sidecar: it holds the exact bytes
+    previously sent for a message, and the next turn substitutes it back into
+    ``content``. Leaving it in place on a message this function rewrote would
+    replay the images the strip just removed — and the recovery cannot re-fire,
+    because it sets ``_vision_supported = False`` and gates itself on that. The
+    session would then send rejected images on every subsequent turn.
+
+    Same contract the other content-rewrite paths follow (stale-confirmation
+    redaction in ``replay_cleanup``, compression rewrites, merge-into-tail):
+    "the cost is one cache boundary miss, never wrong content".
+    """
+
+    @staticmethod
+    def _wire(msg):
+        """What the next turn actually sends for this history message."""
+        from agent.turn_context import substitute_api_content
+
+        api_msg = msg.copy()
+        substitute_api_content(api_msg)
+        return api_msg["content"]
+
+    def _image_msg(self, sidecar="look<IMAGE BYTES SENT LAST TURN>"):
+        return {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "look"},
+                {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
+            ],
+            "api_content": sidecar,
+        }
+
+    def test_stripped_message_loses_its_sidecar(self):
+        msgs = [self._image_msg()]
+        assert _strip_images_from_messages(msgs) is True
+        assert "api_content" not in msgs[0]
+
+    def test_next_turn_does_not_resend_the_stripped_images(self):
+        msgs = [self._image_msg()]
+        _strip_images_from_messages(msgs)
+
+        wire = self._wire(msgs[0])
+        assert "IMAGE BYTES" not in str(wire), (
+            "the stale sidecar replayed the images the strip removed"
+        )
+        assert wire == [{"type": "text", "text": "look"}]
+
+    def test_tool_placeholder_message_also_loses_its_sidecar(self):
+        """An image-only tool result becomes a placeholder — same rewrite."""
+        msgs = [
+            {
+                "role": "tool",
+                "tool_call_id": "call_1",
+                "content": [{"type": "image_url", "image_url": {"url": "x"}}],
+                "api_content": "<SCREENSHOT BYTES>",
+            }
+        ]
+        assert _strip_images_from_messages(msgs) is True
+        assert "api_content" not in msgs[0]
+        assert "image content removed" in msgs[0]["content"]
+
+    def test_untouched_messages_keep_their_sidecar(self):
+        """Only rewritten messages pay the cache boundary — not the whole prefix."""
+        msgs = [
+            {
+                "role": "user",
+                "content": [{"type": "text", "text": "no images here"}],
+                "api_content": "no images here<injected ctx>",
+            },
+            self._image_msg(),
+        ]
+        _strip_images_from_messages(msgs)
+
+        assert msgs[0]["api_content"] == "no images here<injected ctx>"
+        assert "api_content" not in msgs[1]