Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

agent/retry_utils.py

@@ -0,0 +1,208 @@
+"""Retry utilities — jittered backoff for decorrelated retries.
+
+Replaces fixed exponential backoff with jittered delays to prevent
+thundering-herd retry spikes when multiple sessions hit the same
+rate-limited provider concurrently.
+"""
+
+import random
+import threading
+import time
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+from typing import Any, Optional
+
+# Monotonic counter for jitter seed uniqueness within the same process.
+# Protected by a lock to avoid race conditions in concurrent retry paths
+# (e.g. multiple gateway sessions retrying simultaneously).
+_jitter_counter = 0
+_jitter_lock = threading.Lock()
+
+# Z.AI Coding Plan's GLM-5.2 endpoint often returns HTTP 429 code 1305
+# ("The service may be temporarily overloaded...") for otherwise valid
+# Hermes requests. Short retries tend to hammer the same overloaded window;
+# after a few normal retries, progressively widen the wait window. Keep the
+# cap interactive-friendly: a simple TUI message should fail visibly in minutes,
+# not sit silent for 20+ minutes.
+_ZAI_CODING_OVERLOAD_LONG_BACKOFF = (30.0, 60.0, 90.0, 120.0)
+
+# Number of initial short retries before the adaptive long-backoff tier kicks
+# in. Shared by ``adaptive_rate_limit_backoff`` (which walks the long table
+# starting at attempt ``short_attempts + 1``) and
+# ``zai_coding_overload_retry_ceiling`` (which sizes the retry loop so every
+# long-tier entry is reachable). Keeping it a single module constant prevents
+# the two from silently desyncing if the short-retry count is ever tuned.
+_ZAI_CODING_OVERLOAD_SHORT_ATTEMPTS = 3
+
+
+def parse_retry_after_seconds(value_or_headers: Any) -> Optional[float]:
+    """Parse a ``Retry-After`` value into non-negative seconds.
+
+    Accepts either a raw header value (numeric string / HTTP-date / number)
+    or a headers mapping, in which case the ``Retry-After`` key is looked up
+    case-insensitively (``.get`` on dict-like objects tries both common
+    casings; real HTTP header containers like httpx/requests are already
+    case-insensitive).
+
+    Returns:
+        Seconds as a ``float`` (negative deltas clamped to ``0.0``), or
+        ``None`` when the header is absent or unparseable.
+    """
+    raw = value_or_headers
+    if raw is not None and not isinstance(raw, (str, int, float)):
+        # Looks like a headers mapping — pull the header out of it.
+        getter = getattr(raw, "get", None)
+        if callable(getter):
+            try:
+                value = getter("Retry-After")
+                if value is None:
+                    value = getter("retry-after")
+            except Exception:
+                return None
+            raw = value
+        else:
+            return None
+    if raw is None:
+        return None
+    if isinstance(raw, bool):
+        return None
+    if isinstance(raw, (int, float)):
+        return max(0.0, float(raw))
+    text = str(raw).strip()
+    if not text:
+        return None
+    try:
+        return max(0.0, float(text))
+    except (TypeError, ValueError):
+        pass
+    # HTTP-date form (RFC 7231): seconds until that instant, clamped at 0.
+    try:
+        when = parsedate_to_datetime(text)
+    except (TypeError, ValueError):
+        return None
+    if when is None:
+        return None
+    if when.tzinfo is None:
+        when = when.replace(tzinfo=timezone.utc)
+    return max(0.0, (when - datetime.now(timezone.utc)).total_seconds())
+
+
+def jittered_backoff(
+    attempt: int,
+    *,
+    base_delay: float = 5.0,
+    max_delay: float = 120.0,
+    jitter_ratio: float = 0.5,
+) -> float:
+    """Compute a jittered exponential backoff delay.
+
+    Args:
+        attempt: 1-based retry attempt number.
+        base_delay: Base delay in seconds for attempt 1.
+        max_delay: Maximum delay cap in seconds.
+        jitter_ratio: Fraction of computed delay to use as random jitter
+            range.  0.5 means jitter is uniform in [0, 0.5 * delay].
+
+    Returns:
+        Delay in seconds: min(base * 2^(attempt-1), max_delay) + jitter.
+
+    The jitter decorrelates concurrent retries so multiple sessions
+    hitting the same provider don't all retry at the same instant.
+    """
+    global _jitter_counter
+    with _jitter_lock:
+        _jitter_counter += 1
+        tick = _jitter_counter
+
+    exponent = max(0, attempt - 1)
+    if exponent >= 63 or base_delay <= 0:
+        delay = max_delay
+    else:
+        delay = min(base_delay * (2 ** exponent), max_delay)
+
+    # Seed from time + counter for decorrelation even with coarse clocks.
+    seed = (time.time_ns() ^ (tick * 0x9E3779B9)) & 0xFFFFFFFF
+    rng = random.Random(seed)
+    jitter = rng.uniform(0, jitter_ratio * delay)
+
+    return delay + jitter
+
+
+def _error_text(error: Any) -> str:
+    """Best-effort flattened provider error text for retry classification."""
+    parts = [
+        error,
+        getattr(error, "message", None),
+        getattr(error, "body", None),
+        getattr(error, "response", None),
+    ]
+    return " ".join(str(part) for part in parts if part is not None).lower()
+
+
+def is_zai_coding_overload_error(*, base_url: str | None, model: str | None, error: Any) -> bool:
+    """Return True for Z.AI Coding Plan transient overload 429s.
+
+    The coding-plan endpoint reports overload as HTTP 429 with body code 1305
+    and message "The service may be temporarily overloaded...". Treat only
+    that narrow shape specially so ordinary quota/billing 429s still fail fast
+    through the existing classifier.
+    """
+    base = (base_url or "").lower()
+    model_name = (model or "").lower()
+    status = getattr(error, "status_code", None)
+    text = _error_text(error)
+    return (
+        status == 429
+        and "api.z.ai/api/coding/paas/v4" in base
+        and "glm-5.2" in model_name
+        and ("1305" in text or "temporarily overloaded" in text)
+    )
+
+
+def adaptive_rate_limit_backoff(
+    attempt: int,
+    *,
+    base_url: str | None,
+    model: str | None,
+    error: Any,
+    default_wait: float,
+    short_attempts: int = _ZAI_CODING_OVERLOAD_SHORT_ATTEMPTS,
+) -> tuple[float, str | None]:
+    """Provider-aware rate-limit backoff.
+
+    For most providers this returns ``default_wait`` unchanged. For Z.AI
+    Coding Plan GLM-5.2 overloads, keep the first ``short_attempts`` retries on
+    the normal short exponential schedule, then switch to progressively longer
+    waits (30s → 60s → 90s → 120s, capped) plus light jitter.
+
+    ``attempt`` is 1-based, matching the retry loop's logged attempt number.
+    Returns ``(wait_seconds, reason_label)`` where ``reason_label`` is suitable
+    for status/log decoration when a provider-specific policy fired.
+    """
+    if not is_zai_coding_overload_error(base_url=base_url, model=model, error=error):
+        return default_wait, None
+    if attempt <= short_attempts:
+        return default_wait, "zai_coding_overload_short"
+
+    idx = min(attempt - short_attempts - 1, len(_ZAI_CODING_OVERLOAD_LONG_BACKOFF) - 1)
+    base_delay = _ZAI_CODING_OVERLOAD_LONG_BACKOFF[idx]
+    # A smaller jitter ratio keeps long waits readable while still avoiding
+    # synchronized retry storms across concurrent Hermes sessions.
+    return jittered_backoff(1, base_delay=base_delay, max_delay=base_delay, jitter_ratio=0.2), "zai_coding_overload_long"
+
+
+def zai_coding_overload_retry_ceiling(short_attempts: int = _ZAI_CODING_OVERLOAD_SHORT_ATTEMPTS) -> int:
+    """Retry-loop ceiling needed for the full Z.AI overload backoff schedule.
+
+    The adaptive policy runs ``short_attempts`` short retries, then walks the
+    long-backoff table one entry per subsequent attempt. The retry loop gives
+    up as soon as ``retry_count >= ceiling`` — and that check runs *before* the
+    attempt's backoff is computed — so the ceiling must sit one past the final
+    long-backoff entry for every long tier to actually execute.
+
+    With the default ``api_max_retries`` (3) equal to ``short_attempts`` (3),
+    the loop always gave up before reaching the long tier, leaving the whole
+    long-backoff schedule as dead code. Callers extend the ceiling to this
+    value for Z.AI Coding overload 429s so the 30/60/90/120s waits run.
+    """
+    return short_attempts + len(_ZAI_CODING_OVERLOAD_LONG_BACKOFF) + 1