Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

This commit is contained in:
2026-09-05 13:26:46 +03:00
commit 03634b1ca3
11340 changed files with 3442369 additions and 0 deletions
+336
View File
@@ -0,0 +1,336 @@
"""DeepInfra image generation backend.
Exposes DeepInfra's image-gen catalog (FLUX, Qwen-Image-Edit, …) through
the OpenAI-compatible ``/v1/openai/images/generations`` endpoint as an
:class:`ImageGenProvider` implementation.
**Fully dynamic model discovery.** Unlike the other image-gen plugins in
this tree (which ship a hardcoded ``_MODELS`` dict), DeepInfra publishes
a single tagged catalog at
``https://api.deepinfra.com/v1/openai/models?filter=true&sort_by=hermes``
where each entry's ``metadata.tags`` declares its surface (``image-gen``
here). ``list_models()`` filters that catalog via
:func:`hermes_cli.models._fetch_deepinfra_models_by_tag` so newly added
models show up in ``hermes tools`` automatically. No model ids are
hardcoded in this file — if a model is retired upstream, it disappears
from hermes the next time the catalog is fetched, no patch required.
Model selection (first hit wins):
1. ``DEEPINFRA_IMAGE_MODEL`` env var
2. ``image_gen.deepinfra.model`` in ``config.yaml``
3. First model from the live catalog
When all three are absent (catalog unreachable, nothing configured),
``generate()`` returns an :func:`error_response` rather than guessing.
"""
from __future__ import annotations
import logging
import os
from typing import Any, Dict, List, Optional
from agent.secret_scope import get_secret
from agent.image_gen_provider import (
DEFAULT_ASPECT_RATIO,
ImageGenProvider,
error_response,
resolve_aspect_ratio,
save_b64_image,
save_url_image,
success_response,
)
logger = logging.getLogger(__name__)
# DeepInfra accepts standard OpenAI ``size`` strings. Mirrors the
# OpenAI plugin's mapping so aspect_ratio semantics stay consistent
# across the agent's image_generate tool surface.
_SIZES = {
"landscape": "1536x1024",
"square": "1024x1024",
"portrait": "1024x1536",
}
def _load_deepinfra_image_config() -> Dict[str, Any]:
"""Read ``image_gen.deepinfra`` from config.yaml."""
try:
from hermes_cli.config import load_config
cfg = load_config()
section = cfg.get("image_gen") if isinstance(cfg, dict) else None
di_section = section.get("deepinfra") if isinstance(section, dict) else None
return di_section if isinstance(di_section, dict) else {}
except Exception as exc:
logger.debug("Could not load image_gen.deepinfra config: %s", exc)
return {}
def _live_models() -> Optional[List[Dict[str, Any]]]:
"""Fetch ``image-gen``-tagged models from the DeepInfra catalog."""
try:
from hermes_cli.models import _fetch_deepinfra_models_by_tag
except Exception as exc:
logger.debug("Cannot import _fetch_deepinfra_models_by_tag: %s", exc)
return None
return _fetch_deepinfra_models_by_tag("image-gen")
def _format_catalog_row(item: Dict[str, Any]) -> Dict[str, Any]:
"""Format a catalog item into the picker row shape."""
mid = item.get("id", "")
metadata = item.get("metadata") or {}
pricing = metadata.get("pricing") if isinstance(metadata, dict) else None
price = ""
if isinstance(pricing, dict) and pricing.get("per_image_unit") is not None:
try:
price = f"${float(pricing['per_image_unit']):.4f}/image"
except (TypeError, ValueError):
price = ""
row: Dict[str, Any] = {
"id": mid,
"display": mid.split("/", 1)[-1] if "/" in mid else mid,
"strengths": metadata.get("description", "") if isinstance(metadata, dict) else "",
}
if price:
row["price"] = price
if isinstance(metadata, dict):
for key in ("default_width", "default_height", "default_iterations"):
if metadata.get(key) is not None:
row[key] = metadata[key]
return row
def _resolve_model(catalog: List[Dict[str, Any]], cfg: Dict[str, Any]) -> Optional[str]:
"""Pick the model id (env > config > first live result, else None).
Takes the already-loaded ``image_gen.deepinfra`` config so ``generate()``
reads config once instead of via a second ``load_config`` deepcopy.
"""
env_override = os.environ.get("DEEPINFRA_IMAGE_MODEL", "").strip()
if env_override:
return env_override
cfg_model = cfg.get("model") if isinstance(cfg, dict) else None
if isinstance(cfg_model, str) and cfg_model.strip():
return cfg_model.strip()
if catalog:
first = catalog[0].get("id")
if isinstance(first, str) and first:
return first
return None
class DeepInfraImageGenProvider(ImageGenProvider):
"""DeepInfra ``images.generations`` backend.
Catalog is discovered live from the DeepInfra ``/models`` endpoint
filtered by the ``image-gen`` surface tag.
"""
@property
def name(self) -> str:
return "deepinfra"
@property
def display_name(self) -> str:
return "DeepInfra"
def is_available(self) -> bool:
return bool((get_secret("DEEPINFRA_API_KEY", "") or "").strip())
def list_models(self) -> List[Dict[str, Any]]:
live = _live_models()
if not live:
return []
return [_format_catalog_row(item) for item in live]
def default_model(self) -> Optional[str]:
rows = self.list_models()
if rows:
return rows[0].get("id")
return None
def capabilities(self) -> Dict[str, Any]:
"""DeepInfra's OpenAI-compatible generation surface is text-only."""
return {"modalities": ["text"], "max_reference_images": 0}
def get_setup_schema(self) -> Dict[str, Any]:
return {
"name": "DeepInfra",
"badge": "paid",
"tag": "FLUX, Qwen-Image, … — live catalog from api.deepinfra.com",
"env_vars": [
{
"key": "DEEPINFRA_API_KEY",
"prompt": "DeepInfra API key",
"url": "https://deepinfra.com/dash/api_keys",
},
],
}
def generate(
self,
prompt: str,
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
**kwargs: Any,
) -> Dict[str, Any]:
prompt = (prompt or "").strip()
aspect = resolve_aspect_ratio(aspect_ratio)
if kwargs.get("image_url") or kwargs.get("reference_image_urls"):
return error_response(
error=(
"DeepInfra image generation is text-to-image only in this "
"backend; image_url and reference_image_urls are unsupported."
),
error_type="modality_unsupported",
provider="deepinfra",
prompt=prompt,
aspect_ratio=aspect,
)
if not prompt:
return error_response(
error="Prompt is required and must be a non-empty string",
error_type="invalid_argument",
provider="deepinfra",
aspect_ratio=aspect,
)
api_key = (get_secret("DEEPINFRA_API_KEY", "") or "").strip()
if not api_key:
return error_response(
error=(
"DEEPINFRA_API_KEY not set. Run `hermes tools` → Image "
"Generation → DeepInfra to configure, or `hermes setup` "
"to add the key."
),
error_type="auth_required",
provider="deepinfra",
aspect_ratio=aspect,
)
di_cfg = _load_deepinfra_image_config()
catalog = _live_models() or []
model_id = _resolve_model(catalog, di_cfg)
if not model_id:
return error_response(
error=(
"No DeepInfra image-gen model available. Pin one in "
"config.yaml under image_gen.deepinfra.model, set "
"DEEPINFRA_IMAGE_MODEL, or check connectivity to "
"api.deepinfra.com so the live catalog can be fetched."
),
error_type="no_model_available",
provider="deepinfra",
prompt=prompt,
aspect_ratio=aspect,
)
size = _SIZES.get(aspect, _SIZES["square"])
from hermes_cli.models import deepinfra_base_url
base_url = deepinfra_base_url(di_cfg)
# DeepInfra's /images/generations is OpenAI-compatible — use the
# openai SDK so we inherit its retry, timeout, and error mapping
# (mirrors the existing OpenAI image-gen plugin).
try:
import openai
except ImportError:
return error_response(
error="openai Python package not installed (pip install openai)",
error_type="missing_dependency",
provider="deepinfra",
aspect_ratio=aspect,
)
client = openai.OpenAI(api_key=api_key, base_url=base_url)
try:
response = client.images.generate(
model=model_id,
prompt=prompt,
size=size,
n=1,
)
except Exception as exc:
logger.debug("DeepInfra image generation failed", exc_info=True)
return error_response(
error=f"DeepInfra image generation failed: {exc}",
error_type="api_error",
provider="deepinfra",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
finally:
close = getattr(client, "close", None)
if callable(close):
close()
data = getattr(response, "data", None) or []
if not data:
return error_response(
error="DeepInfra returned no image data",
error_type="empty_response",
provider="deepinfra",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
first = data[0]
b64 = getattr(first, "b64_json", None)
url = getattr(first, "url", None)
# Drop the ``vendor/`` prefix and any colons so the saved filename
# stays a single path component on every OS.
short = model_id.split("/", 1)[-1].replace(":", "_")
if b64:
try:
saved_path = save_b64_image(b64, prefix=f"deepinfra_{short}")
except Exception as exc:
return error_response(
error=f"Could not save image to cache: {exc}",
error_type="io_error",
provider="deepinfra",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
image_ref = str(saved_path)
elif url:
# Materialise the (often short-lived) delivery URL locally so a
# downstream consumer (Telegram send_photo, browser fetch) doesn't
# get a dead link — mirrors the openai/xai/krea image plugins.
# Best-effort: fall back to the bare URL if the download fails.
try:
image_ref = str(save_url_image(url, prefix=f"deepinfra_{short}"))
except Exception as exc:
logger.debug("DeepInfra: caching delivery URL failed (%s); returning URL", exc)
image_ref = url
else:
return error_response(
error="DeepInfra response contained neither b64_json nor URL",
error_type="empty_response",
provider="deepinfra",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
return success_response(
image=image_ref,
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
provider="deepinfra",
extra={"size": size},
)
def register(ctx) -> None:
"""Plugin entry point — wire ``DeepInfraImageGenProvider`` into the registry."""
ctx.register_image_gen_provider(DeepInfraImageGenProvider())
+7
View File
@@ -0,0 +1,7 @@
name: deepinfra
version: 1.0.0
description: "DeepInfra image generation backend (FLUX, Qwen-Image, …) via OpenAI-compatible /v1/images/generations. Catalog discovered live from api.deepinfra.com."
author: Georgi Atsev
kind: backend
requires_env:
- DEEPINFRA_API_KEY
+218
View File
@@ -0,0 +1,218 @@
"""FAL.ai image generation backend.
Wraps the 18-model FAL catalog (FLUX 2, Z-Image, Nano Banana, GPT
Image 1.5, Recraft, Imagen 4, Qwen, Ideogram, …) as an
:class:`ImageGenProvider` implementation.
The heavy lifting — model catalog, payload construction, request
submission, managed-Nous-gateway selection, Clarity Upscaler chaining
— lives in :mod:`tools.image_generation_tool`. This plugin reaches into
that module via call-time indirection (``import tools.image_generation_tool as _it``)
so:
* the existing test suite (``tests/tools/test_image_generation.py``,
``tests/tools/test_managed_media_gateways.py``) keeps patching
``image_tool._submit_fal_request`` / ``image_tool.fal_client`` /
``image_tool._managed_fal_client`` without modification, and
* there's exactly one canonical FAL code path on disk — the plugin is a
registration adapter, not a parallel implementation.
See issue #26241 for the migration plan and the
``plugin-extraction-test-patch-compatibility.md`` rules this follows.
"""
from __future__ import annotations
import json
import logging
import os
from typing import Any, Dict, List, Optional
from agent.image_gen_provider import (
DEFAULT_ASPECT_RATIO,
ImageGenProvider,
resolve_aspect_ratio,
)
logger = logging.getLogger(__name__)
class FalImageGenProvider(ImageGenProvider):
"""FAL.ai image generation backend.
Delegates to ``tools.image_generation_tool.image_generate_tool`` so
the in-tree FAL implementation (model catalog, payload builder,
managed-gateway selection, Clarity Upscaler chaining) is the single
source of truth. Everything is resolved at call time via the
``_it`` indirection so tests can monkey-patch the legacy module.
"""
@property
def name(self) -> str:
return "fal"
@property
def display_name(self) -> str:
return "FAL.ai"
def is_available(self) -> bool:
# Available when direct FAL_KEY is set OR the managed Nous
# gateway resolves a fal-queue origin. Both checks come from the
# legacy module so this provider tracks whatever logic ships
# there.
import tools.image_generation_tool as _it
try:
return bool(_it.check_fal_api_key())
except Exception: # noqa: BLE001 — defensive; never break the picker
return False
def list_models(self) -> List[Dict[str, Any]]:
import tools.image_generation_tool as _it
return [
{
"id": model_id,
"display": meta.get("display", model_id),
"speed": meta.get("speed", ""),
"strengths": meta.get("strengths", ""),
"price": meta.get("price", ""),
}
for model_id, meta in _it.FAL_MODELS.items()
]
def default_model(self) -> Optional[str]:
import tools.image_generation_tool as _it
return _it.DEFAULT_MODEL
def get_setup_schema(self) -> Dict[str, Any]:
return {
"name": "FAL.ai",
"badge": "paid",
"tag": "Pick from flux-2-klein, flux-2-pro, gpt-image, nano-banana-2, nano-banana-pro, etc. — text-to-image & image editing",
"env_vars": [
{
"key": "FAL_KEY",
"prompt": "FAL API key",
"url": "https://fal.ai/dashboard/keys",
},
],
}
def capabilities(self) -> Dict[str, Any]:
# Whether image-to-image is available depends on the currently-
# selected FAL model (each model entry declares an edit_endpoint or
# not). Report the active model's actual surface so the dynamic tool
# schema is accurate.
import tools.image_generation_tool as _it
try:
_model_id, meta = _it._resolve_fal_model()
except Exception: # noqa: BLE001
return {"modalities": ["text"], "max_reference_images": 0}
# Clarity Upscaler chains on explicit request for any FAL model.
if meta.get("edit_endpoint"):
return {
"modalities": ["text", "image"],
"max_reference_images": int(meta.get("max_reference_images") or 1),
"supports_upscale": True,
}
return {
"modalities": ["text"],
"max_reference_images": 0,
"supports_upscale": True,
}
def generate(
self,
prompt: str,
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
*,
image_url: Optional[str] = None,
reference_image_urls: Optional[List[str]] = None,
**kwargs: Any,
) -> Dict[str, Any]:
"""Generate or edit an image via the legacy FAL pipeline.
Forwards prompt + aspect_ratio + image_url/reference_image_urls (and
any forward-compat extras the schema supports) into
:func:`tools.image_generation_tool.image_generate_tool`, then reshapes
its JSON-string response into the provider-ABC dict format consumed by
``_dispatch_to_plugin_provider``.
"""
import tools.image_generation_tool as _it
aspect = resolve_aspect_ratio(aspect_ratio)
passthrough = {
key: kwargs[key]
for key in (
"num_inference_steps",
"guidance_scale",
"num_images",
"output_format",
"seed",
"upscale",
)
if key in kwargs and kwargs[key] is not None
}
# Only forward the image-to-image inputs when actually supplied, so a
# plain text-to-image call delegates exactly as it did before (no
# noisy None kwargs).
if image_url is not None:
passthrough["image_url"] = image_url
if reference_image_urls is not None:
passthrough["reference_image_urls"] = reference_image_urls
try:
raw = _it.image_generate_tool(
prompt=prompt,
aspect_ratio=aspect,
**passthrough,
)
except Exception as exc: # noqa: BLE001 — never raise out of generate
logger.warning("FAL image_generate_tool raised: %s", exc, exc_info=True)
return {
"success": False,
"image": None,
"error": f"FAL image generation failed: {exc}",
"error_type": type(exc).__name__,
"provider": "fal",
"prompt": prompt,
"aspect_ratio": aspect,
}
try:
response = json.loads(raw) if isinstance(raw, str) else raw
except Exception: # noqa: BLE001
response = {"success": False, "image": None, "error": "Invalid JSON from FAL pipeline"}
if not isinstance(response, dict):
response = {
"success": False,
"image": None,
"error": "FAL pipeline returned a non-dict response",
"error_type": "provider_contract",
}
# Stamp provider/prompt/aspect_ratio so downstream consumers see
# the uniform shape declared in ``agent.image_gen_provider``.
response.setdefault("provider", "fal")
response.setdefault("prompt", prompt)
response.setdefault("aspect_ratio", aspect)
# Annotate model best-effort — the legacy pipeline resolves it
# internally, so query it after the fact for the response shape.
if "model" not in response:
try:
model_id, _meta = _it._resolve_fal_model()
response["model"] = model_id
except Exception: # noqa: BLE001
pass
return response
# ---------------------------------------------------------------------------
# Plugin entry point
# ---------------------------------------------------------------------------
def register(ctx) -> None:
"""Plugin entry point — wire ``FalImageGenProvider`` into the registry."""
ctx.register_image_gen_provider(FalImageGenProvider())
+7
View File
@@ -0,0 +1,7 @@
name: fal
version: 1.0.0
description: "FAL.ai image generation backend (flux-2-klein, flux-2-pro, nano-banana-2, nano-banana-pro, gpt-image-1.5, recraft-v3, etc.)."
author: NousResearch
kind: backend
requires_env:
- FAL_KEY
+921
View File
@@ -0,0 +1,921 @@
"""Krea image generation backend.
Exposes Krea's `Krea 2` foundation image model family — Krea 2 Medium and
Krea 2 Large — as an :class:`ImageGenProvider` implementation.
Krea's API is asynchronous: the generate endpoint returns a ``job_id``
that you poll at ``GET /jobs/{job_id}``. This provider hides that
roundtrip behind the synchronous ``generate()`` contract: submit, poll
every 2s with light backoff, materialise the result URL to local cache,
return the success/error dict like every other backend.
Selection precedence (first hit wins):
1. ``KREA_IMAGE_MODEL`` env var (escape hatch for scripts / tests)
2. ``image_gen.krea.model`` in ``config.yaml``
3. ``image_gen.model`` in ``config.yaml`` (when it's one of our IDs)
4. :data:`DEFAULT_MODEL` — ``krea-2-medium`` (Krea's "start here" recommendation)
Docs: https://docs.krea.ai/developers/krea-2/overview
API: https://docs.krea.ai/api-reference/krea/krea-2-large
"""
from __future__ import annotations
import logging
import os
import time
import uuid
from typing import Any, Dict, List, Optional, Tuple
import requests
from agent.secret_scope import get_secret
from agent.image_gen_provider import (
DEFAULT_ASPECT_RATIO,
ImageGenProvider,
error_response,
normalize_reference_images,
resolve_aspect_ratio,
save_url_image,
success_response,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
BASE_URL = "https://api.krea.ai"
# Map our short model IDs to Krea's URL path segment.
_MODELS: Dict[str, Dict[str, Any]] = {
"krea-2-medium": {
"display": "Krea 2 Medium",
"speed": "~15-25s",
"strengths": "Illustration, anime, painting, expressive styles. Faster + cheaper.",
"price": "$0.030 (text) / $0.035 (style refs) / $0.040 (moodboards)",
"path": "medium",
# Upscaling is opt-in everywhere (Aug 2026 policy: default-on
# enhance passes degraded output quality).
"upscale": False,
},
"krea-2-large": {
"display": "Krea 2 Large",
"speed": "~25-60s",
"strengths": "Photorealism, raw textured looks (motion blur, grain), expressive styles.",
"price": "$0.060 (text) / $0.065 (style refs) / $0.070 (moodboards)",
"path": "large",
# 2K native — high-res enough out of the box.
"upscale": False,
},
"krea-2-medium-turbo": {
"display": "Krea 2 Medium Turbo",
"speed": "~8-15s",
"strengths": "Fastest Krea 2 — medium quality at lower latency / cost.",
"price": "$0.015 (text) / $0.0175 (style refs)",
"path": "medium-turbo",
# Opt-in only (Aug 2026 policy).
"upscale": False,
},
}
DEFAULT_MODEL = "krea-2-medium"
# Hermes uses 3 abstract aspect ratios. Map to Krea's enum (which is wider).
# Krea accepts: 1:1, 4:3, 3:2, 16:9, 2.35:1, 4:5, 2:3, 9:16
_ASPECT_MAP = {
"landscape": "16:9",
"square": "1:1",
"portrait": "9:16",
}
# Only resolution Krea currently supports.
DEFAULT_RESOLUTION = "1K"
# Krea's image_style_references entries are objects ({"url", "strength"}), not
# bare URL strings. When the caller supplies a URL without an explicit strength
# we apply Krea's recommended starting value. Range per Krea docs is -2..2.
_DEFAULT_STYLE_REFERENCE_STRENGTH = 0.6
# Valid creativity levels per Krea docs. Default is "medium".
_VALID_CREATIVITY = {"raw", "low", "medium", "high"}
# Polling cadence. Krea recommends 2-5s; we start at 2s and back off to 5s
# for long jobs (Large can take ~1min). Total ceiling matches Krea's
# hosted-tool timeout of 3 minutes.
_POLL_INITIAL_INTERVAL = 2.0
_POLL_MAX_INTERVAL = 5.0
_POLL_BACKOFF = 1.3
_POLL_TIMEOUT_SECONDS = 180.0
# HTTP statuses worth retrying during the poll loop. Everything else (401,
# 402, 403, 404, other 4xx) is a permanent failure — surface it immediately
# instead of burning the 180s deadline retrying a request that will never
# succeed.
_RETRYABLE_POLL_STATUSES = frozenset({408, 409, 425, 429, 500, 502, 503, 504})
_TERMINAL_STATES = {"completed", "failed", "cancelled"}
# Krea Enhance — the upscale/enhancer endpoint used for the optional
# ``upscale`` pass after generation ("1.5K native, 4K via Enhancer" is
# Krea's own pipeline shape). Cheap creative enhancer, max 8K.
_ENHANCE_PATH = "/generate/enhance/krea/enhance"
_ENHANCE_SCALE_FACTOR = 2
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
def _load_krea_config() -> Dict[str, Any]:
"""Read ``image_gen.krea`` (with fallthrough to ``image_gen``) from config.yaml."""
try:
from hermes_cli.config import load_config
cfg = load_config()
section = cfg.get("image_gen") if isinstance(cfg, dict) else None
return section if isinstance(section, dict) else {}
except Exception as exc: # noqa: BLE001
logger.debug("Could not load image_gen config: %s", exc)
return {}
def _resolve_model(explicit: Optional[str] = None) -> Tuple[str, Dict[str, Any]]:
"""Decide which model to use and return ``(model_id, meta)``.
Precedence: explicit caller override (e.g. managed-mode routing or a direct
``model`` kwarg) → ``KREA_IMAGE_MODEL`` env → ``image_gen.krea.model`` →
``image_gen.model`` → :data:`DEFAULT_MODEL`.
"""
if isinstance(explicit, str) and explicit.strip() in _MODELS:
return explicit.strip(), _MODELS[explicit.strip()]
env_override = os.environ.get("KREA_IMAGE_MODEL")
if env_override and env_override in _MODELS:
return env_override, _MODELS[env_override]
cfg = _load_krea_config()
krea_cfg = cfg.get("krea") if isinstance(cfg.get("krea"), dict) else {}
candidate: Optional[str] = None
if isinstance(krea_cfg, dict):
value = krea_cfg.get("model")
if isinstance(value, str) and value in _MODELS:
candidate = value
if candidate is None:
top = cfg.get("model")
if isinstance(top, str) and top in _MODELS:
candidate = top
if candidate is not None:
return candidate, _MODELS[candidate]
return DEFAULT_MODEL, _MODELS[DEFAULT_MODEL]
def _resolve_managed_krea_gateway():
"""Return managed Krea gateway config when the user is on the managed path.
Strict selection model: the managed Krea gateway is used when the stored
``image_gen`` selection is ``nous`` (or legacy ``use_gateway: true``), or
on a never-configured install when no direct ``KREA_API_KEY`` exists.
An explicit vendor selection (``krea``, ``fal``, ...) pins the direct
path. Returns ``None`` (direct/BYO path) otherwise, and never raises —
plugin discovery and availability scans must stay robust.
"""
try:
from tools.managed_tool_gateway import resolve_managed_tool_gateway
from tools.tool_backend_helpers import (
NOUS_MANAGED_PROVIDER,
read_selection,
)
except Exception as exc: # noqa: BLE001
logger.debug("Managed Krea gateway resolution unavailable: %s", exc)
return None
try:
selected = read_selection("image_gen")
except Exception: # noqa: BLE001
selected = None
if selected is not None and selected != NOUS_MANAGED_PROVIDER:
# Explicit vendor selection: direct credentials only.
return None
if selected is None and get_secret("KREA_API_KEY"):
return None
try:
return resolve_managed_tool_gateway("krea")
except Exception as exc: # noqa: BLE001
logger.debug("Managed Krea gateway resolution failed: %s", exc)
return None
def _managed_krea_gateway_ready() -> bool:
"""Cheap, offline-friendly probe for managed Krea availability."""
try:
from tools.managed_tool_gateway import is_managed_tool_gateway_ready
except Exception: # noqa: BLE001
return False
try:
return bool(is_managed_tool_gateway_ready("krea"))
except Exception: # noqa: BLE001
return False
def _resolve_creativity(value: Optional[str]) -> str:
"""Coerce ``creativity`` kwarg to a valid Krea value (default ``medium``)."""
if isinstance(value, str):
v = value.strip().lower()
if v in _VALID_CREATIVITY:
return v
cfg = _load_krea_config()
krea_cfg = cfg.get("krea") if isinstance(cfg.get("krea"), dict) else {}
cfg_value = krea_cfg.get("creativity") if isinstance(krea_cfg, dict) else None
if isinstance(cfg_value, str) and cfg_value.strip().lower() in _VALID_CREATIVITY:
return cfg_value.strip().lower()
return "medium"
def _poll_krea_job(
base_url: str,
auth_token: str,
job_id: str,
*,
timeout_seconds: float = _POLL_TIMEOUT_SECONDS,
) -> Optional[Dict[str, Any]]:
"""Poll ``/jobs/{job_id}`` until terminal; return the job dict or None.
Best-effort variant of the main generate() poll loop used for secondary
jobs (the Enhance upscale pass): any failure returns ``None`` so the
caller can fall back instead of failing the whole generation.
"""
job_url = f"{base_url}/jobs/{job_id}"
headers = {
"Authorization": f"Bearer {auth_token}",
"User-Agent": "Hermes-Agent/1.0 (krea-image-gen)",
}
interval = _POLL_INITIAL_INTERVAL
deadline = time.monotonic() + timeout_seconds
while True:
time.sleep(interval)
interval = min(interval * _POLL_BACKOFF, _POLL_MAX_INTERVAL)
try:
resp = requests.get(job_url, headers=headers, timeout=30)
resp.raise_for_status()
job = resp.json()
except requests.HTTPError as exc:
status = exc.response.status_code if exc.response is not None else 0
if status not in _RETRYABLE_POLL_STATUSES or time.monotonic() >= deadline:
logger.warning("Krea enhance poll failed (%d) for job %s", status, job_id)
return None
continue
except Exception as exc: # noqa: BLE001 — timeout/connection/JSON
if time.monotonic() >= deadline:
logger.warning("Krea enhance poll gave up for job %s: %s", job_id, exc)
return None
continue
if isinstance(job, dict):
status_str = job.get("status")
if status_str in _TERMINAL_STATES or job.get("completed_at"):
return job
if time.monotonic() >= deadline:
logger.warning("Krea enhance job %s did not finish in %ds", job_id, int(timeout_seconds))
return None
def _extract_result_url(job: Optional[Dict[str, Any]]) -> Optional[str]:
"""Pull the first result URL out of a terminal Krea job dict."""
if not isinstance(job, dict):
return None
result = job.get("result")
if not isinstance(result, dict):
return None
urls = result.get("urls")
if isinstance(urls, list):
for candidate in urls:
if isinstance(candidate, str) and candidate.strip():
return candidate.strip()
single = result.get("url")
if isinstance(single, str) and single.strip():
return single.strip()
return None
def _enhance_image(
base_url: str,
auth_token: str,
image_url: str,
prompt: str,
*,
managed: bool,
) -> Optional[str]:
"""Run Krea Enhance on ``image_url``; return the enhanced URL or None.
Best-effort: any submit/poll/result failure logs and returns ``None`` so
the caller falls back to the original (un-upscaled) image — an upscale
failure must never destroy an already-successful generation.
"""
headers = {
"Authorization": f"Bearer {auth_token}",
"Content-Type": "application/json",
"User-Agent": "Hermes-Agent/1.0 (krea-image-gen)",
}
if managed:
headers["x-idempotency-key"] = str(uuid.uuid4())
payload: Dict[str, Any] = {
"image_url": image_url,
"image_scaling_factor": _ENHANCE_SCALE_FACTOR,
# Keep the enhancer faithful to the generated composition: the
# original prompt guides detail, and default ai_strength stays
# conservative (Krea default 0.4 adds detail without redrawing).
"prompt": prompt,
}
try:
resp = requests.post(
f"{base_url}{_ENHANCE_PATH}", headers=headers, json=payload, timeout=30,
)
resp.raise_for_status()
job_id = (resp.json() or {}).get("job_id")
except Exception as exc: # noqa: BLE001
logger.warning("Krea Enhance submit failed: %s", exc)
return None
if not isinstance(job_id, str) or not job_id:
logger.warning("Krea Enhance submit response missing job_id")
return None
job = _poll_krea_job(base_url, auth_token, job_id)
if not isinstance(job, dict) or job.get("status") in {"failed", "cancelled"}:
logger.warning("Krea Enhance job %s did not complete successfully", job_id)
return None
return _extract_result_url(job)
# ---------------------------------------------------------------------------
# Provider
# ---------------------------------------------------------------------------
class KreaImageGenProvider(ImageGenProvider):
"""Krea ``Krea 2`` foundation image model backend (Medium + Large)."""
@property
def name(self) -> str:
return "krea"
@property
def display_name(self) -> str:
return "Krea"
def is_available(self) -> bool:
# Available with a direct Krea key OR via the managed Nous gateway
# (Nous Subscription), so portal users with no Krea key can still
# reach Krea 2 through the gateway.
return bool(get_secret("KREA_API_KEY")) or _managed_krea_gateway_ready()
def list_models(self) -> List[Dict[str, Any]]:
return [
{
"id": model_id,
"display": meta["display"],
"speed": meta["speed"],
"strengths": meta["strengths"],
"price": meta["price"],
}
for model_id, meta in _MODELS.items()
]
def default_model(self) -> Optional[str]:
return DEFAULT_MODEL
def get_setup_schema(self) -> Dict[str, Any]:
return {
"name": "Krea",
"badge": "paid",
"tag": "Krea 2 foundation model — Medium ($0.03), Large ($0.06), Medium Turbo ($0.015). Style transfer, moodboards, reference-guided generation. Direct key or managed Nous Subscription gateway.",
"env_vars": [
{
"key": "KREA_API_KEY",
"prompt": "Krea API key",
"url": "https://www.krea.ai/settings/api-tokens",
},
],
}
def capabilities(self) -> Dict[str, Any]:
# Krea supports reference-guided generation (image-to-image style
# transfer) via image_style_references — up to 10 refs — and an
# opt-in Enhance upscale pass (see generate()'s upscale_requested).
return {
"modalities": ["text", "image"],
"max_reference_images": 10,
"supports_upscale": True,
}
# ------------------------------------------------------------------
# generate()
# ------------------------------------------------------------------
def generate(
self,
prompt: str,
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
*,
image_url: Optional[str] = None,
reference_image_urls: Optional[List[str]] = None,
**kwargs: Any,
) -> Dict[str, Any]:
prompt = (prompt or "").strip()
aspect = resolve_aspect_ratio(aspect_ratio)
krea_ar = _ASPECT_MAP.get(aspect, "1:1")
# Collect reference images for reference-guided generation (image-to-
# image style transfer). Sources, in order:
# 1. unified image_url (primary source) + reference_image_urls (strings)
# 2. legacy image_style_references kwarg — may be plain URL strings OR
# Krea's richer ref objects (e.g. {"url": ..., "strength": ...}),
# which are passed through verbatim for backward compatibility.
style_refs: List[Any] = []
if isinstance(image_url, str) and image_url.strip():
style_refs.append(image_url.strip())
for ref in (normalize_reference_images(reference_image_urls) or []):
style_refs.append(ref)
legacy_refs = kwargs.get("image_style_references")
if isinstance(legacy_refs, list):
for ref in legacy_refs:
if isinstance(ref, str):
if ref.strip():
style_refs.append(ref.strip())
elif ref:
# Non-string ref object (dict, etc.) — pass through as-is.
style_refs.append(ref)
# Dedupe string entries while preserving order (dict refs aren't
# hashable, so they're kept verbatim); Krea caps at 10.
seen: set = set()
deduped: List[Any] = []
for r in style_refs:
if isinstance(r, str):
if r in seen:
continue
seen.add(r)
deduped.append(r)
style_refs = deduped[:10]
modality = "image" if style_refs else "text"
if not prompt:
return error_response(
error="Prompt is required and must be a non-empty string",
error_type="invalid_argument",
provider="krea",
aspect_ratio=aspect,
)
# Route through the managed Nous gateway (Nous Subscription) when the
# user is on the managed path; otherwise use the direct Krea API with a
# BYO ``KREA_API_KEY``. The gateway owns the shared Krea credential and
# meters/bills per generation, so the caller token is the Nous access
# token, not a Krea key.
managed = _resolve_managed_krea_gateway()
if managed is not None:
base_url = managed.gateway_origin.rstrip("/")
auth_token = managed.nous_user_token
else:
base_url = BASE_URL
auth_token = get_secret("KREA_API_KEY")
if not auth_token:
return error_response(
error=(
"KREA_API_KEY not set. Run `hermes tools` → Image "
"Generation → Krea to configure, get a key at "
"https://www.krea.ai/settings/api-tokens, or sign in to "
"a Nous account with the managed Krea gateway enabled "
"(`hermes setup`)."
),
error_type="auth_required",
provider="krea",
aspect_ratio=aspect,
)
model_id, meta = _resolve_model(kwargs.get("model"))
creativity = _resolve_creativity(kwargs.get("creativity"))
# The managed gateway only prices base text-to-image and URL
# ``image_style_references`` tiers. Trained styles (LoRAs) and
# moodboards have no managed price and are rejected at the gateway, so
# fail fast here with actionable guidance instead of a raw 400.
if managed is not None:
if isinstance(kwargs.get("styles"), list) and kwargs.get("styles"):
return error_response(
error=(
"Managed Krea (Nous Subscription) does not support "
"trained styles (LoRAs). Set KREA_API_KEY to use Krea "
"directly, or omit `styles`."
),
error_type="unsupported_argument",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
if isinstance(kwargs.get("moodboards"), list) and kwargs.get("moodboards"):
return error_response(
error=(
"Managed Krea (Nous Subscription) does not support "
"moodboards. Set KREA_API_KEY to use Krea directly, or "
"omit `moodboards`."
),
error_type="unsupported_argument",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
payload: Dict[str, Any] = {
"prompt": prompt,
"aspect_ratio": krea_ar,
"resolution": DEFAULT_RESOLUTION,
"creativity": creativity,
}
# Optional forward-compat passthroughs — the Krea API accepts these
# but they're not required and most agent calls won't supply them.
seed = kwargs.get("seed")
if isinstance(seed, int):
payload["seed"] = seed
styles = kwargs.get("styles")
if isinstance(styles, list) and styles:
payload["styles"] = styles
if style_refs:
# Reference-guided generation (image-to-image style transfer).
# Krea requires each entry to be an object ({"url", "strength"}),
# NOT a bare URL string — a string yields a 422 "Expected object,
# received string". Convert URL strings to the object form and pass
# already-object refs through verbatim (clamped to 10 above).
normalized_refs: List[Any] = []
for ref in style_refs:
if isinstance(ref, str):
normalized_refs.append(
{"url": ref, "strength": _DEFAULT_STYLE_REFERENCE_STRENGTH}
)
else:
normalized_refs.append(ref)
payload["image_style_references"] = normalized_refs
moodboards = kwargs.get("moodboards")
if isinstance(moodboards, list) and moodboards:
# Krea currently caps at 1 moodboard per request.
payload["moodboards"] = moodboards[:1]
headers = {
"Authorization": f"Bearer {auth_token}",
"Content-Type": "application/json",
"User-Agent": "Hermes-Agent/1.0 (krea-image-gen)",
}
if managed is not None:
# The gateway derives the per-generation billing idempotency
# boundary from this header (else it falls back to a body
# fingerprint). A fresh key per submit keeps each generation a
# distinct billable execution.
headers["x-idempotency-key"] = str(uuid.uuid4())
# 1. Submit job.
submit_url = f"{base_url}/generate/image/krea/krea-2/{meta['path']}"
try:
response = requests.post(
submit_url,
headers=headers,
json=payload,
timeout=30,
)
response.raise_for_status()
except requests.HTTPError as exc:
resp = exc.response
status = resp.status_code if resp is not None else 0
try:
body = resp.json() if resp is not None else {}
err_msg = (
body.get("error", {}).get("message")
if isinstance(body.get("error"), dict)
else body.get("message") or body.get("detail")
) or (resp.text[:300] if resp is not None else str(exc))
except Exception: # noqa: BLE001
err_msg = resp.text[:300] if resp is not None else str(exc)
logger.error("Krea submit failed (%d): %s", status, err_msg)
# On a managed 4xx, surface actionable remediation mirroring the
# FAL managed gateway path: the model may not be enabled/priced on
# the Nous Portal, or the gateway's shared Krea key hit its
# concurrency cap (429).
if managed is not None and 400 <= status < 500:
hint = (
"Krea's shared-key concurrency cap was hit — retry shortly."
if status == 429
else (
f"Model '{model_id}' may not be enabled/priced on the "
"Nous Portal's Krea gateway. Set KREA_API_KEY to use "
"Krea directly, or pick a different model via "
"`hermes tools` → Image Generation."
)
)
return error_response(
error=(
f"Nous Subscription Krea gateway rejected '{model_id}' "
f"(HTTP {status}): {err_msg}. {hint}"
),
error_type="api_error",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
return error_response(
error=f"Krea image generation failed ({status}): {err_msg}",
error_type="api_error",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
except requests.Timeout:
return error_response(
error="Krea submit timed out (30s)",
error_type="timeout",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
except requests.ConnectionError as exc:
return error_response(
error=f"Krea connection error: {exc}",
error_type="connection_error",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
try:
submit_body = response.json()
except Exception as exc: # noqa: BLE001
return error_response(
error=f"Krea returned invalid JSON on submit: {exc}",
error_type="invalid_response",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
job_id = submit_body.get("job_id")
if not isinstance(job_id, str) or not job_id:
return error_response(
error="Krea submit response missing job_id",
error_type="invalid_response",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
# 2. Poll for completion. Status/result polling is bound to the same
# principal at the gateway, so the managed path polls the gateway's
# ``/jobs/{id}`` with the Nous token (404 on cross-user/unknown jobs).
job_url = f"{base_url}/jobs/{job_id}"
poll_headers = {
"Authorization": f"Bearer {auth_token}",
"User-Agent": "Hermes-Agent/1.0 (krea-image-gen)",
}
interval = _POLL_INITIAL_INTERVAL
deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS
last_status: Optional[str] = None
while True:
time.sleep(interval)
interval = min(interval * _POLL_BACKOFF, _POLL_MAX_INTERVAL)
try:
poll_resp = requests.get(job_url, headers=poll_headers, timeout=30)
poll_resp.raise_for_status()
except requests.HTTPError as exc:
resp = exc.response
status = resp.status_code if resp is not None else 0
logger.error("Krea poll failed (%d) for job %s", status, job_id)
# Fail fast for non-retryable statuses (auth/billing/not-found,
# other permanent 4xx) so callers don't wait the full 180s
# deadline on a request that will never succeed. Only retry
# transient statuses such as 408/409/425/429/5xx.
if status not in _RETRYABLE_POLL_STATUSES or time.monotonic() >= deadline:
return error_response(
error=f"Krea poll failed ({status}) for job {job_id}",
error_type="api_error",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
# Otherwise keep trying — transient 5xx (and a few retryable
# 4xx like 408/409/425/429) are common on async jobs.
continue
except (requests.Timeout, requests.ConnectionError) as exc:
logger.warning("Krea poll transient error for job %s: %s", job_id, exc)
if time.monotonic() >= deadline:
return error_response(
error=f"Krea poll timed out for job {job_id}: {exc}",
error_type="timeout",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
continue
try:
job = poll_resp.json()
except Exception as exc: # noqa: BLE001
logger.warning("Krea poll returned invalid JSON for job %s: %s", job_id, exc)
if time.monotonic() >= deadline:
return error_response(
error=f"Krea poll returned invalid JSON: {exc}",
error_type="invalid_response",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
continue
status_str = job.get("status") if isinstance(job, dict) else None
if isinstance(status_str, str):
last_status = status_str
if status_str in _TERMINAL_STATES:
break
# ``completed_at`` is a backstop terminal marker even when the
# ``status`` enum is unfamiliar (Krea adds new pending states
# over time — backlogged/scheduled/sampling — and we don't
# want to mis-handle a future one).
if isinstance(job, dict) and job.get("completed_at"):
break
if time.monotonic() >= deadline:
return error_response(
error=(
f"Krea job {job_id} did not complete within "
f"{int(_POLL_TIMEOUT_SECONDS)}s (last status: {last_status or 'unknown'})"
),
error_type="timeout",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
# 3. Terminal — extract result.
if not isinstance(job, dict):
return error_response(
error="Krea returned non-dict job body",
error_type="invalid_response",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
if last_status == "failed":
err = (job.get("result") or {}).get("error") if isinstance(job.get("result"), dict) else None
return error_response(
error=f"Krea job {job_id} failed: {err or 'unknown error'}",
error_type="api_error",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
if last_status == "cancelled":
return error_response(
error=f"Krea job {job_id} was cancelled",
error_type="cancelled",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
# Successful path — pull URL out of the result.
result = job.get("result")
if not isinstance(result, dict):
return error_response(
error="Krea job completed but result was missing",
error_type="empty_response",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
# Per Krea's job-lifecycle docs the completed payload exposes
# ``result.urls`` (an array). Fall back to a single ``url`` field
# for forward/backward compatibility.
result_image_url: Optional[str] = None
urls = result.get("urls")
if isinstance(urls, list) and urls:
for candidate in urls:
if isinstance(candidate, str) and candidate.strip():
result_image_url = candidate.strip()
break
if result_image_url is None:
single = result.get("url")
if isinstance(single, str) and single.strip():
result_image_url = single.strip()
if result_image_url is None:
return error_response(
error="Krea result contained no image URL",
error_type="empty_response",
provider="krea",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
# High-resolution pass (Krea Enhance). Precedence: explicit kwarg >
# ``image_gen.krea.upscale`` config > per-model catalog default
# (1.5K-native tiers default on; 2K-native Large stays off). Best-
# effort: failure falls back to the original image rather than
# failing the generation.
upscaled = False
upscale_requested = kwargs.get("upscale")
if not isinstance(upscale_requested, bool):
cfg_krea = _load_krea_config().get("krea")
cfg_upscale = cfg_krea.get("upscale") if isinstance(cfg_krea, dict) else None
if isinstance(cfg_upscale, bool):
upscale_requested = cfg_upscale
else:
upscale_requested = bool(meta.get("upscale", False))
if upscale_requested:
enhanced_url = _enhance_image(
base_url,
auth_token,
result_image_url,
prompt,
managed=managed is not None,
)
if enhanced_url:
result_image_url = enhanced_url
upscaled = True
else:
logger.warning(
"Krea Enhance pass failed — returning native-resolution image"
)
# Materialise locally — Krea result URLs may expire, mirroring
# what we do for xAI / OpenAI URL responses (#26942).
try:
saved_path = save_url_image(result_image_url, prefix=f"krea_{model_id}")
except Exception as exc: # noqa: BLE001
logger.warning(
"Krea image URL %s could not be cached (%s); falling back to bare URL.",
result_image_url,
exc,
)
image_ref = result_image_url
else:
image_ref = str(saved_path)
extra: Dict[str, Any] = {
"krea_aspect_ratio": krea_ar,
"resolution": DEFAULT_RESOLUTION,
"creativity": creativity,
"job_id": job_id,
"upscaled": upscaled,
}
if upscaled:
extra["upscale_factor"] = _ENHANCE_SCALE_FACTOR
if isinstance(job.get("completed_at"), str):
extra["completed_at"] = job["completed_at"]
return success_response(
image=image_ref,
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
provider="krea",
modality=modality,
extra=extra,
)
# ---------------------------------------------------------------------------
# Plugin entry point
# ---------------------------------------------------------------------------
def register(ctx) -> None:
"""Plugin entry point — wire ``KreaImageGenProvider`` into the registry."""
ctx.register_image_gen_provider(KreaImageGenProvider())
+7
View File
@@ -0,0 +1,7 @@
name: krea
version: 1.1.0
description: "Krea image generation backend (Krea 2 Large + Medium + Medium Turbo foundation models). Direct KREA_API_KEY or managed Nous Subscription gateway."
author: NousResearch
kind: backend
requires_env:
- KREA_API_KEY
+297
View File
@@ -0,0 +1,297 @@
"""Meta Model API image generation backend.
Exposes Meta's ``muse-image`` model(s) as an :class:`ImageGenProvider`.
The Meta Model API (https://api.meta.ai/v1) is OpenAI-compatible, so we reuse
the OpenAI Python SDK pointed at Meta's base URL and authenticate with
``META_MODEL_API_KEY``.
Output is base64 JSON (WebP) -> saved under ``$HERMES_HOME/cache/images/``.
Selection precedence (first hit wins):
1. ``model`` kwarg forwarded by the dispatcher (the ``hermes tools`` pick)
2. ``META_IMAGE_MODEL`` env var (escape hatch for scripts / tests)
3. ``image_gen.meta-ai.model`` in ``config.yaml``
4. ``image_gen.model`` in ``config.yaml`` (when it's one of our IDs)
5. :data:`DEFAULT_MODEL`
"""
from __future__ import annotations
import logging
import os
from typing import Any, Dict, List, Optional, Tuple
from agent.secret_scope import get_secret
from agent.image_gen_provider import (
DEFAULT_ASPECT_RATIO,
ImageGenProvider,
error_response,
normalize_reference_images,
resolve_aspect_ratio,
save_b64_image,
save_url_image,
success_response,
)
logger = logging.getLogger(__name__)
DEFAULT_BASE_URL = "https://api.meta.ai/v1"
# Auth env vars, in priority order. Mirrors the bundled ``meta-ai`` chat
# provider (plugins/model-providers/meta-ai): MODEL_API_KEY is Meta's
# documented var; the rest are accepted aliases.
API_KEY_ENVS = ("MODEL_API_KEY", "META_API_KEY", "META_MODEL_API_KEY")
# Primary key shown in setup prompts / error messages.
API_KEY_ENV = "META_MODEL_API_KEY"
# Optional base-url override (same var the chat provider honors).
BASE_URL_ENV = "META_BASE_URL"
def _resolve_api_key() -> Optional[str]:
"""First non-empty auth env var, checked in priority order."""
for env in API_KEY_ENVS:
val = get_secret(env)
if val:
return val
return None
def _resolve_base_url() -> str:
return (os.environ.get(BASE_URL_ENV) or "").strip() or DEFAULT_BASE_URL
# ---------------------------------------------------------------------------
# Model catalog
# ---------------------------------------------------------------------------
# Catalog shown in `hermes tools` and matched against `image_gen.model`.
# The model id is sent verbatim to the Meta Model API (`/v1/images/generations`).
_MODELS: Dict[str, Dict[str, Any]] = {
"muse-image-1.0": {
"display": "Muse Image 1.0",
"speed": "~10s",
"strengths": "Meta Model API image generation",
"price": "$0.01/image",
},
}
DEFAULT_MODEL = "muse-image-1.0"
# aspect_ratio -> OpenAI-style size string
_SIZES: Dict[str, str] = {
"square": "1024x1024",
"landscape": "1536x1024",
"portrait": "1024x1536",
}
def _resolve_model(caller_model: Optional[str] = None) -> Tuple[str, Dict[str, Any]]:
"""Return (model_id, metadata) using the documented precedence chain.
``caller_model`` is the ``model`` kwarg the dispatcher forwards from the
top-level ``image_gen.model`` config key (what ``hermes tools`` writes).
It wins when it names one of our models, mirroring the xai/krea/openrouter
providers, so a user's picker choice is never silently dropped.
"""
if caller_model and caller_model in _MODELS:
return caller_model, _MODELS[caller_model]
env_model = os.environ.get("META_IMAGE_MODEL")
if env_model and env_model in _MODELS:
return env_model, _MODELS[env_model]
try:
from hermes_cli.config import load_config
cfg = load_config() or {}
ig = cfg.get("image_gen") or {}
scoped = (ig.get("meta-ai") or {}).get("model")
if scoped and scoped in _MODELS:
return scoped, _MODELS[scoped]
top = ig.get("model")
if top and top in _MODELS:
return top, _MODELS[top]
except Exception:
logger.debug("Could not read image_gen model from config", exc_info=True)
return DEFAULT_MODEL, _MODELS[DEFAULT_MODEL]
class MetaImageGenProvider(ImageGenProvider):
"""Meta Model API ``images.generate`` backend (muse-image)."""
@property
def name(self) -> str:
return "meta-ai"
@property
def display_name(self) -> str:
return "Meta Model API"
def is_available(self) -> bool:
if not _resolve_api_key():
return False
try:
import openai # noqa: F401
except ImportError:
return False
return True
def list_models(self) -> List[Dict[str, Any]]:
return [
{
"id": mid,
"display": m["display"],
"speed": m["speed"],
"strengths": m["strengths"],
"price": m["price"],
}
for mid, m in _MODELS.items()
]
def default_model(self) -> Optional[str]:
return DEFAULT_MODEL
def get_setup_schema(self) -> Dict[str, Any]:
return {
"name": "Meta Model API",
"badge": "paid",
"tag": "Muse Image via Meta Model API (api.meta.ai)",
"env_vars": [
{
"key": API_KEY_ENV,
"prompt": "Meta Model API key (LLM|... token)",
"url": "https://api.meta.ai",
},
],
}
def capabilities(self) -> Dict[str, Any]:
# Text-to-image only for now. Bump this once image-to-image is verified
# against the Meta endpoint.
return {"modalities": ["text"], "max_reference_images": 0}
def generate(
self,
prompt: str,
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
*,
image_url: Optional[str] = None,
reference_image_urls: Optional[List[str]] = None,
**kwargs: Any,
) -> Dict[str, Any]:
prompt = (prompt or "").strip()
aspect = resolve_aspect_ratio(aspect_ratio)
if not prompt:
return error_response(
error="Prompt is required and must be a non-empty string",
error_type="invalid_argument",
provider="meta-ai",
aspect_ratio=aspect,
)
api_key = _resolve_api_key()
if not api_key:
return error_response(
error=(
f"{API_KEY_ENV} not set. Run `hermes tools` -> Image "
"Generation -> Meta Model API to configure."
),
error_type="auth_required",
provider="meta-ai",
aspect_ratio=aspect,
)
try:
import openai
except ImportError:
return error_response(
error="openai Python package not installed (pip install openai)",
error_type="missing_dependency",
provider="meta-ai",
aspect_ratio=aspect,
)
model_id, _meta = _resolve_model(kwargs.get("model"))
size = _SIZES.get(aspect, _SIZES["square"])
client = openai.OpenAI(api_key=api_key, base_url=_resolve_base_url())
payload: Dict[str, Any] = {
"model": model_id,
"prompt": prompt,
"size": size,
"n": 1,
}
try:
response = client.images.generate(**payload)
except Exception as exc:
logger.debug("Meta image generation failed", exc_info=True)
return error_response(
error=f"Meta image generation failed: {exc}",
error_type="api_error",
provider="meta-ai",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
try:
first = response.data[0]
except (AttributeError, IndexError, TypeError):
return error_response(
error="Meta response contained no image data",
error_type="empty_response",
provider="meta-ai",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
b64 = getattr(first, "b64_json", None)
url = getattr(first, "url", None)
try:
if b64:
path = save_b64_image(b64, prefix="meta", extension="webp")
image_ref = str(path)
elif url:
path = save_url_image(url, prefix="meta")
image_ref = str(path)
else:
return error_response(
error="Meta response contained neither b64_json nor URL",
error_type="empty_response",
provider="meta-ai",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
except Exception as exc:
return error_response(
error=f"Failed to save Meta image: {exc}",
error_type="io_error",
provider="meta-ai",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
revised_prompt = getattr(first, "revised_prompt", None)
extra: Dict[str, Any] = {"size": size}
if revised_prompt:
extra["revised_prompt"] = revised_prompt
return success_response(
image=image_ref,
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
provider="meta-ai",
modality="text",
extra=extra,
)
def register(ctx) -> None:
"""Plugin entry point -- wire ``MetaImageGenProvider`` into the registry."""
ctx.register_image_gen_provider(MetaImageGenProvider())
+7
View File
@@ -0,0 +1,7 @@
name: meta-ai-image-gen
version: 1.0.0
description: "Meta Model API image generation backend (muse-image). OpenAI-compatible /v1/images/generations. Saves images to $HERMES_HOME/cache/images/."
author: Meta Platforms, Inc.
kind: backend
requires_env:
- META_MODEL_API_KEY
+770
View File
@@ -0,0 +1,770 @@
"""OpenAI image generation backend — ChatGPT/Codex OAuth variant.
Identical model catalog and tier semantics to the ``openai`` image-gen plugin
(``gpt-image-2`` at low/medium/high quality), but routes the request through
the Codex Responses API ``image_generation`` tool instead of the
``images.generate`` REST endpoint. This lets users who are already
authenticated with Codex/ChatGPT generate images without configuring a
separate ``OPENAI_API_KEY``.
Selection precedence for the tier (first hit wins):
1. ``OPENAI_IMAGE_MODEL`` env var (escape hatch for scripts / tests)
2. ``image_gen.openai-codex.model`` in ``config.yaml``
3. ``image_gen.model`` in ``config.yaml`` (when it's one of our tier IDs)
4. :data:`DEFAULT_MODEL` — ``gpt-image-2-medium``
Output is saved as PNG under ``$HERMES_HOME/cache/images/``. Source images for
image-to-image/editing are sent as Responses ``input_image`` content parts.
"""
from __future__ import annotations
import base64
import json
import logging
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from agent.image_gen_provider import (
DEFAULT_ASPECT_RATIO,
ImageGenProvider,
error_response,
normalize_reference_images,
resolve_aspect_ratio,
save_b64_image,
success_response,
)
logger = logging.getLogger(__name__)
# NOTE: do NOT reintroduce an "account capability" classifier keyed on
# ``Tool choice 'image_generation' not found in 'tools' parameter``. That HTTP
# 400 is a *request-shape* rejection (the Codex backend resolves tool_choice as
# a function-tool name and never recognizes hosted-tool entries) — it is
# emitted for every account, including accounts where image generation works.
# A previous version of this file translated that 400 into "Image generation is
# not enabled for the current Codex account. Switch the image provider to
# OpenAI API key, FAL, or xAI.", which reported a universal bug in our own
# payload as the user's entitlement problem and sent people away from a
# provider that was never actually tried. The request-shape bug is fixed by
# omitting tool_choice (see ``_build_responses_payload``); any remaining HTTP
# error must surface verbatim so it stays diagnosable. See issues #19505,
# #49008 and #31335.
_MAX_ERROR_BODY_CHARS = 500
def _summarize_error_body(body: str) -> str:
"""Return a bounded, information-preserving summary of an error body.
Prefers the parsed ``error.message`` field, because a blind head-truncation
of the raw body can cut the actual message off entirely — Codex error
payloads sometimes carry hundreds of bytes of leading metadata, so
``body[:500]`` yielded a wall of padding and no diagnosis. Falls back to a
truncated raw body for non-JSON responses.
"""
text = body or ""
try:
payload = json.loads(text)
error = payload.get("error") if isinstance(payload, dict) else None
message = error.get("message") if isinstance(error, dict) else None
if isinstance(message, str) and message.strip():
return message.strip()[:_MAX_ERROR_BODY_CHARS]
except (TypeError, ValueError):
pass
return text[:_MAX_ERROR_BODY_CHARS]
# ---------------------------------------------------------------------------
# Model catalog — mirrors the ``openai`` plugin so the picker UX is identical.
# ---------------------------------------------------------------------------
API_MODEL = "gpt-image-2"
_MODELS: Dict[str, Dict[str, Any]] = {
"gpt-image-2-low": {
"display": "GPT Image 2 (Low)",
"speed": "~15s",
"strengths": "Fast iteration, lowest cost",
"quality": "low",
},
"gpt-image-2-medium": {
"display": "GPT Image 2 (Medium)",
"speed": "~40s",
"strengths": "Balanced — default",
"quality": "medium",
},
"gpt-image-2-high": {
"display": "GPT Image 2 (High)",
"speed": "~2min",
"strengths": "Highest fidelity, strongest prompt adherence",
"quality": "high",
},
}
DEFAULT_MODEL = "gpt-image-2-medium"
_SIZES = {
"landscape": "1536x1024",
"square": "1024x1024",
"portrait": "1024x1536",
}
# Codex Responses surface used for the request. The chat model itself is only
# the host that calls the ``image_generation`` tool; the actual image work is
# done by ``API_MODEL``.
_CODEX_CHAT_MODEL = "gpt-5.5"
_CODEX_BASE_URL = "https://chatgpt.com/backend-api/codex"
_CODEX_INSTRUCTIONS = (
"You are an assistant that must fulfill image generation and image editing "
"requests by using the image_generation tool when provided."
)
_MAX_REFERENCE_IMAGES = 16
_MAX_INPUT_IMAGE_BYTES = 25 * 1024 * 1024
# gpt-image-2's Responses ``input_image`` accepts raster formats only. The
# shared magic-byte sniffer also recognizes SVG/TIFF/ICO, which the API
# rejects server-side — gate to this allowlist so unsupported inputs fail
# locally with a clear error instead of an opaque HTTP 400.
_ACCEPTED_INPUT_MIME = frozenset(
{"image/png", "image/jpeg", "image/gif", "image/webp"}
)
# ---------------------------------------------------------------------------
# Config + auth helpers
# ---------------------------------------------------------------------------
def _load_image_gen_config() -> Dict[str, Any]:
"""Read ``image_gen`` from config.yaml (returns {} on any failure)."""
try:
from hermes_cli.config import load_config
cfg = load_config()
section = cfg.get("image_gen") if isinstance(cfg, dict) else None
return section if isinstance(section, dict) else {}
except Exception as exc:
logger.debug("Could not load image_gen config: %s", exc)
return {}
def _resolve_model() -> Tuple[str, Dict[str, Any]]:
"""Decide which tier to use and return ``(model_id, meta)``."""
import os
env_override = os.environ.get("OPENAI_IMAGE_MODEL")
if env_override and env_override in _MODELS:
return env_override, _MODELS[env_override]
cfg = _load_image_gen_config()
sub = cfg.get("openai-codex") if isinstance(cfg.get("openai-codex"), dict) else {}
candidate: Optional[str] = None
if isinstance(sub, dict):
value = sub.get("model")
if isinstance(value, str) and value in _MODELS:
candidate = value
if candidate is None:
top = cfg.get("model")
if isinstance(top, str) and top in _MODELS:
candidate = top
if candidate is not None:
return candidate, _MODELS[candidate]
return DEFAULT_MODEL, _MODELS[DEFAULT_MODEL]
def _read_codex_access_token() -> Optional[str]:
"""Return a usable Codex OAuth token, or None.
Delegates to the canonical reader in ``agent.auxiliary_client`` so token
expiry, credential pool selection, and JWT decoding stay in one place.
"""
try:
from agent.auxiliary_client import _read_codex_access_token as _reader
token = _reader()
if isinstance(token, str) and token.strip():
return token.strip()
return None
except Exception as exc:
logger.debug("Could not resolve Codex access token: %s", exc)
return None
def _sniff_image_mime(raw: bytes) -> Optional[str]:
"""Return a safe raster image MIME from magic bytes (not filename labels).
Delegates magic-byte detection to the shared sniffer in
``agent.image_routing`` (single source of truth), then gates the result
to :data:`_ACCEPTED_INPUT_MIME` — the raster formats gpt-image-2's
``input_image`` actually accepts. SVG/TIFF/ICO (which the shared sniffer
also recognizes) are rejected here so they fail locally with a clear
error instead of an opaque server-side HTTP 400.
"""
from agent.image_routing import _sniff_mime_from_bytes
mime = _sniff_mime_from_bytes(raw)
if mime in _ACCEPTED_INPUT_MIME:
return mime
return None
def _data_url_to_input_image_url(value: str) -> str:
"""Validate and canonicalize a data:image URL for Responses input_image."""
if "," not in value:
raise ValueError("Image data URL is missing a comma separator")
header, data = value.split(",", 1)
header_lc = header.lower()
if not header_lc.startswith("data:image/") or ";base64" not in header_lc:
raise ValueError("Only base64 data:image URLs are supported as Codex image inputs")
raw = base64.b64decode(data, validate=True)
if len(raw) > _MAX_INPUT_IMAGE_BYTES:
raise ValueError("Image data URL exceeds 25MB cap")
mime = _sniff_image_mime(raw)
if mime is None:
raise ValueError("Image data URL does not contain supported image bytes")
encoded = base64.b64encode(raw).decode("ascii")
return f"data:{mime};base64,{encoded}"
def _local_image_to_data_url(value: str) -> str:
"""Read a local image path and return a validated data:image URL."""
try:
from agent.file_safety import get_read_block_error
blocked = get_read_block_error(value)
if blocked:
raise ValueError(blocked)
except ValueError:
raise
except Exception as exc:
logger.debug("Codex image input read guard unavailable: %s", exc)
path = Path(os.path.expanduser(value)).resolve()
if not path.is_file():
raise ValueError(f"Image input path does not exist or is not a file: {value}")
size = path.stat().st_size
if size <= 0:
raise ValueError(f"Image input path is empty: {value}")
if size > _MAX_INPUT_IMAGE_BYTES:
raise ValueError(f"Image input path exceeds 25MB cap: {value}")
raw = path.read_bytes()
mime = _sniff_image_mime(raw)
if mime is None:
raise ValueError(f"Image input path is not a supported image: {value}")
encoded = base64.b64encode(raw).decode("ascii")
return f"data:{mime};base64,{encoded}"
def _to_input_image_part(value: str) -> Dict[str, str]:
"""Convert a URL/data URL/local path into a Responses input_image part."""
candidate = (value or "").strip()
if not candidate:
raise ValueError("Blank image input")
lowered = candidate.lower()
if lowered.startswith("http://") or lowered.startswith("https://"):
image_url = candidate
elif lowered.startswith("data:"):
image_url = _data_url_to_input_image_url(candidate)
else:
image_url = _local_image_to_data_url(candidate)
return {"type": "input_image", "image_url": image_url}
def _normalize_input_images(
image_url: Optional[str],
reference_image_urls: Optional[List[str]],
) -> List[Dict[str, str]]:
"""Collect primary + reference images as ordered Responses content parts."""
values: List[str] = []
if isinstance(image_url, str) and image_url.strip():
values.append(image_url.strip())
for ref in (normalize_reference_images(reference_image_urls) or []):
values.append(ref)
values = values[:_MAX_REFERENCE_IMAGES]
return [_to_input_image_part(value) for value in values]
# Progressive preview frames (partial_image_b64) are intermediate renders.
# Saving them as finals produced the long-running "smear" failure mode on the
# Codex Responses path. Defense in depth:
# 1) request layer prefers no progressive frames when the backend honors it
# 2) extractor never lets a partial overwrite a final result
# 3) generate() only delivers source=final; partial-only / empty are not success
# Live streams sometimes still emit a partial event even with 0; that is fine as
# long as only a final ``result`` can be saved.
_PARTIAL_IMAGES_REQUESTED = 0
# Content-agnostic retries when the stream does not yield a final result
# (empty stream or progressive-only). No prompt-class branching.
_NONFINAL_RETRIES = 1
def _build_responses_payload(
*,
prompt: str,
size: str,
quality: str,
input_images: Optional[List[Dict[str, str]]] = None,
) -> Dict[str, Any]:
"""Build the Codex Responses request body for an image_generation call."""
content: List[Dict[str, Any]] = [{"type": "input_text", "text": prompt}]
if input_images:
content.extend(input_images)
return {
"model": _CODEX_CHAT_MODEL,
"store": False,
"instructions": _CODEX_INSTRUCTIONS,
"input": [{
"type": "message",
"role": "user",
"content": content,
}],
"tools": [{
"type": "image_generation",
"model": API_MODEL,
"size": size,
"quality": quality,
"output_format": "png",
"background": "opaque",
# Prefer 0 progressive preview frames. Preview frames can arrive
# without a later final ``result`` and look like smeared /
# unfinished images if saved as the deliverable. Even when the
# backend still emits a partial event, generate() refuses to
# deliver anything except source=final.
"partial_images": _PARTIAL_IMAGES_REQUESTED,
}],
# No ``tool_choice`` is sent: the chatgpt.com/backend-api/codex backend
# rejects every shape we have for forcing the hosted ``image_generation``
# tool. ``{"type": "allowed_tools", "mode": "required", "tools": [{"type":
# "image_generation"}]}`` (and the simpler ``{"type": "image_generation"}``
# form) both 400 with ``Tool choice 'image_generation' not found in 'tools'
# parameter`` — the backend looks up tool_choice as a *function* name and
# never recognizes hosted-tool entries. Letting the host model decide is
# the only shape Codex currently accepts; the ``instructions`` above are
# what nudge it toward the tool. See issue #19505.
"stream": True,
}
def _extract_image_candidates(value: Any) -> Tuple[Optional[str], Optional[str]]:
"""Return ``(final_result_b64, latest_partial_b64)`` from a payload tree.
Final ``image_generation_call.result`` and progressive ``partial_image_b64``
are tracked separately so a partial can never overwrite a genuine final,
including when both coexist in the same event payload.
"""
result_b64: Optional[str] = None
partial_b64: Optional[str] = None
def walk(node: Any) -> None:
nonlocal result_b64, partial_b64
if isinstance(node, dict):
if node.get("type") == "image_generation_call":
result = node.get("result")
if isinstance(result, str) and result:
result_b64 = result
partial = node.get("partial_image_b64")
if isinstance(partial, str) and partial:
partial_b64 = partial
for child in node.values():
walk(child)
elif isinstance(node, list):
for child in node:
walk(child)
walk(value)
return result_b64, partial_b64
def _extract_image_b64(value: Any) -> Optional[str]:
"""Return image b64 from a payload, preferring final result over partial.
Progressive ``partial_image_b64`` is only used when no final
``image_generation_call.result`` is present in the same payload tree.
"""
result_b64, partial_b64 = _extract_image_candidates(value)
return result_b64 or partial_b64
def _png_pixel_size(raw: bytes) -> Optional[str]:
"""Return ``\"{w}x{h}\"`` for a PNG payload, or None if not a PNG IHDR."""
import struct
if len(raw) < 24 or raw[:8] != b"\x89PNG\r\n\x1a\n":
return None
# IHDR: length(4) + type(4) + width(4) + height(4)
if raw[12:16] != b"IHDR":
return None
width, height = struct.unpack(">II", raw[16:24])
return f"{width}x{height}"
def _iter_sse_json(response: Any):
"""Yield JSON payloads from an SSE response without OpenAI SDK parsing.
The ChatGPT/Codex backend can emit image-generation events newer than the
pinned Python SDK understands. Parsing raw SSE keeps this provider tolerant
of those event-shape changes.
"""
event_name: Optional[str] = None
data_lines: List[str] = []
def flush():
nonlocal event_name, data_lines
if not data_lines:
event_name = None
return None
raw = "\n".join(data_lines).strip()
event = event_name
event_name = None
data_lines = []
if not raw or raw == "[DONE]":
return None
payload = json.loads(raw)
if isinstance(payload, dict) and event and "type" not in payload:
payload["type"] = event
return payload
for line in response.iter_lines():
if isinstance(line, bytes):
line = line.decode("utf-8", errors="replace")
line = str(line)
if line == "":
payload = flush()
if payload is not None:
yield payload
continue
if line.startswith(":"):
continue
if line.startswith("event:"):
event_name = line[len("event:"):].strip()
elif line.startswith("data:"):
data_lines.append(line[len("data:"):].lstrip())
payload = flush()
if payload is not None:
yield payload
def _collect_image_b64(
token: str,
*,
prompt: str,
size: str,
quality: str,
input_images: Optional[List[Dict[str, str]]] = None,
) -> Optional[Dict[str, str]]:
"""Stream a Codex Responses image_generation call.
Returns ``{\"b64\": ..., \"source\": \"final\"|\"partial\"}`` or ``None``.
Final ``result`` frames are preferred across the whole stream. A progressive
``partial_image_b64`` is retained only when no final result ever arrives;
callers must not treat partial-only as an unconditional success.
"""
import httpx
from agent.codex_headers import codex_cloudflare_headers
headers = codex_cloudflare_headers(token)
headers.update({
"Accept": "text/event-stream",
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
})
payload = _build_responses_payload(
prompt=prompt,
size=size,
quality=quality,
input_images=input_images,
)
timeout = httpx.Timeout(300.0, connect=30.0, read=300.0, write=30.0, pool=30.0)
final_b64: Optional[str] = None
partial_b64: Optional[str] = None
with httpx.Client(timeout=timeout, headers=headers) as http:
with http.stream("POST", f"{_CODEX_BASE_URL}/responses", json=payload) as response:
try:
response.raise_for_status()
except httpx.HTTPStatusError as exc:
exc.response.read()
raise RuntimeError(
f"Codex Responses API returned HTTP {exc.response.status_code}: "
f"{_summarize_error_body(exc.response.text)}"
) from exc
for event in _iter_sse_json(response):
result_b64, event_partial = _extract_image_candidates(event)
if result_b64:
final_b64 = result_b64
if event_partial:
partial_b64 = event_partial
if final_b64:
return {"b64": final_b64, "source": "final"}
if partial_b64:
return {"b64": partial_b64, "source": "partial"}
return None
# ---------------------------------------------------------------------------
# Provider
# ---------------------------------------------------------------------------
class OpenAICodexImageGenProvider(ImageGenProvider):
"""gpt-image-2 routed through ChatGPT/Codex OAuth instead of an API key."""
@property
def name(self) -> str:
return "openai-codex"
@property
def display_name(self) -> str:
return "OpenAI (Codex auth)"
def is_available(self) -> bool:
if not _read_codex_access_token():
return False
try:
import httpx # noqa: F401
except ImportError:
return False
return True
def list_models(self) -> List[Dict[str, Any]]:
return [
{
"id": model_id,
"display": meta["display"],
"speed": meta["speed"],
"strengths": meta["strengths"],
"price": "varies",
}
for model_id, meta in _MODELS.items()
]
def default_model(self) -> Optional[str]:
return DEFAULT_MODEL
def get_setup_schema(self) -> Dict[str, Any]:
return {
"name": "OpenAI (Codex auth)",
"badge": "free",
"tag": "gpt-image-2 via ChatGPT/Codex OAuth — no API key required; supports text and image inputs",
"env_vars": [],
"post_setup_hint": (
"Sign in with `hermes auth codex` (or `hermes setup` → Codex) "
"if you haven't already. No API key needed."
),
}
def capabilities(self) -> Dict[str, Any]:
# The Codex Responses image_generation tool accepts source/reference
# images as `input_image` message content parts. Keep this capability
# honest so the dynamic `image_generate` schema encourages identity-
# preserving edits instead of unrelated text-to-image redraws.
return {"modalities": ["text", "image"], "max_reference_images": _MAX_REFERENCE_IMAGES}
def generate(
self,
prompt: str,
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
*,
image_url: Optional[str] = None,
reference_image_urls: Optional[List[str]] = None,
**kwargs: Any,
) -> Dict[str, Any]:
prompt = (prompt or "").strip()
aspect = resolve_aspect_ratio(aspect_ratio)
if not prompt:
return error_response(
error="Prompt is required and must be a non-empty string",
error_type="invalid_argument",
provider="openai-codex",
aspect_ratio=aspect,
)
if not _read_codex_access_token():
return error_response(
error=(
"No Codex/ChatGPT OAuth credentials available. Run "
"`hermes auth codex` (or `hermes setup` → Codex) to sign in."
),
error_type="auth_required",
provider="openai-codex",
aspect_ratio=aspect,
)
try:
import httpx # noqa: F401
except ImportError:
return error_response(
error="httpx Python package not installed (pip install httpx)",
error_type="missing_dependency",
provider="openai-codex",
aspect_ratio=aspect,
)
tier_id, meta = _resolve_model()
size = _SIZES.get(aspect, _SIZES["square"])
token = _read_codex_access_token()
if not token:
return error_response(
error=(
"No Codex/ChatGPT OAuth credentials available. Run "
"`hermes auth codex` (or `hermes setup` → Codex) to sign in."
),
error_type="auth_required",
provider="openai-codex",
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
)
try:
input_images = _normalize_input_images(image_url, reference_image_urls)
except Exception as exc:
return error_response(
error=f"Invalid image input for Codex image editing: {exc}",
error_type="invalid_image_input",
provider="openai-codex",
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
)
try:
collected: Optional[Dict[str, str]] = None
for attempt in range(_NONFINAL_RETRIES + 1):
collected = _collect_image_b64(
token,
prompt=prompt,
size=size,
quality=meta["quality"],
input_images=input_images or None,
)
if collected and collected.get("source") == "final" and collected.get("b64"):
break
if attempt < _NONFINAL_RETRIES:
kind = (
"progressive-only partial frame"
if collected and collected.get("source") == "partial"
else "no image_generation_call result"
)
logger.warning(
"Codex image stream ended with %s (attempt %s/%s); "
"retrying once before failing closed.",
kind,
attempt + 1,
_NONFINAL_RETRIES + 1,
)
continue
break
except Exception as exc:
logger.debug("Codex image generation failed", exc_info=True)
return error_response(
error=f"OpenAI image generation via Codex auth failed: {exc}",
error_type="api_error",
provider="openai-codex",
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
)
if not collected or not collected.get("b64"):
return error_response(
error=(
"Codex response contained no image_generation_call result "
f"after {_NONFINAL_RETRIES + 1} attempt(s)"
),
error_type="empty_response",
provider="openai-codex",
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
)
image_source = collected.get("source") or "unknown"
b64 = collected["b64"]
# Defense in depth: never deliver a progressive-only frame as success.
# Partials are intermediate previews and have presented as smeared /
# unfinished images when saved as finals.
if image_source != "final":
pixel_hint = None
try:
import base64 as _b64mod
pixel_hint = _png_pixel_size(_b64mod.b64decode(b64, validate=False))
except Exception:
pixel_hint = None
detail = (
"Codex returned only a progressive partial image frame after "
f"{_NONFINAL_RETRIES + 1} attempt(s); refusing to save it "
"as a final deliverable."
)
if pixel_hint:
detail = f"{detail} partial_pixel_size={pixel_hint}."
err = error_response(
error=detail,
error_type="incomplete_image",
provider="openai-codex",
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
)
err["image_source"] = image_source
err["requested_size"] = size
err["partial_pixel_size"] = pixel_hint
err["nonfinal_retries"] = _NONFINAL_RETRIES
return err
try:
import base64 as _b64mod
raw_bytes = _b64mod.b64decode(b64)
pixel_size = _png_pixel_size(raw_bytes)
saved_path = save_b64_image(b64, prefix=f"openai_codex_{tier_id}")
except Exception as exc:
return error_response(
error=f"Could not save image to cache: {exc}",
error_type="io_error",
provider="openai-codex",
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
)
return success_response(
image=str(saved_path),
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
provider="openai-codex",
modality="image" if input_images else "text",
extra={
"size": size,
"quality": meta["quality"],
"input_image_count": len(input_images),
"image_source": image_source,
"requested_size": size,
"pixel_size": pixel_size,
},
)
# ---------------------------------------------------------------------------
# Plugin entry point
# ---------------------------------------------------------------------------
def register(ctx) -> None:
"""Plugin entry point — register the Codex-backed image-gen provider."""
ctx.register_image_gen_provider(OpenAICodexImageGenProvider())
@@ -0,0 +1,5 @@
name: openai-codex
version: 1.0.0
description: "OpenAI image generation backed by ChatGPT/Codex OAuth (gpt-image-2 via the Responses image_generation tool). Saves generated images to $HERMES_HOME/cache/images/."
author: NousResearch
kind: backend
+419
View File
@@ -0,0 +1,419 @@
"""OpenAI image generation backend.
Exposes OpenAI's ``gpt-image-2`` model at three quality tiers as an
:class:`ImageGenProvider` implementation. The tiers are implemented as
three virtual model IDs so the ``hermes tools`` model picker and the
``image_gen.model`` config key behave like any other multi-model backend:
gpt-image-2-low ~15s fastest, good for iteration
gpt-image-2-medium ~40s default — balanced
gpt-image-2-high ~2min slowest, highest fidelity
All three hit the same underlying API model (``gpt-image-2``) with a
different ``quality`` parameter. Output is base64 JSON → saved under
``$HERMES_HOME/cache/images/``.
Selection precedence (first hit wins):
1. ``OPENAI_IMAGE_MODEL`` env var (escape hatch for scripts / tests)
2. ``image_gen.openai.model`` in ``config.yaml``
3. ``image_gen.model`` in ``config.yaml`` (when it's one of our tier IDs)
4. :data:`DEFAULT_MODEL` — ``gpt-image-2-medium``
"""
from __future__ import annotations
import logging
import os
from typing import Any, Dict, List, Optional, Tuple
from agent.secret_scope import get_secret
from agent.image_gen_provider import (
DEFAULT_ASPECT_RATIO,
ImageGenProvider,
error_response,
normalize_reference_images,
resolve_aspect_ratio,
save_b64_image,
save_url_image,
success_response,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Model catalog
# ---------------------------------------------------------------------------
#
# All three IDs resolve to the same underlying API model with a different
# ``quality`` setting. ``api_model`` is what gets sent to OpenAI;
# ``quality`` is the knob that changes generation time and output fidelity.
API_MODEL = "gpt-image-2"
_MODELS: Dict[str, Dict[str, Any]] = {
"gpt-image-2-low": {
"display": "GPT Image 2 (Low)",
"speed": "~15s",
"strengths": "Fast iteration, lowest cost",
"quality": "low",
},
"gpt-image-2-medium": {
"display": "GPT Image 2 (Medium)",
"speed": "~40s",
"strengths": "Balanced — default",
"quality": "medium",
},
"gpt-image-2-high": {
"display": "GPT Image 2 (High)",
"speed": "~2min",
"strengths": "Highest fidelity, strongest prompt adherence",
"quality": "high",
},
}
DEFAULT_MODEL = "gpt-image-2-medium"
_SIZES = {
"landscape": "1536x1024",
"square": "1024x1024",
"portrait": "1024x1536",
}
def _load_openai_config() -> Dict[str, Any]:
"""Read ``image_gen`` from config.yaml (returns {} on any failure)."""
try:
from hermes_cli.config import load_config
cfg = load_config()
section = cfg.get("image_gen") if isinstance(cfg, dict) else None
return section if isinstance(section, dict) else {}
except Exception as exc:
logger.debug("Could not load image_gen config: %s", exc)
return {}
def _resolve_model() -> Tuple[str, Dict[str, Any]]:
"""Decide which tier to use and return ``(model_id, meta)``."""
env_override = os.environ.get("OPENAI_IMAGE_MODEL")
if env_override and env_override in _MODELS:
return env_override, _MODELS[env_override]
cfg = _load_openai_config()
openai_cfg = cfg.get("openai") if isinstance(cfg.get("openai"), dict) else {}
candidate: Optional[str] = None
if isinstance(openai_cfg, dict):
value = openai_cfg.get("model")
if isinstance(value, str) and value in _MODELS:
candidate = value
if candidate is None:
top = cfg.get("model")
if isinstance(top, str) and top in _MODELS:
candidate = top
if candidate is not None:
return candidate, _MODELS[candidate]
return DEFAULT_MODEL, _MODELS[DEFAULT_MODEL]
# ---------------------------------------------------------------------------
# Source-image loading (for image-to-image / edit)
# ---------------------------------------------------------------------------
def _load_image_bytes(ref: str) -> Tuple[bytes, str]:
"""Load image bytes from a URL or local file path.
Returns ``(data, filename)``. Raises on any network / IO error so the
caller can surface a clean error_response.
"""
ref = ref.strip()
lower = ref.lower()
if lower.startswith(("http://", "https://")):
import requests
resp = requests.get(ref, timeout=60)
resp.raise_for_status()
name = ref.split("?", 1)[0].rsplit("/", 1)[-1] or "image.png"
return resp.content, name
if lower.startswith("data:"):
import base64
header, _, b64 = ref.partition(",")
ext = "png"
if "image/" in header:
ext = header.split("image/", 1)[1].split(";", 1)[0] or "png"
return base64.b64decode(b64), f"image.{ext}"
# Local file path — enforce the shared credential-read guard before reading.
from agent.file_safety import raise_if_read_blocked
raise_if_read_blocked(ref)
with open(ref, "rb") as fh:
data = fh.read()
name = os.path.basename(ref) or "image.png"
return data, name
# ---------------------------------------------------------------------------
# Provider
# ---------------------------------------------------------------------------
class OpenAIImageGenProvider(ImageGenProvider):
"""OpenAI ``images.generate`` / ``images.edit`` backend — gpt-image-2."""
@property
def name(self) -> str:
return "openai"
@property
def display_name(self) -> str:
return "OpenAI"
def is_available(self) -> bool:
if not get_secret("OPENAI_API_KEY"):
return False
try:
import openai # noqa: F401
except ImportError:
return False
return True
def list_models(self) -> List[Dict[str, Any]]:
return [
{
"id": model_id,
"display": meta["display"],
"speed": meta["speed"],
"strengths": meta["strengths"],
"price": "varies",
}
for model_id, meta in _MODELS.items()
]
def default_model(self) -> Optional[str]:
return DEFAULT_MODEL
def get_setup_schema(self) -> Dict[str, Any]:
return {
"name": "OpenAI",
"badge": "paid",
"tag": "gpt-image-2 at low/medium/high quality tiers — text-to-image & image editing",
"env_vars": [
{
"key": "OPENAI_API_KEY",
"prompt": "OpenAI API key",
"url": "https://platform.openai.com/api-keys",
},
],
}
def capabilities(self) -> Dict[str, Any]:
# gpt-image-2 supports editing via images.edit() with up to 16 source
# images.
return {"modalities": ["text", "image"], "max_reference_images": 16}
def generate(
self,
prompt: str,
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
*,
image_url: Optional[str] = None,
reference_image_urls: Optional[List[str]] = None,
**kwargs: Any,
) -> Dict[str, Any]:
prompt = (prompt or "").strip()
aspect = resolve_aspect_ratio(aspect_ratio)
if not prompt:
return error_response(
error="Prompt is required and must be a non-empty string",
error_type="invalid_argument",
provider="openai",
aspect_ratio=aspect,
)
api_key = get_secret("OPENAI_API_KEY")
if not api_key:
return error_response(
error=(
"OPENAI_API_KEY not set. Run `hermes tools` → Image "
"Generation → OpenAI to configure, or `hermes setup` "
"to add the key."
),
error_type="auth_required",
provider="openai",
aspect_ratio=aspect,
)
try:
import openai
except ImportError:
return error_response(
error="openai Python package not installed (pip install openai)",
error_type="missing_dependency",
provider="openai",
aspect_ratio=aspect,
)
tier_id, meta = _resolve_model()
size = _SIZES.get(aspect, _SIZES["square"])
# Collect source images (primary + references) for image-to-image.
sources: List[str] = []
if isinstance(image_url, str) and image_url.strip():
sources.append(image_url.strip())
for ref in (normalize_reference_images(reference_image_urls) or []):
sources.append(ref)
sources = sources[:16] # gpt-image-2 edit caps at 16 images
is_edit = bool(sources)
modality = "image" if is_edit else "text"
client = openai.OpenAI(api_key=api_key)
if is_edit:
# images.edit() expects file-like objects. Download/read each
# source into a named BytesIO so the SDK sends correct multipart.
import io
try:
files = []
for ref in sources:
data, fname = _load_image_bytes(ref)
bio = io.BytesIO(data)
bio.name = fname
files.append(bio)
except Exception as exc:
return error_response(
error=f"Could not load source image for editing: {exc}",
error_type="io_error",
provider="openai",
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
)
try:
response = client.images.edit(
model=API_MODEL,
image=files if len(files) > 1 else files[0],
prompt=prompt,
size=size, # type: ignore[arg-type] # _SIZES values are valid gpt-image sizes
quality=meta["quality"],
n=1,
)
except Exception as exc:
logger.debug("OpenAI image edit failed", exc_info=True)
return error_response(
error=f"OpenAI image editing failed: {exc}",
error_type="api_error",
provider="openai",
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
)
else:
# gpt-image-2 returns b64_json unconditionally and REJECTS
# ``response_format`` as an unknown parameter. Don't send it.
payload: Dict[str, Any] = {
"model": API_MODEL,
"prompt": prompt,
"size": size,
"n": 1,
"quality": meta["quality"],
}
try:
response = client.images.generate(**payload)
except Exception as exc:
logger.debug("OpenAI image generation failed", exc_info=True)
return error_response(
error=f"OpenAI image generation failed: {exc}",
error_type="api_error",
provider="openai",
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
)
data = getattr(response, "data", None) or []
if not data:
return error_response(
error="OpenAI returned no image data",
error_type="empty_response",
provider="openai",
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
)
first = data[0]
b64 = getattr(first, "b64_json", None)
url = getattr(first, "url", None)
revised_prompt = getattr(first, "revised_prompt", None)
if b64:
try:
saved_path = save_b64_image(b64, prefix=f"openai_{tier_id}")
except Exception as exc:
return error_response(
error=f"Could not save image to cache: {exc}",
error_type="io_error",
provider="openai",
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
)
image_ref = str(saved_path)
elif url:
# Defensive — gpt-image-2 returns b64 today, but OpenAI's API
# has previously returned URLs. Cache the bytes locally so the
# gateway never tries to fetch an ephemeral / signed URL after
# it expires — same rationale as the xAI provider (#26942).
try:
saved_path = save_url_image(url, prefix=f"openai_{tier_id}")
except Exception as exc:
logger.warning(
"OpenAI image URL %s could not be cached (%s); falling back to bare URL.",
url,
exc,
)
image_ref = url
else:
image_ref = str(saved_path)
else:
return error_response(
error="OpenAI response contained neither b64_json nor URL",
error_type="empty_response",
provider="openai",
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
)
extra: Dict[str, Any] = {"size": size, "quality": meta["quality"]}
if revised_prompt:
extra["revised_prompt"] = revised_prompt
return success_response(
image=image_ref,
model=tier_id,
prompt=prompt,
aspect_ratio=aspect,
provider="openai",
modality=modality,
extra=extra,
)
# ---------------------------------------------------------------------------
# Plugin entry point
# ---------------------------------------------------------------------------
def register(ctx) -> None:
"""Plugin entry point — wire ``OpenAIImageGenProvider`` into the registry."""
ctx.register_image_gen_provider(OpenAIImageGenProvider())
+7
View File
@@ -0,0 +1,7 @@
name: openai
version: 1.0.0
description: "OpenAI image generation backend (gpt-image-2). Saves generated images to $HERMES_HOME/cache/images/."
author: NousResearch
kind: backend
requires_env:
- OPENAI_API_KEY
File diff suppressed because it is too large Load Diff
+7
View File
@@ -0,0 +1,7 @@
name: openrouter
version: 1.1.0
description: "OpenRouter + Nous Portal image generation. Chat-completions image output (reference-grounded) plus OpenRouter's Dedicated Image API (/images/generations) for gpt-image-2, Krea 2, Qwen Image 3 Pro, MAI-Image-2.5 and Grok Imagine — exact per-model aspect ratios, resolution/quality/background/seed/n, up to 16 reference images. Text-to-image and image-to-image."
author: Hermes Agent
kind: backend
requires_env:
- OPENROUTER_API_KEY
+625
View File
@@ -0,0 +1,625 @@
"""xAI image generation backend.
Exposes xAI's ``grok-imagine-image`` model as an
:class:`ImageGenProvider` implementation.
Features:
- Text-to-image generation
- Multiple aspect ratios (1:1, 16:9, 9:16, etc.)
- Multiple resolutions (1K, 2K)
- Base64 output saved to cache
Selection precedence (first hit wins):
1. ``XAI_IMAGE_MODEL`` env var
2. ``image_gen.xai.model`` in ``config.yaml``
3. :data:`DEFAULT_MODEL`
"""
from __future__ import annotations
import logging
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import requests
from agent.image_gen_provider import (
DEFAULT_ASPECT_RATIO,
ImageGenProvider,
error_response,
normalize_reference_images,
resolve_aspect_ratio,
save_b64_image,
save_url_image,
success_response,
)
from tools.xai_http import (
build_xai_storage_options,
hermes_xai_user_agent,
maybe_mark_xai_storage_notice_seen,
read_xai_imagine_storage_config,
resolve_xai_http_credentials,
xai_storage_notice_text,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Model catalog
# ---------------------------------------------------------------------------
_MODELS: Dict[str, Dict[str, Any]] = {
"grok-imagine-image": {
"display": "Grok Imagine Image",
"speed": "~5-10s",
"strengths": "Fast, high-quality",
},
"grok-imagine-image-2.0": {
"display": "Grok Imagine Image 2.0",
"speed": "~10-20s",
"strengths": "Typography/layout-aware; legible small text; strongest quality.",
},
"grok-imagine-image-quality": {
"display": "Grok Imagine Image (Quality)",
"speed": "~10-20s",
"strengths": "Higher fidelity / detail; slower than the standard model.",
},
}
DEFAULT_MODEL = "grok-imagine-image"
# Live catalog cache: (models_dict, fetched_monotonic). xAI's
# ``/image-generation-models`` endpoint is the source of truth so newly
# released Imagine models appear in the picker without a code change;
# the static ``_MODELS`` table is the offline fallback and supplies curated
# speed/strengths text for the models we know about.
_LIVE_CACHE: Optional[Tuple[Dict[str, Dict[str, Any]], float]] = None
_LIVE_CACHE_TTL = 300.0
_LIVE_TIMEOUT = 10.0
def _fetch_live_models() -> Dict[str, Dict[str, Any]]:
"""Fetch image models from xAI's ``/image-generation-models`` endpoint.
Returns ``{model_id: {"input_modalities": [...], "aliases": [...]}}``.
Raises on any failure — callers treat that as "use the static table".
"""
creds = resolve_xai_http_credentials()
api_key = str(creds.get("api_key") or "").strip()
if not api_key:
raise RuntimeError("no xAI credentials")
base_url = str(creds.get("base_url") or "https://api.x.ai/v1").strip().rstrip("/")
response = requests.get(
f"{base_url}/image-generation-models",
headers={
"Authorization": f"Bearer {api_key}",
"User-Agent": hermes_xai_user_agent(),
},
timeout=_LIVE_TIMEOUT,
)
response.raise_for_status()
payload = response.json()
entries = payload.get("models") or payload.get("data") or []
out: Dict[str, Dict[str, Any]] = {}
for entry in entries:
if not isinstance(entry, dict):
continue
model_id = entry.get("id") or entry.get("name")
if not isinstance(model_id, str) or not model_id.strip():
continue
out[model_id.strip()] = {
"input_modalities": entry.get("input_modalities") or [],
"aliases": entry.get("aliases") or [],
}
return out
def _live_models() -> Dict[str, Dict[str, Any]]:
"""Cached live catalog (``{}`` when unreachable)."""
global _LIVE_CACHE
import time
if _LIVE_CACHE is not None and time.monotonic() - _LIVE_CACHE[1] < _LIVE_CACHE_TTL:
return _LIVE_CACHE[0]
try:
live = _fetch_live_models()
except Exception as exc: # noqa: BLE001 - offline/unauth → static fallback
logger.debug("xAI live image model catalog unavailable: %s", exc)
live = {}
_LIVE_CACHE = (live, time.monotonic())
return live
def _catalog() -> Dict[str, Dict[str, Any]]:
"""Merged model catalog: live endpoint IDs + curated static metadata.
Known models keep their curated display/speed/strengths; models xAI
ships after this file was written still show up (with generic metadata)
so users can pick them the day they launch. Static table alone when the
API is unreachable.
"""
live = _live_models()
if not live:
return dict(_MODELS)
merged: Dict[str, Dict[str, Any]] = {}
for model_id in live:
meta = _MODELS.get(model_id)
if meta is None:
meta = {
"display": model_id,
"speed": "",
"strengths": "New xAI Imagine model (from live xAI catalog)",
}
merged[model_id] = dict(meta)
merged[model_id]["input_modalities"] = live[model_id].get("input_modalities") or []
# Keep curated entries that the live list may momentarily omit.
for model_id, meta in _MODELS.items():
merged.setdefault(model_id, dict(meta))
return merged
# xAI aspect ratios (more options than FAL/OpenAI)
_XAI_ASPECT_RATIOS = {
"landscape": "16:9",
"square": "1:1",
"portrait": "9:16",
"4:3": "4:3",
"3:4": "3:4",
"3:2": "3:2",
"2:3": "2:3",
}
# xAI resolutions
_XAI_RESOLUTIONS = {"1k", "2k"}
DEFAULT_RESOLUTION = "1k"
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
def _load_xai_config() -> Dict[str, Any]:
"""Read ``image_gen.xai`` from config.yaml."""
try:
from hermes_cli.config import load_config
cfg = load_config()
section = cfg.get("image_gen") if isinstance(cfg, dict) else None
xai_section = section.get("xai") if isinstance(section, dict) else None
return xai_section if isinstance(xai_section, dict) else {}
except Exception as exc:
logger.debug("Could not load image_gen.xai config: %s", exc)
return {}
def _resolve_model(caller_model: Optional[str] = None) -> Tuple[str, Dict[str, Any]]:
"""Decide which model to use and return ``(model_id, meta)``.
Priority:
1. Caller-supplied ``caller_model`` — the dispatcher forwards top-level
``image_gen.model`` (what ``hermes tools`` writes) as the ``model``
kwarg, mirroring the openrouter provider.
2. ``XAI_IMAGE_MODEL`` env override.
3. Scoped ``image_gen.xai.model`` in config.yaml.
4. :data:`DEFAULT_MODEL`.
Every candidate is validated against the merged live+static catalog,
so a newly released xAI model is selectable the day it appears in the
live catalog — no code change required.
"""
catalog = _catalog()
if caller_model and caller_model in catalog:
return caller_model, catalog[caller_model]
env_override = os.environ.get("XAI_IMAGE_MODEL")
if env_override and env_override in catalog:
return env_override, catalog[env_override]
cfg = _load_xai_config()
candidate = cfg.get("model") if isinstance(cfg.get("model"), str) else None
if candidate and candidate in catalog:
return candidate, catalog[candidate]
return DEFAULT_MODEL, catalog.get(DEFAULT_MODEL, _MODELS[DEFAULT_MODEL])
def _resolve_edit_model(caller_model: Optional[str] = None) -> str:
"""Model for ``/v1/images/edits`` requests.
An explicitly selected model (caller kwarg, env, or config) that accepts
image input is honored for edits; otherwise fall back to the quality
model, which xAI documents as the edit-capable baseline.
"""
catalog = _catalog()
explicit = caller_model or os.environ.get("XAI_IMAGE_MODEL") or (
_load_xai_config().get("model") if isinstance(_load_xai_config().get("model"), str) else None
)
if explicit and explicit in catalog:
modalities = catalog[explicit].get("input_modalities") or []
if "image" in modalities:
return explicit
return "grok-imagine-image-quality"
def _resolve_resolution() -> str:
"""Get configured resolution."""
cfg = _load_xai_config()
res = cfg.get("resolution") if isinstance(cfg.get("resolution"), str) else None
if res and res in _XAI_RESOLUTIONS:
return res
return DEFAULT_RESOLUTION
def _xai_image_field(source: str) -> Dict[str, str]:
"""Build the xAI ``image`` field for an edit request.
xAI's ``/v1/images/edits`` accepts a public HTTPS URL or a base64 data URI.
Local file paths are read and encoded into a ``data:`` URI.
"""
source = source.strip()
lower = source.lower()
if lower.startswith(("http://", "https://", "data:")):
return {"url": source, "type": "image_url"}
# Local file path → base64 data URI.
import base64
import os as _os
# Enforce the shared credential-read guard before reading local bytes
# (same boundary the OpenAI / OpenRouter / Codex image providers apply).
from agent.file_safety import raise_if_read_blocked
raise_if_read_blocked(source)
with open(_os.path.expanduser(source), "rb") as fh: # windows-footgun: ok
raw = fh.read()
ext = (_os.path.splitext(source)[1].lstrip(".") or "png").lower()
if ext == "jpg":
ext = "jpeg"
b64 = base64.b64encode(raw).decode("utf-8")
return {"url": f"data:image/{ext};base64,{b64}", "type": "image_url"}
# ---------------------------------------------------------------------------
# Provider
# ---------------------------------------------------------------------------
class XAIImageGenProvider(ImageGenProvider):
"""xAI ``grok-imagine-image`` backend."""
@property
def name(self) -> str:
return "xai"
@property
def display_name(self) -> str:
return "xAI (Grok)"
def is_available(self) -> bool:
creds = resolve_xai_http_credentials()
return bool(creds.get("api_key"))
def list_models(self) -> List[Dict[str, Any]]:
return [
{
"id": model_id,
"display": meta.get("display", model_id),
"speed": meta.get("speed", ""),
"strengths": meta.get("strengths", ""),
}
for model_id, meta in _catalog().items()
]
def get_setup_schema(self) -> Dict[str, Any]:
# Auth resolution is delegated to the shared ``xai_grok`` post_setup
# hook (``hermes_cli/tools_config.py``); identical to the TTS / video
# gen entries so users see the same OAuth-or-API-key choice for every
# xAI service.
storage_notice = xai_storage_notice_text("image_gen")
tag = (
"grok-imagine-image - text-to-image & image editing; uses xAI "
"Grok OAuth or XAI_API_KEY"
)
if storage_notice:
tag += f". {storage_notice}"
return {
"name": "xAI Grok Imagine (image)",
"badge": "paid",
"tag": tag,
"env_vars": [],
"post_setup": "xai_grok",
}
def capabilities(self) -> Dict[str, Any]:
# xAI's /v1/images/edits supports image editing via grok-imagine-image
# -quality, including up to 3 total source images.
return {
"modalities": ["text", "image"],
"max_reference_images": 2,
"max_source_images": 3,
}
def generate(
self,
prompt: str,
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
*,
image_url: Optional[str] = None,
reference_image_urls: Optional[List[str]] = None,
**kwargs: Any,
) -> Dict[str, Any]:
"""Generate an image (text-to-image) or edit a source image (image-to-image).
Routing: when ``image_url`` is provided, POST to ``/v1/images/edits``
with the source image; otherwise POST to ``/v1/images/generations``.
Per xAI docs, editing uses the ``grok-imagine-image-quality`` model and
a JSON body (the OpenAI SDK's multipart ``images.edit()`` is NOT
supported by xAI).
"""
creds = resolve_xai_http_credentials()
api_key = str(creds.get("api_key") or "").strip()
provider_name = str(creds.get("provider") or "xai").strip() or "xai"
if not api_key:
return error_response(
error="No xAI credentials found. Configure xAI OAuth in `hermes model` or set XAI_API_KEY.",
error_type="missing_api_key",
provider=provider_name,
aspect_ratio=aspect_ratio,
)
model_id, meta = _resolve_model(kwargs.get("model"))
aspect = resolve_aspect_ratio(aspect_ratio)
xai_ar = _XAI_ASPECT_RATIOS.get(aspect, "1:1")
resolution = _resolve_resolution()
xai_res = resolution if resolution in _XAI_RESOLUTIONS else DEFAULT_RESOLUTION
source_images: List[str] = []
if isinstance(image_url, str) and image_url.strip():
source_images.append(image_url.strip())
refs = normalize_reference_images(reference_image_urls)
if refs:
source_images.extend(refs)
if len(source_images) > 3:
return error_response(
error="xAI image editing supports at most 3 source images",
error_type="too_many_references",
provider=provider_name,
model="grok-imagine-image-quality",
prompt=prompt,
aspect_ratio=aspect,
)
for index, source in enumerate(source_images):
field = "image_url" if index == 0 and image_url and image_url.strip() == source else "reference_image_urls"
lower = source.lower()
if not lower.startswith(("http://", "https://", "data:")):
path = Path(source).expanduser()
if not path.is_file():
return error_response(
error=(
f"{field} must be a public HTTPS URL or data URI "
"(e.g. the `image`/`public_url` from a prior Imagine result)"
),
error_type="invalid_image_url",
provider=provider_name,
model="grok-imagine-image-quality",
prompt=prompt,
aspect_ratio=aspect,
)
is_edit = bool(source_images)
modality = "image" if is_edit else "text"
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
"User-Agent": hermes_xai_user_agent(),
}
base_url = str(creds.get("base_url") or "https://api.x.ai/v1").strip().rstrip("/")
storage_options = build_xai_storage_options(
"image_gen",
filename_prefix="hermes-xai-image",
extension="png",
)
storage_notice = maybe_mark_xai_storage_notice_seen("image_gen")
storage_cfg = read_xai_imagine_storage_config("image_gen")
if is_edit:
# Editing needs an image-input-capable model. An explicit user
# selection that accepts image input (e.g. grok-imagine-image-2.0)
# is honored; otherwise the documented quality baseline is used.
# The source image may be a public URL or a base64 data URI;
# local file paths are converted to a data URI here.
edit_model = _resolve_edit_model(kwargs.get("model"))
try:
image_fields = [_xai_image_field(source) for source in source_images]
except Exception as exc:
return error_response(
error=f"Could not load source image for editing: {exc}",
error_type="io_error",
provider=provider_name,
model=edit_model,
prompt=prompt,
aspect_ratio=aspect,
)
payload: Dict[str, Any] = {
"model": edit_model,
"prompt": prompt,
}
if len(image_fields) == 1:
payload["image"] = image_fields[0]
else:
payload["images"] = image_fields
endpoint_url = f"{base_url}/images/edits"
model_id = edit_model
else:
payload = {
"model": model_id,
"prompt": prompt,
"aspect_ratio": xai_ar,
"resolution": xai_res,
}
endpoint_url = f"{base_url}/images/generations"
if storage_options is not None:
payload["storage_options"] = storage_options
try:
response = requests.post(
endpoint_url,
headers=headers,
json=payload,
timeout=120,
)
response.raise_for_status()
except requests.HTTPError as exc:
response = exc.response
status = response.status_code if response is not None else 0
try:
err_msg = response.json().get("error", {}).get("message", response.text[:300])
except Exception:
err_msg = response.text[:300] if response is not None else str(exc)
logger.error("xAI image gen failed (%d): %s", status, err_msg)
return error_response(
error=f"xAI image generation failed ({status}): {err_msg}",
error_type="api_error",
provider=provider_name,
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
except requests.Timeout:
return error_response(
error="xAI image generation timed out (120s)",
error_type="timeout",
provider=provider_name,
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
except requests.ConnectionError as exc:
return error_response(
error=f"xAI connection error: {exc}",
error_type="connection_error",
provider=provider_name,
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
try:
result = response.json()
except Exception as exc:
return error_response(
error=f"xAI returned invalid JSON: {exc}",
error_type="invalid_response",
provider=provider_name,
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
# Parse response - xAI returns data[0].b64_json, data[0].url, and
# optionally data[0].file_output when storage_options were requested.
data = result.get("data", [])
if not data:
return error_response(
error="xAI returned no image data",
error_type="empty_response",
provider=provider_name,
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
first = data[0]
b64 = first.get("b64_json")
url = first.get("url")
file_output = first.get("file_output") if isinstance(first, dict) else None
file_output = file_output if isinstance(file_output, dict) else {}
public_url = file_output.get("public_url") if isinstance(file_output.get("public_url"), str) else None
if public_url:
image_ref = public_url
elif b64:
try:
saved_path = save_b64_image(b64, prefix=f"xai_{model_id}")
except Exception as exc:
return error_response(
error=f"Could not save image to cache: {exc}",
error_type="io_error",
provider="xai",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
image_ref = str(saved_path)
elif url:
# xAI's grok-imagine-image returns ephemeral ``imgen.x.ai/xai-tmp-*``
# URLs that 404 within minutes — by the time Telegram's
# ``send_photo`` or any downstream consumer fetches them, the
# asset is gone (#26942). Materialise the bytes locally at
# tool-completion time so the gateway has a stable file path to
# upload, mirroring the b64 branch above and the audio_cache
# pattern used by text_to_speech.
try:
saved_path = save_url_image(url, prefix=f"xai_{model_id}")
except Exception as exc:
logger.warning(
"xAI image URL %s could not be cached (%s); falling back to bare URL.",
url,
exc,
)
image_ref = url
else:
image_ref = str(saved_path)
else:
return error_response(
error="xAI response contained neither b64_json nor URL",
error_type="empty_response",
provider="xai",
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
)
extra: Dict[str, Any] = {
"storage_enabled": bool(storage_cfg["enabled"]),
}
if not is_edit:
extra["resolution"] = xai_res
if storage_notice:
extra["storage_notice"] = storage_notice
if public_url:
extra["public_url"] = public_url
if file_output:
for key in (
"filename",
"expires_at",
"public_url_expires_at",
"public_url_error",
"storage_error",
):
if key in file_output:
extra[key] = file_output[key]
if result.get("usage"):
extra["usage"] = result["usage"]
return success_response(
image=image_ref,
model=model_id,
prompt=prompt,
aspect_ratio=aspect,
provider="xai",
modality=modality,
extra=extra,
)
# ---------------------------------------------------------------------------
# Plugin registration
# ---------------------------------------------------------------------------
def register(ctx: Any) -> None:
"""Register this provider with the image gen registry."""
ctx.register_image_gen_provider(XAIImageGenProvider())
+7
View File
@@ -0,0 +1,7 @@
name: xai
version: 1.0.0
description: "xAI image generation backend (grok-imagine-image). Text-to-image."
author: Julien Talbot
kind: backend
requires_env:
- XAI_API_KEY