Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,336 @@
|
||||
"""DeepInfra image generation backend.
|
||||
|
||||
Exposes DeepInfra's image-gen catalog (FLUX, Qwen-Image-Edit, …) through
|
||||
the OpenAI-compatible ``/v1/openai/images/generations`` endpoint as an
|
||||
:class:`ImageGenProvider` implementation.
|
||||
|
||||
**Fully dynamic model discovery.** Unlike the other image-gen plugins in
|
||||
this tree (which ship a hardcoded ``_MODELS`` dict), DeepInfra publishes
|
||||
a single tagged catalog at
|
||||
``https://api.deepinfra.com/v1/openai/models?filter=true&sort_by=hermes``
|
||||
where each entry's ``metadata.tags`` declares its surface (``image-gen``
|
||||
here). ``list_models()`` filters that catalog via
|
||||
:func:`hermes_cli.models._fetch_deepinfra_models_by_tag` so newly added
|
||||
models show up in ``hermes tools`` automatically. No model ids are
|
||||
hardcoded in this file — if a model is retired upstream, it disappears
|
||||
from hermes the next time the catalog is fetched, no patch required.
|
||||
|
||||
Model selection (first hit wins):
|
||||
|
||||
1. ``DEEPINFRA_IMAGE_MODEL`` env var
|
||||
2. ``image_gen.deepinfra.model`` in ``config.yaml``
|
||||
3. First model from the live catalog
|
||||
|
||||
When all three are absent (catalog unreachable, nothing configured),
|
||||
``generate()`` returns an :func:`error_response` rather than guessing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from agent.secret_scope import get_secret
|
||||
from agent.image_gen_provider import (
|
||||
DEFAULT_ASPECT_RATIO,
|
||||
ImageGenProvider,
|
||||
error_response,
|
||||
resolve_aspect_ratio,
|
||||
save_b64_image,
|
||||
save_url_image,
|
||||
success_response,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# DeepInfra accepts standard OpenAI ``size`` strings. Mirrors the
|
||||
# OpenAI plugin's mapping so aspect_ratio semantics stay consistent
|
||||
# across the agent's image_generate tool surface.
|
||||
_SIZES = {
|
||||
"landscape": "1536x1024",
|
||||
"square": "1024x1024",
|
||||
"portrait": "1024x1536",
|
||||
}
|
||||
|
||||
|
||||
def _load_deepinfra_image_config() -> Dict[str, Any]:
|
||||
"""Read ``image_gen.deepinfra`` from config.yaml."""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
cfg = load_config()
|
||||
section = cfg.get("image_gen") if isinstance(cfg, dict) else None
|
||||
di_section = section.get("deepinfra") if isinstance(section, dict) else None
|
||||
return di_section if isinstance(di_section, dict) else {}
|
||||
except Exception as exc:
|
||||
logger.debug("Could not load image_gen.deepinfra config: %s", exc)
|
||||
return {}
|
||||
|
||||
|
||||
def _live_models() -> Optional[List[Dict[str, Any]]]:
|
||||
"""Fetch ``image-gen``-tagged models from the DeepInfra catalog."""
|
||||
try:
|
||||
from hermes_cli.models import _fetch_deepinfra_models_by_tag
|
||||
except Exception as exc:
|
||||
logger.debug("Cannot import _fetch_deepinfra_models_by_tag: %s", exc)
|
||||
return None
|
||||
return _fetch_deepinfra_models_by_tag("image-gen")
|
||||
|
||||
|
||||
def _format_catalog_row(item: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Format a catalog item into the picker row shape."""
|
||||
mid = item.get("id", "")
|
||||
metadata = item.get("metadata") or {}
|
||||
pricing = metadata.get("pricing") if isinstance(metadata, dict) else None
|
||||
price = ""
|
||||
if isinstance(pricing, dict) and pricing.get("per_image_unit") is not None:
|
||||
try:
|
||||
price = f"${float(pricing['per_image_unit']):.4f}/image"
|
||||
except (TypeError, ValueError):
|
||||
price = ""
|
||||
row: Dict[str, Any] = {
|
||||
"id": mid,
|
||||
"display": mid.split("/", 1)[-1] if "/" in mid else mid,
|
||||
"strengths": metadata.get("description", "") if isinstance(metadata, dict) else "",
|
||||
}
|
||||
if price:
|
||||
row["price"] = price
|
||||
if isinstance(metadata, dict):
|
||||
for key in ("default_width", "default_height", "default_iterations"):
|
||||
if metadata.get(key) is not None:
|
||||
row[key] = metadata[key]
|
||||
return row
|
||||
|
||||
|
||||
def _resolve_model(catalog: List[Dict[str, Any]], cfg: Dict[str, Any]) -> Optional[str]:
|
||||
"""Pick the model id (env > config > first live result, else None).
|
||||
|
||||
Takes the already-loaded ``image_gen.deepinfra`` config so ``generate()``
|
||||
reads config once instead of via a second ``load_config`` deepcopy.
|
||||
"""
|
||||
env_override = os.environ.get("DEEPINFRA_IMAGE_MODEL", "").strip()
|
||||
if env_override:
|
||||
return env_override
|
||||
cfg_model = cfg.get("model") if isinstance(cfg, dict) else None
|
||||
if isinstance(cfg_model, str) and cfg_model.strip():
|
||||
return cfg_model.strip()
|
||||
if catalog:
|
||||
first = catalog[0].get("id")
|
||||
if isinstance(first, str) and first:
|
||||
return first
|
||||
return None
|
||||
|
||||
|
||||
class DeepInfraImageGenProvider(ImageGenProvider):
|
||||
"""DeepInfra ``images.generations`` backend.
|
||||
|
||||
Catalog is discovered live from the DeepInfra ``/models`` endpoint
|
||||
filtered by the ``image-gen`` surface tag.
|
||||
"""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "deepinfra"
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
return "DeepInfra"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return bool((get_secret("DEEPINFRA_API_KEY", "") or "").strip())
|
||||
|
||||
def list_models(self) -> List[Dict[str, Any]]:
|
||||
live = _live_models()
|
||||
if not live:
|
||||
return []
|
||||
return [_format_catalog_row(item) for item in live]
|
||||
|
||||
def default_model(self) -> Optional[str]:
|
||||
rows = self.list_models()
|
||||
if rows:
|
||||
return rows[0].get("id")
|
||||
return None
|
||||
|
||||
def capabilities(self) -> Dict[str, Any]:
|
||||
"""DeepInfra's OpenAI-compatible generation surface is text-only."""
|
||||
return {"modalities": ["text"], "max_reference_images": 0}
|
||||
|
||||
def get_setup_schema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": "DeepInfra",
|
||||
"badge": "paid",
|
||||
"tag": "FLUX, Qwen-Image, … — live catalog from api.deepinfra.com",
|
||||
"env_vars": [
|
||||
{
|
||||
"key": "DEEPINFRA_API_KEY",
|
||||
"prompt": "DeepInfra API key",
|
||||
"url": "https://deepinfra.com/dash/api_keys",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
prompt = (prompt or "").strip()
|
||||
aspect = resolve_aspect_ratio(aspect_ratio)
|
||||
|
||||
if kwargs.get("image_url") or kwargs.get("reference_image_urls"):
|
||||
return error_response(
|
||||
error=(
|
||||
"DeepInfra image generation is text-to-image only in this "
|
||||
"backend; image_url and reference_image_urls are unsupported."
|
||||
),
|
||||
error_type="modality_unsupported",
|
||||
provider="deepinfra",
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
if not prompt:
|
||||
return error_response(
|
||||
error="Prompt is required and must be a non-empty string",
|
||||
error_type="invalid_argument",
|
||||
provider="deepinfra",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
api_key = (get_secret("DEEPINFRA_API_KEY", "") or "").strip()
|
||||
if not api_key:
|
||||
return error_response(
|
||||
error=(
|
||||
"DEEPINFRA_API_KEY not set. Run `hermes tools` → Image "
|
||||
"Generation → DeepInfra to configure, or `hermes setup` "
|
||||
"to add the key."
|
||||
),
|
||||
error_type="auth_required",
|
||||
provider="deepinfra",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
di_cfg = _load_deepinfra_image_config()
|
||||
catalog = _live_models() or []
|
||||
model_id = _resolve_model(catalog, di_cfg)
|
||||
if not model_id:
|
||||
return error_response(
|
||||
error=(
|
||||
"No DeepInfra image-gen model available. Pin one in "
|
||||
"config.yaml under image_gen.deepinfra.model, set "
|
||||
"DEEPINFRA_IMAGE_MODEL, or check connectivity to "
|
||||
"api.deepinfra.com so the live catalog can be fetched."
|
||||
),
|
||||
error_type="no_model_available",
|
||||
provider="deepinfra",
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
size = _SIZES.get(aspect, _SIZES["square"])
|
||||
from hermes_cli.models import deepinfra_base_url
|
||||
base_url = deepinfra_base_url(di_cfg)
|
||||
|
||||
# DeepInfra's /images/generations is OpenAI-compatible — use the
|
||||
# openai SDK so we inherit its retry, timeout, and error mapping
|
||||
# (mirrors the existing OpenAI image-gen plugin).
|
||||
try:
|
||||
import openai
|
||||
except ImportError:
|
||||
return error_response(
|
||||
error="openai Python package not installed (pip install openai)",
|
||||
error_type="missing_dependency",
|
||||
provider="deepinfra",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
client = openai.OpenAI(api_key=api_key, base_url=base_url)
|
||||
try:
|
||||
response = client.images.generate(
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
size=size,
|
||||
n=1,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("DeepInfra image generation failed", exc_info=True)
|
||||
return error_response(
|
||||
error=f"DeepInfra image generation failed: {exc}",
|
||||
error_type="api_error",
|
||||
provider="deepinfra",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
finally:
|
||||
close = getattr(client, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
|
||||
data = getattr(response, "data", None) or []
|
||||
if not data:
|
||||
return error_response(
|
||||
error="DeepInfra returned no image data",
|
||||
error_type="empty_response",
|
||||
provider="deepinfra",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
first = data[0]
|
||||
b64 = getattr(first, "b64_json", None)
|
||||
url = getattr(first, "url", None)
|
||||
|
||||
# Drop the ``vendor/`` prefix and any colons so the saved filename
|
||||
# stays a single path component on every OS.
|
||||
short = model_id.split("/", 1)[-1].replace(":", "_")
|
||||
|
||||
if b64:
|
||||
try:
|
||||
saved_path = save_b64_image(b64, prefix=f"deepinfra_{short}")
|
||||
except Exception as exc:
|
||||
return error_response(
|
||||
error=f"Could not save image to cache: {exc}",
|
||||
error_type="io_error",
|
||||
provider="deepinfra",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
image_ref = str(saved_path)
|
||||
elif url:
|
||||
# Materialise the (often short-lived) delivery URL locally so a
|
||||
# downstream consumer (Telegram send_photo, browser fetch) doesn't
|
||||
# get a dead link — mirrors the openai/xai/krea image plugins.
|
||||
# Best-effort: fall back to the bare URL if the download fails.
|
||||
try:
|
||||
image_ref = str(save_url_image(url, prefix=f"deepinfra_{short}"))
|
||||
except Exception as exc:
|
||||
logger.debug("DeepInfra: caching delivery URL failed (%s); returning URL", exc)
|
||||
image_ref = url
|
||||
else:
|
||||
return error_response(
|
||||
error="DeepInfra response contained neither b64_json nor URL",
|
||||
error_type="empty_response",
|
||||
provider="deepinfra",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
return success_response(
|
||||
image=image_ref,
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
provider="deepinfra",
|
||||
extra={"size": size},
|
||||
)
|
||||
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Plugin entry point — wire ``DeepInfraImageGenProvider`` into the registry."""
|
||||
ctx.register_image_gen_provider(DeepInfraImageGenProvider())
|
||||
@@ -0,0 +1,7 @@
|
||||
name: deepinfra
|
||||
version: 1.0.0
|
||||
description: "DeepInfra image generation backend (FLUX, Qwen-Image, …) via OpenAI-compatible /v1/images/generations. Catalog discovered live from api.deepinfra.com."
|
||||
author: Georgi Atsev
|
||||
kind: backend
|
||||
requires_env:
|
||||
- DEEPINFRA_API_KEY
|
||||
@@ -0,0 +1,218 @@
|
||||
"""FAL.ai image generation backend.
|
||||
|
||||
Wraps the 18-model FAL catalog (FLUX 2, Z-Image, Nano Banana, GPT
|
||||
Image 1.5, Recraft, Imagen 4, Qwen, Ideogram, …) as an
|
||||
:class:`ImageGenProvider` implementation.
|
||||
|
||||
The heavy lifting — model catalog, payload construction, request
|
||||
submission, managed-Nous-gateway selection, Clarity Upscaler chaining
|
||||
— lives in :mod:`tools.image_generation_tool`. This plugin reaches into
|
||||
that module via call-time indirection (``import tools.image_generation_tool as _it``)
|
||||
so:
|
||||
|
||||
* the existing test suite (``tests/tools/test_image_generation.py``,
|
||||
``tests/tools/test_managed_media_gateways.py``) keeps patching
|
||||
``image_tool._submit_fal_request`` / ``image_tool.fal_client`` /
|
||||
``image_tool._managed_fal_client`` without modification, and
|
||||
* there's exactly one canonical FAL code path on disk — the plugin is a
|
||||
registration adapter, not a parallel implementation.
|
||||
|
||||
See issue #26241 for the migration plan and the
|
||||
``plugin-extraction-test-patch-compatibility.md`` rules this follows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from agent.image_gen_provider import (
|
||||
DEFAULT_ASPECT_RATIO,
|
||||
ImageGenProvider,
|
||||
resolve_aspect_ratio,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FalImageGenProvider(ImageGenProvider):
|
||||
"""FAL.ai image generation backend.
|
||||
|
||||
Delegates to ``tools.image_generation_tool.image_generate_tool`` so
|
||||
the in-tree FAL implementation (model catalog, payload builder,
|
||||
managed-gateway selection, Clarity Upscaler chaining) is the single
|
||||
source of truth. Everything is resolved at call time via the
|
||||
``_it`` indirection so tests can monkey-patch the legacy module.
|
||||
"""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "fal"
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
return "FAL.ai"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
# Available when direct FAL_KEY is set OR the managed Nous
|
||||
# gateway resolves a fal-queue origin. Both checks come from the
|
||||
# legacy module so this provider tracks whatever logic ships
|
||||
# there.
|
||||
import tools.image_generation_tool as _it
|
||||
try:
|
||||
return bool(_it.check_fal_api_key())
|
||||
except Exception: # noqa: BLE001 — defensive; never break the picker
|
||||
return False
|
||||
|
||||
def list_models(self) -> List[Dict[str, Any]]:
|
||||
import tools.image_generation_tool as _it
|
||||
return [
|
||||
{
|
||||
"id": model_id,
|
||||
"display": meta.get("display", model_id),
|
||||
"speed": meta.get("speed", ""),
|
||||
"strengths": meta.get("strengths", ""),
|
||||
"price": meta.get("price", ""),
|
||||
}
|
||||
for model_id, meta in _it.FAL_MODELS.items()
|
||||
]
|
||||
|
||||
def default_model(self) -> Optional[str]:
|
||||
import tools.image_generation_tool as _it
|
||||
return _it.DEFAULT_MODEL
|
||||
|
||||
def get_setup_schema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": "FAL.ai",
|
||||
"badge": "paid",
|
||||
"tag": "Pick from flux-2-klein, flux-2-pro, gpt-image, nano-banana-2, nano-banana-pro, etc. — text-to-image & image editing",
|
||||
"env_vars": [
|
||||
{
|
||||
"key": "FAL_KEY",
|
||||
"prompt": "FAL API key",
|
||||
"url": "https://fal.ai/dashboard/keys",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
def capabilities(self) -> Dict[str, Any]:
|
||||
# Whether image-to-image is available depends on the currently-
|
||||
# selected FAL model (each model entry declares an edit_endpoint or
|
||||
# not). Report the active model's actual surface so the dynamic tool
|
||||
# schema is accurate.
|
||||
import tools.image_generation_tool as _it
|
||||
|
||||
try:
|
||||
_model_id, meta = _it._resolve_fal_model()
|
||||
except Exception: # noqa: BLE001
|
||||
return {"modalities": ["text"], "max_reference_images": 0}
|
||||
# Clarity Upscaler chains on explicit request for any FAL model.
|
||||
if meta.get("edit_endpoint"):
|
||||
return {
|
||||
"modalities": ["text", "image"],
|
||||
"max_reference_images": int(meta.get("max_reference_images") or 1),
|
||||
"supports_upscale": True,
|
||||
}
|
||||
return {
|
||||
"modalities": ["text"],
|
||||
"max_reference_images": 0,
|
||||
"supports_upscale": True,
|
||||
}
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
|
||||
*,
|
||||
image_url: Optional[str] = None,
|
||||
reference_image_urls: Optional[List[str]] = None,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
"""Generate or edit an image via the legacy FAL pipeline.
|
||||
|
||||
Forwards prompt + aspect_ratio + image_url/reference_image_urls (and
|
||||
any forward-compat extras the schema supports) into
|
||||
:func:`tools.image_generation_tool.image_generate_tool`, then reshapes
|
||||
its JSON-string response into the provider-ABC dict format consumed by
|
||||
``_dispatch_to_plugin_provider``.
|
||||
"""
|
||||
import tools.image_generation_tool as _it
|
||||
|
||||
aspect = resolve_aspect_ratio(aspect_ratio)
|
||||
passthrough = {
|
||||
key: kwargs[key]
|
||||
for key in (
|
||||
"num_inference_steps",
|
||||
"guidance_scale",
|
||||
"num_images",
|
||||
"output_format",
|
||||
"seed",
|
||||
"upscale",
|
||||
)
|
||||
if key in kwargs and kwargs[key] is not None
|
||||
}
|
||||
# Only forward the image-to-image inputs when actually supplied, so a
|
||||
# plain text-to-image call delegates exactly as it did before (no
|
||||
# noisy None kwargs).
|
||||
if image_url is not None:
|
||||
passthrough["image_url"] = image_url
|
||||
if reference_image_urls is not None:
|
||||
passthrough["reference_image_urls"] = reference_image_urls
|
||||
|
||||
try:
|
||||
raw = _it.image_generate_tool(
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
**passthrough,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — never raise out of generate
|
||||
logger.warning("FAL image_generate_tool raised: %s", exc, exc_info=True)
|
||||
return {
|
||||
"success": False,
|
||||
"image": None,
|
||||
"error": f"FAL image generation failed: {exc}",
|
||||
"error_type": type(exc).__name__,
|
||||
"provider": "fal",
|
||||
"prompt": prompt,
|
||||
"aspect_ratio": aspect,
|
||||
}
|
||||
|
||||
try:
|
||||
response = json.loads(raw) if isinstance(raw, str) else raw
|
||||
except Exception: # noqa: BLE001
|
||||
response = {"success": False, "image": None, "error": "Invalid JSON from FAL pipeline"}
|
||||
|
||||
if not isinstance(response, dict):
|
||||
response = {
|
||||
"success": False,
|
||||
"image": None,
|
||||
"error": "FAL pipeline returned a non-dict response",
|
||||
"error_type": "provider_contract",
|
||||
}
|
||||
|
||||
# Stamp provider/prompt/aspect_ratio so downstream consumers see
|
||||
# the uniform shape declared in ``agent.image_gen_provider``.
|
||||
response.setdefault("provider", "fal")
|
||||
response.setdefault("prompt", prompt)
|
||||
response.setdefault("aspect_ratio", aspect)
|
||||
# Annotate model best-effort — the legacy pipeline resolves it
|
||||
# internally, so query it after the fact for the response shape.
|
||||
if "model" not in response:
|
||||
try:
|
||||
model_id, _meta = _it._resolve_fal_model()
|
||||
response["model"] = model_id
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return response
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Plugin entry point — wire ``FalImageGenProvider`` into the registry."""
|
||||
ctx.register_image_gen_provider(FalImageGenProvider())
|
||||
@@ -0,0 +1,7 @@
|
||||
name: fal
|
||||
version: 1.0.0
|
||||
description: "FAL.ai image generation backend (flux-2-klein, flux-2-pro, nano-banana-2, nano-banana-pro, gpt-image-1.5, recraft-v3, etc.)."
|
||||
author: NousResearch
|
||||
kind: backend
|
||||
requires_env:
|
||||
- FAL_KEY
|
||||
@@ -0,0 +1,921 @@
|
||||
"""Krea image generation backend.
|
||||
|
||||
Exposes Krea's `Krea 2` foundation image model family — Krea 2 Medium and
|
||||
Krea 2 Large — as an :class:`ImageGenProvider` implementation.
|
||||
|
||||
Krea's API is asynchronous: the generate endpoint returns a ``job_id``
|
||||
that you poll at ``GET /jobs/{job_id}``. This provider hides that
|
||||
roundtrip behind the synchronous ``generate()`` contract: submit, poll
|
||||
every 2s with light backoff, materialise the result URL to local cache,
|
||||
return the success/error dict like every other backend.
|
||||
|
||||
Selection precedence (first hit wins):
|
||||
|
||||
1. ``KREA_IMAGE_MODEL`` env var (escape hatch for scripts / tests)
|
||||
2. ``image_gen.krea.model`` in ``config.yaml``
|
||||
3. ``image_gen.model`` in ``config.yaml`` (when it's one of our IDs)
|
||||
4. :data:`DEFAULT_MODEL` — ``krea-2-medium`` (Krea's "start here" recommendation)
|
||||
|
||||
Docs: https://docs.krea.ai/developers/krea-2/overview
|
||||
API: https://docs.krea.ai/api-reference/krea/krea-2-large
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
from agent.secret_scope import get_secret
|
||||
from agent.image_gen_provider import (
|
||||
DEFAULT_ASPECT_RATIO,
|
||||
ImageGenProvider,
|
||||
error_response,
|
||||
normalize_reference_images,
|
||||
resolve_aspect_ratio,
|
||||
save_url_image,
|
||||
success_response,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
BASE_URL = "https://api.krea.ai"
|
||||
|
||||
# Map our short model IDs to Krea's URL path segment.
|
||||
_MODELS: Dict[str, Dict[str, Any]] = {
|
||||
"krea-2-medium": {
|
||||
"display": "Krea 2 Medium",
|
||||
"speed": "~15-25s",
|
||||
"strengths": "Illustration, anime, painting, expressive styles. Faster + cheaper.",
|
||||
"price": "$0.030 (text) / $0.035 (style refs) / $0.040 (moodboards)",
|
||||
"path": "medium",
|
||||
# Upscaling is opt-in everywhere (Aug 2026 policy: default-on
|
||||
# enhance passes degraded output quality).
|
||||
"upscale": False,
|
||||
},
|
||||
"krea-2-large": {
|
||||
"display": "Krea 2 Large",
|
||||
"speed": "~25-60s",
|
||||
"strengths": "Photorealism, raw textured looks (motion blur, grain), expressive styles.",
|
||||
"price": "$0.060 (text) / $0.065 (style refs) / $0.070 (moodboards)",
|
||||
"path": "large",
|
||||
# 2K native — high-res enough out of the box.
|
||||
"upscale": False,
|
||||
},
|
||||
"krea-2-medium-turbo": {
|
||||
"display": "Krea 2 Medium Turbo",
|
||||
"speed": "~8-15s",
|
||||
"strengths": "Fastest Krea 2 — medium quality at lower latency / cost.",
|
||||
"price": "$0.015 (text) / $0.0175 (style refs)",
|
||||
"path": "medium-turbo",
|
||||
# Opt-in only (Aug 2026 policy).
|
||||
"upscale": False,
|
||||
},
|
||||
}
|
||||
|
||||
DEFAULT_MODEL = "krea-2-medium"
|
||||
|
||||
# Hermes uses 3 abstract aspect ratios. Map to Krea's enum (which is wider).
|
||||
# Krea accepts: 1:1, 4:3, 3:2, 16:9, 2.35:1, 4:5, 2:3, 9:16
|
||||
_ASPECT_MAP = {
|
||||
"landscape": "16:9",
|
||||
"square": "1:1",
|
||||
"portrait": "9:16",
|
||||
}
|
||||
|
||||
# Only resolution Krea currently supports.
|
||||
DEFAULT_RESOLUTION = "1K"
|
||||
|
||||
# Krea's image_style_references entries are objects ({"url", "strength"}), not
|
||||
# bare URL strings. When the caller supplies a URL without an explicit strength
|
||||
# we apply Krea's recommended starting value. Range per Krea docs is -2..2.
|
||||
_DEFAULT_STYLE_REFERENCE_STRENGTH = 0.6
|
||||
|
||||
# Valid creativity levels per Krea docs. Default is "medium".
|
||||
_VALID_CREATIVITY = {"raw", "low", "medium", "high"}
|
||||
|
||||
# Polling cadence. Krea recommends 2-5s; we start at 2s and back off to 5s
|
||||
# for long jobs (Large can take ~1min). Total ceiling matches Krea's
|
||||
# hosted-tool timeout of 3 minutes.
|
||||
_POLL_INITIAL_INTERVAL = 2.0
|
||||
_POLL_MAX_INTERVAL = 5.0
|
||||
_POLL_BACKOFF = 1.3
|
||||
_POLL_TIMEOUT_SECONDS = 180.0
|
||||
|
||||
# HTTP statuses worth retrying during the poll loop. Everything else (401,
|
||||
# 402, 403, 404, other 4xx) is a permanent failure — surface it immediately
|
||||
# instead of burning the 180s deadline retrying a request that will never
|
||||
# succeed.
|
||||
_RETRYABLE_POLL_STATUSES = frozenset({408, 409, 425, 429, 500, 502, 503, 504})
|
||||
|
||||
_TERMINAL_STATES = {"completed", "failed", "cancelled"}
|
||||
|
||||
# Krea Enhance — the upscale/enhancer endpoint used for the optional
|
||||
# ``upscale`` pass after generation ("1.5K native, 4K via Enhancer" is
|
||||
# Krea's own pipeline shape). Cheap creative enhancer, max 8K.
|
||||
_ENHANCE_PATH = "/generate/enhance/krea/enhance"
|
||||
_ENHANCE_SCALE_FACTOR = 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_krea_config() -> Dict[str, Any]:
|
||||
"""Read ``image_gen.krea`` (with fallthrough to ``image_gen``) from config.yaml."""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
cfg = load_config()
|
||||
section = cfg.get("image_gen") if isinstance(cfg, dict) else None
|
||||
return section if isinstance(section, dict) else {}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("Could not load image_gen config: %s", exc)
|
||||
return {}
|
||||
|
||||
|
||||
def _resolve_model(explicit: Optional[str] = None) -> Tuple[str, Dict[str, Any]]:
|
||||
"""Decide which model to use and return ``(model_id, meta)``.
|
||||
|
||||
Precedence: explicit caller override (e.g. managed-mode routing or a direct
|
||||
``model`` kwarg) → ``KREA_IMAGE_MODEL`` env → ``image_gen.krea.model`` →
|
||||
``image_gen.model`` → :data:`DEFAULT_MODEL`.
|
||||
"""
|
||||
if isinstance(explicit, str) and explicit.strip() in _MODELS:
|
||||
return explicit.strip(), _MODELS[explicit.strip()]
|
||||
|
||||
env_override = os.environ.get("KREA_IMAGE_MODEL")
|
||||
if env_override and env_override in _MODELS:
|
||||
return env_override, _MODELS[env_override]
|
||||
|
||||
cfg = _load_krea_config()
|
||||
krea_cfg = cfg.get("krea") if isinstance(cfg.get("krea"), dict) else {}
|
||||
candidate: Optional[str] = None
|
||||
if isinstance(krea_cfg, dict):
|
||||
value = krea_cfg.get("model")
|
||||
if isinstance(value, str) and value in _MODELS:
|
||||
candidate = value
|
||||
if candidate is None:
|
||||
top = cfg.get("model")
|
||||
if isinstance(top, str) and top in _MODELS:
|
||||
candidate = top
|
||||
|
||||
if candidate is not None:
|
||||
return candidate, _MODELS[candidate]
|
||||
|
||||
return DEFAULT_MODEL, _MODELS[DEFAULT_MODEL]
|
||||
|
||||
|
||||
def _resolve_managed_krea_gateway():
|
||||
"""Return managed Krea gateway config when the user is on the managed path.
|
||||
|
||||
Strict selection model: the managed Krea gateway is used when the stored
|
||||
``image_gen`` selection is ``nous`` (or legacy ``use_gateway: true``), or
|
||||
on a never-configured install when no direct ``KREA_API_KEY`` exists.
|
||||
An explicit vendor selection (``krea``, ``fal``, ...) pins the direct
|
||||
path. Returns ``None`` (direct/BYO path) otherwise, and never raises —
|
||||
plugin discovery and availability scans must stay robust.
|
||||
"""
|
||||
try:
|
||||
from tools.managed_tool_gateway import resolve_managed_tool_gateway
|
||||
from tools.tool_backend_helpers import (
|
||||
NOUS_MANAGED_PROVIDER,
|
||||
read_selection,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("Managed Krea gateway resolution unavailable: %s", exc)
|
||||
return None
|
||||
|
||||
try:
|
||||
selected = read_selection("image_gen")
|
||||
except Exception: # noqa: BLE001
|
||||
selected = None
|
||||
if selected is not None and selected != NOUS_MANAGED_PROVIDER:
|
||||
# Explicit vendor selection: direct credentials only.
|
||||
return None
|
||||
if selected is None and get_secret("KREA_API_KEY"):
|
||||
return None
|
||||
|
||||
try:
|
||||
return resolve_managed_tool_gateway("krea")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("Managed Krea gateway resolution failed: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _managed_krea_gateway_ready() -> bool:
|
||||
"""Cheap, offline-friendly probe for managed Krea availability."""
|
||||
try:
|
||||
from tools.managed_tool_gateway import is_managed_tool_gateway_ready
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
try:
|
||||
return bool(is_managed_tool_gateway_ready("krea"))
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_creativity(value: Optional[str]) -> str:
|
||||
"""Coerce ``creativity`` kwarg to a valid Krea value (default ``medium``)."""
|
||||
if isinstance(value, str):
|
||||
v = value.strip().lower()
|
||||
if v in _VALID_CREATIVITY:
|
||||
return v
|
||||
cfg = _load_krea_config()
|
||||
krea_cfg = cfg.get("krea") if isinstance(cfg.get("krea"), dict) else {}
|
||||
cfg_value = krea_cfg.get("creativity") if isinstance(krea_cfg, dict) else None
|
||||
if isinstance(cfg_value, str) and cfg_value.strip().lower() in _VALID_CREATIVITY:
|
||||
return cfg_value.strip().lower()
|
||||
return "medium"
|
||||
|
||||
|
||||
def _poll_krea_job(
|
||||
base_url: str,
|
||||
auth_token: str,
|
||||
job_id: str,
|
||||
*,
|
||||
timeout_seconds: float = _POLL_TIMEOUT_SECONDS,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Poll ``/jobs/{job_id}`` until terminal; return the job dict or None.
|
||||
|
||||
Best-effort variant of the main generate() poll loop used for secondary
|
||||
jobs (the Enhance upscale pass): any failure returns ``None`` so the
|
||||
caller can fall back instead of failing the whole generation.
|
||||
"""
|
||||
job_url = f"{base_url}/jobs/{job_id}"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {auth_token}",
|
||||
"User-Agent": "Hermes-Agent/1.0 (krea-image-gen)",
|
||||
}
|
||||
interval = _POLL_INITIAL_INTERVAL
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
while True:
|
||||
time.sleep(interval)
|
||||
interval = min(interval * _POLL_BACKOFF, _POLL_MAX_INTERVAL)
|
||||
try:
|
||||
resp = requests.get(job_url, headers=headers, timeout=30)
|
||||
resp.raise_for_status()
|
||||
job = resp.json()
|
||||
except requests.HTTPError as exc:
|
||||
status = exc.response.status_code if exc.response is not None else 0
|
||||
if status not in _RETRYABLE_POLL_STATUSES or time.monotonic() >= deadline:
|
||||
logger.warning("Krea enhance poll failed (%d) for job %s", status, job_id)
|
||||
return None
|
||||
continue
|
||||
except Exception as exc: # noqa: BLE001 — timeout/connection/JSON
|
||||
if time.monotonic() >= deadline:
|
||||
logger.warning("Krea enhance poll gave up for job %s: %s", job_id, exc)
|
||||
return None
|
||||
continue
|
||||
|
||||
if isinstance(job, dict):
|
||||
status_str = job.get("status")
|
||||
if status_str in _TERMINAL_STATES or job.get("completed_at"):
|
||||
return job
|
||||
if time.monotonic() >= deadline:
|
||||
logger.warning("Krea enhance job %s did not finish in %ds", job_id, int(timeout_seconds))
|
||||
return None
|
||||
|
||||
|
||||
def _extract_result_url(job: Optional[Dict[str, Any]]) -> Optional[str]:
|
||||
"""Pull the first result URL out of a terminal Krea job dict."""
|
||||
if not isinstance(job, dict):
|
||||
return None
|
||||
result = job.get("result")
|
||||
if not isinstance(result, dict):
|
||||
return None
|
||||
urls = result.get("urls")
|
||||
if isinstance(urls, list):
|
||||
for candidate in urls:
|
||||
if isinstance(candidate, str) and candidate.strip():
|
||||
return candidate.strip()
|
||||
single = result.get("url")
|
||||
if isinstance(single, str) and single.strip():
|
||||
return single.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _enhance_image(
|
||||
base_url: str,
|
||||
auth_token: str,
|
||||
image_url: str,
|
||||
prompt: str,
|
||||
*,
|
||||
managed: bool,
|
||||
) -> Optional[str]:
|
||||
"""Run Krea Enhance on ``image_url``; return the enhanced URL or None.
|
||||
|
||||
Best-effort: any submit/poll/result failure logs and returns ``None`` so
|
||||
the caller falls back to the original (un-upscaled) image — an upscale
|
||||
failure must never destroy an already-successful generation.
|
||||
"""
|
||||
headers = {
|
||||
"Authorization": f"Bearer {auth_token}",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "Hermes-Agent/1.0 (krea-image-gen)",
|
||||
}
|
||||
if managed:
|
||||
headers["x-idempotency-key"] = str(uuid.uuid4())
|
||||
payload: Dict[str, Any] = {
|
||||
"image_url": image_url,
|
||||
"image_scaling_factor": _ENHANCE_SCALE_FACTOR,
|
||||
# Keep the enhancer faithful to the generated composition: the
|
||||
# original prompt guides detail, and default ai_strength stays
|
||||
# conservative (Krea default 0.4 adds detail without redrawing).
|
||||
"prompt": prompt,
|
||||
}
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{base_url}{_ENHANCE_PATH}", headers=headers, json=payload, timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
job_id = (resp.json() or {}).get("job_id")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Krea Enhance submit failed: %s", exc)
|
||||
return None
|
||||
if not isinstance(job_id, str) or not job_id:
|
||||
logger.warning("Krea Enhance submit response missing job_id")
|
||||
return None
|
||||
|
||||
job = _poll_krea_job(base_url, auth_token, job_id)
|
||||
if not isinstance(job, dict) or job.get("status") in {"failed", "cancelled"}:
|
||||
logger.warning("Krea Enhance job %s did not complete successfully", job_id)
|
||||
return None
|
||||
return _extract_result_url(job)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class KreaImageGenProvider(ImageGenProvider):
|
||||
"""Krea ``Krea 2`` foundation image model backend (Medium + Large)."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "krea"
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
return "Krea"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
# Available with a direct Krea key OR via the managed Nous gateway
|
||||
# (Nous Subscription), so portal users with no Krea key can still
|
||||
# reach Krea 2 through the gateway.
|
||||
return bool(get_secret("KREA_API_KEY")) or _managed_krea_gateway_ready()
|
||||
|
||||
def list_models(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": model_id,
|
||||
"display": meta["display"],
|
||||
"speed": meta["speed"],
|
||||
"strengths": meta["strengths"],
|
||||
"price": meta["price"],
|
||||
}
|
||||
for model_id, meta in _MODELS.items()
|
||||
]
|
||||
|
||||
def default_model(self) -> Optional[str]:
|
||||
return DEFAULT_MODEL
|
||||
|
||||
def get_setup_schema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": "Krea",
|
||||
"badge": "paid",
|
||||
"tag": "Krea 2 foundation model — Medium ($0.03), Large ($0.06), Medium Turbo ($0.015). Style transfer, moodboards, reference-guided generation. Direct key or managed Nous Subscription gateway.",
|
||||
"env_vars": [
|
||||
{
|
||||
"key": "KREA_API_KEY",
|
||||
"prompt": "Krea API key",
|
||||
"url": "https://www.krea.ai/settings/api-tokens",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
def capabilities(self) -> Dict[str, Any]:
|
||||
# Krea supports reference-guided generation (image-to-image style
|
||||
# transfer) via image_style_references — up to 10 refs — and an
|
||||
# opt-in Enhance upscale pass (see generate()'s upscale_requested).
|
||||
return {
|
||||
"modalities": ["text", "image"],
|
||||
"max_reference_images": 10,
|
||||
"supports_upscale": True,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# generate()
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
|
||||
*,
|
||||
image_url: Optional[str] = None,
|
||||
reference_image_urls: Optional[List[str]] = None,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
prompt = (prompt or "").strip()
|
||||
aspect = resolve_aspect_ratio(aspect_ratio)
|
||||
krea_ar = _ASPECT_MAP.get(aspect, "1:1")
|
||||
|
||||
# Collect reference images for reference-guided generation (image-to-
|
||||
# image style transfer). Sources, in order:
|
||||
# 1. unified image_url (primary source) + reference_image_urls (strings)
|
||||
# 2. legacy image_style_references kwarg — may be plain URL strings OR
|
||||
# Krea's richer ref objects (e.g. {"url": ..., "strength": ...}),
|
||||
# which are passed through verbatim for backward compatibility.
|
||||
style_refs: List[Any] = []
|
||||
if isinstance(image_url, str) and image_url.strip():
|
||||
style_refs.append(image_url.strip())
|
||||
for ref in (normalize_reference_images(reference_image_urls) or []):
|
||||
style_refs.append(ref)
|
||||
legacy_refs = kwargs.get("image_style_references")
|
||||
if isinstance(legacy_refs, list):
|
||||
for ref in legacy_refs:
|
||||
if isinstance(ref, str):
|
||||
if ref.strip():
|
||||
style_refs.append(ref.strip())
|
||||
elif ref:
|
||||
# Non-string ref object (dict, etc.) — pass through as-is.
|
||||
style_refs.append(ref)
|
||||
# Dedupe string entries while preserving order (dict refs aren't
|
||||
# hashable, so they're kept verbatim); Krea caps at 10.
|
||||
seen: set = set()
|
||||
deduped: List[Any] = []
|
||||
for r in style_refs:
|
||||
if isinstance(r, str):
|
||||
if r in seen:
|
||||
continue
|
||||
seen.add(r)
|
||||
deduped.append(r)
|
||||
style_refs = deduped[:10]
|
||||
modality = "image" if style_refs else "text"
|
||||
|
||||
if not prompt:
|
||||
return error_response(
|
||||
error="Prompt is required and must be a non-empty string",
|
||||
error_type="invalid_argument",
|
||||
provider="krea",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
# Route through the managed Nous gateway (Nous Subscription) when the
|
||||
# user is on the managed path; otherwise use the direct Krea API with a
|
||||
# BYO ``KREA_API_KEY``. The gateway owns the shared Krea credential and
|
||||
# meters/bills per generation, so the caller token is the Nous access
|
||||
# token, not a Krea key.
|
||||
managed = _resolve_managed_krea_gateway()
|
||||
if managed is not None:
|
||||
base_url = managed.gateway_origin.rstrip("/")
|
||||
auth_token = managed.nous_user_token
|
||||
else:
|
||||
base_url = BASE_URL
|
||||
auth_token = get_secret("KREA_API_KEY")
|
||||
if not auth_token:
|
||||
return error_response(
|
||||
error=(
|
||||
"KREA_API_KEY not set. Run `hermes tools` → Image "
|
||||
"Generation → Krea to configure, get a key at "
|
||||
"https://www.krea.ai/settings/api-tokens, or sign in to "
|
||||
"a Nous account with the managed Krea gateway enabled "
|
||||
"(`hermes setup`)."
|
||||
),
|
||||
error_type="auth_required",
|
||||
provider="krea",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
model_id, meta = _resolve_model(kwargs.get("model"))
|
||||
creativity = _resolve_creativity(kwargs.get("creativity"))
|
||||
|
||||
# The managed gateway only prices base text-to-image and URL
|
||||
# ``image_style_references`` tiers. Trained styles (LoRAs) and
|
||||
# moodboards have no managed price and are rejected at the gateway, so
|
||||
# fail fast here with actionable guidance instead of a raw 400.
|
||||
if managed is not None:
|
||||
if isinstance(kwargs.get("styles"), list) and kwargs.get("styles"):
|
||||
return error_response(
|
||||
error=(
|
||||
"Managed Krea (Nous Subscription) does not support "
|
||||
"trained styles (LoRAs). Set KREA_API_KEY to use Krea "
|
||||
"directly, or omit `styles`."
|
||||
),
|
||||
error_type="unsupported_argument",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
if isinstance(kwargs.get("moodboards"), list) and kwargs.get("moodboards"):
|
||||
return error_response(
|
||||
error=(
|
||||
"Managed Krea (Nous Subscription) does not support "
|
||||
"moodboards. Set KREA_API_KEY to use Krea directly, or "
|
||||
"omit `moodboards`."
|
||||
),
|
||||
error_type="unsupported_argument",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"prompt": prompt,
|
||||
"aspect_ratio": krea_ar,
|
||||
"resolution": DEFAULT_RESOLUTION,
|
||||
"creativity": creativity,
|
||||
}
|
||||
|
||||
# Optional forward-compat passthroughs — the Krea API accepts these
|
||||
# but they're not required and most agent calls won't supply them.
|
||||
seed = kwargs.get("seed")
|
||||
if isinstance(seed, int):
|
||||
payload["seed"] = seed
|
||||
|
||||
styles = kwargs.get("styles")
|
||||
if isinstance(styles, list) and styles:
|
||||
payload["styles"] = styles
|
||||
|
||||
if style_refs:
|
||||
# Reference-guided generation (image-to-image style transfer).
|
||||
# Krea requires each entry to be an object ({"url", "strength"}),
|
||||
# NOT a bare URL string — a string yields a 422 "Expected object,
|
||||
# received string". Convert URL strings to the object form and pass
|
||||
# already-object refs through verbatim (clamped to 10 above).
|
||||
normalized_refs: List[Any] = []
|
||||
for ref in style_refs:
|
||||
if isinstance(ref, str):
|
||||
normalized_refs.append(
|
||||
{"url": ref, "strength": _DEFAULT_STYLE_REFERENCE_STRENGTH}
|
||||
)
|
||||
else:
|
||||
normalized_refs.append(ref)
|
||||
payload["image_style_references"] = normalized_refs
|
||||
|
||||
moodboards = kwargs.get("moodboards")
|
||||
if isinstance(moodboards, list) and moodboards:
|
||||
# Krea currently caps at 1 moodboard per request.
|
||||
payload["moodboards"] = moodboards[:1]
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {auth_token}",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "Hermes-Agent/1.0 (krea-image-gen)",
|
||||
}
|
||||
if managed is not None:
|
||||
# The gateway derives the per-generation billing idempotency
|
||||
# boundary from this header (else it falls back to a body
|
||||
# fingerprint). A fresh key per submit keeps each generation a
|
||||
# distinct billable execution.
|
||||
headers["x-idempotency-key"] = str(uuid.uuid4())
|
||||
|
||||
# 1. Submit job.
|
||||
submit_url = f"{base_url}/generate/image/krea/krea-2/{meta['path']}"
|
||||
try:
|
||||
response = requests.post(
|
||||
submit_url,
|
||||
headers=headers,
|
||||
json=payload,
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except requests.HTTPError as exc:
|
||||
resp = exc.response
|
||||
status = resp.status_code if resp is not None else 0
|
||||
try:
|
||||
body = resp.json() if resp is not None else {}
|
||||
err_msg = (
|
||||
body.get("error", {}).get("message")
|
||||
if isinstance(body.get("error"), dict)
|
||||
else body.get("message") or body.get("detail")
|
||||
) or (resp.text[:300] if resp is not None else str(exc))
|
||||
except Exception: # noqa: BLE001
|
||||
err_msg = resp.text[:300] if resp is not None else str(exc)
|
||||
logger.error("Krea submit failed (%d): %s", status, err_msg)
|
||||
# On a managed 4xx, surface actionable remediation mirroring the
|
||||
# FAL managed gateway path: the model may not be enabled/priced on
|
||||
# the Nous Portal, or the gateway's shared Krea key hit its
|
||||
# concurrency cap (429).
|
||||
if managed is not None and 400 <= status < 500:
|
||||
hint = (
|
||||
"Krea's shared-key concurrency cap was hit — retry shortly."
|
||||
if status == 429
|
||||
else (
|
||||
f"Model '{model_id}' may not be enabled/priced on the "
|
||||
"Nous Portal's Krea gateway. Set KREA_API_KEY to use "
|
||||
"Krea directly, or pick a different model via "
|
||||
"`hermes tools` → Image Generation."
|
||||
)
|
||||
)
|
||||
return error_response(
|
||||
error=(
|
||||
f"Nous Subscription Krea gateway rejected '{model_id}' "
|
||||
f"(HTTP {status}): {err_msg}. {hint}"
|
||||
),
|
||||
error_type="api_error",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
return error_response(
|
||||
error=f"Krea image generation failed ({status}): {err_msg}",
|
||||
error_type="api_error",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
except requests.Timeout:
|
||||
return error_response(
|
||||
error="Krea submit timed out (30s)",
|
||||
error_type="timeout",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
except requests.ConnectionError as exc:
|
||||
return error_response(
|
||||
error=f"Krea connection error: {exc}",
|
||||
error_type="connection_error",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
try:
|
||||
submit_body = response.json()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return error_response(
|
||||
error=f"Krea returned invalid JSON on submit: {exc}",
|
||||
error_type="invalid_response",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
job_id = submit_body.get("job_id")
|
||||
if not isinstance(job_id, str) or not job_id:
|
||||
return error_response(
|
||||
error="Krea submit response missing job_id",
|
||||
error_type="invalid_response",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
# 2. Poll for completion. Status/result polling is bound to the same
|
||||
# principal at the gateway, so the managed path polls the gateway's
|
||||
# ``/jobs/{id}`` with the Nous token (404 on cross-user/unknown jobs).
|
||||
job_url = f"{base_url}/jobs/{job_id}"
|
||||
poll_headers = {
|
||||
"Authorization": f"Bearer {auth_token}",
|
||||
"User-Agent": "Hermes-Agent/1.0 (krea-image-gen)",
|
||||
}
|
||||
interval = _POLL_INITIAL_INTERVAL
|
||||
deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS
|
||||
last_status: Optional[str] = None
|
||||
|
||||
while True:
|
||||
time.sleep(interval)
|
||||
interval = min(interval * _POLL_BACKOFF, _POLL_MAX_INTERVAL)
|
||||
|
||||
try:
|
||||
poll_resp = requests.get(job_url, headers=poll_headers, timeout=30)
|
||||
poll_resp.raise_for_status()
|
||||
except requests.HTTPError as exc:
|
||||
resp = exc.response
|
||||
status = resp.status_code if resp is not None else 0
|
||||
logger.error("Krea poll failed (%d) for job %s", status, job_id)
|
||||
# Fail fast for non-retryable statuses (auth/billing/not-found,
|
||||
# other permanent 4xx) so callers don't wait the full 180s
|
||||
# deadline on a request that will never succeed. Only retry
|
||||
# transient statuses such as 408/409/425/429/5xx.
|
||||
if status not in _RETRYABLE_POLL_STATUSES or time.monotonic() >= deadline:
|
||||
return error_response(
|
||||
error=f"Krea poll failed ({status}) for job {job_id}",
|
||||
error_type="api_error",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
# Otherwise keep trying — transient 5xx (and a few retryable
|
||||
# 4xx like 408/409/425/429) are common on async jobs.
|
||||
continue
|
||||
except (requests.Timeout, requests.ConnectionError) as exc:
|
||||
logger.warning("Krea poll transient error for job %s: %s", job_id, exc)
|
||||
if time.monotonic() >= deadline:
|
||||
return error_response(
|
||||
error=f"Krea poll timed out for job {job_id}: {exc}",
|
||||
error_type="timeout",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
job = poll_resp.json()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Krea poll returned invalid JSON for job %s: %s", job_id, exc)
|
||||
if time.monotonic() >= deadline:
|
||||
return error_response(
|
||||
error=f"Krea poll returned invalid JSON: {exc}",
|
||||
error_type="invalid_response",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
continue
|
||||
|
||||
status_str = job.get("status") if isinstance(job, dict) else None
|
||||
if isinstance(status_str, str):
|
||||
last_status = status_str
|
||||
if status_str in _TERMINAL_STATES:
|
||||
break
|
||||
|
||||
# ``completed_at`` is a backstop terminal marker even when the
|
||||
# ``status`` enum is unfamiliar (Krea adds new pending states
|
||||
# over time — backlogged/scheduled/sampling — and we don't
|
||||
# want to mis-handle a future one).
|
||||
if isinstance(job, dict) and job.get("completed_at"):
|
||||
break
|
||||
|
||||
if time.monotonic() >= deadline:
|
||||
return error_response(
|
||||
error=(
|
||||
f"Krea job {job_id} did not complete within "
|
||||
f"{int(_POLL_TIMEOUT_SECONDS)}s (last status: {last_status or 'unknown'})"
|
||||
),
|
||||
error_type="timeout",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
# 3. Terminal — extract result.
|
||||
if not isinstance(job, dict):
|
||||
return error_response(
|
||||
error="Krea returned non-dict job body",
|
||||
error_type="invalid_response",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
if last_status == "failed":
|
||||
err = (job.get("result") or {}).get("error") if isinstance(job.get("result"), dict) else None
|
||||
return error_response(
|
||||
error=f"Krea job {job_id} failed: {err or 'unknown error'}",
|
||||
error_type="api_error",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
if last_status == "cancelled":
|
||||
return error_response(
|
||||
error=f"Krea job {job_id} was cancelled",
|
||||
error_type="cancelled",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
# Successful path — pull URL out of the result.
|
||||
result = job.get("result")
|
||||
if not isinstance(result, dict):
|
||||
return error_response(
|
||||
error="Krea job completed but result was missing",
|
||||
error_type="empty_response",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
# Per Krea's job-lifecycle docs the completed payload exposes
|
||||
# ``result.urls`` (an array). Fall back to a single ``url`` field
|
||||
# for forward/backward compatibility.
|
||||
result_image_url: Optional[str] = None
|
||||
urls = result.get("urls")
|
||||
if isinstance(urls, list) and urls:
|
||||
for candidate in urls:
|
||||
if isinstance(candidate, str) and candidate.strip():
|
||||
result_image_url = candidate.strip()
|
||||
break
|
||||
if result_image_url is None:
|
||||
single = result.get("url")
|
||||
if isinstance(single, str) and single.strip():
|
||||
result_image_url = single.strip()
|
||||
|
||||
if result_image_url is None:
|
||||
return error_response(
|
||||
error="Krea result contained no image URL",
|
||||
error_type="empty_response",
|
||||
provider="krea",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
# High-resolution pass (Krea Enhance). Precedence: explicit kwarg >
|
||||
# ``image_gen.krea.upscale`` config > per-model catalog default
|
||||
# (1.5K-native tiers default on; 2K-native Large stays off). Best-
|
||||
# effort: failure falls back to the original image rather than
|
||||
# failing the generation.
|
||||
upscaled = False
|
||||
upscale_requested = kwargs.get("upscale")
|
||||
if not isinstance(upscale_requested, bool):
|
||||
cfg_krea = _load_krea_config().get("krea")
|
||||
cfg_upscale = cfg_krea.get("upscale") if isinstance(cfg_krea, dict) else None
|
||||
if isinstance(cfg_upscale, bool):
|
||||
upscale_requested = cfg_upscale
|
||||
else:
|
||||
upscale_requested = bool(meta.get("upscale", False))
|
||||
if upscale_requested:
|
||||
enhanced_url = _enhance_image(
|
||||
base_url,
|
||||
auth_token,
|
||||
result_image_url,
|
||||
prompt,
|
||||
managed=managed is not None,
|
||||
)
|
||||
if enhanced_url:
|
||||
result_image_url = enhanced_url
|
||||
upscaled = True
|
||||
else:
|
||||
logger.warning(
|
||||
"Krea Enhance pass failed — returning native-resolution image"
|
||||
)
|
||||
|
||||
# Materialise locally — Krea result URLs may expire, mirroring
|
||||
# what we do for xAI / OpenAI URL responses (#26942).
|
||||
try:
|
||||
saved_path = save_url_image(result_image_url, prefix=f"krea_{model_id}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"Krea image URL %s could not be cached (%s); falling back to bare URL.",
|
||||
result_image_url,
|
||||
exc,
|
||||
)
|
||||
image_ref = result_image_url
|
||||
else:
|
||||
image_ref = str(saved_path)
|
||||
|
||||
extra: Dict[str, Any] = {
|
||||
"krea_aspect_ratio": krea_ar,
|
||||
"resolution": DEFAULT_RESOLUTION,
|
||||
"creativity": creativity,
|
||||
"job_id": job_id,
|
||||
"upscaled": upscaled,
|
||||
}
|
||||
if upscaled:
|
||||
extra["upscale_factor"] = _ENHANCE_SCALE_FACTOR
|
||||
if isinstance(job.get("completed_at"), str):
|
||||
extra["completed_at"] = job["completed_at"]
|
||||
|
||||
return success_response(
|
||||
image=image_ref,
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
provider="krea",
|
||||
modality=modality,
|
||||
extra=extra,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Plugin entry point — wire ``KreaImageGenProvider`` into the registry."""
|
||||
ctx.register_image_gen_provider(KreaImageGenProvider())
|
||||
@@ -0,0 +1,7 @@
|
||||
name: krea
|
||||
version: 1.1.0
|
||||
description: "Krea image generation backend (Krea 2 Large + Medium + Medium Turbo foundation models). Direct KREA_API_KEY or managed Nous Subscription gateway."
|
||||
author: NousResearch
|
||||
kind: backend
|
||||
requires_env:
|
||||
- KREA_API_KEY
|
||||
@@ -0,0 +1,297 @@
|
||||
"""Meta Model API image generation backend.
|
||||
|
||||
Exposes Meta's ``muse-image`` model(s) as an :class:`ImageGenProvider`.
|
||||
The Meta Model API (https://api.meta.ai/v1) is OpenAI-compatible, so we reuse
|
||||
the OpenAI Python SDK pointed at Meta's base URL and authenticate with
|
||||
``META_MODEL_API_KEY``.
|
||||
|
||||
Output is base64 JSON (WebP) -> saved under ``$HERMES_HOME/cache/images/``.
|
||||
|
||||
Selection precedence (first hit wins):
|
||||
1. ``model`` kwarg forwarded by the dispatcher (the ``hermes tools`` pick)
|
||||
2. ``META_IMAGE_MODEL`` env var (escape hatch for scripts / tests)
|
||||
3. ``image_gen.meta-ai.model`` in ``config.yaml``
|
||||
4. ``image_gen.model`` in ``config.yaml`` (when it's one of our IDs)
|
||||
5. :data:`DEFAULT_MODEL`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from agent.secret_scope import get_secret
|
||||
from agent.image_gen_provider import (
|
||||
DEFAULT_ASPECT_RATIO,
|
||||
ImageGenProvider,
|
||||
error_response,
|
||||
normalize_reference_images,
|
||||
resolve_aspect_ratio,
|
||||
save_b64_image,
|
||||
save_url_image,
|
||||
success_response,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_BASE_URL = "https://api.meta.ai/v1"
|
||||
# Auth env vars, in priority order. Mirrors the bundled ``meta-ai`` chat
|
||||
# provider (plugins/model-providers/meta-ai): MODEL_API_KEY is Meta's
|
||||
# documented var; the rest are accepted aliases.
|
||||
API_KEY_ENVS = ("MODEL_API_KEY", "META_API_KEY", "META_MODEL_API_KEY")
|
||||
# Primary key shown in setup prompts / error messages.
|
||||
API_KEY_ENV = "META_MODEL_API_KEY"
|
||||
# Optional base-url override (same var the chat provider honors).
|
||||
BASE_URL_ENV = "META_BASE_URL"
|
||||
|
||||
|
||||
def _resolve_api_key() -> Optional[str]:
|
||||
"""First non-empty auth env var, checked in priority order."""
|
||||
for env in API_KEY_ENVS:
|
||||
val = get_secret(env)
|
||||
if val:
|
||||
return val
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_base_url() -> str:
|
||||
return (os.environ.get(BASE_URL_ENV) or "").strip() or DEFAULT_BASE_URL
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model catalog
|
||||
# ---------------------------------------------------------------------------
|
||||
# Catalog shown in `hermes tools` and matched against `image_gen.model`.
|
||||
# The model id is sent verbatim to the Meta Model API (`/v1/images/generations`).
|
||||
_MODELS: Dict[str, Dict[str, Any]] = {
|
||||
"muse-image-1.0": {
|
||||
"display": "Muse Image 1.0",
|
||||
"speed": "~10s",
|
||||
"strengths": "Meta Model API image generation",
|
||||
"price": "$0.01/image",
|
||||
},
|
||||
}
|
||||
DEFAULT_MODEL = "muse-image-1.0"
|
||||
|
||||
# aspect_ratio -> OpenAI-style size string
|
||||
_SIZES: Dict[str, str] = {
|
||||
"square": "1024x1024",
|
||||
"landscape": "1536x1024",
|
||||
"portrait": "1024x1536",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_model(caller_model: Optional[str] = None) -> Tuple[str, Dict[str, Any]]:
|
||||
"""Return (model_id, metadata) using the documented precedence chain.
|
||||
|
||||
``caller_model`` is the ``model`` kwarg the dispatcher forwards from the
|
||||
top-level ``image_gen.model`` config key (what ``hermes tools`` writes).
|
||||
It wins when it names one of our models, mirroring the xai/krea/openrouter
|
||||
providers, so a user's picker choice is never silently dropped.
|
||||
"""
|
||||
if caller_model and caller_model in _MODELS:
|
||||
return caller_model, _MODELS[caller_model]
|
||||
|
||||
env_model = os.environ.get("META_IMAGE_MODEL")
|
||||
if env_model and env_model in _MODELS:
|
||||
return env_model, _MODELS[env_model]
|
||||
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
cfg = load_config() or {}
|
||||
ig = cfg.get("image_gen") or {}
|
||||
scoped = (ig.get("meta-ai") or {}).get("model")
|
||||
if scoped and scoped in _MODELS:
|
||||
return scoped, _MODELS[scoped]
|
||||
top = ig.get("model")
|
||||
if top and top in _MODELS:
|
||||
return top, _MODELS[top]
|
||||
except Exception:
|
||||
logger.debug("Could not read image_gen model from config", exc_info=True)
|
||||
|
||||
return DEFAULT_MODEL, _MODELS[DEFAULT_MODEL]
|
||||
|
||||
|
||||
class MetaImageGenProvider(ImageGenProvider):
|
||||
"""Meta Model API ``images.generate`` backend (muse-image)."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "meta-ai"
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
return "Meta Model API"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
if not _resolve_api_key():
|
||||
return False
|
||||
try:
|
||||
import openai # noqa: F401
|
||||
except ImportError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def list_models(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": mid,
|
||||
"display": m["display"],
|
||||
"speed": m["speed"],
|
||||
"strengths": m["strengths"],
|
||||
"price": m["price"],
|
||||
}
|
||||
for mid, m in _MODELS.items()
|
||||
]
|
||||
|
||||
def default_model(self) -> Optional[str]:
|
||||
return DEFAULT_MODEL
|
||||
|
||||
def get_setup_schema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": "Meta Model API",
|
||||
"badge": "paid",
|
||||
"tag": "Muse Image via Meta Model API (api.meta.ai)",
|
||||
"env_vars": [
|
||||
{
|
||||
"key": API_KEY_ENV,
|
||||
"prompt": "Meta Model API key (LLM|... token)",
|
||||
"url": "https://api.meta.ai",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
def capabilities(self) -> Dict[str, Any]:
|
||||
# Text-to-image only for now. Bump this once image-to-image is verified
|
||||
# against the Meta endpoint.
|
||||
return {"modalities": ["text"], "max_reference_images": 0}
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
|
||||
*,
|
||||
image_url: Optional[str] = None,
|
||||
reference_image_urls: Optional[List[str]] = None,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
prompt = (prompt or "").strip()
|
||||
aspect = resolve_aspect_ratio(aspect_ratio)
|
||||
|
||||
if not prompt:
|
||||
return error_response(
|
||||
error="Prompt is required and must be a non-empty string",
|
||||
error_type="invalid_argument",
|
||||
provider="meta-ai",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
api_key = _resolve_api_key()
|
||||
if not api_key:
|
||||
return error_response(
|
||||
error=(
|
||||
f"{API_KEY_ENV} not set. Run `hermes tools` -> Image "
|
||||
"Generation -> Meta Model API to configure."
|
||||
),
|
||||
error_type="auth_required",
|
||||
provider="meta-ai",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
try:
|
||||
import openai
|
||||
except ImportError:
|
||||
return error_response(
|
||||
error="openai Python package not installed (pip install openai)",
|
||||
error_type="missing_dependency",
|
||||
provider="meta-ai",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
model_id, _meta = _resolve_model(kwargs.get("model"))
|
||||
size = _SIZES.get(aspect, _SIZES["square"])
|
||||
|
||||
client = openai.OpenAI(api_key=api_key, base_url=_resolve_base_url())
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"model": model_id,
|
||||
"prompt": prompt,
|
||||
"size": size,
|
||||
"n": 1,
|
||||
}
|
||||
|
||||
try:
|
||||
response = client.images.generate(**payload)
|
||||
except Exception as exc:
|
||||
logger.debug("Meta image generation failed", exc_info=True)
|
||||
return error_response(
|
||||
error=f"Meta image generation failed: {exc}",
|
||||
error_type="api_error",
|
||||
provider="meta-ai",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
try:
|
||||
first = response.data[0]
|
||||
except (AttributeError, IndexError, TypeError):
|
||||
return error_response(
|
||||
error="Meta response contained no image data",
|
||||
error_type="empty_response",
|
||||
provider="meta-ai",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
b64 = getattr(first, "b64_json", None)
|
||||
url = getattr(first, "url", None)
|
||||
|
||||
try:
|
||||
if b64:
|
||||
path = save_b64_image(b64, prefix="meta", extension="webp")
|
||||
image_ref = str(path)
|
||||
elif url:
|
||||
path = save_url_image(url, prefix="meta")
|
||||
image_ref = str(path)
|
||||
else:
|
||||
return error_response(
|
||||
error="Meta response contained neither b64_json nor URL",
|
||||
error_type="empty_response",
|
||||
provider="meta-ai",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
except Exception as exc:
|
||||
return error_response(
|
||||
error=f"Failed to save Meta image: {exc}",
|
||||
error_type="io_error",
|
||||
provider="meta-ai",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
revised_prompt = getattr(first, "revised_prompt", None)
|
||||
extra: Dict[str, Any] = {"size": size}
|
||||
if revised_prompt:
|
||||
extra["revised_prompt"] = revised_prompt
|
||||
|
||||
return success_response(
|
||||
image=image_ref,
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
provider="meta-ai",
|
||||
modality="text",
|
||||
extra=extra,
|
||||
)
|
||||
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Plugin entry point -- wire ``MetaImageGenProvider`` into the registry."""
|
||||
ctx.register_image_gen_provider(MetaImageGenProvider())
|
||||
@@ -0,0 +1,7 @@
|
||||
name: meta-ai-image-gen
|
||||
version: 1.0.0
|
||||
description: "Meta Model API image generation backend (muse-image). OpenAI-compatible /v1/images/generations. Saves images to $HERMES_HOME/cache/images/."
|
||||
author: Meta Platforms, Inc.
|
||||
kind: backend
|
||||
requires_env:
|
||||
- META_MODEL_API_KEY
|
||||
@@ -0,0 +1,770 @@
|
||||
"""OpenAI image generation backend — ChatGPT/Codex OAuth variant.
|
||||
|
||||
Identical model catalog and tier semantics to the ``openai`` image-gen plugin
|
||||
(``gpt-image-2`` at low/medium/high quality), but routes the request through
|
||||
the Codex Responses API ``image_generation`` tool instead of the
|
||||
``images.generate`` REST endpoint. This lets users who are already
|
||||
authenticated with Codex/ChatGPT generate images without configuring a
|
||||
separate ``OPENAI_API_KEY``.
|
||||
|
||||
Selection precedence for the tier (first hit wins):
|
||||
|
||||
1. ``OPENAI_IMAGE_MODEL`` env var (escape hatch for scripts / tests)
|
||||
2. ``image_gen.openai-codex.model`` in ``config.yaml``
|
||||
3. ``image_gen.model`` in ``config.yaml`` (when it's one of our tier IDs)
|
||||
4. :data:`DEFAULT_MODEL` — ``gpt-image-2-medium``
|
||||
|
||||
Output is saved as PNG under ``$HERMES_HOME/cache/images/``. Source images for
|
||||
image-to-image/editing are sent as Responses ``input_image`` content parts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from agent.image_gen_provider import (
|
||||
DEFAULT_ASPECT_RATIO,
|
||||
ImageGenProvider,
|
||||
error_response,
|
||||
normalize_reference_images,
|
||||
resolve_aspect_ratio,
|
||||
save_b64_image,
|
||||
success_response,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# NOTE: do NOT reintroduce an "account capability" classifier keyed on
|
||||
# ``Tool choice 'image_generation' not found in 'tools' parameter``. That HTTP
|
||||
# 400 is a *request-shape* rejection (the Codex backend resolves tool_choice as
|
||||
# a function-tool name and never recognizes hosted-tool entries) — it is
|
||||
# emitted for every account, including accounts where image generation works.
|
||||
# A previous version of this file translated that 400 into "Image generation is
|
||||
# not enabled for the current Codex account. Switch the image provider to
|
||||
# OpenAI API key, FAL, or xAI.", which reported a universal bug in our own
|
||||
# payload as the user's entitlement problem and sent people away from a
|
||||
# provider that was never actually tried. The request-shape bug is fixed by
|
||||
# omitting tool_choice (see ``_build_responses_payload``); any remaining HTTP
|
||||
# error must surface verbatim so it stays diagnosable. See issues #19505,
|
||||
# #49008 and #31335.
|
||||
|
||||
_MAX_ERROR_BODY_CHARS = 500
|
||||
|
||||
|
||||
def _summarize_error_body(body: str) -> str:
|
||||
"""Return a bounded, information-preserving summary of an error body.
|
||||
|
||||
Prefers the parsed ``error.message`` field, because a blind head-truncation
|
||||
of the raw body can cut the actual message off entirely — Codex error
|
||||
payloads sometimes carry hundreds of bytes of leading metadata, so
|
||||
``body[:500]`` yielded a wall of padding and no diagnosis. Falls back to a
|
||||
truncated raw body for non-JSON responses.
|
||||
"""
|
||||
text = body or ""
|
||||
try:
|
||||
payload = json.loads(text)
|
||||
error = payload.get("error") if isinstance(payload, dict) else None
|
||||
message = error.get("message") if isinstance(error, dict) else None
|
||||
if isinstance(message, str) and message.strip():
|
||||
return message.strip()[:_MAX_ERROR_BODY_CHARS]
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return text[:_MAX_ERROR_BODY_CHARS]
|
||||
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model catalog — mirrors the ``openai`` plugin so the picker UX is identical.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
API_MODEL = "gpt-image-2"
|
||||
|
||||
_MODELS: Dict[str, Dict[str, Any]] = {
|
||||
"gpt-image-2-low": {
|
||||
"display": "GPT Image 2 (Low)",
|
||||
"speed": "~15s",
|
||||
"strengths": "Fast iteration, lowest cost",
|
||||
"quality": "low",
|
||||
},
|
||||
"gpt-image-2-medium": {
|
||||
"display": "GPT Image 2 (Medium)",
|
||||
"speed": "~40s",
|
||||
"strengths": "Balanced — default",
|
||||
"quality": "medium",
|
||||
},
|
||||
"gpt-image-2-high": {
|
||||
"display": "GPT Image 2 (High)",
|
||||
"speed": "~2min",
|
||||
"strengths": "Highest fidelity, strongest prompt adherence",
|
||||
"quality": "high",
|
||||
},
|
||||
}
|
||||
|
||||
DEFAULT_MODEL = "gpt-image-2-medium"
|
||||
|
||||
_SIZES = {
|
||||
"landscape": "1536x1024",
|
||||
"square": "1024x1024",
|
||||
"portrait": "1024x1536",
|
||||
}
|
||||
|
||||
# Codex Responses surface used for the request. The chat model itself is only
|
||||
# the host that calls the ``image_generation`` tool; the actual image work is
|
||||
# done by ``API_MODEL``.
|
||||
_CODEX_CHAT_MODEL = "gpt-5.5"
|
||||
_CODEX_BASE_URL = "https://chatgpt.com/backend-api/codex"
|
||||
_CODEX_INSTRUCTIONS = (
|
||||
"You are an assistant that must fulfill image generation and image editing "
|
||||
"requests by using the image_generation tool when provided."
|
||||
)
|
||||
|
||||
_MAX_REFERENCE_IMAGES = 16
|
||||
_MAX_INPUT_IMAGE_BYTES = 25 * 1024 * 1024
|
||||
# gpt-image-2's Responses ``input_image`` accepts raster formats only. The
|
||||
# shared magic-byte sniffer also recognizes SVG/TIFF/ICO, which the API
|
||||
# rejects server-side — gate to this allowlist so unsupported inputs fail
|
||||
# locally with a clear error instead of an opaque HTTP 400.
|
||||
_ACCEPTED_INPUT_MIME = frozenset(
|
||||
{"image/png", "image/jpeg", "image/gif", "image/webp"}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config + auth helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_image_gen_config() -> Dict[str, Any]:
|
||||
"""Read ``image_gen`` from config.yaml (returns {} on any failure)."""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
cfg = load_config()
|
||||
section = cfg.get("image_gen") if isinstance(cfg, dict) else None
|
||||
return section if isinstance(section, dict) else {}
|
||||
except Exception as exc:
|
||||
logger.debug("Could not load image_gen config: %s", exc)
|
||||
return {}
|
||||
|
||||
|
||||
def _resolve_model() -> Tuple[str, Dict[str, Any]]:
|
||||
"""Decide which tier to use and return ``(model_id, meta)``."""
|
||||
import os
|
||||
|
||||
env_override = os.environ.get("OPENAI_IMAGE_MODEL")
|
||||
if env_override and env_override in _MODELS:
|
||||
return env_override, _MODELS[env_override]
|
||||
|
||||
cfg = _load_image_gen_config()
|
||||
sub = cfg.get("openai-codex") if isinstance(cfg.get("openai-codex"), dict) else {}
|
||||
candidate: Optional[str] = None
|
||||
if isinstance(sub, dict):
|
||||
value = sub.get("model")
|
||||
if isinstance(value, str) and value in _MODELS:
|
||||
candidate = value
|
||||
if candidate is None:
|
||||
top = cfg.get("model")
|
||||
if isinstance(top, str) and top in _MODELS:
|
||||
candidate = top
|
||||
|
||||
if candidate is not None:
|
||||
return candidate, _MODELS[candidate]
|
||||
|
||||
return DEFAULT_MODEL, _MODELS[DEFAULT_MODEL]
|
||||
|
||||
|
||||
def _read_codex_access_token() -> Optional[str]:
|
||||
"""Return a usable Codex OAuth token, or None.
|
||||
|
||||
Delegates to the canonical reader in ``agent.auxiliary_client`` so token
|
||||
expiry, credential pool selection, and JWT decoding stay in one place.
|
||||
"""
|
||||
try:
|
||||
from agent.auxiliary_client import _read_codex_access_token as _reader
|
||||
|
||||
token = _reader()
|
||||
if isinstance(token, str) and token.strip():
|
||||
return token.strip()
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.debug("Could not resolve Codex access token: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _sniff_image_mime(raw: bytes) -> Optional[str]:
|
||||
"""Return a safe raster image MIME from magic bytes (not filename labels).
|
||||
|
||||
Delegates magic-byte detection to the shared sniffer in
|
||||
``agent.image_routing`` (single source of truth), then gates the result
|
||||
to :data:`_ACCEPTED_INPUT_MIME` — the raster formats gpt-image-2's
|
||||
``input_image`` actually accepts. SVG/TIFF/ICO (which the shared sniffer
|
||||
also recognizes) are rejected here so they fail locally with a clear
|
||||
error instead of an opaque server-side HTTP 400.
|
||||
"""
|
||||
from agent.image_routing import _sniff_mime_from_bytes
|
||||
|
||||
mime = _sniff_mime_from_bytes(raw)
|
||||
if mime in _ACCEPTED_INPUT_MIME:
|
||||
return mime
|
||||
return None
|
||||
|
||||
|
||||
def _data_url_to_input_image_url(value: str) -> str:
|
||||
"""Validate and canonicalize a data:image URL for Responses input_image."""
|
||||
if "," not in value:
|
||||
raise ValueError("Image data URL is missing a comma separator")
|
||||
header, data = value.split(",", 1)
|
||||
header_lc = header.lower()
|
||||
if not header_lc.startswith("data:image/") or ";base64" not in header_lc:
|
||||
raise ValueError("Only base64 data:image URLs are supported as Codex image inputs")
|
||||
raw = base64.b64decode(data, validate=True)
|
||||
if len(raw) > _MAX_INPUT_IMAGE_BYTES:
|
||||
raise ValueError("Image data URL exceeds 25MB cap")
|
||||
mime = _sniff_image_mime(raw)
|
||||
if mime is None:
|
||||
raise ValueError("Image data URL does not contain supported image bytes")
|
||||
encoded = base64.b64encode(raw).decode("ascii")
|
||||
return f"data:{mime};base64,{encoded}"
|
||||
|
||||
|
||||
def _local_image_to_data_url(value: str) -> str:
|
||||
"""Read a local image path and return a validated data:image URL."""
|
||||
try:
|
||||
from agent.file_safety import get_read_block_error
|
||||
|
||||
blocked = get_read_block_error(value)
|
||||
if blocked:
|
||||
raise ValueError(blocked)
|
||||
except ValueError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.debug("Codex image input read guard unavailable: %s", exc)
|
||||
|
||||
path = Path(os.path.expanduser(value)).resolve()
|
||||
if not path.is_file():
|
||||
raise ValueError(f"Image input path does not exist or is not a file: {value}")
|
||||
size = path.stat().st_size
|
||||
if size <= 0:
|
||||
raise ValueError(f"Image input path is empty: {value}")
|
||||
if size > _MAX_INPUT_IMAGE_BYTES:
|
||||
raise ValueError(f"Image input path exceeds 25MB cap: {value}")
|
||||
raw = path.read_bytes()
|
||||
mime = _sniff_image_mime(raw)
|
||||
if mime is None:
|
||||
raise ValueError(f"Image input path is not a supported image: {value}")
|
||||
encoded = base64.b64encode(raw).decode("ascii")
|
||||
return f"data:{mime};base64,{encoded}"
|
||||
|
||||
|
||||
def _to_input_image_part(value: str) -> Dict[str, str]:
|
||||
"""Convert a URL/data URL/local path into a Responses input_image part."""
|
||||
candidate = (value or "").strip()
|
||||
if not candidate:
|
||||
raise ValueError("Blank image input")
|
||||
lowered = candidate.lower()
|
||||
if lowered.startswith("http://") or lowered.startswith("https://"):
|
||||
image_url = candidate
|
||||
elif lowered.startswith("data:"):
|
||||
image_url = _data_url_to_input_image_url(candidate)
|
||||
else:
|
||||
image_url = _local_image_to_data_url(candidate)
|
||||
return {"type": "input_image", "image_url": image_url}
|
||||
|
||||
|
||||
def _normalize_input_images(
|
||||
image_url: Optional[str],
|
||||
reference_image_urls: Optional[List[str]],
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Collect primary + reference images as ordered Responses content parts."""
|
||||
values: List[str] = []
|
||||
if isinstance(image_url, str) and image_url.strip():
|
||||
values.append(image_url.strip())
|
||||
for ref in (normalize_reference_images(reference_image_urls) or []):
|
||||
values.append(ref)
|
||||
values = values[:_MAX_REFERENCE_IMAGES]
|
||||
return [_to_input_image_part(value) for value in values]
|
||||
|
||||
|
||||
# Progressive preview frames (partial_image_b64) are intermediate renders.
|
||||
# Saving them as finals produced the long-running "smear" failure mode on the
|
||||
# Codex Responses path. Defense in depth:
|
||||
# 1) request layer prefers no progressive frames when the backend honors it
|
||||
# 2) extractor never lets a partial overwrite a final result
|
||||
# 3) generate() only delivers source=final; partial-only / empty are not success
|
||||
# Live streams sometimes still emit a partial event even with 0; that is fine as
|
||||
# long as only a final ``result`` can be saved.
|
||||
_PARTIAL_IMAGES_REQUESTED = 0
|
||||
# Content-agnostic retries when the stream does not yield a final result
|
||||
# (empty stream or progressive-only). No prompt-class branching.
|
||||
_NONFINAL_RETRIES = 1
|
||||
|
||||
|
||||
def _build_responses_payload(
|
||||
*,
|
||||
prompt: str,
|
||||
size: str,
|
||||
quality: str,
|
||||
input_images: Optional[List[Dict[str, str]]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build the Codex Responses request body for an image_generation call."""
|
||||
content: List[Dict[str, Any]] = [{"type": "input_text", "text": prompt}]
|
||||
if input_images:
|
||||
content.extend(input_images)
|
||||
return {
|
||||
"model": _CODEX_CHAT_MODEL,
|
||||
"store": False,
|
||||
"instructions": _CODEX_INSTRUCTIONS,
|
||||
"input": [{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": content,
|
||||
}],
|
||||
"tools": [{
|
||||
"type": "image_generation",
|
||||
"model": API_MODEL,
|
||||
"size": size,
|
||||
"quality": quality,
|
||||
"output_format": "png",
|
||||
"background": "opaque",
|
||||
# Prefer 0 progressive preview frames. Preview frames can arrive
|
||||
# without a later final ``result`` and look like smeared /
|
||||
# unfinished images if saved as the deliverable. Even when the
|
||||
# backend still emits a partial event, generate() refuses to
|
||||
# deliver anything except source=final.
|
||||
"partial_images": _PARTIAL_IMAGES_REQUESTED,
|
||||
}],
|
||||
# No ``tool_choice`` is sent: the chatgpt.com/backend-api/codex backend
|
||||
# rejects every shape we have for forcing the hosted ``image_generation``
|
||||
# tool. ``{"type": "allowed_tools", "mode": "required", "tools": [{"type":
|
||||
# "image_generation"}]}`` (and the simpler ``{"type": "image_generation"}``
|
||||
# form) both 400 with ``Tool choice 'image_generation' not found in 'tools'
|
||||
# parameter`` — the backend looks up tool_choice as a *function* name and
|
||||
# never recognizes hosted-tool entries. Letting the host model decide is
|
||||
# the only shape Codex currently accepts; the ``instructions`` above are
|
||||
# what nudge it toward the tool. See issue #19505.
|
||||
"stream": True,
|
||||
}
|
||||
|
||||
|
||||
def _extract_image_candidates(value: Any) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Return ``(final_result_b64, latest_partial_b64)`` from a payload tree.
|
||||
|
||||
Final ``image_generation_call.result`` and progressive ``partial_image_b64``
|
||||
are tracked separately so a partial can never overwrite a genuine final,
|
||||
including when both coexist in the same event payload.
|
||||
"""
|
||||
result_b64: Optional[str] = None
|
||||
partial_b64: Optional[str] = None
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
nonlocal result_b64, partial_b64
|
||||
if isinstance(node, dict):
|
||||
if node.get("type") == "image_generation_call":
|
||||
result = node.get("result")
|
||||
if isinstance(result, str) and result:
|
||||
result_b64 = result
|
||||
partial = node.get("partial_image_b64")
|
||||
if isinstance(partial, str) and partial:
|
||||
partial_b64 = partial
|
||||
for child in node.values():
|
||||
walk(child)
|
||||
elif isinstance(node, list):
|
||||
for child in node:
|
||||
walk(child)
|
||||
|
||||
walk(value)
|
||||
return result_b64, partial_b64
|
||||
|
||||
|
||||
def _extract_image_b64(value: Any) -> Optional[str]:
|
||||
"""Return image b64 from a payload, preferring final result over partial.
|
||||
|
||||
Progressive ``partial_image_b64`` is only used when no final
|
||||
``image_generation_call.result`` is present in the same payload tree.
|
||||
"""
|
||||
result_b64, partial_b64 = _extract_image_candidates(value)
|
||||
return result_b64 or partial_b64
|
||||
|
||||
|
||||
def _png_pixel_size(raw: bytes) -> Optional[str]:
|
||||
"""Return ``\"{w}x{h}\"`` for a PNG payload, or None if not a PNG IHDR."""
|
||||
import struct
|
||||
|
||||
if len(raw) < 24 or raw[:8] != b"\x89PNG\r\n\x1a\n":
|
||||
return None
|
||||
# IHDR: length(4) + type(4) + width(4) + height(4)
|
||||
if raw[12:16] != b"IHDR":
|
||||
return None
|
||||
width, height = struct.unpack(">II", raw[16:24])
|
||||
return f"{width}x{height}"
|
||||
|
||||
|
||||
def _iter_sse_json(response: Any):
|
||||
"""Yield JSON payloads from an SSE response without OpenAI SDK parsing.
|
||||
|
||||
The ChatGPT/Codex backend can emit image-generation events newer than the
|
||||
pinned Python SDK understands. Parsing raw SSE keeps this provider tolerant
|
||||
of those event-shape changes.
|
||||
"""
|
||||
event_name: Optional[str] = None
|
||||
data_lines: List[str] = []
|
||||
|
||||
def flush():
|
||||
nonlocal event_name, data_lines
|
||||
if not data_lines:
|
||||
event_name = None
|
||||
return None
|
||||
raw = "\n".join(data_lines).strip()
|
||||
event = event_name
|
||||
event_name = None
|
||||
data_lines = []
|
||||
if not raw or raw == "[DONE]":
|
||||
return None
|
||||
payload = json.loads(raw)
|
||||
if isinstance(payload, dict) and event and "type" not in payload:
|
||||
payload["type"] = event
|
||||
return payload
|
||||
|
||||
for line in response.iter_lines():
|
||||
if isinstance(line, bytes):
|
||||
line = line.decode("utf-8", errors="replace")
|
||||
line = str(line)
|
||||
if line == "":
|
||||
payload = flush()
|
||||
if payload is not None:
|
||||
yield payload
|
||||
continue
|
||||
if line.startswith(":"):
|
||||
continue
|
||||
if line.startswith("event:"):
|
||||
event_name = line[len("event:"):].strip()
|
||||
elif line.startswith("data:"):
|
||||
data_lines.append(line[len("data:"):].lstrip())
|
||||
|
||||
payload = flush()
|
||||
if payload is not None:
|
||||
yield payload
|
||||
|
||||
|
||||
def _collect_image_b64(
|
||||
token: str,
|
||||
*,
|
||||
prompt: str,
|
||||
size: str,
|
||||
quality: str,
|
||||
input_images: Optional[List[Dict[str, str]]] = None,
|
||||
) -> Optional[Dict[str, str]]:
|
||||
"""Stream a Codex Responses image_generation call.
|
||||
|
||||
Returns ``{\"b64\": ..., \"source\": \"final\"|\"partial\"}`` or ``None``.
|
||||
|
||||
Final ``result`` frames are preferred across the whole stream. A progressive
|
||||
``partial_image_b64`` is retained only when no final result ever arrives;
|
||||
callers must not treat partial-only as an unconditional success.
|
||||
"""
|
||||
import httpx
|
||||
from agent.codex_headers import codex_cloudflare_headers
|
||||
|
||||
headers = codex_cloudflare_headers(token)
|
||||
headers.update({
|
||||
"Accept": "text/event-stream",
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
})
|
||||
payload = _build_responses_payload(
|
||||
prompt=prompt,
|
||||
size=size,
|
||||
quality=quality,
|
||||
input_images=input_images,
|
||||
)
|
||||
timeout = httpx.Timeout(300.0, connect=30.0, read=300.0, write=30.0, pool=30.0)
|
||||
|
||||
final_b64: Optional[str] = None
|
||||
partial_b64: Optional[str] = None
|
||||
with httpx.Client(timeout=timeout, headers=headers) as http:
|
||||
with http.stream("POST", f"{_CODEX_BASE_URL}/responses", json=payload) as response:
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
exc.response.read()
|
||||
raise RuntimeError(
|
||||
f"Codex Responses API returned HTTP {exc.response.status_code}: "
|
||||
f"{_summarize_error_body(exc.response.text)}"
|
||||
) from exc
|
||||
for event in _iter_sse_json(response):
|
||||
result_b64, event_partial = _extract_image_candidates(event)
|
||||
if result_b64:
|
||||
final_b64 = result_b64
|
||||
if event_partial:
|
||||
partial_b64 = event_partial
|
||||
|
||||
if final_b64:
|
||||
return {"b64": final_b64, "source": "final"}
|
||||
if partial_b64:
|
||||
return {"b64": partial_b64, "source": "partial"}
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class OpenAICodexImageGenProvider(ImageGenProvider):
|
||||
"""gpt-image-2 routed through ChatGPT/Codex OAuth instead of an API key."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "openai-codex"
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
return "OpenAI (Codex auth)"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
if not _read_codex_access_token():
|
||||
return False
|
||||
try:
|
||||
import httpx # noqa: F401
|
||||
except ImportError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def list_models(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": model_id,
|
||||
"display": meta["display"],
|
||||
"speed": meta["speed"],
|
||||
"strengths": meta["strengths"],
|
||||
"price": "varies",
|
||||
}
|
||||
for model_id, meta in _MODELS.items()
|
||||
]
|
||||
|
||||
def default_model(self) -> Optional[str]:
|
||||
return DEFAULT_MODEL
|
||||
|
||||
def get_setup_schema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": "OpenAI (Codex auth)",
|
||||
"badge": "free",
|
||||
"tag": "gpt-image-2 via ChatGPT/Codex OAuth — no API key required; supports text and image inputs",
|
||||
"env_vars": [],
|
||||
"post_setup_hint": (
|
||||
"Sign in with `hermes auth codex` (or `hermes setup` → Codex) "
|
||||
"if you haven't already. No API key needed."
|
||||
),
|
||||
}
|
||||
|
||||
def capabilities(self) -> Dict[str, Any]:
|
||||
# The Codex Responses image_generation tool accepts source/reference
|
||||
# images as `input_image` message content parts. Keep this capability
|
||||
# honest so the dynamic `image_generate` schema encourages identity-
|
||||
# preserving edits instead of unrelated text-to-image redraws.
|
||||
return {"modalities": ["text", "image"], "max_reference_images": _MAX_REFERENCE_IMAGES}
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
|
||||
*,
|
||||
image_url: Optional[str] = None,
|
||||
reference_image_urls: Optional[List[str]] = None,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
prompt = (prompt or "").strip()
|
||||
aspect = resolve_aspect_ratio(aspect_ratio)
|
||||
|
||||
if not prompt:
|
||||
return error_response(
|
||||
error="Prompt is required and must be a non-empty string",
|
||||
error_type="invalid_argument",
|
||||
provider="openai-codex",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
if not _read_codex_access_token():
|
||||
return error_response(
|
||||
error=(
|
||||
"No Codex/ChatGPT OAuth credentials available. Run "
|
||||
"`hermes auth codex` (or `hermes setup` → Codex) to sign in."
|
||||
),
|
||||
error_type="auth_required",
|
||||
provider="openai-codex",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
try:
|
||||
import httpx # noqa: F401
|
||||
except ImportError:
|
||||
return error_response(
|
||||
error="httpx Python package not installed (pip install httpx)",
|
||||
error_type="missing_dependency",
|
||||
provider="openai-codex",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
tier_id, meta = _resolve_model()
|
||||
size = _SIZES.get(aspect, _SIZES["square"])
|
||||
|
||||
token = _read_codex_access_token()
|
||||
if not token:
|
||||
return error_response(
|
||||
error=(
|
||||
"No Codex/ChatGPT OAuth credentials available. Run "
|
||||
"`hermes auth codex` (or `hermes setup` → Codex) to sign in."
|
||||
),
|
||||
error_type="auth_required",
|
||||
provider="openai-codex",
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
try:
|
||||
input_images = _normalize_input_images(image_url, reference_image_urls)
|
||||
except Exception as exc:
|
||||
return error_response(
|
||||
error=f"Invalid image input for Codex image editing: {exc}",
|
||||
error_type="invalid_image_input",
|
||||
provider="openai-codex",
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
try:
|
||||
collected: Optional[Dict[str, str]] = None
|
||||
for attempt in range(_NONFINAL_RETRIES + 1):
|
||||
collected = _collect_image_b64(
|
||||
token,
|
||||
prompt=prompt,
|
||||
size=size,
|
||||
quality=meta["quality"],
|
||||
input_images=input_images or None,
|
||||
)
|
||||
if collected and collected.get("source") == "final" and collected.get("b64"):
|
||||
break
|
||||
if attempt < _NONFINAL_RETRIES:
|
||||
kind = (
|
||||
"progressive-only partial frame"
|
||||
if collected and collected.get("source") == "partial"
|
||||
else "no image_generation_call result"
|
||||
)
|
||||
logger.warning(
|
||||
"Codex image stream ended with %s (attempt %s/%s); "
|
||||
"retrying once before failing closed.",
|
||||
kind,
|
||||
attempt + 1,
|
||||
_NONFINAL_RETRIES + 1,
|
||||
)
|
||||
continue
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.debug("Codex image generation failed", exc_info=True)
|
||||
return error_response(
|
||||
error=f"OpenAI image generation via Codex auth failed: {exc}",
|
||||
error_type="api_error",
|
||||
provider="openai-codex",
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
if not collected or not collected.get("b64"):
|
||||
return error_response(
|
||||
error=(
|
||||
"Codex response contained no image_generation_call result "
|
||||
f"after {_NONFINAL_RETRIES + 1} attempt(s)"
|
||||
),
|
||||
error_type="empty_response",
|
||||
provider="openai-codex",
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
image_source = collected.get("source") or "unknown"
|
||||
b64 = collected["b64"]
|
||||
|
||||
# Defense in depth: never deliver a progressive-only frame as success.
|
||||
# Partials are intermediate previews and have presented as smeared /
|
||||
# unfinished images when saved as finals.
|
||||
if image_source != "final":
|
||||
pixel_hint = None
|
||||
try:
|
||||
import base64 as _b64mod
|
||||
|
||||
pixel_hint = _png_pixel_size(_b64mod.b64decode(b64, validate=False))
|
||||
except Exception:
|
||||
pixel_hint = None
|
||||
detail = (
|
||||
"Codex returned only a progressive partial image frame after "
|
||||
f"{_NONFINAL_RETRIES + 1} attempt(s); refusing to save it "
|
||||
"as a final deliverable."
|
||||
)
|
||||
if pixel_hint:
|
||||
detail = f"{detail} partial_pixel_size={pixel_hint}."
|
||||
err = error_response(
|
||||
error=detail,
|
||||
error_type="incomplete_image",
|
||||
provider="openai-codex",
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
err["image_source"] = image_source
|
||||
err["requested_size"] = size
|
||||
err["partial_pixel_size"] = pixel_hint
|
||||
err["nonfinal_retries"] = _NONFINAL_RETRIES
|
||||
return err
|
||||
|
||||
try:
|
||||
import base64 as _b64mod
|
||||
|
||||
raw_bytes = _b64mod.b64decode(b64)
|
||||
pixel_size = _png_pixel_size(raw_bytes)
|
||||
saved_path = save_b64_image(b64, prefix=f"openai_codex_{tier_id}")
|
||||
except Exception as exc:
|
||||
return error_response(
|
||||
error=f"Could not save image to cache: {exc}",
|
||||
error_type="io_error",
|
||||
provider="openai-codex",
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
return success_response(
|
||||
image=str(saved_path),
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
provider="openai-codex",
|
||||
modality="image" if input_images else "text",
|
||||
extra={
|
||||
"size": size,
|
||||
"quality": meta["quality"],
|
||||
"input_image_count": len(input_images),
|
||||
"image_source": image_source,
|
||||
"requested_size": size,
|
||||
"pixel_size": pixel_size,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Plugin entry point — register the Codex-backed image-gen provider."""
|
||||
ctx.register_image_gen_provider(OpenAICodexImageGenProvider())
|
||||
@@ -0,0 +1,5 @@
|
||||
name: openai-codex
|
||||
version: 1.0.0
|
||||
description: "OpenAI image generation backed by ChatGPT/Codex OAuth (gpt-image-2 via the Responses image_generation tool). Saves generated images to $HERMES_HOME/cache/images/."
|
||||
author: NousResearch
|
||||
kind: backend
|
||||
@@ -0,0 +1,419 @@
|
||||
"""OpenAI image generation backend.
|
||||
|
||||
Exposes OpenAI's ``gpt-image-2`` model at three quality tiers as an
|
||||
:class:`ImageGenProvider` implementation. The tiers are implemented as
|
||||
three virtual model IDs so the ``hermes tools`` model picker and the
|
||||
``image_gen.model`` config key behave like any other multi-model backend:
|
||||
|
||||
gpt-image-2-low ~15s fastest, good for iteration
|
||||
gpt-image-2-medium ~40s default — balanced
|
||||
gpt-image-2-high ~2min slowest, highest fidelity
|
||||
|
||||
All three hit the same underlying API model (``gpt-image-2``) with a
|
||||
different ``quality`` parameter. Output is base64 JSON → saved under
|
||||
``$HERMES_HOME/cache/images/``.
|
||||
|
||||
Selection precedence (first hit wins):
|
||||
|
||||
1. ``OPENAI_IMAGE_MODEL`` env var (escape hatch for scripts / tests)
|
||||
2. ``image_gen.openai.model`` in ``config.yaml``
|
||||
3. ``image_gen.model`` in ``config.yaml`` (when it's one of our tier IDs)
|
||||
4. :data:`DEFAULT_MODEL` — ``gpt-image-2-medium``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from agent.secret_scope import get_secret
|
||||
from agent.image_gen_provider import (
|
||||
DEFAULT_ASPECT_RATIO,
|
||||
ImageGenProvider,
|
||||
error_response,
|
||||
normalize_reference_images,
|
||||
resolve_aspect_ratio,
|
||||
save_b64_image,
|
||||
save_url_image,
|
||||
success_response,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model catalog
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# All three IDs resolve to the same underlying API model with a different
|
||||
# ``quality`` setting. ``api_model`` is what gets sent to OpenAI;
|
||||
# ``quality`` is the knob that changes generation time and output fidelity.
|
||||
|
||||
API_MODEL = "gpt-image-2"
|
||||
|
||||
_MODELS: Dict[str, Dict[str, Any]] = {
|
||||
"gpt-image-2-low": {
|
||||
"display": "GPT Image 2 (Low)",
|
||||
"speed": "~15s",
|
||||
"strengths": "Fast iteration, lowest cost",
|
||||
"quality": "low",
|
||||
},
|
||||
"gpt-image-2-medium": {
|
||||
"display": "GPT Image 2 (Medium)",
|
||||
"speed": "~40s",
|
||||
"strengths": "Balanced — default",
|
||||
"quality": "medium",
|
||||
},
|
||||
"gpt-image-2-high": {
|
||||
"display": "GPT Image 2 (High)",
|
||||
"speed": "~2min",
|
||||
"strengths": "Highest fidelity, strongest prompt adherence",
|
||||
"quality": "high",
|
||||
},
|
||||
}
|
||||
|
||||
DEFAULT_MODEL = "gpt-image-2-medium"
|
||||
|
||||
_SIZES = {
|
||||
"landscape": "1536x1024",
|
||||
"square": "1024x1024",
|
||||
"portrait": "1024x1536",
|
||||
}
|
||||
|
||||
|
||||
def _load_openai_config() -> Dict[str, Any]:
|
||||
"""Read ``image_gen`` from config.yaml (returns {} on any failure)."""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
cfg = load_config()
|
||||
section = cfg.get("image_gen") if isinstance(cfg, dict) else None
|
||||
return section if isinstance(section, dict) else {}
|
||||
except Exception as exc:
|
||||
logger.debug("Could not load image_gen config: %s", exc)
|
||||
return {}
|
||||
|
||||
|
||||
def _resolve_model() -> Tuple[str, Dict[str, Any]]:
|
||||
"""Decide which tier to use and return ``(model_id, meta)``."""
|
||||
env_override = os.environ.get("OPENAI_IMAGE_MODEL")
|
||||
if env_override and env_override in _MODELS:
|
||||
return env_override, _MODELS[env_override]
|
||||
|
||||
cfg = _load_openai_config()
|
||||
openai_cfg = cfg.get("openai") if isinstance(cfg.get("openai"), dict) else {}
|
||||
candidate: Optional[str] = None
|
||||
if isinstance(openai_cfg, dict):
|
||||
value = openai_cfg.get("model")
|
||||
if isinstance(value, str) and value in _MODELS:
|
||||
candidate = value
|
||||
if candidate is None:
|
||||
top = cfg.get("model")
|
||||
if isinstance(top, str) and top in _MODELS:
|
||||
candidate = top
|
||||
|
||||
if candidate is not None:
|
||||
return candidate, _MODELS[candidate]
|
||||
|
||||
return DEFAULT_MODEL, _MODELS[DEFAULT_MODEL]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source-image loading (for image-to-image / edit)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_image_bytes(ref: str) -> Tuple[bytes, str]:
|
||||
"""Load image bytes from a URL or local file path.
|
||||
|
||||
Returns ``(data, filename)``. Raises on any network / IO error so the
|
||||
caller can surface a clean error_response.
|
||||
"""
|
||||
ref = ref.strip()
|
||||
lower = ref.lower()
|
||||
if lower.startswith(("http://", "https://")):
|
||||
import requests
|
||||
|
||||
resp = requests.get(ref, timeout=60)
|
||||
resp.raise_for_status()
|
||||
name = ref.split("?", 1)[0].rsplit("/", 1)[-1] or "image.png"
|
||||
return resp.content, name
|
||||
if lower.startswith("data:"):
|
||||
import base64
|
||||
|
||||
header, _, b64 = ref.partition(",")
|
||||
ext = "png"
|
||||
if "image/" in header:
|
||||
ext = header.split("image/", 1)[1].split(";", 1)[0] or "png"
|
||||
return base64.b64decode(b64), f"image.{ext}"
|
||||
# Local file path — enforce the shared credential-read guard before reading.
|
||||
from agent.file_safety import raise_if_read_blocked
|
||||
|
||||
raise_if_read_blocked(ref)
|
||||
with open(ref, "rb") as fh:
|
||||
data = fh.read()
|
||||
name = os.path.basename(ref) or "image.png"
|
||||
return data, name
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class OpenAIImageGenProvider(ImageGenProvider):
|
||||
"""OpenAI ``images.generate`` / ``images.edit`` backend — gpt-image-2."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "openai"
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
return "OpenAI"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
if not get_secret("OPENAI_API_KEY"):
|
||||
return False
|
||||
try:
|
||||
import openai # noqa: F401
|
||||
except ImportError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def list_models(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": model_id,
|
||||
"display": meta["display"],
|
||||
"speed": meta["speed"],
|
||||
"strengths": meta["strengths"],
|
||||
"price": "varies",
|
||||
}
|
||||
for model_id, meta in _MODELS.items()
|
||||
]
|
||||
|
||||
def default_model(self) -> Optional[str]:
|
||||
return DEFAULT_MODEL
|
||||
|
||||
def get_setup_schema(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": "OpenAI",
|
||||
"badge": "paid",
|
||||
"tag": "gpt-image-2 at low/medium/high quality tiers — text-to-image & image editing",
|
||||
"env_vars": [
|
||||
{
|
||||
"key": "OPENAI_API_KEY",
|
||||
"prompt": "OpenAI API key",
|
||||
"url": "https://platform.openai.com/api-keys",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
def capabilities(self) -> Dict[str, Any]:
|
||||
# gpt-image-2 supports editing via images.edit() with up to 16 source
|
||||
# images.
|
||||
return {"modalities": ["text", "image"], "max_reference_images": 16}
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
|
||||
*,
|
||||
image_url: Optional[str] = None,
|
||||
reference_image_urls: Optional[List[str]] = None,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
prompt = (prompt or "").strip()
|
||||
aspect = resolve_aspect_ratio(aspect_ratio)
|
||||
|
||||
if not prompt:
|
||||
return error_response(
|
||||
error="Prompt is required and must be a non-empty string",
|
||||
error_type="invalid_argument",
|
||||
provider="openai",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
api_key = get_secret("OPENAI_API_KEY")
|
||||
if not api_key:
|
||||
return error_response(
|
||||
error=(
|
||||
"OPENAI_API_KEY not set. Run `hermes tools` → Image "
|
||||
"Generation → OpenAI to configure, or `hermes setup` "
|
||||
"to add the key."
|
||||
),
|
||||
error_type="auth_required",
|
||||
provider="openai",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
try:
|
||||
import openai
|
||||
except ImportError:
|
||||
return error_response(
|
||||
error="openai Python package not installed (pip install openai)",
|
||||
error_type="missing_dependency",
|
||||
provider="openai",
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
tier_id, meta = _resolve_model()
|
||||
size = _SIZES.get(aspect, _SIZES["square"])
|
||||
|
||||
# Collect source images (primary + references) for image-to-image.
|
||||
sources: List[str] = []
|
||||
if isinstance(image_url, str) and image_url.strip():
|
||||
sources.append(image_url.strip())
|
||||
for ref in (normalize_reference_images(reference_image_urls) or []):
|
||||
sources.append(ref)
|
||||
sources = sources[:16] # gpt-image-2 edit caps at 16 images
|
||||
is_edit = bool(sources)
|
||||
modality = "image" if is_edit else "text"
|
||||
|
||||
client = openai.OpenAI(api_key=api_key)
|
||||
|
||||
if is_edit:
|
||||
# images.edit() expects file-like objects. Download/read each
|
||||
# source into a named BytesIO so the SDK sends correct multipart.
|
||||
import io
|
||||
|
||||
try:
|
||||
files = []
|
||||
for ref in sources:
|
||||
data, fname = _load_image_bytes(ref)
|
||||
bio = io.BytesIO(data)
|
||||
bio.name = fname
|
||||
files.append(bio)
|
||||
except Exception as exc:
|
||||
return error_response(
|
||||
error=f"Could not load source image for editing: {exc}",
|
||||
error_type="io_error",
|
||||
provider="openai",
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
try:
|
||||
response = client.images.edit(
|
||||
model=API_MODEL,
|
||||
image=files if len(files) > 1 else files[0],
|
||||
prompt=prompt,
|
||||
size=size, # type: ignore[arg-type] # _SIZES values are valid gpt-image sizes
|
||||
quality=meta["quality"],
|
||||
n=1,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("OpenAI image edit failed", exc_info=True)
|
||||
return error_response(
|
||||
error=f"OpenAI image editing failed: {exc}",
|
||||
error_type="api_error",
|
||||
provider="openai",
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
else:
|
||||
# gpt-image-2 returns b64_json unconditionally and REJECTS
|
||||
# ``response_format`` as an unknown parameter. Don't send it.
|
||||
payload: Dict[str, Any] = {
|
||||
"model": API_MODEL,
|
||||
"prompt": prompt,
|
||||
"size": size,
|
||||
"n": 1,
|
||||
"quality": meta["quality"],
|
||||
}
|
||||
|
||||
try:
|
||||
response = client.images.generate(**payload)
|
||||
except Exception as exc:
|
||||
logger.debug("OpenAI image generation failed", exc_info=True)
|
||||
return error_response(
|
||||
error=f"OpenAI image generation failed: {exc}",
|
||||
error_type="api_error",
|
||||
provider="openai",
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
data = getattr(response, "data", None) or []
|
||||
if not data:
|
||||
return error_response(
|
||||
error="OpenAI returned no image data",
|
||||
error_type="empty_response",
|
||||
provider="openai",
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
first = data[0]
|
||||
b64 = getattr(first, "b64_json", None)
|
||||
url = getattr(first, "url", None)
|
||||
revised_prompt = getattr(first, "revised_prompt", None)
|
||||
|
||||
if b64:
|
||||
try:
|
||||
saved_path = save_b64_image(b64, prefix=f"openai_{tier_id}")
|
||||
except Exception as exc:
|
||||
return error_response(
|
||||
error=f"Could not save image to cache: {exc}",
|
||||
error_type="io_error",
|
||||
provider="openai",
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
image_ref = str(saved_path)
|
||||
elif url:
|
||||
# Defensive — gpt-image-2 returns b64 today, but OpenAI's API
|
||||
# has previously returned URLs. Cache the bytes locally so the
|
||||
# gateway never tries to fetch an ephemeral / signed URL after
|
||||
# it expires — same rationale as the xAI provider (#26942).
|
||||
try:
|
||||
saved_path = save_url_image(url, prefix=f"openai_{tier_id}")
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"OpenAI image URL %s could not be cached (%s); falling back to bare URL.",
|
||||
url,
|
||||
exc,
|
||||
)
|
||||
image_ref = url
|
||||
else:
|
||||
image_ref = str(saved_path)
|
||||
else:
|
||||
return error_response(
|
||||
error="OpenAI response contained neither b64_json nor URL",
|
||||
error_type="empty_response",
|
||||
provider="openai",
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
extra: Dict[str, Any] = {"size": size, "quality": meta["quality"]}
|
||||
if revised_prompt:
|
||||
extra["revised_prompt"] = revised_prompt
|
||||
|
||||
return success_response(
|
||||
image=image_ref,
|
||||
model=tier_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
provider="openai",
|
||||
modality=modality,
|
||||
extra=extra,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Plugin entry point — wire ``OpenAIImageGenProvider`` into the registry."""
|
||||
ctx.register_image_gen_provider(OpenAIImageGenProvider())
|
||||
@@ -0,0 +1,7 @@
|
||||
name: openai
|
||||
version: 1.0.0
|
||||
description: "OpenAI image generation backend (gpt-image-2). Saves generated images to $HERMES_HOME/cache/images/."
|
||||
author: NousResearch
|
||||
kind: backend
|
||||
requires_env:
|
||||
- OPENAI_API_KEY
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,7 @@
|
||||
name: openrouter
|
||||
version: 1.1.0
|
||||
description: "OpenRouter + Nous Portal image generation. Chat-completions image output (reference-grounded) plus OpenRouter's Dedicated Image API (/images/generations) for gpt-image-2, Krea 2, Qwen Image 3 Pro, MAI-Image-2.5 and Grok Imagine — exact per-model aspect ratios, resolution/quality/background/seed/n, up to 16 reference images. Text-to-image and image-to-image."
|
||||
author: Hermes Agent
|
||||
kind: backend
|
||||
requires_env:
|
||||
- OPENROUTER_API_KEY
|
||||
@@ -0,0 +1,625 @@
|
||||
"""xAI image generation backend.
|
||||
|
||||
Exposes xAI's ``grok-imagine-image`` model as an
|
||||
:class:`ImageGenProvider` implementation.
|
||||
|
||||
Features:
|
||||
- Text-to-image generation
|
||||
- Multiple aspect ratios (1:1, 16:9, 9:16, etc.)
|
||||
- Multiple resolutions (1K, 2K)
|
||||
- Base64 output saved to cache
|
||||
|
||||
Selection precedence (first hit wins):
|
||||
1. ``XAI_IMAGE_MODEL`` env var
|
||||
2. ``image_gen.xai.model`` in ``config.yaml``
|
||||
3. :data:`DEFAULT_MODEL`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
from agent.image_gen_provider import (
|
||||
DEFAULT_ASPECT_RATIO,
|
||||
ImageGenProvider,
|
||||
error_response,
|
||||
normalize_reference_images,
|
||||
resolve_aspect_ratio,
|
||||
save_b64_image,
|
||||
save_url_image,
|
||||
success_response,
|
||||
)
|
||||
from tools.xai_http import (
|
||||
build_xai_storage_options,
|
||||
hermes_xai_user_agent,
|
||||
maybe_mark_xai_storage_notice_seen,
|
||||
read_xai_imagine_storage_config,
|
||||
resolve_xai_http_credentials,
|
||||
xai_storage_notice_text,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model catalog
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MODELS: Dict[str, Dict[str, Any]] = {
|
||||
"grok-imagine-image": {
|
||||
"display": "Grok Imagine Image",
|
||||
"speed": "~5-10s",
|
||||
"strengths": "Fast, high-quality",
|
||||
},
|
||||
"grok-imagine-image-2.0": {
|
||||
"display": "Grok Imagine Image 2.0",
|
||||
"speed": "~10-20s",
|
||||
"strengths": "Typography/layout-aware; legible small text; strongest quality.",
|
||||
},
|
||||
"grok-imagine-image-quality": {
|
||||
"display": "Grok Imagine Image (Quality)",
|
||||
"speed": "~10-20s",
|
||||
"strengths": "Higher fidelity / detail; slower than the standard model.",
|
||||
},
|
||||
}
|
||||
|
||||
DEFAULT_MODEL = "grok-imagine-image"
|
||||
|
||||
# Live catalog cache: (models_dict, fetched_monotonic). xAI's
|
||||
# ``/image-generation-models`` endpoint is the source of truth so newly
|
||||
# released Imagine models appear in the picker without a code change;
|
||||
# the static ``_MODELS`` table is the offline fallback and supplies curated
|
||||
# speed/strengths text for the models we know about.
|
||||
_LIVE_CACHE: Optional[Tuple[Dict[str, Dict[str, Any]], float]] = None
|
||||
_LIVE_CACHE_TTL = 300.0
|
||||
_LIVE_TIMEOUT = 10.0
|
||||
|
||||
|
||||
def _fetch_live_models() -> Dict[str, Dict[str, Any]]:
|
||||
"""Fetch image models from xAI's ``/image-generation-models`` endpoint.
|
||||
|
||||
Returns ``{model_id: {"input_modalities": [...], "aliases": [...]}}``.
|
||||
Raises on any failure — callers treat that as "use the static table".
|
||||
"""
|
||||
creds = resolve_xai_http_credentials()
|
||||
api_key = str(creds.get("api_key") or "").strip()
|
||||
if not api_key:
|
||||
raise RuntimeError("no xAI credentials")
|
||||
base_url = str(creds.get("base_url") or "https://api.x.ai/v1").strip().rstrip("/")
|
||||
response = requests.get(
|
||||
f"{base_url}/image-generation-models",
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"User-Agent": hermes_xai_user_agent(),
|
||||
},
|
||||
timeout=_LIVE_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
entries = payload.get("models") or payload.get("data") or []
|
||||
out: Dict[str, Dict[str, Any]] = {}
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
model_id = entry.get("id") or entry.get("name")
|
||||
if not isinstance(model_id, str) or not model_id.strip():
|
||||
continue
|
||||
out[model_id.strip()] = {
|
||||
"input_modalities": entry.get("input_modalities") or [],
|
||||
"aliases": entry.get("aliases") or [],
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def _live_models() -> Dict[str, Dict[str, Any]]:
|
||||
"""Cached live catalog (``{}`` when unreachable)."""
|
||||
global _LIVE_CACHE
|
||||
import time
|
||||
|
||||
if _LIVE_CACHE is not None and time.monotonic() - _LIVE_CACHE[1] < _LIVE_CACHE_TTL:
|
||||
return _LIVE_CACHE[0]
|
||||
try:
|
||||
live = _fetch_live_models()
|
||||
except Exception as exc: # noqa: BLE001 - offline/unauth → static fallback
|
||||
logger.debug("xAI live image model catalog unavailable: %s", exc)
|
||||
live = {}
|
||||
_LIVE_CACHE = (live, time.monotonic())
|
||||
return live
|
||||
|
||||
|
||||
def _catalog() -> Dict[str, Dict[str, Any]]:
|
||||
"""Merged model catalog: live endpoint IDs + curated static metadata.
|
||||
|
||||
Known models keep their curated display/speed/strengths; models xAI
|
||||
ships after this file was written still show up (with generic metadata)
|
||||
so users can pick them the day they launch. Static table alone when the
|
||||
API is unreachable.
|
||||
"""
|
||||
live = _live_models()
|
||||
if not live:
|
||||
return dict(_MODELS)
|
||||
merged: Dict[str, Dict[str, Any]] = {}
|
||||
for model_id in live:
|
||||
meta = _MODELS.get(model_id)
|
||||
if meta is None:
|
||||
meta = {
|
||||
"display": model_id,
|
||||
"speed": "",
|
||||
"strengths": "New xAI Imagine model (from live xAI catalog)",
|
||||
}
|
||||
merged[model_id] = dict(meta)
|
||||
merged[model_id]["input_modalities"] = live[model_id].get("input_modalities") or []
|
||||
# Keep curated entries that the live list may momentarily omit.
|
||||
for model_id, meta in _MODELS.items():
|
||||
merged.setdefault(model_id, dict(meta))
|
||||
return merged
|
||||
|
||||
# xAI aspect ratios (more options than FAL/OpenAI)
|
||||
_XAI_ASPECT_RATIOS = {
|
||||
"landscape": "16:9",
|
||||
"square": "1:1",
|
||||
"portrait": "9:16",
|
||||
"4:3": "4:3",
|
||||
"3:4": "3:4",
|
||||
"3:2": "3:2",
|
||||
"2:3": "2:3",
|
||||
}
|
||||
|
||||
# xAI resolutions
|
||||
_XAI_RESOLUTIONS = {"1k", "2k"}
|
||||
|
||||
DEFAULT_RESOLUTION = "1k"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_xai_config() -> Dict[str, Any]:
|
||||
"""Read ``image_gen.xai`` from config.yaml."""
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
cfg = load_config()
|
||||
section = cfg.get("image_gen") if isinstance(cfg, dict) else None
|
||||
xai_section = section.get("xai") if isinstance(section, dict) else None
|
||||
return xai_section if isinstance(xai_section, dict) else {}
|
||||
except Exception as exc:
|
||||
logger.debug("Could not load image_gen.xai config: %s", exc)
|
||||
return {}
|
||||
|
||||
|
||||
def _resolve_model(caller_model: Optional[str] = None) -> Tuple[str, Dict[str, Any]]:
|
||||
"""Decide which model to use and return ``(model_id, meta)``.
|
||||
|
||||
Priority:
|
||||
1. Caller-supplied ``caller_model`` — the dispatcher forwards top-level
|
||||
``image_gen.model`` (what ``hermes tools`` writes) as the ``model``
|
||||
kwarg, mirroring the openrouter provider.
|
||||
2. ``XAI_IMAGE_MODEL`` env override.
|
||||
3. Scoped ``image_gen.xai.model`` in config.yaml.
|
||||
4. :data:`DEFAULT_MODEL`.
|
||||
|
||||
Every candidate is validated against the merged live+static catalog,
|
||||
so a newly released xAI model is selectable the day it appears in the
|
||||
live catalog — no code change required.
|
||||
"""
|
||||
catalog = _catalog()
|
||||
if caller_model and caller_model in catalog:
|
||||
return caller_model, catalog[caller_model]
|
||||
|
||||
env_override = os.environ.get("XAI_IMAGE_MODEL")
|
||||
if env_override and env_override in catalog:
|
||||
return env_override, catalog[env_override]
|
||||
|
||||
cfg = _load_xai_config()
|
||||
candidate = cfg.get("model") if isinstance(cfg.get("model"), str) else None
|
||||
if candidate and candidate in catalog:
|
||||
return candidate, catalog[candidate]
|
||||
|
||||
return DEFAULT_MODEL, catalog.get(DEFAULT_MODEL, _MODELS[DEFAULT_MODEL])
|
||||
|
||||
|
||||
def _resolve_edit_model(caller_model: Optional[str] = None) -> str:
|
||||
"""Model for ``/v1/images/edits`` requests.
|
||||
|
||||
An explicitly selected model (caller kwarg, env, or config) that accepts
|
||||
image input is honored for edits; otherwise fall back to the quality
|
||||
model, which xAI documents as the edit-capable baseline.
|
||||
"""
|
||||
catalog = _catalog()
|
||||
explicit = caller_model or os.environ.get("XAI_IMAGE_MODEL") or (
|
||||
_load_xai_config().get("model") if isinstance(_load_xai_config().get("model"), str) else None
|
||||
)
|
||||
if explicit and explicit in catalog:
|
||||
modalities = catalog[explicit].get("input_modalities") or []
|
||||
if "image" in modalities:
|
||||
return explicit
|
||||
return "grok-imagine-image-quality"
|
||||
|
||||
|
||||
def _resolve_resolution() -> str:
|
||||
"""Get configured resolution."""
|
||||
cfg = _load_xai_config()
|
||||
res = cfg.get("resolution") if isinstance(cfg.get("resolution"), str) else None
|
||||
if res and res in _XAI_RESOLUTIONS:
|
||||
return res
|
||||
return DEFAULT_RESOLUTION
|
||||
|
||||
|
||||
def _xai_image_field(source: str) -> Dict[str, str]:
|
||||
"""Build the xAI ``image`` field for an edit request.
|
||||
|
||||
xAI's ``/v1/images/edits`` accepts a public HTTPS URL or a base64 data URI.
|
||||
Local file paths are read and encoded into a ``data:`` URI.
|
||||
"""
|
||||
source = source.strip()
|
||||
lower = source.lower()
|
||||
if lower.startswith(("http://", "https://", "data:")):
|
||||
return {"url": source, "type": "image_url"}
|
||||
# Local file path → base64 data URI.
|
||||
import base64
|
||||
import os as _os
|
||||
|
||||
# Enforce the shared credential-read guard before reading local bytes
|
||||
# (same boundary the OpenAI / OpenRouter / Codex image providers apply).
|
||||
from agent.file_safety import raise_if_read_blocked
|
||||
|
||||
raise_if_read_blocked(source)
|
||||
with open(_os.path.expanduser(source), "rb") as fh: # windows-footgun: ok
|
||||
raw = fh.read()
|
||||
ext = (_os.path.splitext(source)[1].lstrip(".") or "png").lower()
|
||||
if ext == "jpg":
|
||||
ext = "jpeg"
|
||||
b64 = base64.b64encode(raw).decode("utf-8")
|
||||
return {"url": f"data:image/{ext};base64,{b64}", "type": "image_url"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class XAIImageGenProvider(ImageGenProvider):
|
||||
"""xAI ``grok-imagine-image`` backend."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "xai"
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
return "xAI (Grok)"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
creds = resolve_xai_http_credentials()
|
||||
return bool(creds.get("api_key"))
|
||||
|
||||
def list_models(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": model_id,
|
||||
"display": meta.get("display", model_id),
|
||||
"speed": meta.get("speed", ""),
|
||||
"strengths": meta.get("strengths", ""),
|
||||
}
|
||||
for model_id, meta in _catalog().items()
|
||||
]
|
||||
|
||||
def get_setup_schema(self) -> Dict[str, Any]:
|
||||
# Auth resolution is delegated to the shared ``xai_grok`` post_setup
|
||||
# hook (``hermes_cli/tools_config.py``); identical to the TTS / video
|
||||
# gen entries so users see the same OAuth-or-API-key choice for every
|
||||
# xAI service.
|
||||
storage_notice = xai_storage_notice_text("image_gen")
|
||||
tag = (
|
||||
"grok-imagine-image - text-to-image & image editing; uses xAI "
|
||||
"Grok OAuth or XAI_API_KEY"
|
||||
)
|
||||
if storage_notice:
|
||||
tag += f". {storage_notice}"
|
||||
return {
|
||||
"name": "xAI Grok Imagine (image)",
|
||||
"badge": "paid",
|
||||
"tag": tag,
|
||||
"env_vars": [],
|
||||
"post_setup": "xai_grok",
|
||||
}
|
||||
|
||||
def capabilities(self) -> Dict[str, Any]:
|
||||
# xAI's /v1/images/edits supports image editing via grok-imagine-image
|
||||
# -quality, including up to 3 total source images.
|
||||
return {
|
||||
"modalities": ["text", "image"],
|
||||
"max_reference_images": 2,
|
||||
"max_source_images": 3,
|
||||
}
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
|
||||
*,
|
||||
image_url: Optional[str] = None,
|
||||
reference_image_urls: Optional[List[str]] = None,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
"""Generate an image (text-to-image) or edit a source image (image-to-image).
|
||||
|
||||
Routing: when ``image_url`` is provided, POST to ``/v1/images/edits``
|
||||
with the source image; otherwise POST to ``/v1/images/generations``.
|
||||
Per xAI docs, editing uses the ``grok-imagine-image-quality`` model and
|
||||
a JSON body (the OpenAI SDK's multipart ``images.edit()`` is NOT
|
||||
supported by xAI).
|
||||
"""
|
||||
creds = resolve_xai_http_credentials()
|
||||
api_key = str(creds.get("api_key") or "").strip()
|
||||
provider_name = str(creds.get("provider") or "xai").strip() or "xai"
|
||||
if not api_key:
|
||||
return error_response(
|
||||
error="No xAI credentials found. Configure xAI OAuth in `hermes model` or set XAI_API_KEY.",
|
||||
error_type="missing_api_key",
|
||||
provider=provider_name,
|
||||
aspect_ratio=aspect_ratio,
|
||||
)
|
||||
|
||||
model_id, meta = _resolve_model(kwargs.get("model"))
|
||||
aspect = resolve_aspect_ratio(aspect_ratio)
|
||||
xai_ar = _XAI_ASPECT_RATIOS.get(aspect, "1:1")
|
||||
resolution = _resolve_resolution()
|
||||
xai_res = resolution if resolution in _XAI_RESOLUTIONS else DEFAULT_RESOLUTION
|
||||
|
||||
source_images: List[str] = []
|
||||
if isinstance(image_url, str) and image_url.strip():
|
||||
source_images.append(image_url.strip())
|
||||
refs = normalize_reference_images(reference_image_urls)
|
||||
if refs:
|
||||
source_images.extend(refs)
|
||||
if len(source_images) > 3:
|
||||
return error_response(
|
||||
error="xAI image editing supports at most 3 source images",
|
||||
error_type="too_many_references",
|
||||
provider=provider_name,
|
||||
model="grok-imagine-image-quality",
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
for index, source in enumerate(source_images):
|
||||
field = "image_url" if index == 0 and image_url and image_url.strip() == source else "reference_image_urls"
|
||||
lower = source.lower()
|
||||
if not lower.startswith(("http://", "https://", "data:")):
|
||||
path = Path(source).expanduser()
|
||||
if not path.is_file():
|
||||
return error_response(
|
||||
error=(
|
||||
f"{field} must be a public HTTPS URL or data URI "
|
||||
"(e.g. the `image`/`public_url` from a prior Imagine result)"
|
||||
),
|
||||
error_type="invalid_image_url",
|
||||
provider=provider_name,
|
||||
model="grok-imagine-image-quality",
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
is_edit = bool(source_images)
|
||||
modality = "image" if is_edit else "text"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": hermes_xai_user_agent(),
|
||||
}
|
||||
|
||||
base_url = str(creds.get("base_url") or "https://api.x.ai/v1").strip().rstrip("/")
|
||||
storage_options = build_xai_storage_options(
|
||||
"image_gen",
|
||||
filename_prefix="hermes-xai-image",
|
||||
extension="png",
|
||||
)
|
||||
storage_notice = maybe_mark_xai_storage_notice_seen("image_gen")
|
||||
storage_cfg = read_xai_imagine_storage_config("image_gen")
|
||||
|
||||
if is_edit:
|
||||
# Editing needs an image-input-capable model. An explicit user
|
||||
# selection that accepts image input (e.g. grok-imagine-image-2.0)
|
||||
# is honored; otherwise the documented quality baseline is used.
|
||||
# The source image may be a public URL or a base64 data URI;
|
||||
# local file paths are converted to a data URI here.
|
||||
edit_model = _resolve_edit_model(kwargs.get("model"))
|
||||
try:
|
||||
image_fields = [_xai_image_field(source) for source in source_images]
|
||||
except Exception as exc:
|
||||
return error_response(
|
||||
error=f"Could not load source image for editing: {exc}",
|
||||
error_type="io_error",
|
||||
provider=provider_name,
|
||||
model=edit_model,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
payload: Dict[str, Any] = {
|
||||
"model": edit_model,
|
||||
"prompt": prompt,
|
||||
}
|
||||
if len(image_fields) == 1:
|
||||
payload["image"] = image_fields[0]
|
||||
else:
|
||||
payload["images"] = image_fields
|
||||
endpoint_url = f"{base_url}/images/edits"
|
||||
model_id = edit_model
|
||||
else:
|
||||
payload = {
|
||||
"model": model_id,
|
||||
"prompt": prompt,
|
||||
"aspect_ratio": xai_ar,
|
||||
"resolution": xai_res,
|
||||
}
|
||||
endpoint_url = f"{base_url}/images/generations"
|
||||
if storage_options is not None:
|
||||
payload["storage_options"] = storage_options
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
endpoint_url,
|
||||
headers=headers,
|
||||
json=payload,
|
||||
timeout=120,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except requests.HTTPError as exc:
|
||||
response = exc.response
|
||||
status = response.status_code if response is not None else 0
|
||||
try:
|
||||
err_msg = response.json().get("error", {}).get("message", response.text[:300])
|
||||
except Exception:
|
||||
err_msg = response.text[:300] if response is not None else str(exc)
|
||||
logger.error("xAI image gen failed (%d): %s", status, err_msg)
|
||||
return error_response(
|
||||
error=f"xAI image generation failed ({status}): {err_msg}",
|
||||
error_type="api_error",
|
||||
provider=provider_name,
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
except requests.Timeout:
|
||||
return error_response(
|
||||
error="xAI image generation timed out (120s)",
|
||||
error_type="timeout",
|
||||
provider=provider_name,
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
except requests.ConnectionError as exc:
|
||||
return error_response(
|
||||
error=f"xAI connection error: {exc}",
|
||||
error_type="connection_error",
|
||||
provider=provider_name,
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
try:
|
||||
result = response.json()
|
||||
except Exception as exc:
|
||||
return error_response(
|
||||
error=f"xAI returned invalid JSON: {exc}",
|
||||
error_type="invalid_response",
|
||||
provider=provider_name,
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
# Parse response - xAI returns data[0].b64_json, data[0].url, and
|
||||
# optionally data[0].file_output when storage_options were requested.
|
||||
data = result.get("data", [])
|
||||
if not data:
|
||||
return error_response(
|
||||
error="xAI returned no image data",
|
||||
error_type="empty_response",
|
||||
provider=provider_name,
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
first = data[0]
|
||||
b64 = first.get("b64_json")
|
||||
url = first.get("url")
|
||||
file_output = first.get("file_output") if isinstance(first, dict) else None
|
||||
file_output = file_output if isinstance(file_output, dict) else {}
|
||||
public_url = file_output.get("public_url") if isinstance(file_output.get("public_url"), str) else None
|
||||
|
||||
if public_url:
|
||||
image_ref = public_url
|
||||
elif b64:
|
||||
try:
|
||||
saved_path = save_b64_image(b64, prefix=f"xai_{model_id}")
|
||||
except Exception as exc:
|
||||
return error_response(
|
||||
error=f"Could not save image to cache: {exc}",
|
||||
error_type="io_error",
|
||||
provider="xai",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
image_ref = str(saved_path)
|
||||
elif url:
|
||||
# xAI's grok-imagine-image returns ephemeral ``imgen.x.ai/xai-tmp-*``
|
||||
# URLs that 404 within minutes — by the time Telegram's
|
||||
# ``send_photo`` or any downstream consumer fetches them, the
|
||||
# asset is gone (#26942). Materialise the bytes locally at
|
||||
# tool-completion time so the gateway has a stable file path to
|
||||
# upload, mirroring the b64 branch above and the audio_cache
|
||||
# pattern used by text_to_speech.
|
||||
try:
|
||||
saved_path = save_url_image(url, prefix=f"xai_{model_id}")
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"xAI image URL %s could not be cached (%s); falling back to bare URL.",
|
||||
url,
|
||||
exc,
|
||||
)
|
||||
image_ref = url
|
||||
else:
|
||||
image_ref = str(saved_path)
|
||||
else:
|
||||
return error_response(
|
||||
error="xAI response contained neither b64_json nor URL",
|
||||
error_type="empty_response",
|
||||
provider="xai",
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
)
|
||||
|
||||
extra: Dict[str, Any] = {
|
||||
"storage_enabled": bool(storage_cfg["enabled"]),
|
||||
}
|
||||
if not is_edit:
|
||||
extra["resolution"] = xai_res
|
||||
if storage_notice:
|
||||
extra["storage_notice"] = storage_notice
|
||||
if public_url:
|
||||
extra["public_url"] = public_url
|
||||
if file_output:
|
||||
for key in (
|
||||
"filename",
|
||||
"expires_at",
|
||||
"public_url_expires_at",
|
||||
"public_url_error",
|
||||
"storage_error",
|
||||
):
|
||||
if key in file_output:
|
||||
extra[key] = file_output[key]
|
||||
if result.get("usage"):
|
||||
extra["usage"] = result["usage"]
|
||||
|
||||
return success_response(
|
||||
image=image_ref,
|
||||
model=model_id,
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect,
|
||||
provider="xai",
|
||||
modality=modality,
|
||||
extra=extra,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin registration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def register(ctx: Any) -> None:
|
||||
"""Register this provider with the image gen registry."""
|
||||
ctx.register_image_gen_provider(XAIImageGenProvider())
|
||||
@@ -0,0 +1,7 @@
|
||||
name: xai
|
||||
version: 1.0.0
|
||||
description: "xAI image generation backend (grok-imagine-image). Text-to-image."
|
||||
author: Julien Talbot
|
||||
kind: backend
|
||||
requires_env:
|
||||
- XAI_API_KEY
|
||||
Reference in New Issue
Block a user