Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tools/environments/__init__.py

@@ -0,0 +1,14 @@
+"""Hermes execution environment backends.
+
+Each backend provides the same interface (BaseEnvironment ABC) for running
+shell commands in a specific execution context: local, Docker, SSH,
+Singularity, Modal, Daytona, or Vercel Sandbox. (Modal additionally has
+direct and Nous-managed modes, selected via terminal.modal_mode.)
+
+The terminal_tool.py factory (_create_environment) selects the backend
+based on the TERMINAL_ENV configuration.
+"""
+
+from tools.environments.base import BaseEnvironment
+
+__all__ = ["BaseEnvironment"]

tools/environments/daytona.py

@@ -0,0 +1,270 @@
+"""Daytona cloud execution environment.
+
+Uses the Daytona Python SDK to run commands in cloud sandboxes.
+Supports persistent sandboxes: when enabled, sandboxes are stopped on cleanup
+and resumed on next creation, preserving the filesystem across sessions.
+"""
+
+import logging
+import math
+import os
+import shlex
+import threading
+from pathlib import Path
+
+from tools.environments.base import (
+    BaseEnvironment,
+    _ThreadedProcessHandle,
+)
+from tools.environments.file_sync import (
+    FileSyncManager,
+    iter_sync_files,
+    quoted_mkdir_command,
+    quoted_rm_command,
+    unique_parent_dirs,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class DaytonaEnvironment(BaseEnvironment):
+    """Daytona cloud sandbox execution backend.
+
+    Spawn-per-call via _ThreadedProcessHandle wrapping blocking SDK calls.
+    cancel_fn wired to sandbox.stop() for interrupt support.
+    Shell timeout wrapper preserved (SDK timeout unreliable).
+    """
+
+    _stdin_mode = "heredoc"
+
+    def __init__(
+        self,
+        image: str,
+        cwd: str = "/home/daytona",
+        timeout: int = 60,
+        cpu: int = 1,
+        memory: int = 5120,
+        disk: int = 10240,
+        persistent_filesystem: bool = True,
+        task_id: str = "default",
+    ):
+        requested_cwd = cwd
+        super().__init__(cwd=cwd, timeout=timeout)
+
+        try:
+            from tools.lazy_deps import ensure as _lazy_ensure
+            _lazy_ensure("terminal.daytona", prompt=False)
+        except ImportError:
+            pass
+        except Exception as e:
+            raise ImportError(str(e))
+        from daytona import (
+            Daytona,
+            CreateSandboxFromImageParams,
+            DaytonaError,
+            Resources,
+            SandboxState,
+        )
+
+        self._persistent = persistent_filesystem
+        self._task_id = task_id
+        self._SandboxState = SandboxState
+        self._daytona = Daytona()
+        self._sandbox = None
+        self._lock = threading.Lock()
+
+        memory_gib = max(1, math.ceil(memory / 1024))
+        disk_gib = max(1, math.ceil(disk / 1024))
+        if disk_gib > 10:
+            logger.warning(
+                "Daytona: requested disk (%dGB) exceeds platform limit (10GB). "
+                "Capping to 10GB.", disk_gib,
+            )
+            disk_gib = 10
+        resources = Resources(cpu=cpu, memory=memory_gib, disk=disk_gib)
+
+        labels = {"hermes_task_id": task_id}
+        sandbox_name = f"hermes-{task_id}"
+
+        if self._persistent:
+            try:
+                self._sandbox = self._daytona.get(sandbox_name)
+                self._sandbox.start()
+                logger.info("Daytona: resumed sandbox %s for task %s",
+                            self._sandbox.id, task_id)
+            except DaytonaError:
+                self._sandbox = None
+            except Exception as e:
+                logger.warning("Daytona: failed to resume sandbox for task %s: %s",
+                               task_id, e)
+                self._sandbox = None
+
+            if self._sandbox is None:
+                try:
+                    # Daytona SDK >=0.108.0 uses cursor-based pagination and
+                    # list() returns an iterator. Offset-based pagination
+                    # (page=1) is removed on June 10, 2026.
+                    results = self._daytona.list(labels=labels, limit=1)
+                    legacy = next(iter(results), None)
+                    if legacy is not None:
+                        self._sandbox = legacy
+                        self._sandbox.start()
+                        logger.info("Daytona: resumed legacy sandbox %s for task %s",
+                                    self._sandbox.id, task_id)
+                except Exception as e:
+                    logger.debug("Daytona: no legacy sandbox found for task %s: %s",
+                                 task_id, e)
+                    self._sandbox = None
+
+        if self._sandbox is None:
+            self._sandbox = self._daytona.create(
+                CreateSandboxFromImageParams(
+                    image=image,
+                    name=sandbox_name,
+                    labels=labels,
+                    auto_stop_interval=0,
+                    resources=resources,
+                )
+            )
+            logger.info("Daytona: created sandbox %s for task %s",
+                        self._sandbox.id, task_id)
+
+        # Detect remote home dir
+        self._remote_home = "/root"
+        try:
+            home = self._sandbox.process.exec("echo $HOME").result.strip()
+            if home:
+                self._remote_home = home
+                if requested_cwd in {"~", "/home/daytona"}:
+                    self.cwd = home
+        except Exception:
+            pass
+        logger.info("Daytona: resolved home to %s, cwd to %s", self._remote_home, self.cwd)
+
+        self._sync_manager = FileSyncManager(
+            get_files_fn=lambda: iter_sync_files(f"{self._remote_home}/.hermes"),
+            upload_fn=self._daytona_upload,
+            delete_fn=self._daytona_delete,
+            bulk_upload_fn=self._daytona_bulk_upload,
+            bulk_download_fn=self._daytona_bulk_download,
+        )
+        self._sync_manager.sync(force=True)
+        self.init_session()
+
+    def _daytona_upload(self, host_path: str, remote_path: str) -> None:
+        """Upload a single file via Daytona SDK."""
+        parent = str(Path(remote_path).parent)
+        self._sandbox.process.exec(quoted_mkdir_command([parent]))
+        self._sandbox.fs.upload_file(host_path, remote_path)
+
+    def _daytona_bulk_upload(self, files: list[tuple[str, str]]) -> None:
+        """Upload many files in a single HTTP call via Daytona SDK.
+
+        Uses ``sandbox.fs.upload_files()`` which batches all files into one
+        multipart POST, avoiding per-file TLS/HTTP overhead (~580 files
+        goes from ~5 min to <2 s).
+        """
+        from daytona.common.filesystem import FileUpload
+
+        if not files:
+            return
+
+        parents = unique_parent_dirs(files)
+        if parents:
+            self._sandbox.process.exec(quoted_mkdir_command(parents))
+
+        uploads = [
+            FileUpload(source=host_path, destination=remote_path)
+            for host_path, remote_path in files
+        ]
+        self._sandbox.fs.upload_files(uploads)
+
+    def _daytona_bulk_download(self, dest: Path) -> None:
+        """Download remote .hermes/ as a tar archive."""
+        rel_base = f"{self._remote_home}/.hermes".lstrip("/")
+        # PID-suffixed remote temp path avoids collisions if sync_back fires
+        # concurrently for the same sandbox (e.g. retry after partial failure).
+        remote_tar = f"/tmp/.hermes_sync.{os.getpid()}.tar"
+        self._sandbox.process.exec(
+            f"tar cf {shlex.quote(remote_tar)} -C / {shlex.quote(rel_base)}"
+        )
+        self._sandbox.fs.download_file(remote_tar, str(dest))
+        # Clean up remote temp file
+        try:
+            self._sandbox.process.exec(f"rm -f {shlex.quote(remote_tar)}")
+        except Exception:
+            pass  # best-effort cleanup
+
+    def _daytona_delete(self, remote_paths: list[str]) -> None:
+        """Batch-delete remote files via SDK exec."""
+        self._sandbox.process.exec(quoted_rm_command(remote_paths))
+
+    # ------------------------------------------------------------------
+    # Sandbox lifecycle
+    # ------------------------------------------------------------------
+
+    def _ensure_sandbox_ready(self) -> None:
+        """Restart sandbox if it was stopped (e.g., by a previous interrupt)."""
+        self._sandbox.refresh_data()
+        if self._sandbox.state in {self._SandboxState.STOPPED, self._SandboxState.ARCHIVED}:
+            self._sandbox.start()
+            logger.info("Daytona: restarted sandbox %s", self._sandbox.id)
+
+    def _before_execute(self) -> None:
+        """Ensure sandbox is ready, then sync files via FileSyncManager."""
+        with self._lock:
+            self._ensure_sandbox_ready()
+        self._sync_manager.sync()
+
+    def _run_bash(self, cmd_string: str, *, login: bool = False,
+                  timeout: int = 120,
+                  stdin_data: str | None = None):
+        """Return a _ThreadedProcessHandle wrapping a blocking Daytona SDK call."""
+        sandbox = self._sandbox
+        lock = self._lock
+
+        def cancel():
+            with lock:
+                try:
+                    sandbox.stop()
+                except Exception:
+                    pass
+
+        if login:
+            shell_cmd = f"bash -l -c {shlex.quote(cmd_string)}"
+        else:
+            shell_cmd = f"bash -c {shlex.quote(cmd_string)}"
+
+        def exec_fn() -> tuple[str, int]:
+            response = sandbox.process.exec(shell_cmd, timeout=timeout)
+            return (response.result or "", response.exit_code)
+
+        return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel)
+
+    def cleanup(self):
+        with self._lock:
+            if self._sandbox is None:
+                return
+
+            # Sync remote changes back to host before teardown. Running
+            # inside the lock (and after the _sandbox is None guard) avoids
+            # firing sync_back on an already-cleaned-up env, which would
+            # trigger a 3-attempt retry storm against a nil sandbox.
+            if self._sync_manager:
+                logger.info("Daytona: syncing files from sandbox...")
+                try:
+                    self._sync_manager.sync_back()
+                except Exception as e:
+                    logger.warning("Daytona: sync_back failed: %s", e)
+
+            try:
+                if self._persistent:
+                    self._sandbox.stop()
+                    logger.info("Daytona: stopped sandbox %s (filesystem preserved)",
+                                self._sandbox.id)
+                else:
+                    self._daytona.delete(self._sandbox)
+                    logger.info("Daytona: deleted sandbox %s", self._sandbox.id)
+            except Exception as e:
+                logger.warning("Daytona: cleanup failed: %s", e)
+            self._sandbox = None

tools/environments/file_sync.py

@@ -0,0 +1,484 @@
+"""Shared file sync manager for remote execution backends.
+
+Tracks local file changes via mtime+size, detects deletions, and
+syncs to remote environments transactionally.  Used by SSH, Modal,
+and Daytona.  Docker and Singularity use bind mounts (live host FS
+view) and don't need this.
+"""
+
+import hashlib
+import logging
+import os
+import posixpath
+import shlex
+import shutil
+import signal
+import tarfile
+import tempfile
+import threading
+import time
+
+try:
+    import fcntl
+except ImportError:
+    fcntl = None  # Windows — file locking skipped
+from pathlib import Path
+from typing import Callable
+
+from hermes_constants import get_hermes_home
+from tools.environments.base import _file_mtime_key
+
+logger = logging.getLogger(__name__)
+
+# Keep retry sleeps patchable without mutating the shared stdlib ``time``
+# module. Patching ``tools.environments.file_sync.time.sleep`` replaces
+# ``time.sleep`` globally because ``time`` is the module object; under xdist
+# that lets unrelated background threads inflate retry-test call counts.
+_sleep = time.sleep
+# Same rationale for the rate-limit clock: tests patch ``_monotonic``
+# instead of ``time.monotonic`` on the shared module object.
+_monotonic = time.monotonic
+
+_SYNC_INTERVAL_SECONDS = 5.0
+_FORCE_SYNC_ENV = "HERMES_FORCE_FILE_SYNC"
+
+# Transport callbacks provided by each backend
+UploadFn = Callable[[str, str], None]  # (host_path, remote_path) -> raises on failure
+BulkUploadFn = Callable[[list[tuple[str, str]]], None]  # [(host_path, remote_path), ...] -> raises on failure
+BulkDownloadFn = Callable[[Path], None]  # (dest_tar_path) -> writes tar archive, raises on failure
+DeleteFn = Callable[[list[str]], None]  # (remote_paths) -> raises on failure
+GetFilesFn = Callable[[], list[tuple[str, str]]]  # () -> [(host_path, remote_path), ...]
+
+
+def iter_sync_files(container_base: str = "/root/.hermes") -> list[tuple[str, str]]:
+    """Enumerate all files that should be synced to a remote environment.
+
+    Combines credentials, skills, and cache into a single flat list of
+    (host_path, remote_path) pairs.  Credential paths are remapped from
+    the hardcoded /root/.hermes to *container_base* because the remote
+    user's home may differ (e.g. /home/daytona, /home/user).
+    """
+    # Late import: credential_files imports agent modules that create
+    # circular dependencies if loaded at file_sync module level.
+    from tools.credential_files import (
+        get_credential_file_mounts,
+        iter_cache_files,
+        iter_skills_files,
+    )
+
+    files: list[tuple[str, str]] = []
+    for entry in get_credential_file_mounts():
+        remote = entry["container_path"].replace(
+            "/root/.hermes", container_base, 1
+        )
+        files.append((entry["host_path"], remote))
+    for entry in iter_skills_files(container_base=container_base):
+        files.append((entry["host_path"], entry["container_path"]))
+    for entry in iter_cache_files(container_base=container_base):
+        files.append((entry["host_path"], entry["container_path"]))
+    return files
+
+
+def _credential_host_paths() -> set[str]:
+    """Return credential files that are upload-only for remote sandboxes."""
+    try:
+        from tools.credential_files import get_credential_file_mounts
+    except Exception:
+        return set()
+
+    paths: set[str] = set()
+    try:
+        mounts = get_credential_file_mounts()
+    except Exception:
+        return set()
+    for entry in mounts:
+        host_path = entry.get("host_path") if isinstance(entry, dict) else None
+        if not host_path:
+            continue
+        try:
+            paths.add(str(Path(host_path).expanduser().resolve()))
+        except OSError:
+            paths.add(str(Path(host_path).expanduser()))
+    return paths
+
+
+def quoted_rm_command(remote_paths: list[str]) -> str:
+    """Build a shell ``rm -f`` command for a batch of remote paths."""
+    return "rm -f " + " ".join(shlex.quote(p) for p in remote_paths)
+
+
+def quoted_mkdir_command(dirs: list[str]) -> str:
+    """Build a shell ``mkdir -p`` command for a batch of directories."""
+    return "mkdir -p " + " ".join(shlex.quote(d) for d in dirs)
+
+
+def unique_parent_dirs(files: list[tuple[str, str]]) -> list[str]:
+    """Extract sorted unique parent directories from (host, remote) pairs."""
+    return sorted({posixpath.dirname(remote) for _, remote in files})
+
+
+def _sha256_file(path: str) -> str:
+    """Return hex SHA-256 digest of a file."""
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(65536), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+_SYNC_BACK_MAX_RETRIES = 3
+_SYNC_BACK_BACKOFF = (2, 4, 8)  # seconds between retries
+_SYNC_BACK_MAX_BYTES = 2 * 1024 * 1024 * 1024  # 2 GiB — refuse to extract larger tars
+
+
+class FileSyncManager:
+    """Tracks local file changes and syncs to a remote environment.
+
+    Backends instantiate this with transport callbacks (upload, delete)
+    and a file-source callable.  The manager handles mtime-based change
+    detection, deletion tracking, rate limiting, and transactional state.
+
+    Not used by bind-mount backends (Docker, Singularity) — those get
+    live host FS views and don't need file sync.
+    """
+
+    def __init__(
+        self,
+        get_files_fn: GetFilesFn,
+        upload_fn: UploadFn,
+        delete_fn: DeleteFn,
+        sync_interval: float = _SYNC_INTERVAL_SECONDS,
+        bulk_upload_fn: BulkUploadFn | None = None,
+        bulk_download_fn: BulkDownloadFn | None = None,
+    ):
+        self._get_files_fn = get_files_fn
+        self._upload_fn = upload_fn
+        self._bulk_upload_fn = bulk_upload_fn
+        self._bulk_download_fn = bulk_download_fn
+        self._delete_fn = delete_fn
+        self._transaction_lock = threading.Lock()
+        self._synced_files: dict[str, tuple[float, int]] = {}  # remote_path -> (mtime, size)
+        self._pushed_hashes: dict[str, str] = {}  # remote_path -> sha256 hex digest
+        self._upload_only_host_paths: set[str] = set()
+        self._last_sync_time: float = 0.0  # monotonic; 0 ensures first sync runs
+        self._sync_interval = sync_interval
+
+    def sync(self, *, force: bool = False) -> None:
+        """Run a sync cycle: upload changed files, delete removed files.
+
+        Rate-limited to once per ``sync_interval`` unless *force* is True
+        or ``HERMES_FORCE_FILE_SYNC=1`` is set.
+
+        Transactional: state only committed if ALL operations succeed.
+        On failure, state rolls back so the next cycle retries everything.
+        """
+        with self._transaction_lock:
+            self._sync_transaction(force=force)
+
+    def _sync_transaction(self, *, force: bool = False) -> None:
+        """Execute one sync cycle while holding the per-manager lock."""
+        if not force and not os.environ.get(_FORCE_SYNC_ENV):
+            now = _monotonic()
+            if now - self._last_sync_time < self._sync_interval:
+                return
+
+        current_files = self._get_files_fn()
+        self._upload_only_host_paths.update(_credential_host_paths())
+        current_remote_paths = {remote for _, remote in current_files}
+
+        # --- Uploads: new or changed files ---
+        to_upload: list[tuple[str, str]] = []
+        new_files = dict(self._synced_files)
+        for host_path, remote_path in current_files:
+            file_key = _file_mtime_key(host_path)
+            if file_key is None:
+                continue
+            if self._synced_files.get(remote_path) == file_key:
+                continue
+            to_upload.append((host_path, remote_path))
+            new_files[remote_path] = file_key
+
+        # --- Deletes: synced paths no longer in current set ---
+        to_delete = [p for p in self._synced_files if p not in current_remote_paths]
+
+        if not to_upload and not to_delete:
+            self._last_sync_time = _monotonic()
+            return
+
+        # Snapshot for rollback (only when there's work to do)
+        prev_files = dict(self._synced_files)
+        prev_hashes = dict(self._pushed_hashes)
+
+        if to_upload:
+            logger.debug("file_sync: uploading %d file(s)", len(to_upload))
+        if to_delete:
+            logger.debug("file_sync: deleting %d stale remote file(s)", len(to_delete))
+
+        try:
+            if to_upload and self._bulk_upload_fn is not None:
+                self._bulk_upload_fn(to_upload)
+                logger.debug("file_sync: bulk-uploaded %d file(s)", len(to_upload))
+            else:
+                for host_path, remote_path in to_upload:
+                    self._upload_fn(host_path, remote_path)
+                    logger.debug("file_sync: uploaded %s -> %s", host_path, remote_path)
+
+            if to_delete:
+                self._delete_fn(to_delete)
+                logger.debug("file_sync: deleted %s", to_delete)
+
+            # --- Commit (all succeeded) ---
+            for host_path, remote_path in to_upload:
+                self._pushed_hashes[remote_path] = _sha256_file(host_path)
+
+            for p in to_delete:
+                new_files.pop(p, None)
+                self._pushed_hashes.pop(p, None)
+
+            self._synced_files = new_files
+            self._last_sync_time = _monotonic()
+
+        except Exception as exc:
+            self._synced_files = prev_files
+            self._pushed_hashes = prev_hashes
+            # Do NOT advance _last_sync_time here: a failed cycle rolls state
+            # back so the next cycle can retry. Bumping the rate-limit clock on
+            # failure would make the next non-forced sync() return early (the
+            # guard above), suppressing that retry for up to _sync_interval and
+            # leaving the remote with stale files — contradicting this method's
+            # documented "next cycle retries everything" contract.
+            logger.warning("file_sync: sync failed, rolled back state: %s", exc)
+
+    # ------------------------------------------------------------------
+    # Sync-back: pull remote changes to host on teardown
+    # ------------------------------------------------------------------
+
+    def sync_back(self, hermes_home: Path | None = None) -> None:
+        """Pull remote changes back to the host filesystem.
+
+        Downloads the remote ``.hermes/`` directory as a tar archive,
+        unpacks it, and applies only files that differ from what was
+        originally pushed (based on SHA-256 content hashes).
+
+        Protected against SIGINT (defers the signal until complete) and
+        serialized across concurrent gateway sandboxes via file lock.
+        """
+        with self._transaction_lock:
+            self._sync_back_transaction(hermes_home=hermes_home)
+
+    def _sync_back_transaction(self, hermes_home: Path | None = None) -> None:
+        """Execute sync-back against a stable snapshot of manager state."""
+        if self._bulk_download_fn is None:
+            return
+
+        # Nothing was ever committed through this manager — the initial
+        # push failed or never ran. Skip sync_back to avoid retry storms
+        # against an uninitialized remote .hermes/ directory.
+        if not self._pushed_hashes and not self._synced_files:
+            logger.debug("sync_back: no prior push state — skipping")
+            return
+
+        lock_path = (hermes_home or get_hermes_home()) / ".sync.lock"
+        lock_path.parent.mkdir(parents=True, exist_ok=True)
+
+        last_exc: Exception | None = None
+        for attempt in range(_SYNC_BACK_MAX_RETRIES):
+            try:
+                self._sync_back_once(lock_path)
+                return
+            except Exception as exc:
+                last_exc = exc
+                if attempt < _SYNC_BACK_MAX_RETRIES - 1:
+                    delay = _SYNC_BACK_BACKOFF[attempt]
+                    logger.warning(
+                        "sync_back: attempt %d failed (%s), retrying in %ds",
+                        attempt + 1, exc, delay,
+                    )
+                    _sleep(delay)
+
+        logger.warning("sync_back: all %d attempts failed: %s", _SYNC_BACK_MAX_RETRIES, last_exc)
+
+    def _sync_back_once(self, lock_path: Path) -> None:
+        """Single sync-back attempt with SIGINT protection and file lock."""
+        # signal.signal() only works from the main thread. In gateway
+        # contexts cleanup() may run from a worker thread — skip SIGINT
+        # deferral there rather than crashing.
+        on_main_thread = threading.current_thread() is threading.main_thread()
+
+        deferred_sigint: list[object] = []
+        original_handler = None
+        if on_main_thread:
+            original_handler = signal.getsignal(signal.SIGINT)
+
+            def _defer_sigint(signum, frame):
+                deferred_sigint.append((signum, frame))
+                logger.debug("sync_back: SIGINT deferred until sync completes")
+
+            signal.signal(signal.SIGINT, _defer_sigint)
+        try:
+            self._sync_back_locked(lock_path)
+        finally:
+            if on_main_thread and original_handler is not None:
+                signal.signal(signal.SIGINT, original_handler)
+                if deferred_sigint:
+                    # Re-deliver the deferred Ctrl+C to the just-restored
+                    # handler. ``os.kill(os.getpid(), signal.SIGINT)`` is NOT a
+                    # graceful signal on Windows: os.kill only treats
+                    # CTRL_C_EVENT(0)/CTRL_BREAK_EVENT(1) as console events; any
+                    # other value (SIGINT == 2) routes to TerminateProcess(sig),
+                    # hard-killing the CLI (exit code 2) instead of raising
+                    # KeyboardInterrupt — so a Ctrl+C during a remote-backend
+                    # sync-back would kill the whole session on Windows.
+                    # ``signal.raise_signal`` (3.8+) invokes the handler via C
+                    # ``raise()`` on every platform.
+                    signal.raise_signal(signal.SIGINT)
+
+    def _sync_back_locked(self, lock_path: Path) -> None:
+        """Sync-back under file lock (serializes concurrent gateways)."""
+        if fcntl is None:
+            # Windows: no flock — run without serialization
+            self._sync_back_impl()
+            return
+        lock_fd = open(lock_path, "w", encoding="utf-8")
+        try:
+            fcntl.flock(lock_fd, fcntl.LOCK_EX)
+            self._sync_back_impl()
+        finally:
+            try:
+                fcntl.flock(lock_fd, fcntl.LOCK_UN)
+            except (OSError, IOError):
+                pass
+            lock_fd.close()
+
+    def _sync_back_impl(self) -> None:
+        """Download, diff, and apply remote changes to host."""
+        if self._bulk_download_fn is None:
+            raise RuntimeError("_sync_back_impl called without bulk_download_fn")
+
+        # Cache file mapping once to avoid O(n*m) from repeated iteration
+        try:
+            file_mapping = list(self._get_files_fn())
+        except Exception:
+            file_mapping = []
+
+        with tempfile.NamedTemporaryFile(suffix=".tar") as tf:
+            self._bulk_download_fn(Path(tf.name))
+
+            # Defensive size cap: a misbehaving sandbox could produce an
+            # arbitrarily large tar. Refuse to extract if it exceeds the cap.
+            try:
+                tar_size = os.path.getsize(tf.name)
+            except OSError:
+                tar_size = 0
+            if tar_size > _SYNC_BACK_MAX_BYTES:
+                logger.warning(
+                    "sync_back: remote tar is %d bytes (cap %d) — skipping extraction",
+                    tar_size, _SYNC_BACK_MAX_BYTES,
+                )
+                return
+
+            with tempfile.TemporaryDirectory(prefix="hermes-sync-back-") as staging:
+                with tarfile.open(tf.name) as tar:
+                    tar.extractall(staging, filter="data")
+
+                applied = 0
+                upload_only_host_paths = (
+                    self._upload_only_host_paths | _credential_host_paths()
+                )
+                for dirpath, _dirnames, filenames in os.walk(staging):
+                    for fname in filenames:
+                        staged_file = os.path.join(dirpath, fname)
+                        rel = os.path.relpath(staged_file, staging)
+                        remote_path = "/" + rel
+
+                        pushed_hash = self._pushed_hashes.get(remote_path)
+
+                        # Skip hashing for files unchanged from push
+                        if pushed_hash is not None:
+                            remote_hash = _sha256_file(staged_file)
+                            if remote_hash == pushed_hash:
+                                continue
+                        else:
+                            remote_hash = None  # new remote file
+
+                        # Resolve host path from cached mapping
+                        host_path = self._resolve_host_path(remote_path, file_mapping)
+                        if host_path is None:
+                            host_path = self._infer_host_path(
+                                remote_path,
+                                file_mapping,
+                                upload_only_host_paths=upload_only_host_paths,
+                            )
+                            if host_path is None:
+                                logger.debug(
+                                    "sync_back: skipping %s (no host mapping)",
+                                    remote_path,
+                                )
+                                continue
+
+                        if self._is_upload_only_host_path(host_path, upload_only_host_paths):
+                            logger.debug(
+                                "sync_back: skipping upload-only credential file %s",
+                                remote_path,
+                            )
+                            continue
+
+                        if os.path.exists(host_path) and pushed_hash is not None:
+                            host_hash = _sha256_file(host_path)
+                            if host_hash != pushed_hash:
+                                logger.warning(
+                                    "sync_back: conflict on %s — host modified "
+                                    "since push, remote also changed. Applying "
+                                    "remote version (last-write-wins).",
+                                    remote_path,
+                                )
+
+                        os.makedirs(os.path.dirname(host_path), exist_ok=True)
+                        shutil.copy2(staged_file, host_path)
+                        applied += 1
+
+                if applied:
+                    logger.info("sync_back: applied %d changed file(s)", applied)
+                else:
+                    logger.debug("sync_back: no remote changes detected")
+
+    def _resolve_host_path(self, remote_path: str,
+                           file_mapping: list[tuple[str, str]] | None = None) -> str | None:
+        """Find the host path for a known remote path from the file mapping."""
+        mapping = file_mapping if file_mapping is not None else []
+        for host, remote in mapping:
+            if remote == remote_path:
+                return host
+        return None
+
+    def _infer_host_path(self, remote_path: str,
+                         file_mapping: list[tuple[str, str]] | None = None,
+                         *,
+                         upload_only_host_paths: set[str] | None = None) -> str | None:
+        """Infer a host path for a new remote file by matching path prefixes.
+
+        Uses the existing file mapping to find a remote->host directory
+        pair, then applies the same prefix substitution to the new file.
+        For example, if the mapping has ``/root/.hermes/skills/a.md`` →
+        ``~/.hermes/skills/a.md``, a new remote file at
+        ``/root/.hermes/skills/b.md`` maps to ``~/.hermes/skills/b.md``.
+        """
+        mapping = file_mapping if file_mapping is not None else []
+        upload_only_host_paths = upload_only_host_paths or set()
+        for host, remote in mapping:
+            if self._is_upload_only_host_path(host, upload_only_host_paths):
+                continue
+            remote_dir = str(Path(remote).parent)
+            if remote_path.startswith(remote_dir + "/"):
+                host_dir = str(Path(host).parent)
+                suffix = remote_path[len(remote_dir):]
+                return host_dir + suffix
+        return None
+
+    @staticmethod
+    def _is_upload_only_host_path(host_path: str, upload_only_host_paths: set[str]) -> bool:
+        try:
+            resolved = str(Path(host_path).expanduser().resolve())
+        except OSError:
+            resolved = str(Path(host_path).expanduser())
+        return resolved in upload_only_host_paths

tools/environments/managed_modal.py

@@ -0,0 +1,282 @@
+"""Managed Modal environment backed by tool-gateway."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import requests
+import uuid
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+from tools.environments.modal_utils import (
+    BaseModalExecutionEnvironment,
+    ModalExecStart,
+    PreparedModalExec,
+)
+from tools.managed_tool_gateway import resolve_managed_tool_gateway
+
+logger = logging.getLogger(__name__)
+
+
+def _request_timeout_env(name: str, default: float) -> float:
+    try:
+        value = float(os.getenv(name, str(default)))
+        return value if value > 0 else default
+    except (TypeError, ValueError):
+        return default
+
+
+@dataclass(frozen=True)
+class _ManagedModalExecHandle:
+    exec_id: str
+
+
+class ManagedModalEnvironment(BaseModalExecutionEnvironment):
+    """Gateway-owned Modal sandbox with Hermes-compatible execute/cleanup."""
+
+    _CONNECT_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_CONNECT_TIMEOUT_SECONDS", 1.0)
+    _POLL_READ_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_POLL_READ_TIMEOUT_SECONDS", 5.0)
+    _CANCEL_READ_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_CANCEL_READ_TIMEOUT_SECONDS", 5.0)
+    _client_timeout_grace_seconds = 10.0
+    _interrupt_output = "[Command interrupted - Modal sandbox exec cancelled]"
+    _unexpected_error_prefix = "Managed Modal exec failed"
+
+    def __init__(
+        self,
+        image: str,
+        cwd: str = "/root",
+        timeout: int = 60,
+        modal_sandbox_kwargs: Optional[Dict[str, Any]] = None,
+        persistent_filesystem: bool = True,
+        task_id: str = "default",
+    ):
+        super().__init__(cwd=cwd, timeout=timeout)
+
+        self._guard_unsupported_credential_passthrough()
+
+        gateway = resolve_managed_tool_gateway("modal")
+        if gateway is None:
+            raise ValueError("Managed Modal requires a configured tool gateway and Nous user token")
+
+        self._gateway_origin = gateway.gateway_origin.rstrip("/")
+        self._nous_user_token = gateway.nous_user_token
+        self._task_id = task_id
+        self._persistent = persistent_filesystem
+        self._image = image
+        self._sandbox_kwargs = dict(modal_sandbox_kwargs or {})
+        self._create_idempotency_key = str(uuid.uuid4())
+        self._sandbox_id = self._create_sandbox()
+
+    def _start_modal_exec(self, prepared: PreparedModalExec) -> ModalExecStart:
+        exec_id = str(uuid.uuid4())
+        payload: Dict[str, Any] = {
+            "execId": exec_id,
+            "command": prepared.command,
+            "cwd": prepared.cwd,
+            "timeoutMs": int(prepared.timeout * 1000),
+        }
+        if prepared.stdin_data is not None:
+            payload["stdinData"] = prepared.stdin_data
+
+        try:
+            response = self._request(
+                "POST",
+                f"/v1/sandboxes/{self._sandbox_id}/execs",
+                json=payload,
+                timeout=10,
+            )
+        except Exception as exc:
+            return ModalExecStart(
+                immediate_result=self._error_result(f"Managed Modal exec failed: {exc}")
+            )
+
+        if response.status_code >= 400:
+            return ModalExecStart(
+                immediate_result=self._error_result(
+                    self._format_error("Managed Modal exec failed", response)
+                )
+            )
+
+        body = response.json()
+        status = body.get("status")
+        if status in {"completed", "failed", "cancelled", "timeout"}:
+            return ModalExecStart(
+                immediate_result=self._result(
+                    body.get("output", ""),
+                    body.get("returncode", 1),
+                )
+            )
+
+        if body.get("execId") != exec_id:
+            return ModalExecStart(
+                immediate_result=self._error_result(
+                    "Managed Modal exec start did not return the expected exec id"
+                )
+            )
+
+        return ModalExecStart(handle=_ManagedModalExecHandle(exec_id=exec_id))
+
+    def _poll_modal_exec(self, handle: _ManagedModalExecHandle) -> dict | None:
+        try:
+            status_response = self._request(
+                "GET",
+                f"/v1/sandboxes/{self._sandbox_id}/execs/{handle.exec_id}",
+                timeout=(self._CONNECT_TIMEOUT_SECONDS, self._POLL_READ_TIMEOUT_SECONDS),
+            )
+        except Exception as exc:
+            return self._error_result(f"Managed Modal exec poll failed: {exc}")
+
+        if status_response.status_code == 404:
+            return self._error_result("Managed Modal exec not found")
+
+        if status_response.status_code >= 400:
+            return self._error_result(
+                self._format_error("Managed Modal exec poll failed", status_response)
+            )
+
+        status_body = status_response.json()
+        status = status_body.get("status")
+        if status in {"completed", "failed", "cancelled", "timeout"}:
+            return self._result(
+                status_body.get("output", ""),
+                status_body.get("returncode", 1),
+            )
+        return None
+
+    def _cancel_modal_exec(self, handle: _ManagedModalExecHandle) -> None:
+        self._cancel_exec(handle.exec_id)
+
+    def _timeout_result_for_modal(self, timeout: int) -> dict:
+        return self._result(f"Managed Modal exec timed out after {timeout}s", 124)
+
+    def cleanup(self):
+        if not getattr(self, "_sandbox_id", None):
+            return
+
+        try:
+            self._request(
+                "POST",
+                f"/v1/sandboxes/{self._sandbox_id}/terminate",
+                json={
+                    "snapshotBeforeTerminate": self._persistent,
+                },
+                timeout=60,
+            )
+        except Exception as exc:
+            logger.warning("Managed Modal cleanup failed: %s", exc)
+        finally:
+            self._sandbox_id = None
+
+    def _create_sandbox(self) -> str:
+        cpu = self._coerce_number(self._sandbox_kwargs.get("cpu"), 1)
+        memory = self._coerce_number(
+            self._sandbox_kwargs.get("memoryMiB", self._sandbox_kwargs.get("memory")),
+            5120,
+        )
+        disk = self._coerce_number(
+            self._sandbox_kwargs.get("ephemeral_disk", self._sandbox_kwargs.get("diskMiB")),
+            None,
+        )
+
+        create_payload = {
+            "image": self._image,
+            "cwd": self.cwd,
+            "cpu": cpu,
+            "memoryMiB": memory,
+            "timeoutMs": 3_600_000,
+            "idleTimeoutMs": max(300_000, int(self.timeout * 1000)),
+            "persistentFilesystem": self._persistent,
+            "logicalKey": self._task_id,
+        }
+        if disk is not None:
+            create_payload["diskMiB"] = disk
+
+        response = self._request(
+            "POST",
+            "/v1/sandboxes",
+            json=create_payload,
+            timeout=60,
+            extra_headers={
+                "x-idempotency-key": self._create_idempotency_key,
+            },
+        )
+        if response.status_code >= 400:
+            raise RuntimeError(self._format_error("Managed Modal create failed", response))
+
+        body = response.json()
+        sandbox_id = body.get("id")
+        if not isinstance(sandbox_id, str) or not sandbox_id:
+            raise RuntimeError("Managed Modal create did not return a sandbox id")
+        return sandbox_id
+
+    def _guard_unsupported_credential_passthrough(self) -> None:
+        """Managed Modal does not sync or mount host credential files."""
+        try:
+            from tools.credential_files import get_credential_file_mounts
+        except Exception:
+            return
+
+        mounts = get_credential_file_mounts()
+        if mounts:
+            raise ValueError(
+                "Managed Modal does not support host credential-file passthrough. "
+                "Use TERMINAL_MODAL_MODE=direct when skills or config require "
+                "credential files inside the sandbox."
+            )
+
+    def _request(self, method: str, path: str, *,
+                 json: Dict[str, Any] | None = None,
+                 timeout: int = 30,
+                 extra_headers: Dict[str, str] | None = None) -> requests.Response:
+        headers = {
+            "Authorization": f"Bearer {self._nous_user_token}",
+            "Content-Type": "application/json",
+        }
+        if extra_headers:
+            headers.update(extra_headers)
+
+        return requests.request(
+            method,
+            f"{self._gateway_origin}{path}",
+            headers=headers,
+            json=json,
+            timeout=timeout,
+        )
+
+    def _cancel_exec(self, exec_id: str) -> None:
+        try:
+            self._request(
+                "POST",
+                f"/v1/sandboxes/{self._sandbox_id}/execs/{exec_id}/cancel",
+                timeout=(self._CONNECT_TIMEOUT_SECONDS, self._CANCEL_READ_TIMEOUT_SECONDS),
+            )
+        except Exception as exc:
+            logger.warning("Managed Modal exec cancel failed: %s", exc)
+
+    @staticmethod
+    def _coerce_number(value: Any, default: float) -> float:
+        try:
+            if value is None:
+                return default
+            return float(value)
+        except (TypeError, ValueError):
+            return default
+
+    @staticmethod
+    def _format_error(prefix: str, response: requests.Response) -> str:
+        try:
+            payload = response.json()
+            if isinstance(payload, dict):
+                message = payload.get("error") or payload.get("message") or payload.get("code")
+                if isinstance(message, str) and message:
+                    return f"{prefix}: {message}"
+                return f"{prefix}: {json.dumps(payload, ensure_ascii=False)}"
+        except Exception:
+            pass
+
+        text = response.text.strip()
+        if text:
+            return f"{prefix}: {text}"
+        return f"{prefix}: HTTP {response.status_code}"

tools/environments/modal.py

@@ -0,0 +1,478 @@
+"""Modal cloud execution environment using the native Modal SDK directly.
+
+Uses ``Sandbox.create()`` + ``Sandbox.exec()`` instead of the older runtime
+wrapper, while preserving Hermes' persistent snapshot behavior across sessions.
+"""
+
+import asyncio
+import base64
+import io
+import logging
+import shlex
+import tarfile
+import threading
+from pathlib import Path
+from typing import Any, Optional
+
+from hermes_constants import get_hermes_home
+from tools.environments.base import (
+    BaseEnvironment,
+    _ThreadedProcessHandle,
+    _load_json_store,
+    _save_json_store,
+)
+from tools.environments.file_sync import (
+    FileSyncManager,
+    iter_sync_files,
+    quoted_mkdir_command,
+    quoted_rm_command,
+    unique_parent_dirs,
+)
+
+logger = logging.getLogger(__name__)
+
+_SNAPSHOT_STORE = get_hermes_home() / "modal_snapshots.json"
+_DIRECT_SNAPSHOT_NAMESPACE = "direct"
+
+
+def _load_snapshots() -> dict:
+    return _load_json_store(_SNAPSHOT_STORE)
+
+
+def _save_snapshots(data: dict) -> None:
+    _save_json_store(_SNAPSHOT_STORE, data)
+
+
+def _direct_snapshot_key(task_id: str) -> str:
+    return f"{_DIRECT_SNAPSHOT_NAMESPACE}:{task_id}"
+
+
+def _get_snapshot_restore_candidate(task_id: str) -> tuple[str | None, bool]:
+    snapshots = _load_snapshots()
+    namespaced_key = _direct_snapshot_key(task_id)
+    snapshot_id = snapshots.get(namespaced_key)
+    if isinstance(snapshot_id, str) and snapshot_id:
+        return snapshot_id, False
+    legacy_snapshot_id = snapshots.get(task_id)
+    if isinstance(legacy_snapshot_id, str) and legacy_snapshot_id:
+        return legacy_snapshot_id, True
+    return None, False
+
+
+def _store_direct_snapshot(task_id: str, snapshot_id: str) -> None:
+    snapshots = _load_snapshots()
+    snapshots[_direct_snapshot_key(task_id)] = snapshot_id
+    snapshots.pop(task_id, None)
+    _save_snapshots(snapshots)
+
+
+def _delete_direct_snapshot(task_id: str, snapshot_id: str | None = None) -> None:
+    snapshots = _load_snapshots()
+    updated = False
+    for key in (_direct_snapshot_key(task_id), task_id):
+        value = snapshots.get(key)
+        if value is None:
+            continue
+        if snapshot_id is None or value == snapshot_id:
+            snapshots.pop(key, None)
+            updated = True
+    if updated:
+        _save_snapshots(snapshots)
+
+
+def _ensure_modal_sdk() -> None:
+    """Lazy-install modal on demand. Idempotent — fast no-op once installed."""
+    try:
+        from tools.lazy_deps import ensure as _lazy_ensure
+        _lazy_ensure("terminal.modal", prompt=False)
+    except ImportError:
+        pass
+    except Exception as e:
+        raise ImportError(str(e))
+
+
+def _resolve_modal_image(image_spec: Any) -> Any:
+    """Convert registry references or snapshot ids into Modal image objects.
+
+    Includes add_python support for ubuntu/debian images (absorbed from PR 4511).
+    """
+    _ensure_modal_sdk()
+    import modal as _modal
+
+    if not isinstance(image_spec, str):
+        return image_spec
+
+    if image_spec.startswith("im-"):
+        return _modal.Image.from_id(image_spec)
+
+    # PR 4511: add python to ubuntu/debian images that don't have it
+    lower = image_spec.lower()
+    add_python = any(base in lower for base in ("ubuntu", "debian"))
+
+    setup_commands = [
+        "RUN rm -rf /usr/local/lib/python*/site-packages/pip* 2>/dev/null; "
+        "python -m ensurepip --upgrade --default-pip 2>/dev/null || true",
+    ]
+    if add_python:
+        setup_commands.insert(0,
+            "RUN apt-get update -qq && apt-get install -y -qq python3 python3-venv > /dev/null 2>&1 || true"
+        )
+
+    return _modal.Image.from_registry(
+        image_spec,
+        setup_dockerfile_commands=setup_commands,
+    )
+
+
+class _AsyncWorker:
+    """Background thread with its own event loop for async-safe Modal calls."""
+
+    def __init__(self):
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._thread: Optional[threading.Thread] = None
+        self._started = threading.Event()
+
+    def start(self):
+        self._thread = threading.Thread(target=self._run_loop, daemon=True)
+        self._thread.start()
+        self._started.wait(timeout=30)
+
+    def _run_loop(self):
+        self._loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(self._loop)
+        self._started.set()
+        self._loop.run_forever()
+
+    def run_coroutine(self, coro, timeout=600):
+        from agent.async_utils import safe_schedule_threadsafe
+        if self._loop is None or self._loop.is_closed():
+            if asyncio.iscoroutine(coro):
+                coro.close()
+            raise RuntimeError("AsyncWorker loop is not running")
+        future = safe_schedule_threadsafe(coro, self._loop)
+        if future is None:
+            raise RuntimeError("AsyncWorker loop is not running")
+        return future.result(timeout=timeout)
+
+    def stop(self):
+        if self._loop and self._loop.is_running():
+            self._loop.call_soon_threadsafe(self._loop.stop)
+        if self._thread:
+            self._thread.join(timeout=10)
+
+
+class ModalEnvironment(BaseEnvironment):
+    """Modal cloud execution via native Modal sandboxes.
+
+    Spawn-per-call via _ThreadedProcessHandle wrapping async SDK calls.
+    cancel_fn wired to sandbox.terminate for interrupt support.
+    """
+
+    _stdin_mode = "heredoc"
+    _snapshot_timeout = 60  # Modal cold starts can be slow
+
+    def __init__(
+        self,
+        image: str,
+        cwd: str = "/root",
+        timeout: int = 60,
+        modal_sandbox_kwargs: Optional[dict[str, Any]] = None,
+        persistent_filesystem: bool = True,
+        task_id: str = "default",
+    ):
+        super().__init__(cwd=cwd, timeout=timeout)
+
+        self._persistent = persistent_filesystem
+        self._task_id = task_id
+        self._sandbox = None
+        self._app = None
+        self._worker = _AsyncWorker()
+        self._sync_manager: FileSyncManager | None = None  # initialized after sandbox creation
+
+        sandbox_kwargs = dict(modal_sandbox_kwargs or {})
+
+        restored_snapshot_id = None
+        restored_from_legacy_key = False
+        if self._persistent:
+            restored_snapshot_id, restored_from_legacy_key = _get_snapshot_restore_candidate(
+                self._task_id
+            )
+            if restored_snapshot_id:
+                logger.info("Modal: restoring from snapshot %s", restored_snapshot_id[:20])
+
+        _ensure_modal_sdk()
+        import modal as _modal
+
+        cred_mounts = []
+        try:
+            from tools.credential_files import (
+                get_credential_file_mounts,
+                iter_skills_files,
+                iter_cache_files,
+            )
+
+            for mount_entry in get_credential_file_mounts():
+                cred_mounts.append(
+                    _modal.Mount.from_local_file(
+                        mount_entry["host_path"],
+                        remote_path=mount_entry["container_path"],
+                    )
+                )
+            for entry in iter_skills_files():
+                cred_mounts.append(
+                    _modal.Mount.from_local_file(
+                        entry["host_path"],
+                        remote_path=entry["container_path"],
+                    )
+                )
+            cache_files = iter_cache_files()
+            for entry in cache_files:
+                cred_mounts.append(
+                    _modal.Mount.from_local_file(
+                        entry["host_path"],
+                        remote_path=entry["container_path"],
+                    )
+                )
+        except Exception as e:
+            logger.debug("Modal: could not load credential file mounts: %s", e)
+
+        self._worker.start()
+
+        async def _create_sandbox(image_spec: Any):
+            app = await _modal.App.lookup.aio("hermes-agent", create_if_missing=True)
+            create_kwargs = dict(sandbox_kwargs)
+            if cred_mounts:
+                existing_mounts = list(create_kwargs.pop("mounts", []))
+                existing_mounts.extend(cred_mounts)
+                create_kwargs["mounts"] = existing_mounts
+            sandbox = await _modal.Sandbox.create.aio(
+                "sleep", "infinity",
+                image=image_spec,
+                app=app,
+                timeout=int(create_kwargs.pop("timeout", 3600)),
+                **create_kwargs,
+            )
+            return app, sandbox
+
+        try:
+            target_image_spec = restored_snapshot_id or image
+            try:
+                effective_image = _resolve_modal_image(target_image_spec)
+                self._app, self._sandbox = self._worker.run_coroutine(
+                    _create_sandbox(effective_image), timeout=300,
+                )
+            except Exception as exc:
+                if not restored_snapshot_id:
+                    raise
+                logger.warning(
+                    "Modal: failed to restore snapshot %s, retrying with base image: %s",
+                    restored_snapshot_id[:20], exc,
+                )
+                _delete_direct_snapshot(self._task_id, restored_snapshot_id)
+                base_image = _resolve_modal_image(image)
+                self._app, self._sandbox = self._worker.run_coroutine(
+                    _create_sandbox(base_image), timeout=300,
+                )
+            else:
+                if restored_snapshot_id and restored_from_legacy_key:
+                    _store_direct_snapshot(self._task_id, restored_snapshot_id)
+        except Exception:
+            self._worker.stop()
+            raise
+
+        logger.info("Modal: sandbox created (task=%s)", self._task_id)
+
+        self._sync_manager = FileSyncManager(
+            get_files_fn=lambda: iter_sync_files("/root/.hermes"),
+            upload_fn=self._modal_upload,
+            delete_fn=self._modal_delete,
+            bulk_upload_fn=self._modal_bulk_upload,
+            bulk_download_fn=self._modal_bulk_download,
+        )
+        self._sync_manager.sync(force=True)
+        self.init_session()
+
+    def _modal_upload(self, host_path: str, remote_path: str) -> None:
+        """Upload a single file via base64 piped through stdin."""
+        content = Path(host_path).read_bytes()
+        b64 = base64.b64encode(content).decode("ascii")
+        container_dir = str(Path(remote_path).parent)
+        cmd = (
+            f"mkdir -p {shlex.quote(container_dir)} && "
+            f"base64 -d > {shlex.quote(remote_path)}"
+        )
+
+        async def _write():
+            proc = await self._sandbox.exec.aio("bash", "-c", cmd)
+            offset = 0
+            chunk_size = self._STDIN_CHUNK_SIZE
+            while offset < len(b64):
+                proc.stdin.write(b64[offset:offset + chunk_size])
+                await proc.stdin.drain.aio()
+                offset += chunk_size
+            proc.stdin.write_eof()
+            await proc.stdin.drain.aio()
+            await proc.wait.aio()
+
+        self._worker.run_coroutine(_write(), timeout=30)
+
+    # Modal SDK stdin buffer limit (legacy server path).  The command-router
+    # path allows 16 MB, but we must stay under the smaller 2 MB cap for
+    # compatibility.  Chunks are written below this threshold and flushed
+    # individually via drain().
+    _STDIN_CHUNK_SIZE = 1 * 1024 * 1024  # 1 MB — safe for both transport paths
+
+    def _modal_bulk_upload(self, files: list[tuple[str, str]]) -> None:
+        """Upload many files via tar archive piped through stdin.
+
+        Builds a gzipped tar archive in memory and streams it into a
+        ``base64 -d | tar xzf -`` pipeline via the process's stdin,
+        avoiding the Modal SDK's 64 KB ``ARG_MAX_BYTES`` exec-arg limit.
+        """
+        if not files:
+            return
+
+        buf = io.BytesIO()
+        with tarfile.open(fileobj=buf, mode="w:gz") as tar:
+            for host_path, remote_path in files:
+                tar.add(host_path, arcname=remote_path.lstrip("/"))
+        payload = base64.b64encode(buf.getvalue()).decode("ascii")
+
+        parents = unique_parent_dirs(files)
+        mkdir_part = quoted_mkdir_command(parents)
+        cmd = f"{mkdir_part} && base64 -d | tar xzf - -C /"
+
+        async def _bulk():
+            proc = await self._sandbox.exec.aio("bash", "-c", cmd)
+
+            # Stream payload through stdin in chunks to stay under the
+            # SDK's per-write buffer limit (2 MB legacy / 16 MB router).
+            offset = 0
+            chunk_size = self._STDIN_CHUNK_SIZE
+            while offset < len(payload):
+                proc.stdin.write(payload[offset:offset + chunk_size])
+                await proc.stdin.drain.aio()
+                offset += chunk_size
+
+            proc.stdin.write_eof()
+            await proc.stdin.drain.aio()
+
+            exit_code = await proc.wait.aio()
+            if exit_code != 0:
+                stderr_text = await proc.stderr.read.aio()
+                raise RuntimeError(
+                    f"Modal bulk upload failed (exit {exit_code}): {stderr_text}"
+                )
+
+        self._worker.run_coroutine(_bulk(), timeout=120)
+
+    def _modal_bulk_download(self, dest: Path) -> None:
+        """Download remote .hermes/ as a tar archive.
+
+        Modal sandboxes always run as root, so /root/.hermes is hardcoded
+        (consistent with iter_sync_files call on line 269).
+        """
+        async def _download():
+            proc = await self._sandbox.exec.aio(
+                "bash", "-c", "tar cf - -C / root/.hermes"
+            )
+            data = await proc.stdout.read.aio()
+            exit_code = await proc.wait.aio()
+            if exit_code != 0:
+                raise RuntimeError(f"Modal bulk download failed (exit {exit_code})")
+            return data
+
+        tar_bytes = self._worker.run_coroutine(_download(), timeout=120)
+        if isinstance(tar_bytes, str):
+            tar_bytes = tar_bytes.encode()
+        dest.write_bytes(tar_bytes)
+
+    def _modal_delete(self, remote_paths: list[str]) -> None:
+        """Batch-delete remote files via exec."""
+        rm_cmd = quoted_rm_command(remote_paths)
+
+        async def _rm():
+            proc = await self._sandbox.exec.aio("bash", "-c", rm_cmd)
+            await proc.wait.aio()
+
+        self._worker.run_coroutine(_rm(), timeout=15)
+
+    def _before_execute(self) -> None:
+        """Sync files to sandbox via FileSyncManager (rate-limited internally)."""
+        self._sync_manager.sync()
+
+    # ------------------------------------------------------------------
+    # Execution
+    # ------------------------------------------------------------------
+
+    def _run_bash(self, cmd_string: str, *, login: bool = False,
+                  timeout: int = 120,
+                  stdin_data: str | None = None):
+        """Return a _ThreadedProcessHandle wrapping an async Modal sandbox exec."""
+        sandbox = self._sandbox
+        worker = self._worker
+
+        def cancel():
+            worker.run_coroutine(sandbox.terminate.aio(), timeout=15)
+
+        def exec_fn() -> tuple[str, int]:
+            async def _do():
+                args = ["bash"]
+                if login:
+                    args.extend(["-l", "-c", cmd_string])
+                else:
+                    args.extend(["-c", cmd_string])
+                process = await sandbox.exec.aio(*args, timeout=timeout)
+                stdout = await process.stdout.read.aio()
+                stderr = await process.stderr.read.aio()
+                exit_code = await process.wait.aio()
+                if isinstance(stdout, bytes):
+                    stdout = stdout.decode("utf-8", errors="replace")
+                if isinstance(stderr, bytes):
+                    stderr = stderr.decode("utf-8", errors="replace")
+                output = stdout
+                if stderr:
+                    output = f"{stdout}\n{stderr}" if stdout else stderr
+                return output, exit_code
+
+            return worker.run_coroutine(_do(), timeout=timeout + 30)
+
+        return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel)
+
+    def cleanup(self):
+        """Snapshot the filesystem (if persistent) then stop the sandbox."""
+        if self._sandbox is None:
+            return
+
+        if self._sync_manager:
+            logger.info("Modal: syncing files from sandbox...")
+            self._sync_manager.sync_back()
+
+        if self._persistent:
+            try:
+                async def _snapshot():
+                    img = await self._sandbox.snapshot_filesystem.aio()
+                    return img.object_id
+
+                try:
+                    snapshot_id = self._worker.run_coroutine(_snapshot(), timeout=60)
+                except Exception:
+                    snapshot_id = None
+
+                if snapshot_id:
+                    _store_direct_snapshot(self._task_id, snapshot_id)
+                    logger.info(
+                        "Modal: saved filesystem snapshot %s for task %s",
+                        snapshot_id[:20], self._task_id,
+                    )
+            except Exception as e:
+                logger.warning("Modal: filesystem snapshot failed: %s", e)
+
+        try:
+            self._worker.run_coroutine(self._sandbox.terminate.aio(), timeout=15)
+        except Exception:
+            pass
+        finally:
+            self._worker.stop()
+            self._sandbox = None
+            self._app = None

tools/environments/modal_utils.py

@@ -0,0 +1,210 @@
+"""Shared Hermes-side execution flow for Modal transports.
+
+This module deliberately stops at the Hermes boundary:
+- command preparation
+- cwd/timeout normalization
+- stdin/sudo shell wrapping
+- common result shape
+- interrupt/cancel polling
+
+Direct Modal and managed Modal keep separate transport logic, persistence, and
+trust-boundary decisions in their own modules.
+"""
+
+from __future__ import annotations
+
+import shlex
+import time
+import uuid
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+from tools.environments.base import BaseEnvironment
+from tools.interrupt import is_interrupted
+
+
+@dataclass(frozen=True)
+class PreparedModalExec:
+    """Normalized command data passed to a transport-specific exec runner."""
+
+    command: str
+    cwd: str
+    timeout: int
+    stdin_data: str | None = None
+
+
+@dataclass(frozen=True)
+class ModalExecStart:
+    """Transport response after starting an exec."""
+
+    handle: Any | None = None
+    immediate_result: dict | None = None
+
+
+def wrap_modal_stdin_heredoc(command: str, stdin_data: str) -> str:
+    """Append stdin as a shell heredoc for transports without stdin piping."""
+    marker = f"HERMES_EOF_{uuid.uuid4().hex[:8]}"
+    while marker in stdin_data:
+        marker = f"HERMES_EOF_{uuid.uuid4().hex[:8]}"
+    return f"{command} << '{marker}'\n{stdin_data}\n{marker}"
+
+
+def wrap_modal_sudo_pipe(command: str, sudo_stdin: str) -> str:
+    """Feed sudo via a shell pipe for transports without direct stdin piping."""
+    return f"printf '%s\\n' {shlex.quote(sudo_stdin.rstrip())} | {command}"
+
+
+class BaseModalExecutionEnvironment(BaseEnvironment):
+    """Execution flow for the *managed* Modal transport (gateway-owned sandbox).
+
+    This deliberately overrides :meth:`BaseEnvironment.execute` because the
+    tool-gateway handles command preparation, CWD tracking, and env-snapshot
+    management on the server side.  The base class's ``_wrap_command`` /
+    ``_wait_for_process`` / snapshot machinery does not apply here — the
+    gateway owns that responsibility.  See ``ManagedModalEnvironment`` for the
+    concrete subclass.
+    """
+
+    _stdin_mode = "payload"
+    _poll_interval_seconds = 0.25
+    _client_timeout_grace_seconds: float | None = None
+    _interrupt_output = "[Command interrupted]"
+    _unexpected_error_prefix = "Modal execution error"
+
+    def execute(
+        self,
+        command: str,
+        cwd: str = "",
+        *,
+        timeout: int | None = None,
+        stdin_data: str | None = None,
+        rewrite_compound_background: bool = True,
+        bounded_capture: bool = False,
+    ) -> dict:
+        # Managed/remote modal transports execute commands via explicit transport
+        # and do not rely on shell background rewriters. Keep parameter for
+        # compatibility with BaseEnvironment callers.
+        _ = rewrite_compound_background
+        # bounded_capture: accepted for BaseEnvironment.execute() signature
+        # parity (the terminal tool passes it). Modal transports return the
+        # remote function's result in one payload, so streaming-time bounding
+        # does not apply; the terminal tool's final truncation still caps it.
+        _ = bounded_capture
+        self._before_execute()
+        prepared = self._prepare_modal_exec(
+            command,
+            cwd=cwd,
+            timeout=timeout,
+            stdin_data=stdin_data,
+        )
+
+        try:
+            start = self._start_modal_exec(prepared)
+        except Exception as exc:
+            return self._error_result(f"{self._unexpected_error_prefix}: {exc}")
+
+        if start.immediate_result is not None:
+            return start.immediate_result
+
+        if start.handle is None:
+            return self._error_result(
+                f"{self._unexpected_error_prefix}: transport did not return an exec handle"
+            )
+
+        deadline = None
+        if self._client_timeout_grace_seconds is not None:
+            deadline = time.monotonic() + prepared.timeout + self._client_timeout_grace_seconds
+
+        _now = time.monotonic()
+        _activity_state = {
+            "last_touch": _now,
+            "start": _now,
+        }
+
+        while True:
+            if is_interrupted():
+                try:
+                    self._cancel_modal_exec(start.handle)
+                except Exception:
+                    pass
+                return self._result(self._interrupt_output, 130)
+
+            try:
+                result = self._poll_modal_exec(start.handle)
+            except Exception as exc:
+                return self._error_result(f"{self._unexpected_error_prefix}: {exc}")
+
+            if result is not None:
+                return result
+
+            if deadline is not None and time.monotonic() >= deadline:
+                try:
+                    self._cancel_modal_exec(start.handle)
+                except Exception:
+                    pass
+                return self._timeout_result_for_modal(prepared.timeout)
+
+            # Periodic activity touch so the gateway knows we're alive
+            try:
+                from tools.environments.base import touch_activity_if_due
+                touch_activity_if_due(_activity_state, "modal command running")
+            except Exception:
+                pass
+
+            time.sleep(self._poll_interval_seconds)
+
+    def _before_execute(self) -> None:
+        """Hook for backends that need pre-exec sync or validation."""
+        pass
+
+    def _prepare_modal_exec(
+        self,
+        command: str,
+        *,
+        cwd: str = "",
+        timeout: int | None = None,
+        stdin_data: str | None = None,
+    ) -> PreparedModalExec:
+        effective_cwd = cwd or self.cwd
+        effective_timeout = timeout or self.timeout
+
+        exec_command = command
+        exec_stdin = stdin_data if self._stdin_mode == "payload" else None
+        if stdin_data is not None and self._stdin_mode == "heredoc":
+            exec_command = wrap_modal_stdin_heredoc(exec_command, stdin_data)
+
+        exec_command, sudo_stdin = self._prepare_command(exec_command)
+        if sudo_stdin is not None:
+            exec_command = wrap_modal_sudo_pipe(exec_command, sudo_stdin)
+
+        return PreparedModalExec(
+            command=exec_command,
+            cwd=effective_cwd,
+            timeout=effective_timeout,
+            stdin_data=exec_stdin,
+        )
+
+    def _result(self, output: str, returncode: int) -> dict:
+        return {
+            "output": output,
+            "returncode": returncode,
+        }
+
+    def _error_result(self, output: str) -> dict:
+        return self._result(output, 1)
+
+    def _timeout_result_for_modal(self, timeout: int) -> dict:
+        return self._result(f"Command timed out after {timeout}s", 124)
+
+    @abstractmethod
+    def _start_modal_exec(self, prepared: PreparedModalExec) -> ModalExecStart:
+        """Begin a transport-specific exec."""
+
+    @abstractmethod
+    def _poll_modal_exec(self, handle: Any) -> dict | None:
+        """Return a final result dict when complete, else ``None``."""
+
+    @abstractmethod
+    def _cancel_modal_exec(self, handle: Any) -> None:
+        """Cancel or terminate the active transport exec."""

tools/environments/path_utils.py

@@ -0,0 +1,44 @@
+"""Path-component helpers shared by execution environment backends.
+
+Kept separate from the base environment class so lazy backend imports do not
+depend on newly added exports from a large module cached earlier in a long-lived
+process.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+
+
+# A persistent sandbox's host directory is named after task_id, and that name
+# then becomes the source half of a Docker bind spec or a writable Singularity
+# overlay directory. Keep every backend on one collision-safe mapping.
+_SANDBOX_DIR_UNSAFE_RE = re.compile(r"[^A-Za-z0-9._-]")
+_SANDBOX_DIR_MAX_LEN = 128
+_SANDBOX_DIR_HASH_LEN = 12
+
+
+def sanitize_task_id_for_path(task_id: str) -> str:
+    """Return a bind-mountable directory name for *task_id*'s sandbox.
+
+    Names that are already safe are returned verbatim, preserving existing
+    sandbox locations. Rewritten names carry a digest because substitution
+    alone is not injective: ``a:b`` and ``a_b`` must not share state.
+    """
+    value = task_id if isinstance(task_id, str) else ""
+    if not value:
+        return "default"
+
+    cleaned = _SANDBOX_DIR_UNSAFE_RE.sub("_", value)
+    if (
+        cleaned == value
+        and len(value) <= _SANDBOX_DIR_MAX_LEN
+        and value not in {".", ".."}
+        and not value.endswith((".", " "))
+    ):
+        return value
+
+    digest = hashlib.sha256(value.encode("utf-8")).hexdigest()[:_SANDBOX_DIR_HASH_LEN]
+    stem = cleaned[: _SANDBOX_DIR_MAX_LEN - _SANDBOX_DIR_HASH_LEN - 1].strip("._")
+    return f"{stem or 'task'}-{digest}"

tools/environments/singularity.py

@@ -0,0 +1,273 @@
+"""Singularity/Apptainer persistent container environment.
+
+Security-hardened with --containall, --no-home, capability dropping.
+Supports configurable resource limits and optional filesystem persistence
+via writable overlay directories that survive across sessions.
+"""
+
+import logging
+import os
+import shutil
+import subprocess
+import threading
+import uuid
+from pathlib import Path
+from typing import Optional
+
+from hermes_constants import get_hermes_home
+from tools.environments.base import (
+    BaseEnvironment,
+    _load_json_store,
+    _popen_bash,
+    _save_json_store,
+)
+from tools.environments.path_utils import sanitize_task_id_for_path
+
+logger = logging.getLogger(__name__)
+
+_SNAPSHOT_STORE = get_hermes_home() / "singularity_snapshots.json"
+
+
+def _find_singularity_executable() -> str:
+    """Locate the apptainer or singularity CLI binary."""
+    if shutil.which("apptainer"):
+        return "apptainer"
+    if shutil.which("singularity"):
+        return "singularity"
+    raise RuntimeError(
+        "Neither 'apptainer' nor 'singularity' was found in PATH. "
+        "Install Apptainer (https://apptainer.org/docs/admin/main/installation.html) "
+        "or Singularity and ensure the CLI is available."
+    )
+
+
+def _ensure_singularity_available() -> str:
+    """Preflight check: resolve the executable and verify it responds."""
+    exe = _find_singularity_executable()
+    try:
+        result = subprocess.run(
+            [exe, "version"], capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=10,
+            stdin=subprocess.DEVNULL,
+        )
+    except FileNotFoundError:
+        raise RuntimeError(
+            f"Singularity backend selected but '{exe}' could not be executed."
+        )
+    except subprocess.TimeoutExpired:
+        raise RuntimeError(f"'{exe} version' timed out.")
+
+    if result.returncode != 0:
+        stderr = result.stderr.strip()[:200]
+        raise RuntimeError(f"'{exe} version' failed (exit code {result.returncode}): {stderr}")
+    return exe
+
+
+def _load_snapshots() -> dict:
+    return _load_json_store(_SNAPSHOT_STORE)
+
+
+def _save_snapshots(data: dict) -> None:
+    _save_json_store(_SNAPSHOT_STORE, data)
+
+
+def _get_scratch_dir() -> Path:
+    custom_scratch = os.getenv("TERMINAL_SCRATCH_DIR")
+    if custom_scratch:
+        scratch_path = Path(custom_scratch)
+        scratch_path.mkdir(parents=True, exist_ok=True)
+        return scratch_path
+
+    from tools.environments.base import get_sandbox_dir
+    sandbox = get_sandbox_dir() / "singularity"
+
+    scratch = Path("/scratch")
+    if scratch.exists() and os.access(scratch, os.W_OK):
+        user_scratch = scratch / os.getenv("USER", "hermes") / "hermes-agent"
+        user_scratch.mkdir(parents=True, exist_ok=True)
+        logger.info("Using /scratch for sandboxes: %s", user_scratch)
+        return user_scratch
+
+    sandbox.mkdir(parents=True, exist_ok=True)
+    return sandbox
+
+
+def _get_apptainer_cache_dir() -> Path:
+    cache_dir = os.getenv("APPTAINER_CACHEDIR")
+    if cache_dir:
+        cache_path = Path(cache_dir)
+        cache_path.mkdir(parents=True, exist_ok=True)
+        return cache_path
+    scratch = _get_scratch_dir()
+    cache_path = scratch / ".apptainer"
+    cache_path.mkdir(parents=True, exist_ok=True)
+    return cache_path
+
+
+_sif_build_lock = threading.Lock()
+
+
+def _get_or_build_sif(image: str, executable: str = "apptainer") -> str:
+    if image.endswith('.sif') and Path(image).exists():
+        return image
+    if not image.startswith('docker://'):
+        return image
+
+    image_name = image.replace('docker://', '').replace('/', '-').replace(':', '-')
+    cache_dir = _get_apptainer_cache_dir()
+    sif_path = cache_dir / f"{image_name}.sif"
+
+    if sif_path.exists():
+        return str(sif_path)
+
+    with _sif_build_lock:
+        if sif_path.exists():
+            return str(sif_path)
+
+        logger.info("Building SIF image (one-time setup)...")
+        logger.info("  Source: %s", image)
+        logger.info("  Target: %s", sif_path)
+
+        tmp_dir = cache_dir / "tmp"
+        tmp_dir.mkdir(parents=True, exist_ok=True)
+
+        # apptainer/singularity build: external tool, may need registry
+        # credentials from the user env — exact preservation.
+        from tools.environments.local import build_subprocess_env
+        env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=False)
+        env["APPTAINER_TMPDIR"] = str(tmp_dir)
+        env["APPTAINER_CACHEDIR"] = str(cache_dir)
+
+        try:
+            result = subprocess.run(
+                [executable, "build", str(sif_path), image],
+                capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=600, env=env,
+                stdin=subprocess.DEVNULL,
+            )
+            if result.returncode != 0:
+                logger.warning("SIF build failed, falling back to docker:// URL")
+                logger.warning("  Error: %s", result.stderr[:500])
+                return image
+            logger.info("SIF image built successfully")
+            return str(sif_path)
+        except subprocess.TimeoutExpired:
+            logger.warning("SIF build timed out, falling back to docker:// URL")
+            if sif_path.exists():
+                sif_path.unlink()
+            return image
+        except Exception as e:
+            logger.warning("SIF build error: %s, falling back to docker:// URL", e)
+            return image
+
+
+class SingularityEnvironment(BaseEnvironment):
+    """Hardened Singularity/Apptainer container with resource limits and persistence.
+
+    Spawn-per-call: every execute() spawns a fresh ``apptainer exec ... bash -c`` process.
+    Session snapshot preserves env vars across calls.
+    CWD persists via in-band stdout markers.
+    """
+
+    def __init__(
+        self,
+        image: str,
+        cwd: str = "~",
+        timeout: int = 60,
+        cpu: float = 0,
+        memory: int = 0,
+        disk: int = 0,
+        persistent_filesystem: bool = False,
+        task_id: str = "default",
+    ):
+        super().__init__(cwd=cwd, timeout=timeout)
+        self.executable = _ensure_singularity_available()
+        self.image = _get_or_build_sif(image, self.executable)
+        self.instance_id = f"hermes_{uuid.uuid4().hex[:12]}"
+        self._instance_started = False
+        self._persistent = persistent_filesystem
+        self._task_id = task_id
+        self._overlay_dir: Optional[Path] = None
+        self._cpu = cpu
+        self._memory = memory
+
+        if self._persistent:
+            overlay_base = _get_scratch_dir() / "hermes-overlays"
+            overlay_base.mkdir(parents=True, exist_ok=True)
+            # A raw session-key task_id carries colons and other characters
+            # that are unsafe in host path components (same class of bug as
+            # the docker -v mount failure); route it through the shared
+            # sanitizer so all backends agree on the mapping.
+            self._overlay_dir = overlay_base / f"overlay-{sanitize_task_id_for_path(task_id)}"
+            self._overlay_dir.mkdir(parents=True, exist_ok=True)
+
+        self._start_instance()
+        self.init_session()
+
+    def _start_instance(self):
+        cmd = [self.executable, "instance", "start"]
+        cmd.extend(["--containall", "--no-home"])
+
+        if self._persistent and self._overlay_dir:
+            cmd.extend(["--overlay", str(self._overlay_dir)])
+        else:
+            cmd.append("--writable-tmpfs")
+
+        try:
+            from tools.credential_files import get_credential_file_mounts, get_skills_directory_mount
+            for mount_entry in get_credential_file_mounts():
+                cmd.extend(["--bind", f"{mount_entry['host_path']}:{mount_entry['container_path']}:ro"])
+            for skills_mount in get_skills_directory_mount():
+                cmd.extend(["--bind", f"{skills_mount['host_path']}:{skills_mount['container_path']}:ro"])
+        except Exception as e:
+            logger.debug("Singularity: could not load credential/skills mounts: %s", e)
+
+        if self._memory > 0:
+            cmd.extend(["--memory", f"{self._memory}M"])
+        if self._cpu > 0:
+            cmd.extend(["--cpus", str(self._cpu)])
+
+        cmd.extend([str(self.image), self.instance_id])
+
+        try:
+            result = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=120, stdin=subprocess.DEVNULL)
+            if result.returncode != 0:
+                raise RuntimeError(f"Failed to start instance: {result.stderr}")
+            self._instance_started = True
+            logger.info("Singularity instance %s started (persistent=%s)",
+                        self.instance_id, self._persistent)
+        except subprocess.TimeoutExpired:
+            raise RuntimeError("Instance start timed out")
+
+    def _run_bash(self, cmd_string: str, *, login: bool = False,
+                  timeout: int = 120,
+                  stdin_data: str | None = None) -> subprocess.Popen:
+        """Spawn a bash process inside the Singularity instance."""
+        if not self._instance_started:
+            raise RuntimeError("Singularity instance not started")
+
+        cmd = [self.executable, "exec",
+               f"instance://{self.instance_id}"]
+        if login:
+            cmd.extend(["bash", "-l", "-c", cmd_string])
+        else:
+            cmd.extend(["bash", "-c", cmd_string])
+
+        return _popen_bash(cmd, stdin_data)
+
+    def cleanup(self):
+        """Stop the instance. If persistent, the overlay dir survives."""
+        if self._instance_started:
+            try:
+                subprocess.run(
+                    [self.executable, "instance", "stop", self.instance_id],
+                    capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=30,
+                    stdin=subprocess.DEVNULL,
+                )
+                logger.info("Singularity instance %s stopped", self.instance_id)
+            except Exception as e:
+                logger.warning("Failed to stop Singularity instance %s: %s", self.instance_id, e)
+            self._instance_started = False
+
+        if self._persistent and self._overlay_dir:
+            snapshots = _load_snapshots()
+            snapshots[self._task_id] = str(self._overlay_dir)
+            _save_snapshots(snapshots)

tools/environments/ssh.py

@@ -0,0 +1,435 @@
+"""SSH remote execution environment with ControlMaster connection persistence."""
+
+import hashlib
+import logging
+import os
+
+# Windows OpenSSH has no Unix-domain-socket ControlMaster support —
+# passing ControlPath/ControlMaster options fails the connection outright
+# ('getsockname failed: Not a socket', #73927). Skip multiplexing there;
+# each command pays a fresh connection but the backend works.
+_SSH_MULTIPLEX = os.name != "nt"
+import shlex
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+from tools.environments.base import (
+    BaseEnvironment,
+    EnvironmentConnectionError,
+    _popen_bash,
+)
+from tools.environments.file_sync import (
+    FileSyncManager,
+    iter_sync_files,
+    quoted_mkdir_command,
+    quoted_rm_command,
+    unique_parent_dirs,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _ensure_ssh_available() -> None:
+    """Fail fast with a clear error when the SSH client is unavailable."""
+    if not shutil.which("ssh"):
+        raise RuntimeError(
+            "SSH is not installed or not in PATH. Install OpenSSH client: apt install openssh-client"
+        )
+    if not shutil.which("scp"):
+        raise RuntimeError(
+            "SCP is not installed or not in PATH. Install OpenSSH client: apt install openssh-client"
+        )
+
+
+class SSHEnvironment(BaseEnvironment):
+    """Run commands on a remote machine over SSH.
+
+    Spawn-per-call: every execute() spawns a fresh ``ssh ... bash -c`` process.
+    Session snapshot preserves env vars across calls.
+    CWD persists via in-band stdout markers.
+    Uses SSH ControlMaster for connection reuse.
+    """
+
+    def __init__(self, host: str, user: str, cwd: str = "~",
+                 timeout: int = 60, port: int = 22, key_path: str = ""):
+        super().__init__(cwd=cwd, timeout=timeout)
+        self.host = host
+        self.user = user
+        self.port = port
+        self.key_path = key_path
+
+        self.control_dir = Path(tempfile.gettempdir()) / "hermes-ssh"
+        self.control_dir.mkdir(parents=True, exist_ok=True)
+        # Keep the socket filename short and deterministic so the full path
+        # stays under the 104-byte sun_path limit that macOS enforces on
+        # Unix domain sockets. A raw ``user@host:port`` — especially with an
+        # IPv6 host — plus the 16-byte random suffix SSH appends in
+        # ControlMaster mode easily exceeds the limit under macOS's
+        # deeply-nested $TMPDIR (e.g. /var/folders/xx/yy/T/). Hashing the
+        # triple keeps the path stable across reconnects so ControlMaster
+        # reuse still works.
+        _socket_id = hashlib.sha256(
+            f"{user}@{host}:{port}".encode()
+        ).hexdigest()[:16]
+        self.control_socket = self.control_dir / f"{_socket_id}.sock"
+        _ensure_ssh_available()
+        self._establish_connection()
+        self._remote_home = self._detect_remote_home()
+
+        self._ensure_remote_dirs()
+        self._sync_manager = FileSyncManager(
+            get_files_fn=lambda: iter_sync_files(f"{self._remote_home}/.hermes"),
+            upload_fn=self._scp_upload,
+            delete_fn=self._ssh_delete,
+            bulk_upload_fn=self._ssh_bulk_upload,
+            bulk_download_fn=self._ssh_bulk_download,
+        )
+        self._sync_manager.sync(force=True)
+
+        self.init_session()
+
+    def _build_ssh_command(self, extra_args: list | None = None) -> list:
+        cmd = ["ssh"]
+        if _SSH_MULTIPLEX:
+            cmd.extend(["-o", f"ControlPath={self.control_socket}"])
+            cmd.extend(["-o", "ControlMaster=auto"])
+            cmd.extend(["-o", "ControlPersist=300"])
+        cmd.extend(["-o", "BatchMode=yes"])
+        cmd.extend(["-o", "StrictHostKeyChecking=accept-new"])
+        cmd.extend(["-o", "ConnectTimeout=10"])
+        if self.port != 22:
+            cmd.extend(["-p", str(self.port)])
+        if self.key_path:
+            cmd.extend(["-i", self.key_path])
+        if extra_args:
+            cmd.extend(extra_args)
+        cmd.append(f"{self.user}@{self.host}")
+        return cmd
+
+    def _establish_connection(self):
+        cmd = self._build_ssh_command()
+        cmd.append("echo 'SSH connection established'")
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True, encoding='utf-8', errors='replace',
+                timeout=15,
+                stdin=subprocess.DEVNULL,
+            )
+            if result.returncode != 0:
+                error_msg = result.stderr.strip() or result.stdout.strip()
+                raise EnvironmentConnectionError(
+                    f"SSH connection failed: {error_msg}",
+                    retry_hint=(
+                        f"Verify {self.user}@{self.host}:{self.port} is reachable "
+                        "(host up, sshd running, key/agent auth working), then "
+                        "retry — the connection is re-established automatically."
+                    ),
+                )
+        except subprocess.TimeoutExpired:
+            raise EnvironmentConnectionError(
+                f"SSH connection to {self.user}@{self.host} timed out",
+                retry_hint=(
+                    f"Check network connectivity to {self.host}:{self.port} "
+                    "and that sshd is accepting connections, then retry."
+                ),
+            )
+
+    def _detect_remote_home(self) -> str:
+        """Detect the remote user's home directory."""
+        try:
+            cmd = self._build_ssh_command()
+            cmd.append("echo $HOME")
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True, encoding='utf-8', errors='replace',
+                timeout=10,
+                stdin=subprocess.DEVNULL,
+            )
+            home = result.stdout.strip()
+            if home and result.returncode == 0:
+                logger.debug("SSH: remote home = %s", home)
+                return home
+        except Exception:
+            pass
+        if self.user == "root":
+            return "/root"
+        return f"/home/{self.user}"
+
+    # ------------------------------------------------------------------
+    # File sync (via FileSyncManager)
+    # ------------------------------------------------------------------
+
+    def _ensure_remote_dirs(self) -> None:
+        """Create base ~/.hermes directory tree on remote in one SSH call."""
+        base = f"{self._remote_home}/.hermes"
+        dirs = [base, f"{base}/skills", f"{base}/credentials", f"{base}/cache"]
+        cmd = self._build_ssh_command()
+        cmd.append(quoted_mkdir_command(dirs))
+        subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True, encoding='utf-8', errors='replace',
+            timeout=10,
+            stdin=subprocess.DEVNULL,
+        )
+
+    # _get_sync_files provided via iter_sync_files in FileSyncManager init
+
+    def _scp_upload(self, host_path: str, remote_path: str) -> None:
+        """Upload a single file via scp over ControlMaster."""
+        parent = str(Path(remote_path).parent)
+        mkdir_cmd = self._build_ssh_command()
+        mkdir_cmd.append(f"mkdir -p {shlex.quote(parent)}")
+        subprocess.run(
+            mkdir_cmd,
+            capture_output=True,
+            text=True, encoding='utf-8', errors='replace',
+            timeout=10,
+            stdin=subprocess.DEVNULL,
+        )
+
+        scp_cmd = ["scp"]
+        if _SSH_MULTIPLEX:
+            scp_cmd.extend(["-o", f"ControlPath={self.control_socket}"])
+        if self.port != 22:
+            scp_cmd.extend(["-P", str(self.port)])
+        if self.key_path:
+            scp_cmd.extend(["-i", self.key_path])
+        scp_cmd.extend([host_path, f"{self.user}@{self.host}:{remote_path}"])
+        result = subprocess.run(
+            scp_cmd,
+            capture_output=True,
+            text=True, encoding='utf-8', errors='replace',
+            timeout=30,
+            stdin=subprocess.DEVNULL,
+        )
+        if result.returncode != 0:
+            raise EnvironmentConnectionError(
+                f"scp failed: {result.stderr.strip()}",
+                retry_hint=(
+                    f"File sync to {self.user}@{self.host} failed — verify the "
+                    "SSH connection is healthy, then retry."
+                ),
+            )
+
+    def _ssh_bulk_upload(self, files: list[tuple[str, str]]) -> None:
+        """Upload many files in a single tar-over-SSH stream.
+
+        Pipes ``tar c`` on the local side through an SSH connection to
+        ``tar x`` on the remote, transferring all files in one TCP stream
+        instead of spawning a subprocess per file.  Directory creation is
+        batched into a single ``mkdir -p`` call beforehand.
+
+        Typical improvement: ~580 files goes from O(N) scp round-trips
+        to a single streaming transfer.
+        """
+        if not files:
+            return
+
+        base = f"{self._remote_home}/.hermes"
+        parents = unique_parent_dirs(files)
+        if parents:
+            cmd = self._build_ssh_command()
+            cmd.append(quoted_mkdir_command(parents))
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True, encoding='utf-8', errors='replace',
+                timeout=30,
+                stdin=subprocess.DEVNULL,
+            )
+            if result.returncode != 0:
+                raise EnvironmentConnectionError(
+                    f"remote mkdir failed: {result.stderr.strip()}",
+                    retry_hint=(
+                        f"Remote directory setup on {self.host} failed — verify "
+                        "the SSH connection is healthy, then retry."
+                    ),
+                )
+
+        # Symlink staging avoids fragile GNU tar --transform rules.
+        # On Windows without Developer Mode, symlink creation raises
+        # OSError with winerror 1314 (privilege not held).  Catch only
+        # that specific error and fall back to a plain copy; all other
+        # OSErrors (e.g. disk full, bad path) are re-raised as normal.
+        with tempfile.TemporaryDirectory(prefix="hermes-ssh-bulk-") as staging:
+            for host_path, remote_path in files:
+                try:
+                    rel_remote = os.path.relpath(remote_path, base)
+                except ValueError as exc:
+                    raise RuntimeError(
+                        f"remote path {remote_path!r} is not under sync base {base!r}"
+                    ) from exc
+
+                if rel_remote == "." or rel_remote.startswith("../"):
+                    raise RuntimeError(
+                        f"remote path {remote_path!r} escapes sync base {base!r}"
+                    )
+
+                staged = os.path.join(staging, rel_remote)
+                os.makedirs(os.path.dirname(staged), exist_ok=True)
+                try:
+                    os.symlink(os.path.abspath(host_path), staged)
+                except OSError as e:
+                    # WinError 1314: symlink privilege not held (Windows without Dev Mode)
+                    if getattr(e, "winerror", None) == 1314:
+                        shutil.copy2(host_path, staged)
+                    else:
+                        raise
+
+            tar_cmd = ["tar", "-chf", "-", "-C", staging, "."]
+            ssh_cmd = self._build_ssh_command()
+            # --no-overwrite-dir prevents tar from overwriting the mode of
+            # existing directories (e.g. /home/<user>) with the staging
+            # directory's mode.  Without this, a umask 002 produces 0775
+            # dirs which breaks sshd StrictModes (refuses authorized_keys).
+            ssh_cmd.append(f"tar xf - --no-overwrite-dir -C {shlex.quote(base)}")
+
+            tar_proc = subprocess.Popen(
+                tar_cmd,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            try:
+                ssh_proc = subprocess.Popen(
+                    ssh_cmd, stdin=tar_proc.stdout, stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                )
+            except Exception:
+                tar_proc.kill()
+                tar_proc.wait()
+                raise
+
+            # Allow tar_proc to receive SIGPIPE if ssh_proc exits early
+            tar_proc.stdout.close()
+
+            try:
+                _, ssh_stderr = ssh_proc.communicate(timeout=120)
+                # Use communicate() instead of wait() to drain stderr and
+                # avoid deadlock if tar produces more than PIPE_BUF of errors.
+                tar_stderr_raw = b""
+                if tar_proc.poll() is None:
+                    _, tar_stderr_raw = tar_proc.communicate(timeout=10)
+                else:
+                    tar_stderr_raw = tar_proc.stderr.read() if tar_proc.stderr else b""
+            except subprocess.TimeoutExpired:
+                tar_proc.kill()
+                ssh_proc.kill()
+                tar_proc.wait()
+                ssh_proc.wait()
+                raise EnvironmentConnectionError(
+                    "SSH bulk upload timed out",
+                    retry_hint=(
+                        f"Bulk file sync to {self.host} timed out — check the "
+                        "connection and retry."
+                    ),
+                )
+
+            if tar_proc.returncode != 0:
+                raise RuntimeError(
+                    f"tar create failed (rc={tar_proc.returncode}): "
+                    f"{tar_stderr_raw.decode(errors='replace').strip()}"
+                )
+            if ssh_proc.returncode != 0:
+                raise EnvironmentConnectionError(
+                    f"tar extract over SSH failed (rc={ssh_proc.returncode}): "
+                    f"{ssh_stderr.decode(errors='replace').strip()}",
+                    retry_hint=(
+                        f"File sync over SSH to {self.host} failed — verify the "
+                        "connection is healthy, then retry."
+                    ),
+                )
+
+        logger.debug("SSH: bulk-uploaded %d file(s) via tar pipe", len(files))
+
+    def _ssh_bulk_download(self, dest: Path) -> None:
+        """Download remote .hermes/ as a tar archive."""
+        # Tar from / with the full path so archive entries preserve absolute
+        # paths (e.g. home/user/.hermes/skills/f.py), matching _pushed_hashes keys.
+        rel_base = f"{self._remote_home}/.hermes".lstrip("/")
+        ssh_cmd = self._build_ssh_command()
+        ssh_cmd.append(f"tar cf - -C / {shlex.quote(rel_base)}")
+        with open(dest, "wb") as f:
+            result = subprocess.run(
+                ssh_cmd,
+                stdin=subprocess.DEVNULL,
+                stdout=f,
+                stderr=subprocess.PIPE,
+                timeout=120,
+            )
+        if result.returncode != 0:
+            raise EnvironmentConnectionError(
+                f"SSH bulk download failed: {result.stderr.decode(errors='replace').strip()}",
+                retry_hint=(
+                    f"File sync from {self.host} failed — verify the SSH "
+                    "connection is healthy, then retry."
+                ),
+            )
+
+    def _ssh_delete(self, remote_paths: list[str]) -> None:
+        """Batch-delete remote files in one SSH call."""
+        cmd = self._build_ssh_command()
+        cmd.append(quoted_rm_command(remote_paths))
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True, encoding='utf-8', errors='replace',
+            timeout=10,
+            stdin=subprocess.DEVNULL,
+        )
+        if result.returncode != 0:
+            raise EnvironmentConnectionError(
+                f"remote rm failed: {result.stderr.strip()}",
+                retry_hint=(
+                    f"Remote file cleanup on {self.host} failed — verify the "
+                    "SSH connection is healthy, then retry."
+                ),
+            )
+
+    def _before_execute(self) -> None:
+        """Sync files to remote via FileSyncManager (rate-limited internally)."""
+        self._sync_manager.sync()
+
+    # ------------------------------------------------------------------
+    # Execution
+    # ------------------------------------------------------------------
+
+    def _run_bash(self, cmd_string: str, *, login: bool = False,
+                  timeout: int = 120,
+                  stdin_data: str | None = None) -> subprocess.Popen:
+        """Spawn an SSH process that runs bash on the remote host."""
+        cmd = self._build_ssh_command()
+        if login:
+            cmd.extend(["bash", "-l", "-c", shlex.quote(cmd_string)])
+        else:
+            cmd.extend(["bash", "-c", shlex.quote(cmd_string)])
+
+        return _popen_bash(cmd, stdin_data)
+
+    def cleanup(self):
+        if self._sync_manager:
+            logger.info("SSH: syncing files from sandbox...")
+            self._sync_manager.sync_back()
+
+        if self.control_socket.exists():
+            try:
+                cmd = ["ssh", "-o", f"ControlPath={self.control_socket}",
+                       "-O", "exit", f"{self.user}@{self.host}"]
+                subprocess.run(
+                    cmd,
+                    capture_output=True,
+                    timeout=5,
+                    stdin=subprocess.DEVNULL,
+                )
+            except (OSError, subprocess.SubprocessError):
+                pass
+            try:
+                self.control_socket.unlink()
+            except OSError:
+                pass

tools/environments/vercel_sandbox.py

@@ -0,0 +1,662 @@
+"""Vercel Sandbox execution environment.
+
+Uses the Vercel Python SDK to run commands in cloud sandboxes through Hermes'
+shared ``BaseEnvironment`` shell contract. When persistence is enabled, the
+backend stores task-scoped snapshot metadata under ``HERMES_HOME`` and restores
+new sandboxes from those snapshots on later task reuse.
+"""
+
+from __future__ import annotations
+
+from functools import cache
+from dataclasses import dataclass
+from datetime import timedelta
+import logging
+import math
+import os
+import shlex
+import threading
+import time
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import httpx
+
+from hermes_constants import get_hermes_home
+from tools.environments.base import (
+    BaseEnvironment,
+    _ThreadedProcessHandle,
+    _load_json_store,
+    _save_json_store,
+)
+from tools.environments.file_sync import (
+    FileSyncManager,
+    iter_sync_files,
+    quoted_rm_command,
+)
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from vercel.sandbox import Resources, Sandbox, SandboxStatus, WriteFile
+
+DEFAULT_VERCEL_CWD = "/vercel/sandbox"
+_DEFAULT_CONTAINER_DISK_MB = 51200
+
+
+def _ensure_vercel_sdk() -> None:
+    """Lazy-install vercel SDK on demand. Idempotent."""
+    # The vercel SDK (>=0.7) ships default-on usage telemetry
+    # (vercel-internal-telemetry posts to telemetry.vercel.com). Hermes
+    # policy is no outbound telemetry without explicit user opt-in, so
+    # disable it before the SDK is ever imported. Users who genuinely want
+    # it can re-enable by exporting VERCEL_TELEMETRY_DISABLED=0 after this
+    # module loads — we only set the default, never override an explicit
+    # user value.
+    os.environ.setdefault("VERCEL_TELEMETRY_DISABLED", "1")
+    try:
+        from tools.lazy_deps import ensure as _lazy_ensure
+        _lazy_ensure("terminal.vercel", prompt=False)
+    except ImportError:
+        pass
+    except Exception as e:
+        raise ImportError(str(e))
+
+
+_CREATE_RETRY_ATTEMPTS = 3
+_WRITE_RETRY_ATTEMPTS = 3
+_TRANSIENT_STATUS_CODES = frozenset({408, 425, 429, 500, 502, 503, 504})
+_RETRY_BACKOFF_STEP = timedelta(milliseconds=100)
+_MIN_SANDBOX_TIMEOUT = timedelta(minutes=5)
+_MIN_RUNNING_WAIT = timedelta(seconds=1)
+_RUNNING_WAIT_TIMEOUT = timedelta(seconds=30)
+_RUNNING_WAIT_POLL_INTERVAL = timedelta(milliseconds=250)
+_STOP_TIMEOUT = timedelta(seconds=15)
+_STOP_POLL_INTERVAL = timedelta(milliseconds=500)
+_SNAPSHOT_STORE_NAME = "vercel_sandbox_snapshots.json"
+
+
+def _exception_chain(exc: BaseException) -> list[BaseException]:
+    chain: list[BaseException] = []
+    current: BaseException | None = exc
+    seen: set[int] = set()
+    while current is not None and id(current) not in seen:
+        chain.append(current)
+        seen.add(id(current))
+        current = current.__cause__ or current.__context__
+    return chain
+
+
+def _extract_status_code(exc: BaseException) -> int | None:
+    response = getattr(exc, "response", None)
+    for value in (getattr(exc, "status_code", None), getattr(response, "status_code", None)):
+        if isinstance(value, int):
+            return value
+    return None
+
+
+def _is_transient_vercel_error(exc: BaseException) -> bool:
+    for error in _exception_chain(exc):
+        status_code = _extract_status_code(error)
+        if status_code in _TRANSIENT_STATUS_CODES:
+            return True
+        if isinstance(
+            error,
+            (httpx.NetworkError, httpx.ProtocolError, httpx.ReadError),
+        ):
+            return True
+        error_name = type(error).__name__.lower()
+        if "ratelimit" in error_name or "servererror" in error_name:
+            return True
+    return False
+
+
+def _retry_vercel_call(
+    label: str,
+    callback,
+    *,
+    attempts: int,
+):
+    backoff_seconds = _RETRY_BACKOFF_STEP.total_seconds()
+    for attempt in range(1, attempts + 1):
+        try:
+            return callback()
+        except Exception as exc:
+            if attempt >= attempts or not _is_transient_vercel_error(exc):
+                raise
+            logger.warning(
+                "Vercel: %s failed (%s); retrying %d/%d",
+                label,
+                exc,
+                attempt,
+                attempts,
+            )
+            time.sleep(backoff_seconds * attempt)
+
+
+def _coerce_text(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, bytes):
+        return value.decode("utf-8", errors="replace")
+    return str(value)
+
+
+def _extract_result_output(result: Any) -> str:
+    try:
+        return _coerce_text(result.output())
+    except (AttributeError, TypeError):
+        return _coerce_text(result)
+
+
+def _extract_result_returncode(result: Any) -> int:
+    try:
+        exit_code = result.exit_code
+    except AttributeError:
+        try:
+            exit_code = result.returncode
+        except AttributeError:
+            return 1
+    return exit_code if isinstance(exit_code, int) else 1
+
+
+def _snapshot_store_path() -> Path:
+    return get_hermes_home() / _SNAPSHOT_STORE_NAME
+
+
+def _load_snapshots() -> dict:
+    return _load_json_store(_snapshot_store_path())
+
+
+def _save_snapshots(data: dict) -> None:
+    _save_json_store(_snapshot_store_path(), data)
+
+
+def _get_snapshot_id(task_id: str) -> str | None:
+    if not task_id:
+        return None
+    snapshot_id = _load_snapshots().get(task_id)
+    return snapshot_id if isinstance(snapshot_id, str) and snapshot_id else None
+
+
+def _store_snapshot(task_id: str, snapshot_id: str) -> None:
+    if not task_id or not snapshot_id:
+        return
+    snapshots = _load_snapshots()
+    snapshots[task_id] = snapshot_id
+    _save_snapshots(snapshots)
+
+
+def _delete_snapshot(task_id: str, snapshot_id: str | None = None) -> None:
+    if not task_id:
+        return
+    snapshots = _load_snapshots()
+    existing = snapshots.get(task_id)
+    if existing is None:
+        return
+    if snapshot_id is not None and existing != snapshot_id:
+        return
+    snapshots.pop(task_id, None)
+    _save_snapshots(snapshots)
+
+
+def _extract_snapshot_id(snapshot: Any) -> str | None:
+    for attr in ("snapshot_id", "snapshotId", "id"):
+        value = getattr(snapshot, attr, None)
+        if isinstance(value, str) and value:
+            return value
+    if isinstance(snapshot, dict):
+        for key in ("snapshot_id", "snapshotId", "id"):
+            value = snapshot.get(key)
+            if isinstance(value, str) and value:
+                return value
+    return None
+
+
+@cache
+def _sandbox_status_type() -> type[SandboxStatus]:
+    _ensure_vercel_sdk()
+    from vercel.sandbox import SandboxStatus
+
+    return SandboxStatus
+
+
+@cache
+def _terminal_sandbox_states() -> frozenset[SandboxStatus]:
+    SandboxStatus = _sandbox_status_type()
+    return frozenset(
+        {
+            SandboxStatus.ABORTED,
+            SandboxStatus.FAILED,
+            SandboxStatus.STOPPED,
+        }
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class _SandboxCreateParams:
+    timeout: timedelta
+    runtime: str | None = None
+    resources: Resources | None = None
+
+
+class VercelSandboxEnvironment(BaseEnvironment):
+    """Vercel cloud sandbox backend."""
+
+    _stdin_mode = "heredoc"
+
+    def __init__(
+        self,
+        runtime: str | None = None,
+        cwd: str = DEFAULT_VERCEL_CWD,
+        timeout: int = 60,
+        cpu: float = 1,
+        memory: int = 5120,
+        disk: int = _DEFAULT_CONTAINER_DISK_MB,
+        persistent_filesystem: bool = True,
+        task_id: str = "default",
+    ):
+        requested_cwd = cwd
+        super().__init__(cwd=cwd, timeout=timeout)
+
+        self._runtime = runtime or None
+        self._persistent = persistent_filesystem
+        self._task_id = task_id
+        self._requested_cwd = requested_cwd
+        self._lock = threading.Lock()
+        self._sandbox: Sandbox | None = None
+        self._workspace_root = DEFAULT_VERCEL_CWD
+        self._remote_home = DEFAULT_VERCEL_CWD
+        self._sync_manager: FileSyncManager | None = None
+        self._create_params = self._build_create_params(cpu=cpu, memory=memory, disk=disk)
+
+        self._sandbox = self._create_sandbox()
+        self._configure_attached_sandbox(requested_cwd=requested_cwd)
+        self._sync_manager.sync(force=True)
+        self.init_session()
+
+    def _build_create_params(self, *, cpu: float, memory: int, disk: int) -> _SandboxCreateParams:
+        if disk not in {0, _DEFAULT_CONTAINER_DISK_MB}:
+            raise ValueError(
+                "Vercel Sandbox does not support configurable container_disk. "
+                "Use the default shared setting."
+            )
+
+        _ensure_vercel_sdk()
+        from vercel.sandbox import Resources
+
+        sandbox_timeout = max(
+            timedelta(seconds=max(self.timeout, 0)),
+            _MIN_SANDBOX_TIMEOUT,
+        )
+        vcpus = math.floor(cpu) if cpu > 0 else None
+        memory_mb = memory if memory > 0 else None
+        resources = (
+            Resources(vcpus=vcpus, memory=memory_mb)
+            if vcpus is not None or memory_mb is not None
+            else None
+        )
+
+        return _SandboxCreateParams(
+            timeout=sandbox_timeout,
+            runtime=self._runtime,
+            resources=resources,
+        )
+
+    def _create_sandbox(self) -> Sandbox:
+        _ensure_vercel_sdk()
+        from vercel.sandbox import Sandbox
+
+        snapshot_id = _get_snapshot_id(self._task_id) if self._persistent else None
+        if snapshot_id:
+            try:
+                return _retry_vercel_call(
+                    "sandbox restore",
+                    lambda: Sandbox.create(
+                        timeout=self._create_params.timeout,
+                        runtime=self._create_params.runtime,
+                        resources=self._create_params.resources,
+                        source={"type": "snapshot", "snapshot_id": snapshot_id},
+                    ),
+                    attempts=_CREATE_RETRY_ATTEMPTS,
+                )
+            except Exception as exc:
+                logger.warning(
+                    "Vercel: failed to restore snapshot %s for task %s; "
+                    "falling back to a fresh sandbox: %s",
+                    snapshot_id,
+                    self._task_id,
+                    exc,
+                )
+                _delete_snapshot(self._task_id, snapshot_id)
+
+        params = self._create_params
+        return _retry_vercel_call(
+            "sandbox create",
+            lambda: Sandbox.create(
+                timeout=params.timeout,
+                runtime=params.runtime,
+                resources=params.resources,
+            ),
+            attempts=_CREATE_RETRY_ATTEMPTS,
+        )
+
+    def _configure_attached_sandbox(self, *, requested_cwd: str) -> None:
+        self._wait_for_running()
+        self._workspace_root = self._detect_workspace_root()
+        self._remote_home = self._detect_remote_home()
+
+        if self._remote_home == "/":
+            container_base = "/.hermes"
+        else:
+            container_base = f"{self._remote_home.rstrip('/')}/.hermes"
+        self._sync_manager = FileSyncManager(
+            get_files_fn=lambda: iter_sync_files(container_base),
+            upload_fn=self._vercel_upload,
+            delete_fn=self._vercel_delete,
+            bulk_upload_fn=self._vercel_bulk_upload,
+            bulk_download_fn=self._vercel_bulk_download,
+        )
+
+        if requested_cwd == "~":
+            self.cwd = self._remote_home
+        elif requested_cwd in {"", DEFAULT_VERCEL_CWD}:
+            self.cwd = self._workspace_root
+        else:
+            self.cwd = requested_cwd
+
+    def _detect_workspace_root(self) -> str:
+        sandbox = self._sandbox
+        if sandbox is None:
+            raise RuntimeError("Vercel sandbox is not attached")
+        cwd = sandbox.sandbox.cwd
+        return cwd if cwd.startswith("/") else DEFAULT_VERCEL_CWD
+
+    def _detect_remote_home(self) -> str:
+        sandbox = self._sandbox
+        if sandbox is None:
+            raise RuntimeError("Vercel sandbox is not attached")
+        try:
+            result = sandbox.run_command(
+                "sh",
+                ["-lc", 'printf %s "$HOME"'],
+                cwd=self._workspace_root,
+            )
+        except Exception as exc:
+            logger.debug(
+                "Vercel: home detection failed for task %s: %s",
+                self._task_id,
+                exc,
+            )
+            return self._workspace_root
+
+        home = _extract_result_output(result).strip()
+        if home.startswith("/"):
+            return home
+        return self._workspace_root
+
+    def _wait_for_running(self, timeout: timedelta = _RUNNING_WAIT_TIMEOUT) -> None:
+        sandbox = self._sandbox
+        if sandbox is None:
+            raise RuntimeError("Vercel sandbox is not attached")
+        SandboxStatus = _sandbox_status_type()
+        status = sandbox.status
+        if status is None or status == SandboxStatus.RUNNING:
+            return
+        if status in _terminal_sandbox_states():
+            raise RuntimeError(f"Sandbox entered terminal state: {status}")
+
+        try:
+            sandbox.wait_for_status(
+                SandboxStatus.RUNNING,
+                timeout=max(timeout, _MIN_RUNNING_WAIT),
+                poll_interval=_RUNNING_WAIT_POLL_INTERVAL,
+            )
+        except TimeoutError as exc:
+            status = sandbox.status
+            if status in _terminal_sandbox_states():
+                raise RuntimeError(f"Sandbox entered terminal state: {status}") from exc
+            raise RuntimeError(
+                f"Sandbox did not reach running state (last status: {status})"
+            ) from exc
+
+    def _close_sandbox_client(self, sandbox: Sandbox | None) -> None:
+        if sandbox is None:
+            return
+        try:
+            sandbox.client.close()
+        except Exception:
+            pass
+
+    def _stop_sandbox(self, sandbox: Sandbox | None) -> None:
+        if sandbox is None:
+            return
+        try:
+            sandbox.stop(
+                blocking=True,
+                timeout=_STOP_TIMEOUT,
+                poll_interval=_STOP_POLL_INTERVAL,
+            )
+        except TypeError:
+            try:
+                sandbox.stop()
+            except Exception:
+                pass
+        except Exception:
+            pass
+
+    def _snapshot_sandbox(self, sandbox: Sandbox) -> str | None:
+        if not self._persistent or not self._task_id:
+            return None
+        try:
+            snapshot = sandbox.snapshot()
+        except Exception as exc:
+            logger.warning(
+                "Vercel: filesystem snapshot failed for task %s: %s",
+                self._task_id,
+                exc,
+            )
+            return None
+
+        snapshot_id = _extract_snapshot_id(snapshot)
+        if not snapshot_id:
+            logger.warning(
+                "Vercel: filesystem snapshot for task %s did not return a snapshot id",
+                self._task_id,
+            )
+            return None
+
+        _store_snapshot(self._task_id, snapshot_id)
+        logger.info(
+            "Vercel: saved filesystem snapshot %s for task %s",
+            snapshot_id,
+            self._task_id,
+        )
+        return snapshot_id
+
+    def _ensure_sandbox_ready(self) -> None:
+        sandbox = self._sandbox
+        requested_cwd = self.cwd or self._requested_cwd or DEFAULT_VERCEL_CWD
+
+        if sandbox is None:
+            self._sandbox = self._create_sandbox()
+            self._configure_attached_sandbox(requested_cwd=requested_cwd)
+            return
+
+        try:
+            sandbox.refresh()
+        except Exception as exc:
+            logger.warning(
+                "Vercel: sandbox refresh failed for task %s: %s; recreating",
+                self._task_id,
+                exc,
+            )
+            self._close_sandbox_client(sandbox)
+            self._sandbox = self._create_sandbox()
+            self._configure_attached_sandbox(requested_cwd=requested_cwd)
+            return
+
+        status = sandbox.status
+        if status in _terminal_sandbox_states():
+            logger.warning(
+                "Vercel: sandbox entered state %s for task %s; recreating",
+                status,
+                self._task_id,
+            )
+            self._close_sandbox_client(sandbox)
+            self._sandbox = self._create_sandbox()
+            self._configure_attached_sandbox(requested_cwd=requested_cwd)
+            return
+
+        self._wait_for_running()
+
+    def _vercel_upload(self, host_path: str, remote_path: str) -> None:
+        self._vercel_bulk_upload([(host_path, remote_path)])
+
+    def _vercel_bulk_upload(self, files: list[tuple[str, str]]) -> None:
+        if not files:
+            return
+
+        payload: list[WriteFile] = [
+            {
+                "path": remote_path,
+                "content": Path(host_path).read_bytes(),
+            }
+            for host_path, remote_path in files
+        ]
+
+        sandbox = self._sandbox
+        if sandbox is None:
+            raise RuntimeError("Vercel sandbox is not attached")
+        _retry_vercel_call(
+            "write_files",
+            lambda: sandbox.write_files(payload),
+            attempts=_WRITE_RETRY_ATTEMPTS,
+        )
+
+    def _vercel_delete(self, remote_paths: list[str]) -> None:
+        if not remote_paths:
+            return
+
+        sandbox = self._sandbox
+        if sandbox is None:
+            raise RuntimeError("Vercel sandbox is not attached")
+        result = sandbox.run_command(
+            "bash",
+            ["-lc", quoted_rm_command(remote_paths)],
+            cwd=self._workspace_root,
+        )
+        if _extract_result_returncode(result) != 0:
+            raise RuntimeError(
+                f"Vercel delete failed: {_extract_result_output(result).strip()}"
+            )
+
+    def _vercel_bulk_download(self, dest_tar_path: Path) -> None:
+        remote_hermes = (
+            "/.hermes"
+            if self._remote_home == "/"
+            else f"{self._remote_home.rstrip('/')}/.hermes"
+        )
+        archive_member = remote_hermes.lstrip("/")
+        remote_tar = f"/tmp/.hermes_sync.{os.getpid()}.tar"
+        sandbox = self._sandbox
+        if sandbox is None:
+            raise RuntimeError("Vercel sandbox is not attached")
+
+        try:
+            result = sandbox.run_command(
+                "bash",
+                [
+                    "-lc",
+                    f"tar cf {shlex.quote(remote_tar)} -C / {shlex.quote(archive_member)}",
+                ],
+                cwd=self._workspace_root,
+            )
+            if _extract_result_returncode(result) != 0:
+                raise RuntimeError(
+                    f"Vercel bulk download failed: {_extract_result_output(result).strip()}"
+                )
+
+            sandbox.download_file(remote_tar, dest_tar_path)
+        finally:
+            try:
+                sandbox.run_command(
+                    "bash",
+                    ["-lc", f"rm -f {shlex.quote(remote_tar)}"],
+                    cwd=self._workspace_root,
+                )
+            except Exception:
+                pass
+
+    def _before_execute(self) -> None:
+        with self._lock:
+            self._ensure_sandbox_ready()
+            if self._sync_manager is not None:
+                self._sync_manager.sync()
+
+    def _run_bash(
+        self,
+        cmd_string: str,
+        *,
+        login: bool = False,
+        timeout: int = 120,
+        stdin_data: str | None = None,
+    ):
+        """Run a bash command in the Vercel sandbox.
+
+        ``timeout`` is not forwarded to the Vercel SDK (which does not expose
+        a per-exec timeout parameter); the base class ``_wait_for_process``
+        enforces timeout by killing the sandbox via ``cancel_fn``.
+
+        ``stdin_data`` is intentionally discarded here because
+        ``_stdin_mode = "heredoc"`` causes the base class ``execute()`` to
+        embed any stdin payload into the command string before calling this
+        method.
+        """
+        del timeout
+        del stdin_data
+
+        sandbox = self._sandbox
+        if sandbox is None:
+            raise RuntimeError("Vercel sandbox is not attached")
+        workspace_root = self._workspace_root
+        lock = self._lock
+
+        def cancel() -> None:
+            with lock:
+                self._stop_sandbox(sandbox)
+
+        def exec_fn() -> tuple[str, int]:
+            result = sandbox.run_command(
+                "bash",
+                ["-lc" if login else "-c", cmd_string],
+                cwd=workspace_root,
+            )
+            return _extract_result_output(result), _extract_result_returncode(result)
+
+        return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel)
+
+    def cleanup(self):
+        with self._lock:
+            sandbox = self._sandbox
+            sync_manager = self._sync_manager
+            if sandbox is not None and sync_manager is not None:
+                try:
+                    sync_manager.sync_back()
+                except Exception as exc:
+                    logger.warning(
+                        "Vercel: sync_back failed for task %s: %s",
+                        self._task_id,
+                        exc,
+                    )
+            self._sandbox = None
+            self._sync_manager = None
+
+        if sandbox is None:
+            return
+
+        snapshot_id = self._snapshot_sandbox(sandbox)
+        # Always stop the sandbox during cleanup to avoid resource leaks,
+        # matching the Modal and Daytona patterns.
+        self._stop_sandbox(sandbox)
+        self._close_sandbox_client(sandbox)