Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,576 @@
|
||||
"""Session-persistent kernels for REMOTE terminal backends (docker/ssh/modal).
|
||||
|
||||
Closes the gap tracked in hermes-agent#96873: local execute_code holds a
|
||||
persistent kernel child (tools/code_kernel.py); remote backends previously
|
||||
re-shipped and re-ran a fresh script per call, losing all interpreter state.
|
||||
|
||||
The remote transport offers exactly one primitive — ``env.execute(cmd)``,
|
||||
run-to-completion — so the three things the local kernel gets from owning a
|
||||
child process are rebuilt on top of it:
|
||||
|
||||
1. **A process that outlives one env.execute():** the kernel runner is
|
||||
started detached (``nohup ... &``) and its PID recorded; each later cell
|
||||
first probes liveness with ``kill -0``.
|
||||
2. **A conversation channel:** a file-based CELL protocol in the kernel dir
|
||||
(``cell_req_NNNNNN.json`` / ``cell_res_NNNNNN.json``), sibling to the
|
||||
existing file-based TOOL-RPC protocol (req_/res_ files) which is reused
|
||||
unchanged — the host-side ``_rpc_poll_loop`` is started per cell with the
|
||||
calling thread's context, which is what gives per-cell tool authority.
|
||||
3. **Death detection:** a failed liveness probe (transport drop, container
|
||||
restart, OOM-killed runner) reads as *kernel died: state lost*; the next
|
||||
call respawns fresh and says so — never a hung poll loop, because every
|
||||
wait is bounded by the cell timeout.
|
||||
|
||||
Same invariants as local: owner = approval session key with the
|
||||
``::child::{id}`` qualifier for delegated children (imported from
|
||||
tools.code_kernel — one resolver, cannot drift), same generated tool stubs,
|
||||
same output post-processing in the caller. ``reset=true`` kills and
|
||||
respawns. Spawn failure fails OPEN to the per-call path with a note, so a
|
||||
degraded remote host never blocks execution entirely.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import shlex
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# One lock guards the registry; teardown runs outside it (mirrors code_kernel).
|
||||
_REMOTE_KERNELS: Dict[Tuple, "RemoteKernel"] = {}
|
||||
_REMOTE_KERNELS_LOCK = threading.Lock()
|
||||
|
||||
# How often the host polls the remote for a cell result file. Each poll is
|
||||
# one env.execute round-trip (typically 0.1-0.4s on ssh/docker), so this is
|
||||
# a floor, not a rate.
|
||||
_CELL_POLL_INTERVAL = 0.5
|
||||
|
||||
# The remote runner: a tiny forever-loop that polls for cell request files,
|
||||
# execs them in one persistent namespace, and writes response files. It is
|
||||
# deliberately transport-agnostic (pure files) and stdlib-only. Cells and
|
||||
# tool-RPC share the kernel dir but use distinct prefixes.
|
||||
REMOTE_KERNEL_RUNNER_SOURCE = '''\
|
||||
"""Auto-generated Hermes REMOTE session-kernel runner (file cell protocol)."""
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
|
||||
KDIR = os.environ["HERMES_KERNEL_DIR"]
|
||||
CELLS = os.path.join(KDIR, "cells")
|
||||
CAPTURE_LIMIT = {capture_limit}
|
||||
IDLE_EXIT_SECONDS = {idle_exit}
|
||||
|
||||
GLOBALS = {{"__name__": "__main__", "__builtins__": __builtins__}}
|
||||
|
||||
|
||||
def _bounded(text):
|
||||
if len(text) <= CAPTURE_LIMIT:
|
||||
return text, False
|
||||
return text[:CAPTURE_LIMIT], True
|
||||
|
||||
|
||||
def main():
|
||||
execution_count = 0
|
||||
last_activity = time.time()
|
||||
while True:
|
||||
pending = sorted(
|
||||
f for f in os.listdir(CELLS)
|
||||
if f.startswith("cell_req_") and f.endswith(".json")
|
||||
)
|
||||
if not pending:
|
||||
if time.time() - last_activity > IDLE_EXIT_SECONDS:
|
||||
return # self-reap: nobody is talking to us anymore
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
for name in pending:
|
||||
req_path = os.path.join(CELLS, name)
|
||||
try:
|
||||
with open(req_path, "r", encoding="utf-8") as f:
|
||||
request = json.load(f)
|
||||
except Exception:
|
||||
# Partially-written request (ship in progress): retry next tick.
|
||||
continue
|
||||
os.remove(req_path)
|
||||
last_activity = time.time()
|
||||
execution_count += 1
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
status = "ok"
|
||||
trace = ""
|
||||
try:
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
||||
exec(compile(request["code"], "<cell>", "exec"), GLOBALS)
|
||||
except SystemExit as exc:
|
||||
status = "exit"
|
||||
trace = "SystemExit: " + repr(exc.code)
|
||||
except BaseException:
|
||||
status = "error"
|
||||
trace = traceback.format_exc()
|
||||
stdout_text, stdout_clipped = _bounded(out.getvalue())
|
||||
stderr_text, stderr_clipped = _bounded(err.getvalue())
|
||||
payload = {{
|
||||
"id": request.get("id", ""),
|
||||
"status": status,
|
||||
"stdout": stdout_text,
|
||||
"stderr": stderr_text,
|
||||
"stdout_clipped": stdout_clipped,
|
||||
"stderr_clipped": stderr_clipped,
|
||||
"traceback": trace,
|
||||
"execution_count": execution_count,
|
||||
}}
|
||||
res_name = name.replace("cell_req_", "cell_res_")
|
||||
tmp = os.path.join(CELLS, res_name + ".tmp")
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, ensure_ascii=False)
|
||||
os.replace(tmp, os.path.join(CELLS, res_name))
|
||||
if status == "exit":
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'''
|
||||
|
||||
|
||||
@dataclass
|
||||
class RemoteKernel:
|
||||
"""Host-side record of one detached remote kernel process."""
|
||||
|
||||
env: Any
|
||||
env_type: str
|
||||
kernel_dir: str
|
||||
pid: str
|
||||
rpc_token: str
|
||||
owner: str
|
||||
created: float = field(default_factory=time.monotonic)
|
||||
last_used: float = field(default_factory=time.monotonic)
|
||||
execution_count: int = 0
|
||||
cell_seq: int = 0
|
||||
# Cells currently running on this kernel. Reap/evict skip attached
|
||||
# kernels: killing one mid-cell tears the runner out from under a live
|
||||
# poll loop (same guard as tools.code_kernel, hermes-agent#101861).
|
||||
attached: int = 0
|
||||
|
||||
|
||||
def _kernel_key(owner: str, env_type: str, task_env_id: str) -> Tuple:
|
||||
return (owner, "remote", env_type, task_env_id)
|
||||
|
||||
|
||||
def _is_alive(kernel: RemoteKernel) -> bool:
|
||||
"""Bounded liveness probe: kill -0 through the transport.
|
||||
|
||||
Any transport failure counts as dead — the caller respawns. This is the
|
||||
"death detection" leg: a dropped ssh connection and a dead runner are
|
||||
indistinguishable from here, and both have the same correct answer.
|
||||
"""
|
||||
try:
|
||||
probe = kernel.env.execute(
|
||||
f"kill -0 {shlex.quote(kernel.pid)} 2>/dev/null && echo ALIVE",
|
||||
cwd="/", timeout=15,
|
||||
)
|
||||
return "ALIVE" in (probe.get("output", "") or "")
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _kill(kernel: RemoteKernel) -> None:
|
||||
"""Best-effort kill of the runner and its subprocesses, then rm -rf."""
|
||||
try:
|
||||
kernel.env.execute(
|
||||
# Kill the runner's process group if the shell gave it one,
|
||||
# falling back to the single PID.
|
||||
f"pkill -TERM -P {shlex.quote(kernel.pid)} 2>/dev/null; "
|
||||
f"kill {shlex.quote(kernel.pid)} 2>/dev/null; true",
|
||||
cwd="/", timeout=15,
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("remote kernel kill failed (transport?)", exc_info=True)
|
||||
try:
|
||||
kernel.env.execute(
|
||||
f"rm -rf {shlex.quote(kernel.kernel_dir)}", cwd="/", timeout=15,
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("remote kernel dir cleanup failed", exc_info=True)
|
||||
|
||||
|
||||
def shutdown_all_remote_kernels() -> None:
|
||||
with _REMOTE_KERNELS_LOCK:
|
||||
kernels = list(_REMOTE_KERNELS.values())
|
||||
_REMOTE_KERNELS.clear()
|
||||
for kernel in kernels:
|
||||
_kill(kernel)
|
||||
|
||||
|
||||
def shutdown_remote_kernels_for_owner(owner: str) -> None:
|
||||
"""Session-boundary disposal — wired to the same clear_session hook as
|
||||
local kernels, so /new and session close reap both kinds."""
|
||||
if not owner:
|
||||
return
|
||||
with _REMOTE_KERNELS_LOCK:
|
||||
doomed = [k for k in _REMOTE_KERNELS if k[0] == owner]
|
||||
kernels = [_REMOTE_KERNELS.pop(k) for k in doomed]
|
||||
for kernel in kernels:
|
||||
_kill(kernel)
|
||||
|
||||
|
||||
def _reap_unlocked(idle_timeout: int) -> List["RemoteKernel"]:
|
||||
"""Pop idle-expired remote kernels; caller tears them down outside the lock.
|
||||
|
||||
Mirrors tools.code_kernel._reap_unlocked. The remote runner itself
|
||||
self-exits after the same idle window (REMOTE_KERNEL_RUNNER_SOURCE's
|
||||
IDLE_EXIT_SECONDS), so this only needs to clear the HOST-side
|
||||
bookkeeping entry — without it, _REMOTE_KERNELS grows one entry per
|
||||
distinct (owner, env_type, task_env_id) that is never revisited, for
|
||||
the life of the gateway process.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
doomed = [
|
||||
key
|
||||
for key, kernel in _REMOTE_KERNELS.items()
|
||||
if kernel.attached == 0 and now - kernel.last_used > idle_timeout
|
||||
]
|
||||
return [_REMOTE_KERNELS.pop(key) for key in doomed]
|
||||
|
||||
|
||||
def _evict_over_cap_unlocked(keep: Tuple) -> List["RemoteKernel"]:
|
||||
"""Pop least-recently-used remote kernels beyond the process-wide cap.
|
||||
|
||||
Mirrors tools.code_kernel._evict_over_cap_unlocked, reusing the same
|
||||
max_session_kernels config as an independent bound on _REMOTE_KERNELS.
|
||||
"""
|
||||
from tools.code_kernel import _lifecycle_limits
|
||||
|
||||
cap, _ = _lifecycle_limits()
|
||||
if len(_REMOTE_KERNELS) <= cap:
|
||||
return []
|
||||
by_age = sorted(
|
||||
(key for key in _REMOTE_KERNELS if key != keep and _REMOTE_KERNELS[key].attached == 0),
|
||||
key=lambda key: _REMOTE_KERNELS[key].last_used,
|
||||
)
|
||||
doomed = by_age[: len(_REMOTE_KERNELS) - cap]
|
||||
return [_REMOTE_KERNELS.pop(key) for key in doomed]
|
||||
|
||||
|
||||
atexit.register(shutdown_all_remote_kernels)
|
||||
|
||||
|
||||
def _spawn_remote_kernel(env, env_type: str, owner: str, task_env_id: str,
|
||||
sandbox_tools: frozenset, *,
|
||||
idle_exit: int) -> Optional[RemoteKernel]:
|
||||
"""Start a detached kernel runner on the remote. None on failure."""
|
||||
from tools.code_execution_tool import (
|
||||
MAX_STDOUT_BYTES,
|
||||
_ship_file_to_remote,
|
||||
_env_temp_dir,
|
||||
generate_hermes_tools_module,
|
||||
)
|
||||
import secrets as _secrets
|
||||
|
||||
kernel_dir = f"{_env_temp_dir(env)}/hermes_rkernel_{uuid.uuid4().hex[:12]}"
|
||||
q_dir = shlex.quote(kernel_dir)
|
||||
try:
|
||||
env.execute(f"mkdir -p {q_dir}/cells {q_dir}/rpc", cwd="/", timeout=15)
|
||||
|
||||
rpc_token = _secrets.token_urlsafe(32)
|
||||
runner_src = REMOTE_KERNEL_RUNNER_SOURCE.format(
|
||||
capture_limit=MAX_STDOUT_BYTES,
|
||||
idle_exit=idle_exit,
|
||||
)
|
||||
_ship_file_to_remote(env, f"{kernel_dir}/kernel_runner.py", runner_src)
|
||||
tools_src = generate_hermes_tools_module(
|
||||
list(sandbox_tools), transport="file",
|
||||
)
|
||||
_ship_file_to_remote(env, f"{kernel_dir}/hermes_tools.py", tools_src)
|
||||
|
||||
env_prefix = (
|
||||
f"HERMES_KERNEL_DIR={q_dir} "
|
||||
f"HERMES_RPC_DIR={shlex.quote(kernel_dir + '/rpc')} "
|
||||
f"HERMES_RPC_TOKEN={shlex.quote(rpc_token)} "
|
||||
f"PYTHONDONTWRITEBYTECODE=1 PYTHONPATH={q_dir}"
|
||||
)
|
||||
started = env.execute(
|
||||
f"cd {q_dir} && nohup env {env_prefix} python3 kernel_runner.py "
|
||||
f"> {q_dir}/runner.log 2>&1 & echo PID:$!",
|
||||
cwd="/", timeout=20,
|
||||
)
|
||||
pid = ""
|
||||
for line in (started.get("output", "") or "").splitlines():
|
||||
if line.strip().startswith("PID:"):
|
||||
pid = line.strip()[4:].strip()
|
||||
break
|
||||
if not pid.isdigit():
|
||||
logger.warning("remote kernel spawn returned no PID: %r",
|
||||
started.get("output", ""))
|
||||
env.execute(f"rm -rf {q_dir}", cwd="/", timeout=15)
|
||||
return None
|
||||
|
||||
kernel = RemoteKernel(
|
||||
env=env, env_type=env_type, kernel_dir=kernel_dir,
|
||||
pid=pid, rpc_token=rpc_token, owner=owner,
|
||||
)
|
||||
if not _is_alive(kernel):
|
||||
# Died instantly (missing python3 was pre-checked by the caller,
|
||||
# so this is unexpected) — surface the runner log at debug.
|
||||
try:
|
||||
log = env.execute(f"cat {q_dir}/runner.log", cwd="/", timeout=10)
|
||||
logger.warning("remote kernel died at spawn: %s",
|
||||
(log.get("output", "") or "")[:500])
|
||||
except Exception:
|
||||
pass
|
||||
env.execute(f"rm -rf {q_dir}", cwd="/", timeout=15)
|
||||
return None
|
||||
return kernel
|
||||
except Exception:
|
||||
logger.warning("remote kernel spawn failed", exc_info=True)
|
||||
try:
|
||||
env.execute(f"rm -rf {q_dir}", cwd="/", timeout=15)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def execute_in_remote_kernel(
|
||||
code: str,
|
||||
*,
|
||||
env,
|
||||
env_type: str,
|
||||
task_env_id: str,
|
||||
sandbox_tools: frozenset,
|
||||
timeout: int,
|
||||
max_tool_calls: int,
|
||||
reset: bool,
|
||||
idle_exit: int = 1800,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Run one cell in the owner's remote kernel.
|
||||
|
||||
Returns the raw cell result dict (caller does output post-processing),
|
||||
or ``None`` when no kernel could be spawned — the caller falls open to
|
||||
the per-call path. ``state_lost`` / ``state_reset`` / ``reused`` ride in
|
||||
the ``kernel`` sub-dict, matching the local kernel's result shape.
|
||||
"""
|
||||
from tools.code_kernel import _resolve_owner
|
||||
|
||||
owner = _resolve_owner(task_env_id)
|
||||
key = _kernel_key(owner, env_type, task_env_id)
|
||||
state_lost = False
|
||||
state_reset = False
|
||||
|
||||
with _REMOTE_KERNELS_LOCK:
|
||||
expired = _reap_unlocked(idle_exit)
|
||||
kernel = _REMOTE_KERNELS.get(key)
|
||||
for doomed in expired:
|
||||
_kill(doomed)
|
||||
|
||||
if kernel is not None and reset:
|
||||
with _REMOTE_KERNELS_LOCK:
|
||||
_REMOTE_KERNELS.pop(key, None)
|
||||
_kill(kernel)
|
||||
kernel = None
|
||||
state_reset = True
|
||||
|
||||
if kernel is not None and not _is_alive(kernel):
|
||||
# Transport drop, container restart, self-reaped on idle, OOM — all
|
||||
# the same answer: report the loss, respawn fresh.
|
||||
with _REMOTE_KERNELS_LOCK:
|
||||
_REMOTE_KERNELS.pop(key, None)
|
||||
_kill(kernel) # best-effort dir cleanup; process is already gone
|
||||
kernel = None
|
||||
state_lost = True
|
||||
|
||||
reused = kernel is not None
|
||||
if kernel is None:
|
||||
kernel = _spawn_remote_kernel(
|
||||
env, env_type, owner, task_env_id, sandbox_tools,
|
||||
idle_exit=idle_exit,
|
||||
)
|
||||
if kernel is None:
|
||||
return None # fail open to per-call
|
||||
with _REMOTE_KERNELS_LOCK:
|
||||
_REMOTE_KERNELS[key] = kernel
|
||||
|
||||
kernel.last_used = time.monotonic()
|
||||
with _REMOTE_KERNELS_LOCK:
|
||||
kernel.attached += 1
|
||||
evicted = _evict_over_cap_unlocked(keep=key)
|
||||
for doomed in evicted:
|
||||
_kill(doomed)
|
||||
try:
|
||||
return _run_remote_cell(
|
||||
kernel, key, code, env=env, task_env_id=task_env_id,
|
||||
sandbox_tools=sandbox_tools, timeout=timeout,
|
||||
max_tool_calls=max_tool_calls, reused=reused,
|
||||
state_reset=state_reset, state_lost=state_lost,
|
||||
)
|
||||
finally:
|
||||
with _REMOTE_KERNELS_LOCK:
|
||||
kernel.attached -= 1
|
||||
kernel.last_used = time.monotonic()
|
||||
|
||||
|
||||
def _run_remote_cell(
|
||||
kernel: RemoteKernel,
|
||||
key: Tuple,
|
||||
code: str,
|
||||
*,
|
||||
env,
|
||||
task_env_id: str,
|
||||
sandbox_tools: frozenset,
|
||||
timeout: int,
|
||||
max_tool_calls: int,
|
||||
reused: bool,
|
||||
state_reset: bool,
|
||||
state_lost: bool,
|
||||
) -> Dict[str, Any]:
|
||||
from tools.code_execution_tool import (
|
||||
_rpc_poll_loop,
|
||||
_ship_file_to_remote,
|
||||
)
|
||||
from tools.thread_context import propagate_context_to_thread
|
||||
|
||||
kernel.cell_seq += 1
|
||||
seq = f"{kernel.cell_seq:06d}"
|
||||
q_cells = shlex.quote(f"{kernel.kernel_dir}/cells")
|
||||
|
||||
# Clean stale tool-RPC requests from a previous cell before arming this
|
||||
# cell's poll loop, so a background thread the last cell leaked cannot
|
||||
# smuggle a call into this cell's authority window.
|
||||
try:
|
||||
env.execute(
|
||||
f"rm -f {shlex.quote(kernel.kernel_dir + '/rpc')}/req_* "
|
||||
f"{shlex.quote(kernel.kernel_dir + '/rpc')}/res_*",
|
||||
cwd="/", timeout=10,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
tool_call_log: list = []
|
||||
tool_call_counter = [0]
|
||||
stop_event = threading.Event()
|
||||
# Per-cell RPC thread carrying THIS call's approval/session context —
|
||||
# the remote analogue of CellAuthority: authority lives exactly as long
|
||||
# as the cell's poll loop.
|
||||
rpc_thread = threading.Thread(
|
||||
target=propagate_context_to_thread(_rpc_poll_loop),
|
||||
args=(
|
||||
env, f"{kernel.kernel_dir}/rpc", task_env_id,
|
||||
tool_call_log, tool_call_counter, max_tool_calls,
|
||||
sandbox_tools, stop_event, kernel.rpc_token,
|
||||
),
|
||||
daemon=True,
|
||||
)
|
||||
rpc_thread.start()
|
||||
|
||||
cell_status = "no-result"
|
||||
cell_payload: Dict[str, Any] = {}
|
||||
try:
|
||||
request = json.dumps({"id": seq, "code": code}, ensure_ascii=False)
|
||||
_ship_file_to_remote(
|
||||
env, f"{kernel.kernel_dir}/cells/cell_req_{seq}.json.tmp", request,
|
||||
)
|
||||
env.execute(
|
||||
f"mv {q_cells}/cell_req_{seq}.json.tmp {q_cells}/cell_req_{seq}.json",
|
||||
cwd="/", timeout=10,
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + timeout
|
||||
res_name = f"cell_res_{seq}.json"
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
probe = env.execute(
|
||||
f"cat {q_cells}/{shlex.quote(res_name)} 2>/dev/null",
|
||||
cwd="/", timeout=20,
|
||||
)
|
||||
except Exception:
|
||||
# One flaky round-trip is not kernel death; liveness decides.
|
||||
time.sleep(_CELL_POLL_INTERVAL)
|
||||
continue
|
||||
body = (probe.get("output", "") or "").strip()
|
||||
if body:
|
||||
try:
|
||||
cell_payload = json.loads(body)
|
||||
cell_status = cell_payload.get("status", "error")
|
||||
except ValueError:
|
||||
cell_status = "protocol-error"
|
||||
env.execute(
|
||||
f"rm -f {q_cells}/{shlex.quote(res_name)}",
|
||||
cwd="/", timeout=10,
|
||||
)
|
||||
break
|
||||
time.sleep(_CELL_POLL_INTERVAL)
|
||||
else:
|
||||
cell_status = "timeout"
|
||||
finally:
|
||||
stop_event.set()
|
||||
rpc_thread.join(timeout=5)
|
||||
|
||||
if cell_status in ("timeout", "protocol-error", "no-result"):
|
||||
# No safe way to interrupt one cell in place (same contract as
|
||||
# local): kill the kernel, report the loss, respawn next call.
|
||||
with _REMOTE_KERNELS_LOCK:
|
||||
_REMOTE_KERNELS.pop(key, None)
|
||||
_kill(kernel)
|
||||
return {
|
||||
"status": "timeout" if cell_status == "timeout" else "error",
|
||||
"stdout": "",
|
||||
"stderr": "",
|
||||
"traceback": "",
|
||||
"tool_calls_made": tool_call_counter[0],
|
||||
"kernel": {
|
||||
"reused": reused,
|
||||
"remote": True,
|
||||
"ended": True,
|
||||
"state_lost": True,
|
||||
"note": (
|
||||
"Cell timed out; the remote session kernel was killed and "
|
||||
"its state was lost. The next call starts a fresh kernel."
|
||||
if cell_status == "timeout" else
|
||||
"Remote kernel protocol failure; kernel killed, state lost."
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
if cell_status == "exit":
|
||||
with _REMOTE_KERNELS_LOCK:
|
||||
_REMOTE_KERNELS.pop(key, None)
|
||||
_kill(kernel)
|
||||
|
||||
kernel.execution_count = int(cell_payload.get("execution_count", 0) or 0)
|
||||
|
||||
result: Dict[str, Any] = {
|
||||
"status": "success" if cell_status in ("ok", "exit") else "error",
|
||||
"stdout": cell_payload.get("stdout", ""),
|
||||
"stderr": cell_payload.get("stderr", ""),
|
||||
"traceback": cell_payload.get("traceback", ""),
|
||||
"stdout_clipped": bool(cell_payload.get("stdout_clipped")),
|
||||
"stderr_clipped": bool(cell_payload.get("stderr_clipped")),
|
||||
"tool_calls_made": tool_call_counter[0],
|
||||
"kernel": {
|
||||
"reused": reused,
|
||||
"remote": True,
|
||||
"execution_count": kernel.execution_count,
|
||||
},
|
||||
}
|
||||
if cell_status == "exit":
|
||||
result["kernel"]["ended"] = True
|
||||
if state_reset:
|
||||
result["kernel"]["state_reset"] = True
|
||||
if state_lost:
|
||||
result["kernel"]["state_lost"] = True
|
||||
result["kernel"]["note"] = (
|
||||
"The previous remote kernel was gone (transport drop, container "
|
||||
"restart, or idle self-exit); state from earlier calls was lost "
|
||||
"and a fresh kernel was started."
|
||||
)
|
||||
if cell_status == "error" and result["traceback"]:
|
||||
result["error"] = result["traceback"].strip().splitlines()[-1]
|
||||
return result
|
||||
Reference in New Issue
Block a user