"""Hermes update pipeline — extracted from ``hermes_cli/main.py``. Mechanical move (main.py decomposition): ``_cmd_update_impl``, ``_cmd_update_check`` and every module-level helper used only by the update path, plus the update-only constants they read. Function bodies are lifted verbatim; the only mechanical change is that references to helpers/constants that STAY in ``hermes_cli.main`` (and to moved-but-test-patched siblings) are routed through ``_m()`` — a lazy ``hermes_cli.main`` reference — so existing call sites and test monkeypatches that target ``hermes_cli.main.`` (``PROJECT_ROOT``, ``_is_windows``, ``_run_pre_update_backup``, ...) keep working unchanged. ``main.py`` re-imports every public-ish name from here (``# noqa: F401``) so the argparse wiring and the test-patch surface still resolve on ``hermes_cli.main``. Three self-contained closures nested inside ``_cmd_update_impl`` (``_print_items``, ``_wait_for_service_active``, ``_service_restart_sec``) were hoisted to module level; they capture no enclosing state (verified via ``symtable``). ``_restart_one_systemd_gateway_unit``, ``_resolve_manage_cmd`` and ``_on_unit_timeout`` DO capture enclosing locals and stay nested, byte-identical. Imports are one-way: ``hermes_cli.main`` imports this module, never the reverse at import time (``_m()`` resolves lazily at call time, when main.py is fully loaded, so there is no import cycle). """ import hashlib import json import logging import os import shlex import shutil import subprocess import sys import time as _time from datetime import datetime from pathlib import Path from typing import Optional from hermes_cli.config import get_hermes_home from hermes_constants import get_default_hermes_root, venv_python_path logger = logging.getLogger(__name__) def _m(): """Lazy ``hermes_cli.main`` reference. Lets callers keep patching ``hermes_cli.main.`` (the historical test surface) and have those patches reach this code path, and defers the import so ``hermes_cli.main`` -> ``hermes_cli.update_cmd`` stays one-way at import time. """ from hermes_cli import main return main def _no_prompt_git_kwargs() -> dict: """``subprocess.run`` kwargs for the updater's network git calls. GitHub answers anonymous fetches with HTTP 401 during outages (and for unreachable repos); git then prompts ``Username for 'https://github.com':`` on the inherited terminal and the update sits there forever. Disable the prompt so the fetch fails fast into ``_classify_fetch_failure``. Only the *prompt* is disabled — a configured credential helper / askpass still runs, so a private-fork origin keeps authenticating non-interactively. """ env = dict(os.environ) env["GIT_TERMINAL_PROMPT"] = "0" env["GCM_INTERACTIVE"] = "Never" return {"stdin": subprocess.DEVNULL, "env": env} _UPDATE_RUNTIME_RELOAD_MODULES = ( "hermes_constants", "tools.environments.local", "tools.lazy_deps", ) #: Package prefixes whose cached modules become stale the moment the checkout #: changes under this process. Purged (not reloaded) by #: ``_purge_stale_hermes_modules`` so any LATER import chain resolves against #: fresh on-disk source only. _STALE_PURGE_PREFIXES = ( "hermes_cli", "gateway", "tools", "tui_gateway", "agent", ) #: Modules that must survive the purge: they are (or are referenced by) the #: code currently EXECUTING the update, so evicting them buys nothing — the #: running frames keep their module objects alive regardless — and reloading #: them mid-flight is the one genuinely unsafe move. _STALE_PURGE_PROTECTED = frozenset( { "hermes_cli", "hermes_cli.main", "hermes_cli.update_cmd", "hermes_cli.hermes_logging", } ) def _purge_stale_hermes_modules() -> None: """Evict every cached Hermes module after the checkout changed in-place. ``hermes update`` keeps running in the pre-pull Python process. The gateway auto-restart phase that follows does function-level ``from hermes_cli.gateway import ...`` — executing NEW source inside an OLD ``sys.modules`` world. The moment new source references a symbol that was added to an already-cached module, the import dies (2026-08-20 field failure: freshly-pulled ``hermes_cli.gateway`` does ``from hermes_cli.cli_output import line_input``, but ``cli_output`` was cached from before d0132b582 which introduced ``line_input`` → the whole restart phase aborted and the gateway kept serving pre-update code). ``_UPDATE_RUNTIME_RELOAD_MODULES`` handled this per-symptom — three hardcoded module names, re-fixed every time a new module grew a new export. This is the class fix: drop EVERY cached module under the Hermes package prefixes so subsequent lazy imports rebuild a self-consistent, all-new module graph from the updated checkout. Old module objects referenced by the running updater frames stay alive and functional (a purge only removes the ``sys.modules`` cache entry); only genuinely executing modules are exempted, because reloading-in-place — not purging — is the operation that can pull code out from under a running frame. Best-effort: never raises. """ try: import importlib importlib.invalidate_caches() purged = [] for name in list(_m().sys.modules): if name in _STALE_PURGE_PROTECTED: continue if not name.startswith(_STALE_PURGE_PREFIXES): continue root = name.split(".", 1)[0] if root not in _STALE_PURGE_PREFIXES: # Prefix-string match caught an unrelated package # (e.g. ``gateway_foo``) — leave it alone. continue if _m().sys.modules.pop(name, None) is not None: purged.append(name) if purged: logger.debug( "Purged %d stale Hermes module(s) after checkout update", len(purged) ) except Exception as exc: logger.debug("Could not purge stale Hermes modules: %s", exc) def _reload_updated_runtime_modules() -> None: """Reload update-sensitive modules after the checkout changes in-place. ``hermes update`` keeps running in the pre-pull Python process. After a large update, modules already present in ``sys.modules`` can still expose old symbols even though their source files on disk are new. Refresh the small module set used by lazy-backend refresh before that step imports newly-updated code paths. """ try: import importlib importlib.invalidate_caches() for module_name in _UPDATE_RUNTIME_RELOAD_MODULES: module = _m().sys.modules.get(module_name) if module is None: continue try: importlib.reload(module) except Exception as exc: logger.debug("Could not reload updated module %s: %s", module_name, exc) except Exception as exc: logger.debug("Could not refresh update runtime modules: %s", exc) def _reload_config_modules() -> None: """Force-reload modules from disk after git pull. ``hermes update`` runs in the PRE-pull Python process. After ``git pull`` updates the source files on disk, modules already in ``sys.modules`` still hold the OLD code. Function-level imports return the cached module, so ``DEFAULT_CONFIG["_config_version"]`` is the OLD value and ``check_config_version()`` reports ``(33, 33)`` — "up to date" — even though the freshly-pulled code has v34 with a migration to run. This function force-reloads ``hermes_cli.config_defaults``, ``hermes_cli.config``, and ``hermes_cli.config_migrations`` from disk so subsequent imports read the UPDATED code. It also reloads ``hermes_cli._subprocess_compat`` and ``hermes_cli.dashboard_procs`` so that post-update dashboard cleanup (``_finish_dashboard_update_cleanup`` → ``_scan_dashboard_processes``) uses the freshly-pulled code. Without this, a new symbol added to ``_subprocess_compat`` (e.g. ``bounded_probe_run``) is invisible to the cached module object, causing ``ImportError`` during the cleanup step that runs later in the same process. """ import importlib importlib.invalidate_caches() for mod_name in ( "hermes_cli.config_defaults", "hermes_cli.config", "hermes_cli.config_migrations", "hermes_cli._subprocess_compat", "hermes_cli.dashboard_procs", ): mod = sys.modules.get(mod_name) if mod is not None: try: importlib.reload(mod) except Exception as exc: logger.debug("Could not reload %s for fresh post-update code: %s", mod_name, exc) def _run_config_check_fresh() -> tuple: """Check config version using freshly-reloaded modules. See ``_reload_config_modules`` for why this is necessary. Returns ``(current_ver, latest_ver)``. """ _reload_config_modules() from hermes_cli.config import check_config_version return check_config_version(raise_on_parse_error=True) def _run_migrate_config_fresh(*, interactive: bool = False, quiet: bool = False) -> dict: """Run config migration using freshly-reloaded modules. See ``_reload_config_modules`` for why this is necessary. Returns the migration results dict. """ _reload_config_modules() from hermes_cli.config import migrate_config return migrate_config(interactive=interactive, quiet=quiet) def _migrate_sibling_profile_configs() -> list[tuple[str, int, int]]: """Migrate every SIBLING profile's config.yaml to the current version. #91277 Phase 2 (fleet-wide config migration; #20438/#54926/#79048): the shared checkout serves every profile, but ``hermes update`` historically migrated only the active profile's config — siblings drifted versions until their gateway hit a config the new code couldn't read. Per profile home (skipping the active one, already migrated by the caller): scope config reads/writes via the context-local HERMES_HOME override (thread-safe — never ``os.environ``), check the version, and run the NON-INTERACTIVE, quiet migration. Prompt-requiring settings are left for the profile's own next interactive session, identical to the gateway-mode contract for the active profile. Returns ``[(profile_name, from_version, to_version), ...]`` for profiles actually migrated. Never raises; a failing profile is skipped (its own startup migration remains the fallback). """ migrated: list[tuple[str, int, int]] = [] try: from hermes_constants import ( get_process_hermes_home, reset_hermes_home_override, set_hermes_home_override, ) from hermes_cli.profiles import _get_profiles_root, _PROFILE_ID_RE active_home = get_process_hermes_home() root = _get_profiles_root() if not root.is_dir(): return migrated for entry in sorted(root.iterdir()): if not entry.is_dir() or not _PROFILE_ID_RE.match(entry.name): continue try: if entry.resolve() == Path(active_home).resolve(): continue except OSError: continue if not (entry / "config.yaml").is_file(): continue # profile never configured — nothing to migrate token = set_hermes_home_override(entry) try: current_ver, latest_ver = _run_config_check_fresh() if current_ver >= latest_ver: continue _run_migrate_config_fresh(interactive=False, quiet=True) after_ver, _ = _run_config_check_fresh() if after_ver > current_ver: migrated.append((entry.name, current_ver, after_ver)) except Exception as exc: logger.debug( "Config migration for profile %s failed: %s", entry.name, exc ) finally: reset_hermes_home_override(token) except Exception as exc: logger.debug("Sibling profile enumeration failed: %s", exc) return migrated def _check_and_apply_config_migration( *, assume_yes: bool = False, gateway_mode: bool = False, pre_update_snapshot_id: str | None = None, ) -> None: """Check and apply configuration migrations on an update completion path (#91360). CRITICAL: ``check_config_version`` and ``migrate_config`` must use freshly-reloaded modules, not the ``sys.modules`` cache (see ``_reload_config_modules``). This must run on EVERY update completion path — the normal post-pull path, the venv-repair retry and the Node-deps repair on the ``commit_count == 0`` "Already up to date" branch — so an interrupted update that previously pulled new code does not strand the user on an older config version. """ print() print("→ Checking configuration for new options...") # Reload config modules BEFORE any config reads so get_missing_*, # check_config_version, and migrate_config all use the updated code. _reload_config_modules() from hermes_cli.config import ( get_missing_env_vars, get_missing_config_fields, ) # Defensive (#91360): this helper runs on repair/retry completion paths # too — a config-check failure must not break an otherwise-successful # update. Log, point at the manual command, and return. try: missing_env = get_missing_env_vars(required_only=True) missing_config = get_missing_config_fields() current_ver, latest_ver = _run_config_check_fresh() except Exception as exc: logger.debug("Config check during update failed: %s", exc) print(" ⚠️ Could not check config version.") print(" Run 'hermes config migrate' to check manually.") return has_new_options = bool(missing_env or missing_config) version_bump_only = ( not has_new_options and current_ver < latest_ver ) needs_migration = has_new_options or current_ver < latest_ver if version_bump_only: # Nothing for the user to fill in — only the config format version # changed (new defaults already merge in transparently). Asking # "configure new options now?" here is misleading: saying yes just # bumps the version and looks like a no-op (issue: ScottFive / # Tt2021). Apply it silently and say what actually happened. print() print( f" ℹ Updating config format (v{current_ver} → v{latest_ver})…" ) try: _mig_results = _run_migrate_config_fresh( interactive=False, quiet=True ) print(" ✓ Config format updated (no new settings to configure)") # quiet=True also mutes migration steps that RESET or REMOVE an # existing setting (e.g. the v33→v34 personality reset from # #81946, which records its note only in the results dict). # Re-surface those notes so an unattended update never silently # changes user configuration (#86656). In this branch # missing_config is empty, so config_added can only contain # migration-step mutations, not missing-key listings. for _note in _mig_results.get("config_added") or []: print(f" ℹ {_note}") for _warn in _mig_results.get("warnings") or []: print(f" ⚠️ {_warn}") except Exception as _mig_err: print(f" ⚠️ Config format update failed: {_mig_err}") print(" Run 'hermes config migrate' to retry.") elif needs_migration: print() # Show WHAT changed, not just a count, so the user can make an # informed yes/no decision (previously the prompt named nothing). def _print_items(items, label, key, fallback_key=None): if not items: return print(f" {label}:") shown = items[:8] for it in shown: if isinstance(it, dict): name = it.get(key) or (fallback_key and it.get(fallback_key)) or "?" desc = (it.get("description") or "").strip() else: # Defensive: some callers/mocks pass bare name strings. name = str(it) desc = "" if desc: print(f" • {name} — {desc}") else: print(f" • {name}") extra = len(items) - len(shown) if extra > 0: print(f" … and {extra} more") if missing_env: print( f" ⚠️ {len(missing_env)} new required setting(s) need configuration" ) _print_items(missing_env, "New settings", "name") if missing_config: print(f" ℹ️ {len(missing_config)} new config option(s) available") _print_items(missing_config, "New options", "key") print() if assume_yes: print( " ℹ --yes: auto-applying config migration (skipping API-key prompts)." ) response = "y" elif gateway_mode: response = ( _gateway_prompt( "Would you like to configure new options now? [Y/n]", "n" ) .strip() .lower() ) elif not (sys.stdin.isatty() and sys.stdout.isatty()): print(" ℹ Non-interactive session — applying safe config migrations.") response = "auto" else: try: response = ( input("Would you like to configure them now? [Y/n]: ") .strip() .lower() ) except EOFError: response = "n" except UnicodeDecodeError: # input() can raise this when the terminal encoding can't # decode the byte sequence (e.g. a non-UTF-8 locale, or an # embedded terminal). Without this, the exception escapes # here and crashes the update at this prompt. print( " ⚠ Could not read input (encoding issue). Skipping. " "Run 'hermes config migrate' manually to configure." ) response = "n" if response in {"", "y", "yes", "auto"}: print() # Gateway mode, --yes, and non-interactive update contexts # (dashboard / web server actions) cannot prompt for API keys. # Still run the non-interactive migration pass before restarting # so new default config fields and version bumps are written # before the freshly updated gateway validates config at startup. interactive_migration = not ( gateway_mode or assume_yes or response == "auto" ) results = _run_migrate_config_fresh(interactive=interactive_migration, quiet=False) if results["env_added"] or results["config_added"]: print() print("✓ Configuration updated!") if (gateway_mode or assume_yes or response == "auto") and missing_env: print(" ℹ API keys require manual entry: hermes config migrate") else: print() print("Skipped. Run 'hermes config migrate' later to configure.") else: print(" ✓ Configuration is up to date") # Fleet-wide config migration (#91277 Phase 2; #20438 earliest report, # #54926, #79048): the shared checkout serves EVERY profile, but the # migration above only touched the active profile's config.yaml. # Sibling profiles kept their old _config_version and silently # drifted (field repro: sibling gateway restarted onto new code but # stayed at config v33 vs v37). Run the same NON-INTERACTIVE safe # migration for every sibling profile home, scoped via the # context-local HERMES_HOME override (never os.environ — other # threads must not see it). try: _migrated_siblings = _migrate_sibling_profile_configs() for _name, _from_ver, _to_ver in _migrated_siblings: print( f" ✓ Profile '{_name}': config format updated " f"(v{_from_ver} → v{_to_ver})" ) except Exception as exc: logger.debug("Sibling config migration failed: %s", exc) # Safety net: config-version migrations have been observed to leave # cron/jobs.json valid-but-empty, silently dropping every scheduled # job (issue #34600). The desktop scheduler can also overwrite with # its own small set, causing partial loss (issue #52144). If the # live file now has fewer jobs than the pre-update snapshot, restore # it and warn loudly. try: from hermes_cli.backup import restore_cron_jobs_if_emptied cron_restore = restore_cron_jobs_if_emptied(pre_update_snapshot_id) if cron_restore: print() print( " ⚠️ cron/jobs.json lost jobs during this update — " f"restored {cron_restore['job_count']} job(s) from " f"pre-update snapshot {cron_restore['snapshot_id']}." ) except Exception as exc: # Never let the cron safety net break an otherwise-good update. logger.debug("Cron jobs auto-restore check failed: %s", exc) # #64160: config.yaml model/provider + MoA safety net. Desktop # update/repair cycles have rewritten user-set model.provider / # model.default and dropped the moa: section — settings the gateway # and cron jobs also consume. Compare the live config against the # same pre-update snapshot and restore only the protected keys. try: from hermes_cli.backup import restore_config_model_settings_if_rewritten cfg_restore = restore_config_model_settings_if_rewritten( pre_update_snapshot_id ) if cfg_restore: print() print( " ⚠️ config.yaml user model settings were rewritten during " f"this update — restored {', '.join(cfg_restore['keys'])} " f"from pre-update snapshot {cfg_restore['snapshot_id']}." ) except Exception as exc: # Never let the config safety net break an otherwise-good update. logger.debug("Config model-settings auto-restore check failed: %s", exc) # #66140: run the same cron-jobs safety net for every sibling # profile against ITS OWN pre-update snapshot (same-generation by # construction — both taken by this run). try: from hermes_cli.backup import restore_cron_jobs_all_profiles for _restored in restore_cron_jobs_all_profiles( _LAST_SIBLING_SNAPSHOTS ): print() print( f" ⚠️ Profile '{_restored['profile']}': cron/jobs.json " f"lost jobs during this update — restored " f"{_restored['job_count']} job(s) from pre-update " f"snapshot {_restored['snapshot_id']}." ) except Exception as exc: logger.debug("Sibling cron auto-restore check failed: %s", exc) # #64160: same config model-settings safety net for sibling profiles. try: from hermes_cli.backup import restore_config_model_settings_all_profiles for _cfg_restored in restore_config_model_settings_all_profiles( _LAST_SIBLING_SNAPSHOTS ): print() print( f" ⚠️ Profile '{_cfg_restored['profile']}': config.yaml " f"user model settings were rewritten during this update — " f"restored {', '.join(_cfg_restored['keys'])} from " f"pre-update snapshot {_cfg_restored['snapshot_id']}." ) except Exception as exc: logger.debug("Sibling config auto-restore check failed: %s", exc) # Critical files that Hermes must be able to import immediately after an # update/install. Most are imported on every CLI startup; ``web_server.py`` # is the desktop/dashboard backend path that a fresh Windows install launches # right away. If any of these fail to parse after a pull, the user can be # left with a bricked CLI or desktop backend. The post-pull syntax guard # validates these and auto-rolls-back on failure. _UPDATE_CRITICAL_FILES = ( "hermes_cli/main.py", "hermes_cli/config.py", "hermes_cli/__init__.py", "hermes_cli/web_server.py", "cli.py", "run_agent.py", "model_tools.py", "toolsets.py", "hermes_constants.py", ) def _capture_head_sha(git_cmd, cwd) -> str | None: """Return the current HEAD SHA, or None if it can't be resolved.""" try: result = subprocess.run( git_cmd + ["rev-parse", "HEAD"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", check=True, ) return result.stdout.strip() or None except (subprocess.CalledProcessError, OSError): return None _ORPHAN_RESCUE_REFS_TO_KEEP = 10 _ORPHAN_RESCUE_REF_MAX_AGE_DAYS = 30 def _prune_orphan_rescue_refs( git_cmd, cwd, branch, keep=_ORPHAN_RESCUE_REFS_TO_KEEP, max_age_days=_ORPHAN_RESCUE_REF_MAX_AGE_DAYS, ) -> None: """Expire old orphan rescue refs so backups stay bounded. Each orphan-history divergence (#87694) parks the pre-reset HEAD under ``refs/hermes-update-backups/orphan---``. A rescue ref pins every object reachable from that commit against ``git gc`` — and in the incident shape those objects include a full working-tree snapshot (the autostash orphan commit), which can be multi-GB when the tree holds large stray files. Left alone, a repeatedly corrupted install would grow ``.git`` without bound. Two independent limits, both enforced on every orphan incident: - **Count cap:** keep only the ``keep`` most-recent refs. - **Age expiry:** drop any ref older than ``max_age_days``, parsed from the ``YYYYMMDD-HHMMSS`` timestamp embedded in the ref name (refs with unparseable names are left alone rather than guessed at). Ref names sort chronologically (timestamp prefix), so lexicographic order from ``for-each-ref`` is also creation order. Deleting a ref makes its objects eligible for ``git gc``; actual disk reclaim happens on the next gc (git auto-gc, or the user running ``git gc``). Best-effort: any failure here must not block the update itself. """ try: list_result = subprocess.run( git_cmd + [ "for-each-ref", "--format=%(refname)", "--sort=refname", f"refs/hermes-update-backups/orphan-{branch}-*", ], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if list_result.returncode != 0: return refs = [line.strip() for line in list_result.stdout.splitlines() if line.strip()] stale = set(refs[:-keep] if keep > 0 else refs) # Age expiry: ref names embed a UTC YYYYMMDD-HHMMSS timestamp right # after the branch segment; anything older than max_age_days goes. if max_age_days > 0: from datetime import timedelta, timezone cutoff = datetime.now(timezone.utc) - timedelta(days=max_age_days) prefix = f"refs/hermes-update-backups/orphan-{branch}-" for ref in refs: stamp = ref[len(prefix):][:15] # "YYYYMMDD-HHMMSS" try: ref_time = datetime.strptime(stamp, "%Y%m%d-%H%M%S").replace( tzinfo=timezone.utc ) except ValueError: continue if ref_time < cutoff: stale.add(ref) for ref in sorted(stale): subprocess.run( git_cmd + ["update-ref", "-d", ref], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) except OSError: pass # Files that define the editable install. A pull that touches none of them # cannot have invalidated it. _INSTALL_DEFINING_FILES = ( "pyproject.toml", "setup.py", "setup.cfg", "MANIFEST.in", "uv.lock", ) def _editable_install_is_current(git_cmd, cwd, pre_pull_sha: str | None) -> bool: """True when the pulled commits cannot have invalidated the editable install. ``uv pip install -e .`` never audits an editable target — it reinstalls on every invocation, and every reinstall rewrites the console-script shims. On Windows that rewrite is the only reason the running ``hermes.exe`` has to be quarantined, and a quarantine that loses its race is the whole ``os error 32`` family. Not reinstalling when the reinstall provably cannot change anything removes that risk outright for the common update, rather than trying to make the rename win more often. Skipping is safe because Hermes pins its editable finder to a *static* module list (``[tool.setuptools] py-modules`` plus ``packages.find.include``). The one source-only change that would stale that finder is a new top-level module or package, and it cannot land without a ``pyproject.toml`` diff. Dependencies and ``[project.scripts]`` live there too. New submodules inside an already-mapped package resolve through the real package directory and need no reinstall. Fails closed: an unresolvable pre-pull SHA (shallow checkout, ZIP swap) or a failed ``git diff`` returns False and the install runs as before. """ if not pre_pull_sha: return False try: result = subprocess.run( git_cmd + ["diff", "--name-only", f"{pre_pull_sha}..HEAD", "--"] + list(_INSTALL_DEFINING_FILES), cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) except OSError: return False if result.returncode != 0: return False return not result.stdout.strip() def _validate_python_files_syntax( root, relpaths ) -> tuple[bool, str | None, str | None]: """Compile *relpaths* under *root* without writing bytecode into the tree.""" import py_compile import tempfile root = Path(root) with tempfile.TemporaryDirectory(prefix="hermes-syntax-check-") as tmpdir: for relpath in relpaths: path = root / relpath if not path.exists(): continue cfile = Path(tmpdir) / (str(relpath).replace("/", "__") + "c") try: py_compile.compile(str(path), cfile=str(cfile), doraise=True) except py_compile.PyCompileError as exc: return False, str(path), str(exc) except OSError as exc: return False, str(path), f"could not read: {exc}" return True, None, None def _validate_critical_files_syntax(root) -> tuple[bool, str | None, str | None]: """Compile each file in ``_UPDATE_CRITICAL_FILES`` to catch SyntaxErrors. These are the files imported on every ``hermes`` startup; if any of them has a syntax error (orphan merge-conflict markers, bad ref to a name that no longer exists, etc.) the CLI can't bootstrap at all. We validate them after a successful ``git pull`` so we can auto-roll-back instead of leaving the user with a bricked install. The compiled ``.pyc`` is written to a temp directory rather than the source tree's ``__pycache__/`` so we don't race with concurrent test workers that walk the same dir, and so we don't leave a stale pyc behind in production if the next interpreter run picks a different Python version. The pyc is discarded on function return either way — we only care about the compile-or-not signal. Returns ``(ok, failing_path, error_message)``. ``ok=True`` means every file parsed cleanly. """ return _validate_python_files_syntax(root, _UPDATE_CRITICAL_FILES) # Modules imported on every agent startup. Unlike _UPDATE_CRITICAL_FILES (which # is only parsed), these are actually *imported* so that cross-module breakage # is caught — a file can be syntactically perfect and still fail to import # because a name it pulls from a sibling module no longer exists. _UPDATE_CRITICAL_MODULES = ( "hermes_cli.main", "run_agent", "model_tools", "toolsets", ) def _critical_module_import_failures( root, *, report_runtime_errors: bool = False ) -> dict[str, tuple[str, str]]: """Import each module in ``_UPDATE_CRITICAL_MODULES`` in a subprocess. ``_validate_critical_files_syntax`` only *parses* files, so it cannot see cross-module breakage: a partially-updated tree where ``agent/`` is new but ``tools/`` is old parses perfectly and still dies at startup with ``ImportError: cannot import name 'TODO_INJECTION_HEADER' from 'tools.todo_tool'``. Every file is valid Python; the *combination* is not. That skew is reachable on the Windows ZIP-update path, whose copy loop walks top-level entries in ``os.listdir`` order and replaces each one independently — ``agent/`` lands long before ``tools/``, so a failure or interruption between them leaves exactly that mismatch on disk. Runs in a subprocess because importing these modules into the running updater would pollute ``sys.modules`` and execute import-time side effects against the half-updated tree. Costs ~0.4s. Uses the project venv's interpreter when there is one (matching ``_venv_core_imports_healthy``): ``hermes update`` can be driven by a different Python than the install's own, and probing the wrong interpreter would test a tree the user never runs. Returns every failing module in probe order. Generic import-time exceptions remain tolerated by default because they can depend on local config or environment. ``report_runtime_errors=True`` exposes them so a caller can compare two states of the same checkout without an earlier failure masking a later one. """ from hermes_constants import FIRST_PARTY_MODULE_ROOTS import secrets marker = f"__HERMES_IMPORT_HEALTH_{secrets.token_hex(16)}__" probe = ( "import importlib, json, sys\n" "failures = []\n" "for name in %r:\n" " try:\n" " importlib.import_module(name)\n" " except ModuleNotFoundError as exc:\n" # A missing *third-party* module means dependencies aren't installed # yet, not a skewed checkout. Only our own packages count as breakage. # The root set is injected from hermes_constants so this can't drift # from the hint the user is shown (they disagreed once already). " missing = (getattr(exc, 'name', '') or '').split('.')[0]\n" " if missing in %r or missing.startswith('hermes_') or %r:\n" " failures.append((name, type(exc).__name__, str(exc)))\n" " except ImportError as exc:\n" " failures.append((name, type(exc).__name__, str(exc)))\n" " except Exception as exc:\n" " if %r:\n" " failures.append((name, type(exc).__name__, str(exc)))\n" " except BaseException as exc:\n" " failures.append((name, type(exc).__name__, str(exc)))\n" "sys.stdout.write('\\n%s' + json.dumps(failures))\n" % ( _UPDATE_CRITICAL_MODULES, tuple(sorted(FIRST_PARTY_MODULE_ROOTS)), report_runtime_errors, report_runtime_errors, marker, ) ) try: interpreter = sys.executable try: venv_python = venv_python_path( Path(root) / "venv", windows=_m()._is_windows() ) if venv_python.exists(): interpreter = str(venv_python) except Exception: pass # fall back to the running interpreter result = subprocess.run( [interpreter, "-c", probe], cwd=str(root), capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=120, ) except subprocess.TimeoutExpired: return { "critical-module probe": ( "TimeoutExpired", "timed out before reporting import health", ) } except (OSError, subprocess.SubprocessError): # Can't run the probe — don't block the update on our own tooling. return {} output = result.stdout or "" if marker not in output: return { "critical-module probe": ( "ProbeTerminated", "terminated before reporting import health " f"(exit code {result.returncode})", ) } try: import json failures = json.loads(output.rsplit(marker, 1)[1]) if not isinstance(failures, list) or any( not isinstance(item, list) or len(item) != 3 or not all(isinstance(value, str) for value in item) for item in failures ): raise ValueError("invalid import-health payload") return { str(module): (str(kind), str(detail)) for module, kind, detail in failures } except (TypeError, ValueError): return { "critical-module probe": ( "MalformedPayload", "reported malformed import health data", ) } def _validate_critical_modules_import( root, *, report_runtime_errors: bool = False ) -> tuple[bool, str | None, str | None]: """Return the first critical-module import failure, if any.""" failures = _critical_module_import_failures( root, report_runtime_errors=report_runtime_errors ) if failures: module = next(iter(failures)) return False, module, failures[module][1] return True, None, None def _gateway_prompt(prompt_text: str, default: str = "", timeout: float = 300.0) -> str: """File-based IPC prompt for gateway mode. Writes a prompt marker file so the gateway can forward the question to the user, then polls for a response file. Falls back to *default* on timeout. Used by ``hermes update --gateway`` so interactive prompts (stash restore, config migration) are forwarded to the messenger instead of being silently skipped. """ import json as _json import uuid as _uuid from hermes_constants import get_hermes_home home = get_hermes_home() prompt_path = home / ".update_prompt.json" response_path = home / ".update_response" # Clean any stale response file response_path.unlink(missing_ok=True) payload = { "prompt": prompt_text, "default": default, "id": str(_uuid.uuid4()), } tmp = prompt_path.with_suffix(".tmp") tmp.write_text(_json.dumps(payload), encoding="utf-8") tmp.replace(prompt_path) # Poll for response deadline = _time.monotonic() + timeout while _time.monotonic() < deadline: if response_path.exists(): try: answer = response_path.read_text(encoding="utf-8").strip() response_path.unlink(missing_ok=True) prompt_path.unlink(missing_ok=True) return answer if answer else default except (OSError, ValueError): pass _time.sleep(0.5) # Timeout — clean up and use default prompt_path.unlink(missing_ok=True) response_path.unlink(missing_ok=True) print(f" (no response after {int(timeout)}s, using default: {default!r})") return default def _npm_bin_exists(bin_dir: Path, name: str) -> bool: """True when an npm bin shim for *name* exists (POSIX or Windows).""" return any( (bin_dir / candidate).exists() for candidate in (name, f"{name}.cmd", f"{name}.ps1", f"{name}.exe") ) def _web_build_toolchain_ready(*roots: Path) -> bool: """True when ``tsc`` and ``vite`` shims are reachable from any of *roots*. Callers must pass every root the build would search; checking only one reports a healthy tree as broken. """ bin_dirs = [ bin_dir for bin_dir in (root / "node_modules" / ".bin" for root in roots) if bin_dir.is_dir() ] return bool(bin_dirs) and all( any(_npm_bin_exists(bin_dir, tool) for bin_dir in bin_dirs) for tool in ("tsc", "vite") ) def _web_toolchain_roots(web_dir: Path) -> tuple[Path, ...]: """Roots whose ``node_modules/.bin`` can satisfy the web build. ``npm run build`` prepends ``node_modules/.bin`` for the package and each of its ancestors, so shims hoisted to the workspace root and shims nested under a package that owns its lockfile (#42973) are equally valid. """ return (web_dir, web_dir.parent) def _print_curator_first_run_notice() -> None: """Print a short heads-up about the skill curator after `hermes update`. Only fires when the curator is enabled AND has no recorded run yet, which is exactly the window where the gateway ticker used to fire Curator against a fresh skill library immediately after an update. We defer the first real pass by one ``interval_hours``; this notice tells the user how to preview or disable before then. Silent on steady state. """ try: from agent import curator except Exception: return try: if not curator.is_enabled(): return state = curator.load_state() except Exception: return if state.get("last_run_at"): # Curator has run before (real or already seeded) — no notice needed. return try: hours = curator.get_interval_hours() except Exception: hours = 24 * 7 days = max(1, hours // 24) print() print("ℹ Skill curator") print( f" Background skill maintenance is enabled. First pass is deferred " f"~{days}d after installation; only agent-created skills are in " f"scope and nothing is ever auto-deleted (archive is recoverable)." ) print(" Preview now: hermes curator run --dry-run") print(" Pause it: hermes curator pause") print( " Docs: https://hermes-agent.nousresearch.com/docs/user-guide/features/curator" ) def _print_fts_optimize_available_notice() -> None: """Advertise the opt-in v23 search-index optimization after `hermes update`. Only fires when the current profile's state.db still needs an FTS storage rebuild. Leads with the reclaimable-space figure and points at the exact command. Honors ``sessions.fts_optimize_notice``: ``advise`` (default) prints an advisory notice, ``require`` prints a firmer required-upgrade notice, ``off`` suppresses it. Silent for fresh/already-optimized installs. """ mode = "advise" try: from hermes_cli.config import load_config mode = str( ((load_config() or {}).get("sessions") or {}).get( "fts_optimize_notice", "advise" ) ).strip().lower() except Exception: mode = "advise" if mode == "off": return try: from hermes_constants import get_hermes_home from hermes_state import SessionDB except Exception: return db_path = get_hermes_home() / "state.db" if not db_path.exists(): return try: size_gb = db_path.stat().st_size / (1024 ** 3) except OSError: return # Skip the notice for trivially small DBs — the win isn't worth the nag. if size_gb < 0.5: return db = None interrupted = False needs_upgrade = False try: db = SessionDB(db_path=db_path, read_only=True) # read_only opens skip schema init, so probe the stored layout directly. row = db._conn.execute( "SELECT sql FROM sqlite_master " "WHERE type = 'table' AND name = 'messages_fts'" ).fetchone() needs_upgrade = bool(row) and getattr( db, "_db_needs_fts_storage_upgrade" )(db._conn) # An interrupted `optimize-storage` run: the table is already the # v23 shape, but backfill markers / demoted trash tables remain. # Offer the command again — re-running resumes and finishes it. interrupted = bool( db._conn.execute( "SELECT 1 FROM state_meta " "WHERE key = 'fts_rebuild_high_water' LIMIT 1" ).fetchone() or db._conn.execute( "SELECT 1 FROM sqlite_master WHERE type = 'table' " "AND name LIKE 'fts\\_v22\\_trash\\_%' ESCAPE '\\' LIMIT 1" ).fetchone() or db._conn.execute( "SELECT 1 FROM state_meta WHERE key IN " "('fts_cjk_rebuild_high_water', 'fts_cjk_stale') LIMIT 1" ).fetchone() ) except Exception: return finally: if db is not None: try: db.close() except Exception: pass if not needs_upgrade and not interrupted: # Current layout already present (fresh/optimized) — nothing to offer. return if interrupted: print() print("◆ Session database optimization incomplete") print( " A previous `hermes sessions optimize-storage` run was " "interrupted. Search still works; re-run the command to resume " "and finish reclaiming disk:" ) print(" hermes sessions optimize-storage") return # Concrete size framing — lead with the savings the user cares about. est_reclaim = size_gb * 0.6 print() if mode == "require": print("◆ Session database upgrade required") print( f" Your search index uses the OLD storage layout and should be " f"upgraded. The new layout typically frees ~60% of state.db " f"(≈{est_reclaim:.1f} GB of your current {size_gb:.1f} GB) and is " f"required for continued optimal operation." ) else: print("◆ Reclaim ~60% of your session database disk") print( f" Your search index uses the old storage layout. Upgrading it " f"typically frees ~60% of state.db — about {est_reclaim:.1f} GB " f"of your current {size_gb:.1f} GB." ) print(" Run when convenient: hermes sessions optimize-storage") print( " It runs in the foreground with a progress bar, is safe to " "interrupt/re-run, and never changes your conversations." ) def _print_curator_recent_run_notice() -> None: """Print the most recent curator run summary, exactly once. The curator runs in the background (gateway tick + CLI session start), so users learn about skill consolidations only by stumbling into a rename. ``hermes update`` is a high-attention surface — surface the most recent run's rename map here, once. Show-once: state stamps ``last_run_summary_shown_at`` after printing. Subsequent ``hermes update`` invocations skip the block until a newer curator run lands. Silent when the curator has never run, when the most recent summary has already been shown, or when the summary has no rename information to display (no archives). """ try: from agent import curator except Exception: return try: state = curator.load_state() except Exception: return last_run_at = state.get("last_run_at") if not last_run_at: return # no curator run yet — first-run notice handles this case if state.get("last_run_summary_shown_at") == last_run_at: return # already shown for this run summary = state.get("last_run_summary") or "" if not summary: return # Only print when there's something interesting to show — i.e. the # rename map block was appended (multi-line summary). A bare "auto: # no changes; llm: no change" doesn't warrant interrupting the # update flow. if "\n" not in summary: # Still stamp it shown so we don't reconsider it on every update. try: state["last_run_summary_shown_at"] = last_run_at curator.save_state(state) except Exception: pass return # Format the timestamp as "Xh ago" for readability. when = _format_time_ago(last_run_at) print() print(f"ℹ Skill curator — last run {when}") for line in summary.splitlines(): print(f" {line}") print( " (This message shows once per curator run. " "View anytime: hermes curator status)" ) # Stamp shown so we don't repeat on the next update. try: state["last_run_summary_shown_at"] = last_run_at curator.save_state(state) except Exception: pass def _format_time_ago(iso_ts: str) -> str: """Render an ISO timestamp as `Xh ago` / `Xd ago` / `Xm ago`. Best effort.""" try: from datetime import datetime, timezone ts = datetime.fromisoformat(iso_ts.replace("Z", "+00:00")) if ts.tzinfo is None: ts = ts.replace(tzinfo=timezone.utc) delta = datetime.now(timezone.utc) - ts secs = int(delta.total_seconds()) if secs < 60: return "just now" if secs < 3600: return f"{secs // 60}m ago" if secs < 86400: return f"{secs // 3600}h ago" return f"{secs // 86400}d ago" except Exception: return "recently" def _reload_process_scan_modules() -> None: """Force-reload the process-scan modules from disk after an update. ``_finish_dashboard_update_cleanup`` runs in the PRE-update Python process, but ``_scan_dashboard_processes`` does a function-level ``from hermes_cli._subprocess_compat import bounded_probe_run``. If the update added a new symbol to ``_subprocess_compat`` (as #87134 did with ``bounded_probe_run``), the cached OLD module object doesn't have it and the cleanup step crashes with ImportError — after the code update itself already succeeded. Reload dependency-first so ``dashboard_procs`` binds against the fresh ``_subprocess_compat``. Lives here (called from the cleanup entry point) rather than only in ``_reload_config_modules`` so EVERY caller — the git-update path, the Windows ZIP fallback path, and any future one — is covered. """ import importlib importlib.invalidate_caches() for mod_name in ( "hermes_cli._subprocess_compat", "hermes_cli.dashboard_procs", ): mod = sys.modules.get(mod_name) if mod is not None: try: importlib.reload(mod) except Exception as exc: # warning, not debug: a failed reload here surfaces seconds # later as an ImportError in the same process — leave a trail. logger.warning( "Could not reload %s for post-update cleanup: %s", mod_name, exc, ) def _finish_dashboard_update_cleanup( node_failures: list[str], already_restarted_units: "set[str] | None" = None ) -> None: """Refresh managed dashboards or stop stale manual ones after an update. *already_restarted_units* forwards the systemd unit names (no ``.service`` suffix) that the fleet-restart loop already restarted directly, so a Serve-only install's freshly restarted process isn't found and restarted a second time here (review on #83595). """ if node_failures: print() print(" ℹ Leaving running dashboard process(es) untouched because the") print(" Node.js dependency refresh did not complete.") return # The scan path lazy-imports symbols from _subprocess_compat; make sure # both modules reflect the freshly-updated source before touching them. _reload_process_scan_modules() stop_result = _m()._kill_stale_dashboard_processes( restart_managed=True, already_restarted_units=already_restarted_units ) if not stop_result.get("unrecovered"): return print() print( "⚠ A web dashboard/serve process was stopped during update and could " "not be auto-restarted." ) print(" Re-launch it when you want the web UI back:") print(" hermes dashboard --port ") def _atomic_replace_dir(src: str, dst: str) -> None: """Replace directory *dst* with *src* without leaving *dst* half-deleted. The naive ``rmtree(dst); copytree(src, dst)`` has a destructive window: if the copy fails partway (common on the Windows ZIP-update path, which only runs because file I/O is already flaky on that machine), the old directory is already gone and nothing replaced it — the install is left with a deleted tree (issue #49145, where ``ui-tui/`` vanished and broke the TUI). Now a thin single-entry alias over the two-phase helpers below, which generalise the same stage-then-swap discipline across every entry the ZIP update touches (#76104). Retained because it is part of the mechanical ``hermes_cli.main`` re-export surface and guards the #49145 regression. """ _commit_staged_replacements([(_stage_replacement(src, dst), dst)]) def _stage_replacement(src: str, dst: str) -> str: """Copy *src* to a sibling staging path for *dst*; return the staging path. Phase 1 of the two-phase replace. Handles both directories and plain files. Touches nothing live, so a failure here leaves the whole install untouched. """ staging = f"{dst}.hermes-update-staging" backup = f"{dst}.hermes-update-old" # A previous run may have died between "move dst aside" and "move staging # in" — leaving dst missing and the backup as the ONLY copy of that entry. # Restore it before clearing leftovers: deleting the backup first and then # failing to stage (disk exhaustion is likely right after writing a full # staging copy) would leave a hole in the install with nothing to roll # back to. The restore is a same-filesystem rename — instant and safe. if not os.path.exists(dst) and os.path.exists(backup): os.rename(backup, dst) for leftover in (staging, backup): if os.path.isdir(leftover): shutil.rmtree(leftover, ignore_errors=True) elif os.path.exists(leftover): os.remove(leftover) if os.path.isdir(src): shutil.copytree(src, staging) else: shutil.copy2(src, staging) return staging def _discard_staged(staged) -> None: """Remove staging paths for entries that were never committed. Without this a phase-1 failure (typically disk exhaustion) orphans one staging copy per entry already processed — up to a full second copy of the tree. The user then follows the "re-run `hermes update`" advice with *less* free space than before and the retry fails harder than the original attempt. """ for staging, _dst in staged: try: if os.path.isdir(staging): shutil.rmtree(staging, ignore_errors=True) elif os.path.exists(staging): os.remove(staging) except OSError as exc: # best-effort cleanup, never fatal logger.warning("could not remove staging path %s: %s", staging, exc) def _commit_staged_replacements(staged) -> None: """Phase 2: swap every staged entry into place, rolling back all on failure. ``_atomic_replace_dir`` makes each *individual* directory swap safe, but the ZIP update replaces ~90 top-level entries in a loop, and nothing made the loop atomic *as a whole*. A failure partway left some entries at the new version and the rest at the old one — every file valid Python, the combination unbootable (issue #76104; the ``ImportError`` in #76091 and the field report in #63717 are both this). This covers plain files as well as directories: the repo root holds 20 first-party modules (``run_agent.py``, ``cli.py``, ``hermes_constants.py`` …), so a files-only failure reproduces exactly the bug class we are closing. Every swap is an ``os.rename`` onto a path that was just moved aside — a same-filesystem rename is atomic on POSIX and NTFS alike, so a file swap can never leave a half-written module the way ``copy2`` onto a live path can. Splitting stage-all-then-swap-all shrinks the failure window from "the duration of a full tree copy" to "the duration of N renames", and makes the remaining window recoverable: if a swap fails we restore every entry already swapped, so the tree lands wholly new or wholly old. """ swapped: list[tuple[str, str]] = [] # (dst, backup) in swap order; "" = absent try: for staging, dst in staged: backup = f"{dst}.hermes-update-old" if os.path.exists(dst): os.rename(dst, backup) swapped.append((dst, backup)) else: swapped.append((dst, "")) os.rename(staging, dst) except OSError: # Undo every swap already made so the install stays self-consistent. for dst, backup in reversed(swapped): try: if os.path.isdir(dst): shutil.rmtree(dst, ignore_errors=True) elif os.path.exists(dst): os.remove(dst) if backup and os.path.exists(backup): os.rename(backup, dst) except OSError as exc: # Keep restoring the rest — a silent failure here is the one # thing that turns a recoverable rollback into a mixed tree, # so say so rather than swallowing it. logger.warning("rollback failed for %s: %s", dst, exc) raise # All swaps succeeded — drop the backups (best-effort, never fatal). for _dst, backup in swapped: if backup and os.path.isdir(backup): shutil.rmtree(backup, ignore_errors=True) elif backup and os.path.exists(backup): try: os.remove(backup) except OSError: pass def _branch_head_label(git_cmd=None, cwd=None) -> str | None: """``" @ "`` for the checkout, or None when unknown. Appended to the update summary lines so branch drift is visible at a glance (live incident 2026-08-17: a checkout parked on a stale feature branch got "✓ Update complete!" with nothing on the line saying WHERE the checkout actually sat). Never raises — summary decoration must not break an update. """ try: cmd = list(git_cmd) if git_cmd else ["git"] root = cwd if cwd is not None else _m().PROJECT_ROOT branch = subprocess.run( cmd + ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root, capture_output=True, text=True, encoding="utf-8", errors="replace", ) sha = subprocess.run( cmd + ["rev-parse", "--short", "HEAD"], cwd=root, capture_output=True, text=True, encoding="utf-8", errors="replace", ) branch_name = branch.stdout.strip() sha_text = sha.stdout.strip() if branch.returncode != 0 or sha.returncode != 0 or not sha_text: return None if not branch_name: return None label = "detached" if branch_name == "HEAD" else branch_name return f"{label} @ {sha_text}" except Exception: return None def _branch_head_suffix(git_cmd=None, cwd=None) -> str: """`` [ @ ]`` suffix for summary lines ("" when unknown).""" label = _branch_head_label(git_cmd, cwd) return f" [{label}]" if label else "" def _assess_parked_branch_switch( git_cmd: list[str], cwd: Path, current_branch: str, target_branch: str ) -> tuple[bool, str]: """Decide whether it is safe to auto-switch a parked feature branch back to the update target. Live incident (2026-08-17, Teknium's box): the source checkout sat on a stale feature branch left behind by earlier tooling; ``hermes update`` autostashed, ran its post-update steps and printed "✓ Code updated!" while the running code stayed days behind main. The guard's contract: - (True, "") when the working tree + index are clean AND every commit on the parked branch is already contained in ``origin/`` (``git cherry`` reports no ``+`` lines). - (True, "unmerged:") when the tree is clean but the branch has commits not yet in the target. Switching is safe — ``git checkout`` never discards committed work and the branch keeps the commits — but the caller must print a LOUD notice naming the branch and count so the work is not forgotten. This is what non-interactive callers (desktop update button, gateway /update, cron) rely on: they have no way to resolve a skip, so a clean checkout must always reach the target. - (False, ) — dirty tree, git errors, or the ``updates.auto_switch_parked_branch: false`` config opt-out — and the caller must NOT touch the branch. A dirty tree is the one genuinely unsafe case: uncommitted work would have to ride an autostash across branches, which is how the 2026-08-17 incident started. Block reasons: "disabled", "dirty", "unverifiable". """ try: from hermes_cli.config import load_config _update_cfg = (load_config() or {}).get("updates", {}) if isinstance(_update_cfg, dict) and not bool( _update_cfg.get("auto_switch_parked_branch", True) ): return False, "disabled" except Exception as exc: # A config read failure must not disable the guard's safety checks — # fall through to them with the default (auto-switch allowed). logger.debug("Could not read updates.auto_switch_parked_branch: %s", exc) status = subprocess.run( git_cmd + ["status", "--porcelain"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if status.returncode != 0: return False, "unverifiable" if status.stdout.strip(): return False, "dirty" cherry = subprocess.run( git_cmd + ["cherry", f"origin/{target_branch}"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if cherry.returncode != 0: return False, "unverifiable" unmerged = [ line for line in cherry.stdout.splitlines() if line.startswith("+") ] if unmerged: # Clean tree: switching is safe (checkout keeps the commits on the # branch). The reason string tells the caller to print the loud # "branch kept with N unmerged commit(s)" notice. return True, f"unmerged:{len(unmerged)}" return True, "" def _print_parked_branch_skip_warning( git_cmd: list[str], cwd: Path, current_branch: str, target_branch: str, reason: str, ) -> None: """LOUD block explaining why the code update was skipped on a parked branch, with the behind-count and the exact commands to resolve.""" behind = None try: behind_result = subprocess.run( git_cmd + ["rev-list", f"HEAD..origin/{target_branch}", "--count"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if behind_result.returncode == 0 and behind_result.stdout.strip(): behind = int(behind_result.stdout.strip()) except Exception: behind = None if reason == "dirty": why = "the working tree has uncommitted changes" elif reason == "disabled": why = "updates.auto_switch_parked_branch is set to false in config.yaml" else: why = ( f"the branch state could not be verified against " f"origin/{target_branch}" ) bar = "=" * 68 print() print(bar) print(f"⚠ CODE UPDATE SKIPPED — checkout is parked on '{current_branch}'") print(f" Not auto-switching to {target_branch}: {why}.") if behind is not None and behind > 0: print( f" This checkout is {behind} commit(s) BEHIND " f"origin/{target_branch} — the code you are running is stale." ) print() print(" To resolve, inspect the branch and switch back yourself:") print(f" git -C {cwd} status") print(f" git -C {cwd} checkout {target_branch} && hermes update") print( " (commit or stash your work on the branch first if you want to " "keep it)" ) print(bar) def _print_parked_branch_kept_notice( current_branch: str, target_branch: str, unmerged_count: str ) -> None: """LOUD notice printed when a clean parked branch with unmerged commits is auto-switched back to the update target. Non-interactive callers (desktop update button, gateway /update, cron) cannot resolve a skip, so a clean checkout always proceeds to the target — but the unmerged work must be impossible to miss. The commits are untouched: ``git checkout`` never discards committed work; the branch keeps them until the user returns. """ bar = "=" * 68 print() print(bar) print( f"⚠ Checkout was parked on '{current_branch}' with " f"{unmerged_count} commit(s) not merged into origin/{target_branch}." ) print( f" Switching to {target_branch} so the update can proceed — your " f"commit(s) are safe on '{current_branch}'." ) print() print(" To pick the work back up later:") print(f" git checkout {current_branch}") print(bar) def _print_update_completion(message: str) -> None: """Print an update outcome plus, when the dashboard launched this run with an action id, a terminal receipt line the Desktop can match after the dashboard restarts (see #47359 / #58764). The outcome line carries the checkout's actual branch + HEAD short-sha so branch drift is visible at a glance (2026-08-17 parked-branch incident).""" print(f"{message}{_branch_head_suffix()}") action_id = os.environ.get("HERMES_ACTION_ID", "") if len(action_id) == 32 and all(char in "0123456789abcdef" for char in action_id): print(f"=== hermes-update completed {action_id} ===") def _called_process_error_cmd_parts(exc: subprocess.CalledProcessError) -> list[str]: """Normalize ``CalledProcessError.cmd`` into argv-style tokens.""" cmd = exc.cmd if cmd is None: return [] if isinstance(cmd, (str, bytes)): text = cmd.decode("utf-8", "replace") if isinstance(cmd, bytes) else cmd try: return shlex.split(text, posix=os.name != "nt") except ValueError: return text.split() return [str(part) for part in cmd] def _called_process_error_is_git(exc: subprocess.CalledProcessError) -> bool: """True when the failed subprocess was git itself.""" parts = _called_process_error_cmd_parts(exc) if not parts: return False # Windows argv may use backslashes; basename() on POSIX would otherwise # keep the whole path. Normalize separators before taking the name. name = os.path.basename(parts[0].replace("\\", "/")).lower() return name in {"git", "git.exe"} def _called_process_error_is_python_dep_install( exc: subprocess.CalledProcessError, ) -> bool: """True when the failed subprocess was a uv/pip (or ensurepip) install.""" parts = [part.lower() for part in _called_process_error_cmd_parts(exc)] if not parts: return False exe = os.path.basename(parts[0].replace("\\", "/")) if "ensurepip" in parts: return True if "install" in parts and ( "pip" in parts or exe in {"pip", "pip.exe", "pip3", "pip3.exe", "uv", "uv.exe"} ): return True return False def _format_update_failure_stage(exc: subprocess.CalledProcessError) -> str: """Name the update stage that actually failed. The git pull and the Python-dependency install share one ``try`` in ``_cmd_update_impl``. Calling every ``CalledProcessError`` a git failure (the historical Windows message) sent users hunting in the wrong place and, worse, keyed the ZIP overlay on exception *type* rather than on git actually having failed (#87304, #85840). """ if _called_process_error_is_python_dep_install(exc): return "Python dependency install failed" if _called_process_error_is_git(exc): return "Git update failed" return "Update step failed" def _shim_quarantine_error_type() -> "type[BaseException]": """The strict-quarantine refusal type, resolved lazily through ``_m()``. Falls back to a never-raised private type when main.py lacks it (torn mid-update tree), so the ``except`` clause stays valid. """ cls = getattr(_m(), "ShimQuarantineError", None) if isinstance(cls, type) and issubclass(cls, BaseException): return cls class _Never(Exception): pass return _Never def _refuse_update_for_contended_shims(exc: BaseException) -> None: """Refuse the dependency sync when live shims could not be quarantined. #87331 fail-closed half: a shim rename that failed every retry proves a process holds the venv without FILE_SHARE_DELETE — running the installer anyway is exactly how the venv ends up stranded between versions. The code swap (when one happened) is already committed; only the dependency install is deferred, via the update-incomplete marker, to the next fresh launch after the holder exits. Exits 2 (refused) so the command-boundary receipt net records it as a refusal, not a failure. """ print("✗ Cannot continue the update: live Hermes launcher(s) could not be") print(" moved aside:") for name in getattr(exc, "failed_shims", []) or ["hermes.exe"]: print(f" {name}") print(" Another process is holding this install's venv — typically Hermes") print(" Desktop, a gateway, or another hermes REPL — and mutating the venv") print(" now would strand it half-updated.") print(" The dependency install has been deferred: close the process(es)") print(" above, then run any `hermes` command to finish it automatically.") # Idempotent: the git path already dropped the marker before the sync; # this covers the ZIP/repair paths so the deferral is never silent. _write_update_incomplete_marker() sys.exit(2) def _should_zip_fallback_on_update_error(exc: BaseException) -> bool: """ZIP fallback is for Windows git file-I/O breakage, not later stages. A dependency-install failure (locked ``hermes.exe`` / ``uv pip install`` exit 2) is not a git failure. The pull has already succeeded by then, so re-downloading the source ZIP cannot fix the install and would replace every top-level entry except ``venv`` / ``node_modules`` / ``.git`` / ``.env`` — permanently deleting uncommitted edits and untracked files. """ return ( isinstance(exc, subprocess.CalledProcessError) and _m()._is_windows() and _called_process_error_is_git(exc) ) def _print_called_process_error_tail( exc: subprocess.CalledProcessError, *, limit: int = 12 ) -> None: """Print a captured stderr/stdout tail when the failing call recorded one.""" blob = exc.stderr or exc.stdout or "" if isinstance(blob, bytes): blob = blob.decode("utf-8", "replace") lines = [line for line in str(blob).splitlines() if line.strip()] if not lines: return print(" Last output:") for line in lines[-limit:]: print(f" {line}") def _zip_overlay_block_reason( root: Path, *, ignore_staging_artifacts: bool = False ) -> Optional[str]: """Why overlaying a ZIP onto ``root`` would destroy work, or None if safe. The ZIP path swaps every top-level entry (except a tiny preserve set) and then deletes the backups, so uncommitted edits and untracked files under a replaced directory are gone. Fail closed when git status cannot run: unknown dirtiness is not a license to clobber the tree (#87304). ``ignore_staging_artifacts`` is for the pre-swap re-check: phase 1 of the two-phase replace creates ``*.hermes-update-staging`` siblings inside the checkout, which git reports as untracked. Those are our own artifacts, not user work — without the filter the re-check would always refuse. """ if not (root / ".git").exists(): return None git_cmd = ["git"] if sys.platform == "win32": git_cmd = ["git", "-c", "windows.appendAtomically=false"] result = subprocess.run( # -uall: a user-level ``status.showUntrackedFiles = no`` git config # would otherwise hide untracked files and silently blind this guard. # --ignored=matching: gitignored files are still USER DATA the ZIP # overlay would permanently delete (logs, scratch files, local data) # — a .gitignore entry must not blind the guard either (#87392). # ``matching`` reports an ignored directory as one ``dir/`` line # instead of enumerating its contents (cheaper, same verdict for the # top-level filter below). NOTE: ``--ignored=all`` is NOT a valid # git mode — it exits 128 and would fail-close every ZIP update. git_cmd + ["status", "--porcelain", "--untracked-files=all", "--ignored=matching"], cwd=root, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if result.returncode != 0: detail = (result.stderr or result.stdout or "").strip().splitlines() suffix = f" ({detail[0]})" if detail else "" return f"could not check the working tree{suffix}" lines = [line for line in (result.stdout or "").splitlines() if line.strip()] # --ignored=all reports the ZIP path's own preserved entries (venv, # node_modules are gitignored on every normal install). The swap never # touches those top-level entries, so they must not turn into a false # dirty-tree refusal. Everything else — including ignored files — blocks. lines = [line for line in lines if not _is_zip_preserved_entry_status_line(line)] if ignore_staging_artifacts: lines = [ line for line in lines if not _is_zip_staging_artifact_status_line(line) ] if lines: return "the working tree has uncommitted changes or untracked files" return None _ZIP_STAGING_ARTIFACT_SUFFIXES = (".hermes-update-staging", ".hermes-update-old") # Single source of truth for the top-level entries the ZIP swap preserves — # consumed by both the dirty-tree filter below and _update_via_zip's swap loop. _ZIP_PRESERVED_TOP_LEVEL = {"venv", "node_modules", ".git", ".env"} def _is_zip_preserved_entry_status_line(line: str) -> bool: """True when every path on a porcelain status line sits under a top-level entry the ZIP swap preserves. The ``" -> "`` two-path split applies ONLY to rename/copy status codes (R/C): porcelain v1 does not quote a plain filename containing spaces, so an ignored file literally named ``venv -> node_modules`` on an ``!!``/``??`` line must be treated as ONE path — splitting it would filter it as two preserved tops and fail-open into the destructive swap. Requiring EVERY path preserved keeps renames leaving a preserved dir (``R venv/x -> src/x``) blocking, fail-closed. """ status, payload = (line[:2], line[3:]) if len(line) >= 3 else ("", line) is_rename = any(code in "RC" for code in status) paths = payload.split(" -> ") if is_rename else [payload] for path in paths: top_level = ( path.strip().strip('"').replace("\\", "/").rstrip("/").split("/", 1)[0] ) if top_level not in _ZIP_PRESERVED_TOP_LEVEL: return False return True def _is_zip_staging_artifact_status_line(line: str) -> bool: """True when a porcelain status line is our own two-phase-swap artifact.""" payload = line[3:] if len(line) >= 3 else line top_level = ( payload.strip().strip('"').replace("\\", "/").rstrip("/").split("/", 1)[0] ) return top_level.endswith(_ZIP_STAGING_ARTIFACT_SUFFIXES) def _abort_zip_update_if_dirty_tree() -> None: """Refuse to overlay a ZIP onto a dirty git checkout (#87304).""" reason = _zip_overlay_block_reason(_m().PROJECT_ROOT) if reason is None: return print(f"✗ ZIP fallback refused: {reason}.") print( " Overlaying the ZIP would overwrite uncommitted edits and permanently " "delete untracked files." ) print(" Stash or commit your changes, then rerun `hermes update`.") print(" To inspect: git status --porcelain") _m().sys.exit(1) def _read_project_version() -> str | None: """Read the ``version`` field from the checkout's pyproject.toml. Reads the on-disk file (not importlib.metadata) because after a git pull the installed distribution metadata still describes the OLD version; the file is the only source that reflects what was just pulled. Returns None on any failure — version reporting is cosmetic and must never break an update. """ try: import tomllib with open(_m().PROJECT_ROOT / "pyproject.toml", "rb") as fh: # windows-footgun: ok — binary mode, tomllib requires bytes version = tomllib.load(fh).get("project", {}).get("version") return str(version) if version else None except Exception: return None def _update_complete_message(pre_version: str | None) -> str: """Completion line with the version transition when it is known. Ported from PrimeIntellect-ai/prime-agent#630: after a successful self-update, show both versions (``v0.19.4 → v0.20.0``) so the user can see what they actually got. Falls back to the plain message when either side is unknown or the version did not change (e.g. several commits landed within one release). """ post_version = _read_project_version() if pre_version and post_version and pre_version != post_version: return f"✓ Update complete! (v{pre_version} → v{post_version})" if post_version: return f"✓ Update complete! (v{post_version})" return "✓ Update complete!" def _post_update_sqlite_runtime_status(): """Return whether the interpreter used after update has safe SQLite.""" from hermes_constants import project_venv_dir from hermes_cli.sqlite_runtime import probe_sqlite_runtime venv_dir = project_venv_dir(_m().PROJECT_ROOT) python = ( venv_python_path(venv_dir, windows=_m()._is_windows()) if venv_dir is not None else Path(sys.executable) ) info = probe_sqlite_runtime(python) return info is not None and not info.wal_reset_vulnerable, info def _print_verified_update_completion(message: str) -> bool: """Print a success completion only after probing the next Hermes runtime.""" if not message.startswith("✓"): _print_update_completion(message) return False sqlite_runtime_ok, sqlite_info = _post_update_sqlite_runtime_status() if sqlite_info is None: # Grace path: an unprobeable interpreter (no venv in a dev checkout, # probe subprocess unavailable) must not fail an otherwise-successful # update — only a POSITIVE vulnerable probe withholds success # (same contract as _venv_core_imports_healthy's unknown states). logger.debug("Post-update SQLite runtime probe unavailable; not blocking") _print_update_completion(message) return True if sqlite_runtime_ok: _print_update_completion(message) return True print() detail = ( f"SQLite {sqlite_info.sqlite_version_string} still has the " "WAL-reset corruption bug" ) print(f"⚠ Update partially complete — {detail}.") print( " Rebuild the Hermes venv with a uv-managed Python, restart Hermes, " "then verify with `hermes doctor`." ) return False def _clear_stale_sqlite_sidecars(db_path: Path) -> None: """Delete the WAL / shared-memory / rollback-journal files next to *db_path*. Call this immediately before overwriting a database file with a snapshot image. Quick snapshots are produced by ``backup._safe_copy_db`` through ``sqlite3.backup()``, so the image is already checkpointed and owns no WAL — which is exactly why ``backup._EXCLUDED_SUFFIXES`` refuses to ship sidecars inside a snapshot. Copying the image over the destination replaces only the main database file, so any ``-wal`` / ``-shm`` left behind by the *old* database (a crashed writer, or a second Hermes process the updater's drain did not stop) survives and is replayed over the fresh image on the next open. The result passes ``PRAGMA integrity_check`` while serving the old database's contents, and the first checkpoint folds it in permanently. Removing them is safe here specifically: they belong to a database the caller has already declared corrupt and is about to discard. """ for suffix in ("-wal", "-shm", "-journal"): db_path.with_name(db_path.name + suffix).unlink(missing_ok=True) def _print_update_summary( *, node_failures: list, desktop_build_ok: bool, pre_update_version: str | None, ) -> bool: """Final update banner. A failed Desktop rebuild is non-fatal for the Python side, but must not print ``✓ Update complete!`` (#88251).""" sqlite_runtime_ok, sqlite_info = _post_update_sqlite_runtime_status() if sqlite_info is None: # Grace path: an unprobeable interpreter must not fail the update — # only a POSITIVE vulnerable probe demotes success to partial. sqlite_runtime_ok = True print() if node_failures or not desktop_build_ok or not sqlite_runtime_ok: parts = [] if node_failures: parts.append( f"Node.js dependencies for {', '.join(node_failures)} did not refresh" ) if not desktop_build_ok: parts.append( "the desktop app was not rebuilt and is still on the previous build" ) if not sqlite_runtime_ok and sqlite_info is not None: parts.append( f"SQLite {sqlite_info.sqlite_version_string} still has the " "WAL-reset corruption bug" ) print("⚠ Update partially complete — " + "; ".join(parts) + ".") if node_failures: print(" Code and Python deps are updated, but the dashboard/TUI may") print(" be in a mixed state until the Node deps are rebuilt.") if not desktop_build_ok: print(" Run `hermes desktop` to retry the desktop rebuild.") if not sqlite_runtime_ok: print( " The Python runtime remediation did not complete. Run `hermes " "update` again; if SQLite is unchanged, rebuild the Hermes venv " "with a uv-managed Python, restart Hermes, then verify with " "`hermes doctor`." ) else: _print_update_completion(_update_complete_message(pre_update_version)) return desktop_build_ok and sqlite_runtime_ok def _write_gateway_update_exit_code(ok: bool) -> None: path = get_hermes_home() / ".update_exit_code" try: path.write_text("0" if ok else "1", encoding="utf-8") except OSError: pass def _restore_state_db_from_snapshot(state_path: Path, snap_state: Path) -> bool: """Replace *state_path* with the snapshot image at *snap_state*. Shared by both post-update auto-restore paths (the ZIP update and the git pull). The destination's stale sidecars are cleared before the copy, so the restored image cannot be silently overwritten by the corrupt database's WAL replay — see :func:`_clear_stale_sqlite_sidecars`. Refuses (returns ``False``) while another process — or a live connection in THIS process — still holds the database or its sidecars open: copying a snapshot over a live writer's inode makes the writer's page cache and WAL index disagree with the file bytes, and its next checkpoint writes pages at offsets that no longer mean what it thinks — the #90950 page-1 clobber. ``None`` (scan unavailable) proceeds: the updater has already drained gateways, and refusing on "unknown" would disable auto-restore on every non-Linux host. Returns ``True`` when the restored file passes an integrity check. Raises ``OSError`` if the copy itself fails, which callers already report. """ from hermes_cli.backup import _foreign_db_holder_pids, verify_sqlite_integrity from hermes_cli.sqlite_safe_read import LiveConnectionError, offline_file_access holders = _foreign_db_holder_pids(state_path) if holders: print( f" ✗ Auto-restore refused: process(es) {holders} still hold " "state.db or its WAL open. Stop them (hermes gateway stop), " "then restore manually with /snapshot restore." ) return False # The foreign-pid scan excludes THIS process on purpose, but an # in-process SessionDB handle is exactly as much of a live holder: # unlinking the -wal/-shm and copy2-ing over the main file under it # leaves this process checkpointing through deleted-inode sidecars — # the #90950 split brain produced first-party (proven live on main: # `/proc/self/fd` shows `state.db-wal (deleted)` right after this ran # under a tracked connection). ``offline_file_access`` fails CLOSED on # any tracked live connection and holds the connection-lifecycle lock # across the sidecar clear + copy so none can appear mid-swap. try: with offline_file_access(state_path, what="restore a snapshot over"): _clear_stale_sqlite_sidecars(state_path) shutil.copy2(snap_state, state_path) except LiveConnectionError as exc: print( f" ✗ Auto-restore refused: {exc} Close the in-process database " "handles (or restart Hermes) and retry." ) return False restored = verify_sqlite_integrity( state_path, check_header=True, run_pragma=True ) return bool(restored.get("valid")) def _verify_and_restore_one_state_db(home: Path, *, label: str) -> None: """Post-update integrity check + auto-restore for ONE home's state.db. Shared by the root-DB and sibling-profile guards (ZIP update path and git-pull path both route here). A corrupt live DB is restored from the most recent valid snapshot under that home's own state-snapshots dir. Never raises: a guard that crashes the update tail would be worse than the corruption it detects. """ try: from hermes_cli.backup import _quick_snapshot_root, verify_sqlite_integrity state_path = home / "state.db" if not state_path.exists(): return ok = verify_sqlite_integrity(state_path, check_header=True, run_pragma=True) if ok.get("valid"): logger.debug( "Post-update state.db integrity OK (%s): %s", label, ok.get("message"), ) return print() print( f"⚠ state.db is corrupted after update ({label}): " + ok.get("message", "unknown error") ) snap_root = _quick_snapshot_root(home) if not snap_root.exists(): print(" ⚠ No pre-update snapshot for this home") return for snap_dir in sorted( (d for d in snap_root.iterdir() if d.is_dir()), reverse=True ): snap_state = snap_dir / "state.db" if not snap_state.exists(): continue snap_ok = verify_sqlite_integrity( snap_state, check_header=True, run_pragma=True ) if not snap_ok.get("valid"): continue try: if _restore_state_db_from_snapshot(state_path, snap_state): print( f" ✓ Auto-restored from snapshot {snap_dir.name} ({label})" ) else: print( " ✗ Auto-restore FAILED — restored copy also failed " "integrity" ) except OSError as exc: print(f" ✗ Auto-restore file copy failed: {exc}") return print(" ⚠ No valid pre-update snapshot found for this home") except Exception as exc: logger.debug( "Post-update state.db guard (%s) failed: %s", label, exc ) def _verify_and_restore_state_dbs_post_update() -> None: """Post-update integrity guard for the ROOT state.db AND every sibling profile's state.db (#97994). The pre-update snapshot already covers every sibling profile (#66140 create_pre_update_snapshots_all_profiles), but the post-update guard only ever verified the root DB — a profile database corrupted by the update was never detected and never auto-restored, leaving that profile's sessions silently gone while the root DB passed. """ home = get_hermes_home() _verify_and_restore_one_state_db(home, label="default home") try: from hermes_cli.backup import _sibling_profile_homes for name, profile_home in _sibling_profile_homes(home): _verify_and_restore_one_state_db(profile_home, label=f"profile {name}") except Exception as exc: logger.debug("Sibling-profile state.db guard sweep failed: %s", exc) def _update_via_zip(args, *, had_desktop_app_before_update: bool = False) -> bool: """Update Hermes Agent by downloading a ZIP archive. Used on Windows when git file I/O is broken (antivirus, NTFS filter drivers causing 'Invalid argument' errors on file creation). Returns ``False`` when a Desktop rebuild ran and failed; ``True`` otherwise. """ active_tool_dependencies = _m()._capture_active_tool_dependencies() import tempfile import zipfile from urllib.request import urlretrieve # Snapshot the pre-update version before files are replaced so the # completion line can report the transition (prime-agent#630 port). pre_update_version = _read_project_version() # The ZIP fallback exists for Windows git-file-I/O breakage. It pulls a # static archive from GitHub, which is fine for the default "main" # channel but would silently ignore --branch and update from main even # if the user asked for something else — exactly the silent-divergence # bug --branch was added to prevent. Refuse to proceed in that case # rather than lie. branch = _m()._resolve_update_branch(args) if branch != "main": print( f"✗ --branch={branch} is not supported on the Windows ZIP-fallback " "update path." ) print( " This path runs when git file I/O is broken on the system. " "Either resolve the git-side breakage (typically an antivirus " "or NTFS filter holding files open) and rerun `hermes update " f"--branch {branch}`, or update against main with `hermes update`." ) _m().sys.exit(1) _abort_zip_update_if_dirty_tree() zip_url = ( f"https://github.com/NousResearch/hermes-agent/archive/refs/heads/{branch}.zip" ) print("→ Downloading latest version...") tmp_dir = tempfile.mkdtemp(prefix="hermes-update-") try: zip_path = os.path.join(tmp_dir, f"hermes-agent-{branch}.zip") urlretrieve(zip_url, zip_path) print("→ Extracting...") import stat as _stat with zipfile.ZipFile(zip_path, "r") as zf: # Validate paths to prevent zip-slip (path traversal) AND reject # symlink members. A GitHub source ZIP for hermes-agent itself # should never contain symlinks — they'd point outside the # extracted tree and let an attacker who can compromise the # update mirror plant arbitrary files via the update path. tmp_dir_real = os.path.realpath(tmp_dir) for member in zf.infolist(): member_path = os.path.realpath(os.path.join(tmp_dir, member.filename)) if ( not member_path.startswith(tmp_dir_real + os.sep) and member_path != tmp_dir_real ): raise ValueError( f"Zip-slip detected: {member.filename} escapes extraction directory" ) # Unix mode lives in the upper 16 bits of external_attr; # mask to the file-type bits. mode = (member.external_attr >> 16) & 0o170000 if _stat.S_ISLNK(mode): raise ValueError( f"ZIP contains unsupported symlink member: {member.filename}" ) zf.extractall(tmp_dir) # GitHub ZIPs extract to hermes-agent-/ extracted = os.path.join(tmp_dir, f"hermes-agent-{branch}") if not os.path.isdir(extracted): # Try to find it for d in os.listdir(tmp_dir): candidate = os.path.join(tmp_dir, d) if os.path.isdir(candidate) and d != "__MACOSX": extracted = candidate break # Copy updated files over existing installation, preserving venv/node_modules/.git preserve = _ZIP_PRESERVED_TOP_LEVEL entries = [i for i in os.listdir(extracted) if i not in preserve] # Two-phase replace (#76104). Phase 1 copies every entry — directories # AND top-level files — to a sibling staging path without touching # anything live; phase 2 swaps them all in with same-filesystem # renames and rolls back every swap if any one fails. Replacing # entries one-at-a-time (the previous shape) meant an interruption # partway left `agent/` new and `tools/` stale — all files valid, the # tree unbootable. Files matter as much as directories here: the repo # root holds 20 first-party modules (run_agent.py, cli.py, # hermes_constants.py, ...). # # Staging costs one extra copy of the tree on disk. Check up front so # we fail with a clear message instead of running out mid-copy. need = sum( os.path.getsize(os.path.join(dirpath, f)) for entry in entries for dirpath, _dirs, files in os.walk(os.path.join(extracted, entry)) for f in files ) + sum( os.path.getsize(os.path.join(extracted, e)) for e in entries if os.path.isfile(os.path.join(extracted, e)) ) # Only the staging copy is new — the live tree already occupies its # space and the swaps are renames, not copies. Ask for the staging # copy plus 20% headroom rather than a full 2x, which would block # updates that would have succeeded on exactly the space-constrained # machines most likely to hit this path. required = int(need * 1.2) free = shutil.disk_usage(str(_m().PROJECT_ROOT)).free if free < required: raise RuntimeError( f"not enough free disk space to stage the update safely " f"(need ~{required // (1024 * 1024)} MB, have " f"{free // (1024 * 1024)} MB)" ) staged: list[tuple[str, str]] = [] try: for item in entries: src = os.path.join(extracted, item) dst = os.path.join(str(_m().PROJECT_ROOT), item) staged.append((_stage_replacement(src, dst), dst)) # #70337/#87331: the GitHub source ZIP contains only source — # apps/desktop/release/ (the BUILT desktop app, win-unpacked/ # Hermes.exe) exists only in the LIVE tree. Swapping `apps` # without it deletes the desktop build and breaks the # shortcut. Graft the live release dir into the staged copy # BEFORE the swap so the commit preserves it atomically. if item == "apps": live_release = os.path.join(dst, "desktop", "release") staged_release = os.path.join( staged[-1][0], "desktop", "release" ) if os.path.isdir(live_release) and not os.path.exists( staged_release ): os.makedirs(os.path.dirname(staged_release), exist_ok=True) shutil.copytree(live_release, staged_release) except Exception: # Nothing is live yet; drop the partial staging copies so a retry # starts from the same free space this attempt did. _discard_staged(staged) raise try: # Re-check the tree right before the swap (#87304 TOCTOU): the # download + extract + staging window above can take minutes, and # work created in it would be destroyed by the commit below. Our # own phase-1 staging siblings are filtered out — they are the # expected artifacts of getting here, not user work. recheck_reason = _zip_overlay_block_reason( _m().PROJECT_ROOT, ignore_staging_artifacts=True ) if recheck_reason is not None: _discard_staged(staged) print(f"✗ ZIP fallback aborted before the swap: {recheck_reason}.") print( " Files appeared in the checkout while the update was " "downloading; committing the swap would delete them." ) print(" Stash or commit your changes, then rerun `hermes update`.") _m().sys.exit(1) _commit_staged_replacements(staged) except Exception: # The rollback already restored every swapped entry, but staging # copies for the not-yet-swapped entries (potentially most of a # full tree) are still on disk. Drop them, or the retry's # up-front free-space check — which runs BEFORE the lazy # per-entry leftover cleanup — fails on litter this attempt # left behind: the exact "retry fails harder" failure mode # _discard_staged exists to prevent. Safe post-rollback: swapped # entries' staging paths were renamed away, and _discard_staged # skips paths that no longer exist. _discard_staged(staged) raise update_count = len(staged) print(f"✓ Updated {update_count} items from ZIP") except Exception as e: print(f"✗ ZIP update failed: {e}") # The two-phase replace either commits every entry or rolls them all # back, so a failure here does not leave a mixed-version tree — don't # scare the user toward a reinstall they don't need. print(" Your existing install was left in place.") print( " Re-run `hermes update` to retry; if the agent won't start, " "reinstall from https://hermes-agent.nousresearch.com" ) _m().sys.exit(1) finally: shutil.rmtree(tmp_dir, ignore_errors=True) # Clear stale bytecode after ZIP extraction removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT) if removed: print( f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}" ) _m()._record_bytecode_fingerprint() _m()._refresh_bootstrap_cache_scripts(branch) # Reinstall Python dependencies. Prefer .[all], but if one optional extra # breaks on this machine, keep base deps and reinstall the remaining extras # individually so update does not silently strip working capabilities. # # Self-lock deferral (relocated preflight — #86735): the ZIP code swap # above is already committed; defer only the dependency sync when this # process holds a native extension the sync must rewrite. _m()._abort_dependency_sync_if_self_locked() print("→ Updating Python dependencies...") from hermes_cli.managed_uv import ensure_uv, update_managed_uv # Keep managed uv current — runs `uv self update` if we already have one. update_managed_uv() uv_bin = ensure_uv() pip_cmd = [_m().sys.executable, "-m", "pip"] if not uv_bin: uv_bin = _ensure_uv_for_termux(pip_cmd) if uv_bin: # Same third-party UV-env isolation as the main update path (#83914): # a user-level UV_PYTHON_INSTALL_DIR / UV_PYTHON from unrelated # software must not steer which interpreter uv resolves here. from hermes_cli.managed_uv import managed_python_env uv_env = managed_python_env() uv_env["VIRTUAL_ENV"] = str(_m().PROJECT_ROOT / "venv") if _m()._is_termux_env(uv_env): uv_env.pop("PYTHONPATH", None) uv_env.pop("PYTHONHOME", None) try: _m()._install_python_dependencies_with_optional_fallback([uv_bin, "pip"], env=uv_env) except _shim_quarantine_error_type() as _sqe: # #87331: this runs inside the ZIP-fallback error handler, so the # boundary except clause in cmd_update cannot catch it — refuse # here with the same defer-via-marker contract. _refuse_update_for_contended_shims(_sqe) else: # Use sys.executable to explicitly call the venv's pip module, # avoiding PEP 668 'externally-managed-environment' errors on Debian/Ubuntu. # Some environments lose pip inside the venv; bootstrap it back with # ensurepip before trying the editable install. try: subprocess.run( pip_cmd + ["--version"], cwd=_m().PROJECT_ROOT, check=True, capture_output=True, ) except subprocess.CalledProcessError: subprocess.run( [_m().sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"], cwd=_m().PROJECT_ROOT, check=True, ) _m()._install_python_dependencies_with_optional_fallback(pip_cmd) install_prefix = [uv_bin, "pip"] if uv_bin else pip_cmd install_env = uv_env if uv_bin else None _m()._restore_active_tool_dependencies( active_tool_dependencies, install_prefix, env=install_env, ) # ZIP path parity: heal the active memory provider's bridge packages # after the dependency reinstall, same as the git-pull path (#53272, # #70636). _m()._refresh_active_memory_provider_dependencies() # Now that dependencies are installed, verify the tree actually imports. # The copy loop above replaces top-level entries one at a time in # os.listdir order, so an interruption between (say) `agent/` and `tools/` # leaves a tree whose files all parse but cannot be imported together — # the ImportError-on-startup class this guard exists to catch. Deliberately # placed *after* the dependency reinstall so a genuinely-new third-party # requirement isn't misreported as a partial copy. There is no SHA to roll # back to here, so surface it with a concrete recovery step rather than # reporting a successful update over a bricked install. import_ok, failing_module, import_error = _validate_critical_modules_import( _m().PROJECT_ROOT ) if not import_ok: print() print("✗ Update left the install in an unimportable state:") print(f" {failing_module}: {import_error}") print() print(" This usually means the copy was interrupted partway through.") print(" Re-run `hermes update` to complete it.") _m().sys.exit(1) node_failures = _update_node_dependencies() _m()._build_web_ui(_m().PROJECT_ROOT / "web") desktop_build_ok = _rebuild_desktop_after_update( _m().PROJECT_ROOT / "apps" / "desktop", had_desktop_app_before_update=had_desktop_app_before_update, ) # Sync skills try: from tools.skills_sync import sync_skills print("→ Syncing bundled skills...") result = sync_skills(quiet=True) if result["copied"]: print(f" + {len(result['copied'])} new: {', '.join(result['copied'])}") if result.get("updated"): print( f" ↑ {len(result['updated'])} updated: {', '.join(result['updated'])}" ) if result.get("user_modified"): print(f" ~ {len(result['user_modified'])} user-modified (kept)") print( " → see them: hermes skills list-modified " "(diff/reset to resume updates)" ) if result.get("cleaned"): print(f" − {len(result['cleaned'])} removed from manifest") if result.get("relocated"): print( f" → {len(result['relocated'])} moved to new upstream paths: " f"{', '.join(result['relocated'])}" ) if not result["copied"] and not result.get("updated"): print(" ✓ Skills are up to date") except Exception: pass # Seed the model-catalog disk cache from the freshly-unpacked checkout # (same rationale as the git-pull path in _cmd_update_impl). Non-fatal. try: from hermes_cli.model_catalog import seed_cache_from_checkout if seed_cache_from_checkout(_m().PROJECT_ROOT): print(" ✓ Model catalog cache refreshed from checkout") except Exception as e: logger.debug("Model catalog seed during zip update failed: %s", e) # ── Post-update state.db integrity guard (#68474, #97994) ──────────── # Verify state.db survived the ZIP update in the root home AND every # sibling profile, auto-restoring each from its own most recent valid # pre-update snapshot when needed. try: _verify_and_restore_state_dbs_post_update() except Exception as exc: logger.debug( "Post-update state.db integrity check (zip path) failed: %s", exc ) update_complete = _print_update_summary( node_failures=node_failures, desktop_build_ok=desktop_build_ok, pre_update_version=pre_update_version, ) try: _print_curator_first_run_notice() except Exception as e: logger.debug("Curator first-run notice failed: %s", e) try: _print_curator_recent_run_notice() except Exception as e: logger.debug("Curator recent-run notice failed: %s", e) # Don't stop a working dashboard when the Node refresh failed — see the # git-update path for rationale (#30271). _finish_dashboard_update_cleanup(node_failures) try: from hermes_cli.update_receipt import finalize_update_receipt finalize_update_receipt( "success" if update_complete and not node_failures else "partial" ) except Exception as _receipt_exc: logger.debug("Update receipt finalize (zip path) failed: %s", _receipt_exc) return update_complete def _stash_local_changes_if_needed(git_cmd: list[str], cwd: Path) -> Optional[str]: status = subprocess.run( git_cmd + ["status", "--porcelain"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", check=True, ) if not status.stdout.strip(): return None # If the index has unmerged entries (e.g. from an interrupted merge/rebase), # git stash will fail with "needs merge / could not write index". Clear the # conflict state with `git reset` so the stash can proceed. Working-tree # changes are preserved; only the index conflict markers are dropped. unmerged = subprocess.run( git_cmd + ["ls-files", "--unmerged"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if unmerged.stdout.strip(): print("→ Clearing unmerged index entries from a previous conflict...") subprocess.run(git_cmd + ["reset"], cwd=cwd, capture_output=True) from datetime import datetime, timezone stash_name = datetime.now(timezone.utc).strftime( f"{_AUTOSTASH_NAME_PREFIX}%Y%m%d-%H%M%S" ) print("→ Local changes detected — stashing before update...") prev_stash = subprocess.run( git_cmd + ["rev-parse", "--verify", "refs/stash"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ).stdout.strip() push = subprocess.run( git_cmd + ["stash", "push", "--include-untracked", "-m", stash_name], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if push.stdout.strip(): print(push.stdout.strip()) stash_probe = subprocess.run( git_cmd + ["rev-parse", "--verify", "refs/stash"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) stash_ref = stash_probe.stdout.strip() stash_created = ( stash_probe.returncode == 0 and bool(stash_ref) and stash_ref != prev_stash ) if push.returncode != 0: if stash_created: # git stash push exits non-zero when it saved everything but could # not delete some swept untracked files from the working tree # (e.g. a root-owned directory: "warning: failed to remove ...: # Permission denied"). The stash entry is complete — the changes # are safe — so this is not a failure. Leave the undeletable # files in place and continue the update. if push.stderr.strip(): print(push.stderr.strip()) print( " ⚠ Some untracked files could not be removed from the " "working tree (permission denied)." ) print( " They were still saved to the stash and were left in " "place — the update will continue." ) # A partially-failed stash push also aborts its working-tree # cleanup for TRACKED modifications — they are saved in the stash # but still dirty the tree, which would break the checkout/pull # that follows. Safe to reset: everything is in the stash entry. subprocess.run( git_cmd + ["reset", "--hard", "HEAD"], cwd=cwd, capture_output=True, ) else: # No stash entry was created: the changes were NOT saved. This # is a real failure — bail out before the update touches HEAD. print("✗ Could not stash local changes — update aborted.") if push.stderr.strip(): print(f" {push.stderr.strip().splitlines()[0]}") print( " Commit, stash, or clean up your local changes manually, " "then re-run `hermes update`." ) raise subprocess.CalledProcessError( push.returncode, push.args, output=push.stdout, stderr=push.stderr ) return stash_ref def _resolve_stash_selector( git_cmd: list[str], cwd: Path, stash_ref: str ) -> Optional[str]: stash_list = subprocess.run( git_cmd + ["stash", "list", "--format=%gd %H"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", check=True, ) for line in stash_list.stdout.splitlines(): selector, _, commit = line.partition(" ") if commit.strip() == stash_ref: return selector.strip() return None #: Producer/consumer contract for update autostash names: the stash subject is #: this prefix + a UTC YYYYMMDD-HHMMSS stamp (see _stash_local_changes_if_needed #: and _warn_orphaned_update_autostashes). _AUTOSTASH_NAME_PREFIX = "hermes-update-autostash-" #: Age past which a leftover ``hermes-update-autostash-*`` entry is called out #: at update time. Entries younger than this are normal (a parked stash from #: the desktop updater's --keep-stash run minutes ago); older ones are almost #: always forgotten (#63717 problem 6: an orphan persisted 9+ days unnoticed). _AUTOSTASH_WARN_AGE_DAYS = 7 def _warn_orphaned_update_autostashes(git_cmd: list[str], cwd: Path) -> int: """Surface leftover update autostashes older than the warn threshold. Autostash entries legitimately outlive an update run (``--keep-stash`` parks them; a conflicted or failed restore preserves them for safety), but nothing ever re-surfaces them afterwards — they sit in ``git stash`` invisibly for weeks (#63717 problem 6). This prints a short notice naming the stale entries with recovery/cleanup guidance. Deliberately NOT a GC: a stash entry can be the only copy of the user's uncommitted work, so Hermes never drops one automatically. Best-effort — any git failure returns 0 and must not block the update. Returns the number of stale entries warned about. """ from datetime import timedelta, timezone try: stash_list = subprocess.run( git_cmd + ["stash", "list", "--format=%gd %s"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if stash_list.returncode != 0: return 0 cutoff = datetime.now(timezone.utc) - timedelta( days=_AUTOSTASH_WARN_AGE_DAYS ) marker = _AUTOSTASH_NAME_PREFIX stale: list[tuple[str, str]] = [] for line in stash_list.stdout.splitlines(): selector, _, subject = line.strip().partition(" ") pos = subject.find(marker) if pos < 0: continue stamp = subject[pos + len(marker):][:15] # "YYYYMMDD-HHMMSS" try: stash_time = datetime.strptime(stamp, "%Y%m%d-%H%M%S").replace( tzinfo=timezone.utc ) except ValueError: # Unparseable name — age unknown; leave it alone rather than # guess (same posture as _prune_orphan_rescue_refs). continue if stash_time < cutoff: stale.append((selector, stamp)) if not stale: return 0 print() print( f"⚠ {len(stale)} leftover update autostash entr" f"{'y is' if len(stale) == 1 else 'ies are'} more than " f"{_AUTOSTASH_WARN_AGE_DAYS} days old:" ) for selector, stamp in stale: print(f" {selector} ({_AUTOSTASH_NAME_PREFIX}{stamp})") print(" These hold local changes stashed by earlier updates and never") print(" restored. Review with: git stash show -p ") print(" Restore with: git stash apply Discard with: git stash drop ") return len(stale) except Exception as exc: logger.debug("Autostash age check failed: %s", exc) return 0 def _print_stash_cleanup_guidance( stash_ref: str, stash_selector: Optional[str] = None ) -> None: print( " Check `git status` first so you don't accidentally reapply the same change twice." ) print(" Find the saved entry with: git stash list --format='%gd %H %s'") if stash_selector: print(f" Remove it with: git stash drop {stash_selector}") else: print( f" Look for commit {stash_ref}, then drop its selector with: git stash drop stash@{{N}}" ) def _stash_apply_failed_only_on_existing_untracked(stderr: str) -> bool: """True when a ``git stash apply`` failure is ONLY about untracked files that already exist in the working tree. This is the tail end of the permission-denied autostash class: ``git stash push --include-untracked`` swept undeletable files (e.g. a root-owned ``packaging/`` directory) into the stash but could not remove them from disk. On restore, git applies all tracked changes, then refuses to overwrite those still-present files (``already exists, no checkout`` / ``could not restore untracked files from stash``) and exits non-zero even though nothing was lost. Any other error line (e.g. ``would be overwritten by merge`` / ``Aborting``) means the tracked apply itself failed and this returns False. """ lines = [ln.strip() for ln in (stderr or "").splitlines() if ln.strip()] if not lines: return False saw_untracked_error = False for ln in lines: if "already exists, no checkout" in ln: saw_untracked_error = True elif "could not restore untracked files from stash" in ln: saw_untracked_error = True elif ln.startswith(("warning:", "hint:")): continue else: return False return saw_untracked_error def _park_stashed_changes(stash_ref: str) -> None: """Leave a pre-update autostash parked instead of re-applying it. Used by ``hermes update --keep-stash`` (the desktop updater's mode): the stash made the update possible on a dirty tree, but local source edits must never be silently re-applied onto the updated code. Nothing is lost — the entry stays in ``git stash`` with printed recovery guidance. """ print() print("ℹ️ Local changes were stashed before updating and were NOT re-applied (--keep-stash).") print(f" Stash ref: {stash_ref}") print(f" Restore manually with: git stash apply {stash_ref}") def _git_untracked_paths(git_cmd: list[str], cwd: Path) -> set[str] | None: """Return untracked paths, or ``None`` when Git cannot enumerate them.""" try: result = subprocess.run( git_cmd + ["ls-files", "--others", "--exclude-standard", "-z"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="surrogateescape", ) except (OSError, subprocess.SubprocessError): result = None if result is None or result.returncode != 0: print( " ⚠ Could not enumerate untracked files while validating the " "restored stash." ) return None return {path for path in result.stdout.split("\0") if path} def _restored_python_paths( git_cmd: list[str], cwd: Path ) -> tuple[str, ...] | None: """Return restored ``.py`` paths changed from ``HEAD``. This deliberately validates Python source only; non-Python entry scripts remain outside the executable import-health check. """ try: changed = subprocess.run( git_cmd + ["diff", "--name-only", "-z", "HEAD", "--", "*.py"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="surrogateescape", ) except (OSError, subprocess.SubprocessError): changed = None if changed is None or changed.returncode != 0: print(" ⚠ Could not enumerate tracked Python files restored from the stash.") return None paths = set(changed.stdout.split("\0")) untracked = _git_untracked_paths(git_cmd, cwd) if untracked is None: return None paths.update(path for path in untracked if path.endswith(".py")) paths.discard("") return tuple(sorted(paths)) def _reject_unsafe_stash_restore( git_cmd: list[str], cwd: Path, stash_ref: str, preexisting_untracked: set[str], failing_target: str, detail: str | None, ) -> None: """Restore the clean updated tree, preserve the stash, and abort the update.""" print() print("✗ Restored local changes made the Hermes agent unexecutable.") print(f" Health check failed: {failing_target}") if detail: for line in str(detail).splitlines()[:6]: print(f" {line}") current_untracked = _git_untracked_paths(git_cmd, cwd) restored_untracked = ( current_untracked - preexisting_untracked if current_untracked is not None else set() ) try: reset = subprocess.run( git_cmd + ["reset", "--hard", "HEAD"], cwd=cwd, capture_output=True ) except (OSError, subprocess.SubprocessError): reset = None clean = None if restored_untracked: try: clean = subprocess.run( git_cmd + ["clean", "-fd", "--", *sorted(restored_untracked)], cwd=cwd, capture_output=True, ) except (OSError, subprocess.SubprocessError): clean = None cleanup_ok = ( current_untracked is not None and reset is not None and reset.returncode == 0 and (not restored_untracked or (clean is not None and clean.returncode == 0)) ) if cleanup_ok: try: verify = subprocess.run( git_cmd + ["diff", "--quiet", "HEAD", "--"], cwd=cwd, capture_output=True, ) cleanup_ok = verify.returncode == 0 except (OSError, subprocess.SubprocessError): cleanup_ok = False if cleanup_ok: print(" The clean updated tree has been restored; the gateway was not restarted.") else: print(" ⚠ The clean updated tree could not be fully restored automatically.") print(" Inspect `git status` and run `git reset --hard HEAD` before retrying.") print(" Platform connectivity alone does not mean the agent can execute turns.") print(f" Your local changes remain preserved in stash: {stash_ref}") print(f" Inspect them with: git stash show --stat {stash_ref}") print(f" Restore manually after fixing them: git stash apply {stash_ref}") raise SystemExit(1) def _restore_stashed_changes( git_cmd: list[str], cwd: Path, stash_ref: str, prompt_user: bool = False, input_fn=None, ) -> bool: if prompt_user: remote_prompt = input_fn is not None prompt_suffix = "[y/N]" if remote_prompt else "[Y/n]" print() print("⚠ Local changes were stashed before updating.") print( " Restoring them may reapply local customizations onto the updated codebase." ) print(" Review the result afterward if Hermes behaves unexpectedly.") print(f"Restore local changes now? {prompt_suffix}") if input_fn is not None: response = input_fn(f"Restore local changes now? {prompt_suffix}", "n") else: try: response = input().strip().lower() except (EOFError, UnicodeDecodeError): # Mirror the config-migration prompt's fix: don't let a # terminal-encoding issue or a closed stdin crash the # update mid-restore. Falls through to the existing # skip-restore path below, which already explains how to # restore manually from git stash. response = "n" accepted = response in {"y", "yes"} or (not remote_prompt and response == "") if not accepted: print("Skipped restoring local changes.") print("Your changes are still preserved in git stash.") print(f"Restore manually with: git stash apply {stash_ref}") return False preexisting_untracked = _git_untracked_paths(git_cmd, cwd) if preexisting_untracked is None: print(" The stash was not restored because its cleanup baseline is unknown.") print(f" Restore manually with: git stash apply {stash_ref}") return False clean_import_failures = _critical_module_import_failures( cwd, report_runtime_errors=True ) print("→ Restoring local changes...") restore = subprocess.run( git_cmd + ["stash", "apply", stash_ref], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) # Check for unmerged (conflicted) files — can happen even when returncode is 0 unmerged = subprocess.run( git_cmd + ["diff", "--name-only", "--diff-filter=U"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) has_conflicts = bool(unmerged.stdout.strip()) if restore.returncode != 0 and not has_conflicts and ( _stash_apply_failed_only_on_existing_untracked(restore.stderr) ): # Permission-denied autostash tail end: the tracked changes applied # cleanly; the only "failure" is untracked files that never left the # working tree (git could not delete them at stash time, so it now # refuses to overwrite them). Their content was never touched — # nothing is lost. Treat as restored. print( " ⚠ Some stashed untracked files already exist in the working " "tree and were kept as-is." ) elif restore.returncode != 0 or has_conflicts: print("✗ Update pulled new code, but restoring local changes hit conflicts.") if restore.stdout.strip(): print(restore.stdout.strip()) if restore.stderr.strip(): print(restore.stderr.strip()) # Show which files conflicted conflicted_files = unmerged.stdout.strip() if conflicted_files: print("\nConflicted files:") for f in conflicted_files.splitlines(): print(f" • {f}") print("\nYour stashed changes are preserved — nothing is lost.") print(f" Stash ref: {stash_ref}") # Always reset to clean state — leaving conflict markers in source # files makes hermes completely unrunnable (SyntaxError on import). # The user's changes are safe in the stash for manual recovery. subprocess.run( git_cmd + ["reset", "--hard", "HEAD"], cwd=cwd, capture_output=True, ) print("Working tree reset to clean state.") print(f"Restore your changes later with: git stash apply {stash_ref}") # Don't sys.exit — the code update itself succeeded, only the stash # restore had conflicts. Let cmd_update continue with pip install, # skill sync, and gateway restart. return False restored_python = _restored_python_paths(git_cmd, cwd) if restored_python is None: _reject_unsafe_stash_restore( git_cmd, cwd, stash_ref, preexisting_untracked, "restored Python source discovery", "could not determine which restored Python files require validation", ) syntax_ok, failing_path, syntax_error = _validate_python_files_syntax( cwd, restored_python ) if not syntax_ok: _reject_unsafe_stash_restore( git_cmd, cwd, stash_ref, preexisting_untracked, failing_path or "restored Python source", syntax_error, ) restored_import_failures = _critical_module_import_failures( cwd, report_runtime_errors=True ) changed_import_failure = next( ( (module, error) for module, error in restored_import_failures.items() if clean_import_failures.get(module) != error ), None, ) if changed_import_failure is not None: failing_module, import_error = changed_import_failure _reject_unsafe_stash_restore( git_cmd, cwd, stash_ref, preexisting_untracked, f"agent import {failing_module or 'unknown'}", import_error[1], ) stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref) if stash_selector is None: print( "⚠ Local changes were restored, but Hermes couldn't find the stash entry to drop." ) print( " The stash was left in place. You can remove it manually after checking the result." ) _print_stash_cleanup_guidance(stash_ref) else: drop = subprocess.run( git_cmd + ["stash", "drop", stash_selector], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if drop.returncode != 0: print( "⚠ Local changes were restored, but Hermes couldn't drop the saved stash entry." ) if drop.stdout.strip(): print(drop.stdout.strip()) if drop.stderr.strip(): print(drop.stderr.strip()) print( " The stash was left in place. You can remove it manually after checking the result." ) _print_stash_cleanup_guidance(stash_ref, stash_selector) print("⚠ Local changes were restored on top of the updated codebase.") print(" Review `git diff` / `git status` if Hermes behaves unexpectedly.") return True def _discard_stashed_changes( git_cmd: list[str], cwd: Path, stash_ref: str, ) -> bool: """Throw away a stash created before an update, without applying it. Used only on a NON-interactive update when the user has set ``updates.non_interactive_local_changes: discard`` — i.e. they've opted out of keeping local source edits on this machine. Drops the stash entry instead of re-applying it, so the working tree stays clean at the freshly pulled HEAD. Unlike ``git reset --hard`` + ``git clean -fd``, this only affects what was stashed (tracked changes + the untracked files we explicitly captured) — ignored paths like node_modules/venv/build outputs are never touched, since they were never stashed. Returns True if the stash was dropped, False on a git failure (in which case the stash is left in place for safety). """ stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref) if stash_selector is None: print( "⚠ Configured to discard local changes on non-interactive update, " "but Hermes couldn't find the stash entry to drop." ) _print_stash_cleanup_guidance(stash_ref) return False drop = subprocess.run( git_cmd + ["stash", "drop", stash_selector], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if drop.returncode != 0: print( "⚠ Configured to discard local changes, but Hermes couldn't drop " "the saved stash entry." ) if drop.stderr.strip(): print(f" {drop.stderr.strip().splitlines()[0]}") _print_stash_cleanup_guidance(stash_ref, stash_selector) return False print("→ Discarded local source changes (updates.non_interactive_local_changes=discard).") return True OFFICIAL_REPO_URLS = { "https://github.com/NousResearch/hermes-agent.git", "git@github.com:NousResearch/hermes-agent.git", "https://github.com/NousResearch/hermes-agent", "git@github.com:NousResearch/hermes-agent", } OFFICIAL_REPO_URL = "https://github.com/NousResearch/hermes-agent.git" SKIP_UPSTREAM_PROMPT_FILE = ".skip_upstream_prompt" def _get_origin_url(git_cmd: list[str], cwd: Path) -> Optional[str]: """Get the URL of the origin remote, or None if not set.""" try: result = subprocess.run( git_cmd + ["remote", "get-url", "origin"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if result.returncode == 0: return result.stdout.strip() except Exception: pass return None def _is_fork(origin_url: Optional[str]) -> bool: """Check if the origin remote points to a fork (not the official repo).""" if not origin_url: return False # Normalize URL for comparison (strip trailing .git if present) normalized = origin_url.rstrip("/") if normalized.endswith(".git"): normalized = normalized[:-4] for official in OFFICIAL_REPO_URLS: official_normalized = official.rstrip("/") if official_normalized.endswith(".git"): official_normalized = official_normalized[:-4] if normalized == official_normalized: return False return True def _has_upstream_remote(git_cmd: list[str], cwd: Path) -> bool: """Check if an 'upstream' remote already exists.""" try: result = subprocess.run( git_cmd + ["remote", "get-url", "upstream"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) return result.returncode == 0 except Exception: return False def _add_upstream_remote(git_cmd: list[str], cwd: Path) -> bool: """Add the official repo as the 'upstream' remote. Returns True on success.""" try: result = subprocess.run( git_cmd + ["remote", "add", "upstream", OFFICIAL_REPO_URL], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) return result.returncode == 0 except Exception: return False def _count_commits_between(git_cmd: list[str], cwd: Path, base: str, head: str) -> int: """Count commits on `head` that are not on `base`. Returns -1 on error.""" try: result = subprocess.run( git_cmd + ["rev-list", "--count", f"{base}..{head}"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if result.returncode == 0: return int(result.stdout.strip()) except Exception: pass return -1 def _should_skip_upstream_prompt() -> bool: """Check if user previously declined to add upstream.""" from hermes_constants import get_hermes_home return (get_hermes_home() / SKIP_UPSTREAM_PROMPT_FILE).exists() def _mark_skip_upstream_prompt(): """Create marker file to skip future upstream prompts.""" try: from hermes_constants import get_hermes_home (get_hermes_home() / SKIP_UPSTREAM_PROMPT_FILE).touch() except Exception: pass def _sync_fork_with_upstream(git_cmd: list[str], cwd: Path) -> bool: """Attempt to push updated main to origin (sync fork). Returns True if push succeeded, False otherwise. """ try: result = subprocess.run( git_cmd + ["push", "origin", "main", "--force-with-lease"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", **_no_prompt_git_kwargs(), ) return result.returncode == 0 except Exception: return False def _sync_with_upstream_if_needed( git_cmd: list[str], cwd: Path, *, assume_yes: bool = False, input_fn=None, ) -> bool: """Check if fork is behind upstream and sync if safe. This implements the fork upstream sync logic: - If upstream remote doesn't exist, ask user if they want to add it - Compare origin/main with upstream/main - If origin/main is strictly behind upstream/main, pull from upstream - Try to sync fork back to origin if possible Returns True when origin/main was actually verified against the official upstream/main, False when the check never happened (prompt skipped or declined, remote add failed, fetch or compare failed) so the caller can avoid reporting the checkout as up to date on the strength of an origin comparison alone (#97052 review). """ has_upstream = _has_upstream_remote(git_cmd, cwd) if not has_upstream: # Check if user previously declined if _should_skip_upstream_prompt(): return False print() print("ℹ Your fork is not tracking the official Hermes repository.") print(" This means you may miss updates from NousResearch/hermes-agent.") print() if assume_yes or ( input_fn is None and not (sys.stdin.isatty() and sys.stdout.isatty()) ): # --yes means "don't block", not "mutate my git remotes". Skip # without persisting the decline so interactive runs still get asked. print(" Skipping upstream setup (non-interactive run).") print( " Add it later with: git remote add upstream https://github.com/NousResearch/hermes-agent.git" ) return False # Ask user if they want to add upstream if input_fn is not None: response = ( input_fn("Add official repo as 'upstream' remote? [y/N]", "n") .strip() .lower() ) else: try: response = ( input("Add official repo as 'upstream' remote? [Y/n]: ") .strip() .lower() ) except (EOFError, KeyboardInterrupt, UnicodeDecodeError): print() response = "n" if response in {"", "y", "yes"}: print("→ Adding upstream remote...") if _add_upstream_remote(git_cmd, cwd): print( " ✓ Added upstream: https://github.com/NousResearch/hermes-agent.git" ) has_upstream = True else: print(" ✗ Failed to add upstream remote. Skipping upstream sync.") return False else: print( " Skipped. Run 'git remote add upstream https://github.com/NousResearch/hermes-agent.git' to add later." ) _mark_skip_upstream_prompt() return False # Fetch upstream main only. This sync compares upstream/main with # origin/main, so there's no reason to pull every upstream ref — and a bare # fetch drags in thousands of auto-generated branches. print() print("→ Fetching upstream...") try: subprocess.run( git_cmd + ["fetch", "upstream", "main", "--quiet"], cwd=cwd, capture_output=True, check=True, **_no_prompt_git_kwargs(), ) except subprocess.CalledProcessError: print(" ✗ Failed to fetch upstream. Skipping upstream sync.") return False # Compare origin/main with upstream/main origin_ahead = _count_commits_between(git_cmd, cwd, "upstream/main", "origin/main") upstream_ahead = _count_commits_between( git_cmd, cwd, "origin/main", "upstream/main" ) if origin_ahead < 0 or upstream_ahead < 0: print(" ✗ Could not compare branches. Skipping upstream sync.") return False # If origin/main has commits not on upstream, don't trample if origin_ahead > 0: print() print(f"ℹ Your fork has {origin_ahead} commit(s) not on upstream.") print(" Skipping upstream sync to preserve your changes.") print(" If you want to merge upstream changes, run:") print(" git pull upstream main") return True # If upstream is not ahead, fork is up to date if upstream_ahead == 0: print(" ✓ Fork is up to date with upstream") return True # origin/main is strictly behind upstream/main (can fast-forward) print() print(f"→ Fork is {upstream_ahead} commit(s) behind upstream") print("→ Pulling from upstream...") try: subprocess.run( git_cmd + ["pull", "--ff-only", "upstream", "main"], cwd=cwd, check=True, **_no_prompt_git_kwargs(), ) except subprocess.CalledProcessError: print( " ✗ Failed to pull from upstream. You may need to resolve conflicts manually." ) return False print(" ✓ Updated from upstream") # Try to sync fork back to origin print("→ Syncing fork...") if _sync_fork_with_upstream(git_cmd, cwd): print(" ✓ Fork synced with upstream") else: print( " ℹ Got updates from upstream but couldn't push to fork (no write access?)" ) print(" Your local repo is updated, but your fork on GitHub may be behind.") return True def _invalidate_update_cache(): """Delete the update-check cache for ALL profiles so no banner reports a stale "commits behind" count after a successful update. The git repo is shared across profiles — when one profile runs ``hermes update``, every profile is now current. """ homes = [] # Default profile home (Docker-aware — uses /opt/data in Docker) from hermes_constants import get_default_hermes_root default_home = get_default_hermes_root() homes.append(default_home) # Named profiles under /profiles/ profiles_root = default_home / "profiles" if profiles_root.is_dir(): for entry in profiles_root.iterdir(): if entry.is_dir(): homes.append(entry) for home in homes: try: cache_file = home / ".update_check" if cache_file.exists(): cache_file.unlink() except Exception: pass def _write_marker_file(path: Path, *, label: str) -> None: """Drop an update-recovery breadcrumb. Never raises.""" if _m()._pytest_owns_live_checkout(path.parent): logger.debug("Skipping %s marker under pytest (live checkout)", label) return try: path.write_text( f"started={_time.time()}\npid={os.getpid()}\n", encoding="utf-8" ) except OSError as exc: logger.debug("Could not write %s marker: %s", label, exc) def _write_update_incomplete_marker() -> None: """Drop the interrupted core-install breadcrumb. Never raises.""" _write_marker_file(_m()._update_marker_path(), label="update-incomplete") def _write_lazy_refresh_incomplete_marker() -> None: """Drop the interrupted lazy-refresh breadcrumb. Never raises.""" _write_marker_file(_m()._lazy_refresh_marker_path(), label="lazy-refresh-incomplete") # ``fleet_restart_pending`` lives under HERMES_HOME (not next to the venv). # The existing ``.update-incomplete`` / ``.lazy-refresh-incomplete`` markers # gate dependency/venv repair; this one is the fleet-restart obligation after # a git pull that advanced HEAD (#95294). Cleared only when the restart phase # completes or there were no running services to restart. _FLEET_RESTART_PENDING_NAME = "fleet_restart_pending" def _fleet_restart_pending_marker_path() -> Path: """HERMES_HOME breadcrumb for a pull that has not yet restarted the fleet.""" return get_hermes_home() / _FLEET_RESTART_PENDING_NAME def _write_fleet_restart_pending_marker(*, expected_sha: str = "") -> None: """Drop the pull→restart obligation breadcrumb. Never raises.""" path = _fleet_restart_pending_marker_path() if _m()._pytest_owns_live_checkout(path.parent): logger.debug("Skipping fleet-restart-pending marker under pytest (live checkout)") return try: lines = [f"started={_time.time()}", f"pid={os.getpid()}"] if expected_sha: lines.append(f"expected_sha={expected_sha}") path.write_text("\n".join(lines) + "\n", encoding="utf-8") except OSError as exc: logger.debug("Could not write fleet-restart-pending marker: %s", exc) def _clear_fleet_restart_pending_marker() -> None: """Remove the pull→restart obligation breadcrumb. Never raises.""" _m()._clear_marker_file( _fleet_restart_pending_marker_path(), label="fleet-restart-pending" ) def _current_checkout_sha() -> str | None: """Current on-disk checkout HEAD, or None if it cannot be resolved.""" try: from hermes_cli.build_info import get_code_identity sha = (get_code_identity(refresh=True) or {}).get("sha") return str(sha) if sha else None except Exception: return _capture_head_sha(["git"], _m().PROJECT_ROOT) def _receipt_looks_unfinished(receipt: dict) -> bool: """True when *receipt* is from an update that did not finish cleanly.""" if receipt.get("stop_reason"): return True exit_code = receipt.get("exit_code") if exit_code not in (0, None): return True outcome = receipt.get("outcome") if outcome in ("failed", "partial", "running"): return True gateway_restart = receipt.get("gateway_restart") if isinstance(gateway_restart, dict) and gateway_restart.get("incomplete"): return True return False def _receipt_reports_stale_runtime(expected_sha: str | None = None) -> bool: """True when ``update_receipts/latest.json`` records a runtime SHA skew. ``plan.runtimes[].code_sha`` is captured *before* the pull of that run, so a successful update's receipt always shows pre-update runtime SHAs. Those must not retrigger a restart on the next invocation. Use the post-restart ``fleet`` matrix when present; fall back to the plan only for an unfinished receipt (interrupt / failed / incomplete restart) — the #95294 smoking-gun shape. """ try: from hermes_cli.update_receipt import read_latest_receipt receipt = read_latest_receipt() except Exception: receipt = None if not isinstance(receipt, dict): return False if not expected_sha: expected_sha = _current_checkout_sha() if not expected_sha: return False def _sha_mismatch(code_sha) -> bool: return bool(code_sha) and str(code_sha) != str(expected_sha) fleet = receipt.get("fleet") if isinstance(fleet, list) and fleet: for entry in fleet: if not isinstance(entry, dict): continue if entry.get("state") == "stale": return True if _sha_mismatch(entry.get("code_sha")): return True return False if not _receipt_looks_unfinished(receipt): return False plan = receipt.get("plan") if not isinstance(plan, dict): return False for runtime in plan.get("runtimes") or []: if isinstance(runtime, dict) and _sha_mismatch(runtime.get("code_sha")): return True return False def _pending_fleet_restart_needed() -> bool: """True when a prior pull still owes the fleet a restart (#95294).""" try: if _fleet_restart_pending_marker_path().is_file(): return True except OSError: pass return _receipt_reports_stale_runtime() def _warn_pending_fleet_restart(*, startup: bool = False) -> None: """Print the specific interrupted-update fleet-restart warning.""" stream = sys.stderr if startup else sys.stdout print( "⚠ A previous `hermes update` pulled new code but did not " "restart running gateways.", file=stream, ) print( " Gateways may still be serving pre-update modules (mixed sys.modules).", file=stream, ) if startup: print( " Run `hermes update` or `hermes gateway restart`.", file=stream, ) def _warn_pending_fleet_restart_on_startup() -> None: """Cheap CLI-startup hint. Never restarts; never raises.""" try: if not _pending_fleet_restart_needed(): return _warn_pending_fleet_restart(startup=True) except Exception: pass def _restart_systemd_gateway_units_best_effort(failed: list) -> None: """Best-effort ``systemctl restart`` of every hermes-gateway/serve unit.""" for scope, scope_cmd in ( ("user", ["systemctl", "--user"]), ("system", ["systemctl"]), ): try: result = subprocess.run( scope_cmd + [ "list-units", "hermes-gateway*", "hermes-serve*", "--plain", "--no-legend", "--no-pager", ], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, ) except (FileNotFoundError, subprocess.TimeoutExpired): continue if result.returncode != 0: continue def process_unit(svc_name: str, _scope=scope, _cmd=scope_cmd) -> None: restart_cmd = list(_cmd) + ["--no-ask-password", "restart", svc_name] if ( _scope == "system" and hasattr(os, "geteuid") and os.geteuid() != 0 # windows-footgun: ok — systemd path, Linux-only ): restart_cmd = ["sudo", "-n"] + restart_cmd subprocess.run( restart_cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, ) def on_timeout(svc_name: str, exc: subprocess.TimeoutExpired) -> None: failed.append(svc_name) _for_each_systemd_gateway_unit( result.stdout, process_unit=process_unit, on_unit_timeout=on_timeout, ) def _run_pending_fleet_restart() -> bool: """Catch-up restart for gateways left on pre-update code (#95294). Returns True when restart completed or no services were running. Returns False if restart was incomplete. Never raises. """ print("→ Restarting gateways left on pre-update code...") try: _m()._purge_stale_hermes_modules() except Exception: pass try: from hermes_cli.gateway import ( find_gateway_pids, is_macos, is_windows, kill_gateway_processes, supports_systemd_services, _wait_for_gateway_exit, ) except Exception as exc: _warn_gateway_restart_phase_aborted(exc, None) return False try: pids = list(find_gateway_pids(all_profiles=True)) except Exception as exc: logger.debug("Pending fleet restart: gateway probe failed: %s", exc) pids = None if pids == []: print(" ✓ No running gateways — nothing to restart.") return True failed: list = [] try: if supports_systemd_services(): _restart_systemd_gateway_units_best_effort(failed) if is_macos(): restarted: list = [] try: _restart_macos_launchd_gateways(restarted, failed, 45.0) except Exception as exc: logger.debug("Pending fleet restart: launchd failed: %s", exc) failed.append("launchd") if is_windows(): try: from hermes_cli import gateway_windows if gateway_windows.is_installed(): gateway_windows.restart() except Exception as exc: logger.debug("Pending fleet restart: Windows failed: %s", exc) failed.append("windows-gateway") leftover: list = [] try: leftover = list(find_gateway_pids(all_profiles=True)) except Exception: leftover = list(pids or []) if leftover: try: kill_gateway_processes(all_profiles=True) _wait_for_gateway_exit(timeout=5.0, force_after=None) except Exception as exc: logger.debug("Pending fleet restart: PID stop failed: %s", exc) if failed: _warn_incomplete_gateway_fleet_restart(failed) return False print(" ✓ Pending fleet restart completed.") return True except Exception as exc: surviving = None try: surviving = list(find_gateway_pids(all_profiles=True)) except Exception: surviving = pids _warn_gateway_restart_phase_aborted(exc, surviving) return False def _apply_pending_fleet_restart_catchup() -> None: """On an already-up-to-date ``hermes update``, finish a skipped restart. No-op when nothing is pending. Exits 1 when the catch-up restart is incomplete so automation does not treat the fleet as healthy. """ if not _pending_fleet_restart_needed(): return print() _warn_pending_fleet_restart() print("→ Running the pending fleet restart...") if _run_pending_fleet_restart(): _clear_fleet_restart_pending_marker() return print(" ⚠ Fleet restart incomplete. Recover with: hermes gateway restart") sys.exit(1) def _format_concurrent_instances_message( matches: list[tuple[int, str]], scripts_dir: Path ) -> str: """Build a human-readable explanation + remediation hint for the user.""" shim = scripts_dir / "hermes.exe" lines = ["✗ Another hermes.exe is running:"] for pid, name in matches: lines.append(f" PID {pid} {name}") lines.append("") lines.append(f" Updating now would fail to overwrite {shim} because") lines.append(" Windows blocks REPLACE on a running executable.") lines.append("") lines.append(" Close Hermes Desktop, exit any open `hermes` REPLs, and") lines.append(" stop the gateway (`hermes gateway stop`) before retrying.") lines.append("") if matches: pid_args = " ".join(f"/PID {pid}" for pid, _ in matches) lines.append(" If you've already closed everything and these PIDs are") lines.append(" stale, terminate them directly, then retry the update:") lines.append(f" taskkill {pid_args} /F") lines.append("") lines.append(" Override with `hermes update --force` if you've already") lines.append(" confirmed those processes will not write to the venv.") return "\n".join(lines) def _classify_concurrent_instance(pid: int) -> str: """Return ``"gateway"`` when ``pid``'s command line is a gateway runtime. Delegates to ``_is_pausable_gateway`` — the same canonical ``gateway run`` matcher (``gateway.status.looks_like_gateway_command_line``, shlex-tokenized, profile-selector aware) used by the Desktop preflight exemption and the venv-holder guard fallback — so a PID classified as ``"gateway"`` here is exactly the set the pause/kill+restart machinery downstream will stop. That symmetry is what lets the pre-update concurrent gate skip the abort for gateway-only matches: the gateway is going to be stopped by ``_pause_windows_gateways_for_update()`` moments later anyway, so refusing the update just to make the user kill it manually is friction without benefit. Returns ``"non-gateway"`` when the cmdline doesn't match, and ``"unknown"`` when psutil can't read it (process gone, access denied, psutil missing). The gate treats ``"unknown"`` as non-gateway — we'd rather block an update we could have completed than proceed against a process we couldn't positively identify as a gateway. """ try: import psutil # noqa: PLC0415 except Exception: return "unknown" try: proc = psutil.Process(int(pid)) cmdline_list = proc.cmdline() except Exception: return "unknown" from hermes_cli._scan_venv_blockers import _is_pausable_gateway # noqa: PLC0415 cmdline = " ".join(cmdline_list or []) if _is_pausable_gateway(cmdline): return "gateway" return "non-gateway" def _filter_non_gateway_concurrent_instances( matches: list[tuple[int, str]], ) -> list[tuple[int, str]]: """Return only the concurrent-instance matches that are NOT the gateway. Used by the pre-update concurrent gate to decide whether to abort ``hermes update``. If every concurrent instance is a gateway, the pause machinery (``_pause_windows_gateways_for_update``) and the post-update kill+restart block handle it — the update proceeds. If anything else (a TUI shell, a Hermes Desktop backend child, an unrelated ``hermes`` REPL) is in the list, the gate still aborts with the existing message, since those have no pause machinery downstream. """ non_gateway: list[tuple[int, str]] = [] for pid, name in matches: if _classify_concurrent_instance(pid) != "gateway": non_gateway.append((pid, name)) return non_gateway def _upgrade_pip_before_lazy_refresh( install_cmd_prefix: list[str], *, env: dict[str, str] | None = None, ) -> None: """Upgrade pip before lazy-backend refreshes. Older pip (e.g. 24.0 on Python 3.11) can fail setuptools-backed source builds during lazy installs and leave a partially-written venv (#57828). Never raises. """ try: _m()._run_package_only_install( install_cmd_prefix + ["install", "--upgrade", "pip"], env=env, ) except subprocess.CalledProcessError as exc: logger.debug("pip upgrade before lazy refresh failed: %s", exc) def _capture_active_lazy_features() -> list[str]: """Snapshot active lazy backends before a managed runtime is replaced.""" try: from tools import lazy_deps return lazy_deps.active_features() except Exception as exc: logger.debug("Could not snapshot active lazy features: %s", exc) return [] def _capture_active_tool_dependencies() -> list[str]: """Snapshot Python dependencies installed explicitly through ``hermes tools``.""" try: from hermes_cli import tools_config return tools_config.active_restorable_python_tool_dependencies() except Exception as exc: logger.debug("Could not snapshot active Hermes Tools dependencies: %s", exc) return [] def _restore_active_tool_dependencies( dependencies: list[str], install_cmd_prefix: list[str], *, env: dict[str, str] | None = None, ) -> None: """Restore allowlisted ``hermes tools`` dependencies into a rebuilt venv. The dependency names came from a pre-rebuild import probe and are resolved through a static package allowlist. Never raises: a failed optional tool must not block the core update, but the user must be told what stayed unavailable. """ if not dependencies: return try: from hermes_cli import tools_config except Exception as exc: logger.debug("Hermes Tools dependency restore skipped (import failed): %s", exc) return target_python = _m()._resolve_install_target_python(install_cmd_prefix, env) missing: list[tuple[str, tuple[str, ...]]] = [] for name in dependencies: spec = tools_config.restorable_python_tool_dependency(name) if spec is None: continue module_name, install_args = spec if target_python is not None: try: probe = subprocess.run( [ str(target_python), "-c", "import importlib.util,sys; " "raise SystemExit(0 if importlib.util.find_spec(sys.argv[1]) else 1)", module_name, ], capture_output=True, env=env, check=False, ) if probe.returncode == 0: continue except (subprocess.SubprocessError, OSError): # An indeterminate probe is safer to repair than to treat as # proof that a pre-rebuild dependency survived. pass missing.append((name, install_args)) if not missing: return print() print(f"→ Restoring {len(missing)} Hermes Tools dependency set(s)...") restored: list[str] = [] failed: list[tuple[str, str]] = [] for name, install_args in missing: try: _m()._run_package_only_install( install_cmd_prefix + ["install", *install_args, "--quiet"], env=env, ) restored.append(name) except Exception as exc: # This is best-effort recovery for optional tooling. Unexpected # installer failures must be surfaced without aborting the core # runtime update. failed.append((name, str(exc))) if restored: print(f" ✓ {len(restored)} restored: {', '.join(restored)}") for name, reason in failed: if len(reason) > 200: reason = reason[:200] + "..." print(f" ⚠ {name} failed to restore: {reason}") def _refresh_active_lazy_features( install_cmd_prefix: list[str] | None = None, *, env: dict[str, str] | None = None, features: list[str] | None = None, ) -> bool: """Refresh lazy-installed backends after a code update. When pyproject.toml's ``[all]`` extra was slimmed down (May 2026), most optional backends moved to ``tools/lazy_deps.py`` and only install on first use. ``hermes update`` runs ``uv pip install -e .[all]`` which leaves those packages untouched — so if we bump a pin in :data:`LAZY_DEPS` (CVE response, transitive bug fix), users who already activated the backend keep the stale version forever. This function asks lazy_deps which features the user has previously activated and reinstalls them under the current pins. Features the user never enabled stay quiet — no churn for cold backends. Returns True when the venv is safe to use (refresh succeeded, or no active lazy backends, or post-failure import repair succeeded). Returns False when a failed lazy install left broken core imports that automatic repair could not fix (#57828). Never raises. A failure here must not block the rest of the update. """ try: from tools import lazy_deps except Exception as exc: logger.debug("Lazy refresh skipped (import failed): %s", exc) return True if features is None: try: active = lazy_deps.active_features() except Exception as exc: logger.debug("Lazy refresh skipped (active_features failed): %s", exc) return True else: active = features if not active: return True print() print(f"→ Refreshing {len(active)} active lazy backend(s)...") unexpected_failure = False try: if features is None: results = lazy_deps.refresh_active_features(prompt=False) else: results = lazy_deps.restore_features(active) except Exception as exc: # refresh_active_features is documented as never-raise, but defend # the update flow against future regressions. print(f" ⚠ Lazy refresh failed unexpectedly: {exc}") results = {} unexpected_failure = True refreshed = [f for f, s in results.items() if s in {"refreshed", "restored"}] current = [f for f, s in results.items() if s == "current"] failed = [(f, s) for f, s in results.items() if s.startswith("failed:")] skipped = [(f, s) for f, s in results.items() if s.startswith("skipped:")] if refreshed: print(f" ↑ {len(refreshed)} refreshed: {', '.join(refreshed)}") if current: print(f" ✓ {len(current)} already current") if skipped: # Most common reason: security.allow_lazy_installs=false. Show one # line so the user knows why; not an error. names = ", ".join(f for f, _ in skipped) reason = skipped[0][1].split(": ", 1)[-1] print(f" · {len(skipped)} skipped ({reason}): {names}") if not failed and not unexpected_failure: return True for feature, status in failed: reason = status.split(": ", 1)[-1] # Clip noisy pip stderr to keep update output legible. if len(reason) > 200: reason = reason[:200] + "..." print(f" ⚠ {feature} failed to refresh: {reason}") if install_cmd_prefix is None: print(" ⚠ Lazy refresh failed; rerun `hermes update` once resolved.") return False # Immediate import-based recovery — metadata-only verifiers miss the case # where DISTRIBUTION-INFO remains but import files were wiped (#57828). # Unavailable probes are indeterminate, not healthy — keep the lazy marker. status = _m()._repair_venv_via_import_probes(install_cmd_prefix, env=env) if status == "repaired": print( " Lazy backend(s) keep their previous version until refresh succeeds." ) return True if status == "healthy": print( " Lazy backend(s) keep their previous version; probed packages look intact." ) print(" Rerun `hermes update` once the upstream issue is resolved.") return True if status == "indeterminate": print( " ⚠ Leaving `.lazy-refresh-incomplete` until import probes can confirm health." ) return False def _refresh_active_memory_provider_dependencies() -> None: """Refresh pip dependencies for the configured external memory provider. Memory-provider bridge packages are declared in each provider's ``plugin.yaml`` (plus mode-dependent extras like Hindsight's ``hindsight-all``), NOT in Hermes' editable-install extras or ``LAZY_DEPS`` alone — so the core dependency reinstall above can strip or downgrade them (#53272 mem0ai, #70636 hindsight-embed). Re-run the provider's declared install for the ACTIVE provider only, after the core install and lazy refresh, so the last write to any shared package is the one the active provider needs. Never raises. A failure here must not block the rest of the update. """ try: from hermes_cli.config import load_config cfg = load_config() except Exception as exc: logger.debug("Memory provider refresh skipped (config load failed): %s", exc) return provider = "" if isinstance(cfg, dict): memory_cfg = cfg.get("memory") if isinstance(memory_cfg, dict): if memory_cfg.get("enabled") is False: return provider = str(memory_cfg.get("provider") or "").strip() # "default" / empty is the built-in file-backed store — no pip deps. if not provider or provider in {"default", "builtin", "none"}: return try: from hermes_cli.memory_setup import _install_dependencies except Exception as exc: logger.debug("Memory provider refresh skipped (import failed): %s", exc) return print() print(f"→ Refreshing active memory provider dependencies ({provider})...") try: _install_dependencies(provider, force=True) except Exception as exc: print(f" ⚠ {provider} dependencies failed to refresh: {exc}") def _is_android_python() -> bool: return _m().sys.platform == "android" def _install_psutil_android_compat( install_cmd_prefix: list[str], *, env: dict[str, str] | None = None, ) -> None: """Install psutil on Android by patching upstream platform detection. psutil's setup currently gates Linux sources behind ``sys.platform.startswith('linux')``. On Termux Python reports ``sys.platform == 'android'``, so setup aborts with "platform android is not supported" despite compiling fine when using the Linux source path. We patch only the extracted build tree used for this install attempt; nothing is persisted in the repository. Stopgap: remove this once https://github.com/giampaolo/psutil/pull/2762 merges and ships in a release. The standalone installer script uses the same shared helper and should be removed together. """ import tempfile import urllib.request from hermes_cli.psutil_android import PSUTIL_URL, prepare_patched_psutil_sdist with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) archive = tmp_path / "psutil.tar.gz" urllib.request.urlretrieve(PSUTIL_URL, archive) src_root = prepare_patched_psutil_sdist(archive, tmp_path) _m()._run_install_with_heartbeat( install_cmd_prefix + ["install", "--no-build-isolation", str(src_root)], env=env, ) def _ensure_uv_for_termux(pip_cmd: list[str]) -> str | None: """Best-effort uv bootstrap on Termux for faster update installs. The normal path (``ensure_uv()`` in managed_uv) installs the managed standalone uv into ``$HERMES_HOME/bin/uv``, but on Termux the official installer may not work (glibc vs bionic). Prefer a uv already on PATH (e.g. ``pkg install uv``); only if there is none do we fall back to a wheel-only ``pip install uv`` so we never source-build the Rust crate. """ from hermes_cli.managed_uv import resolve_uv existing = resolve_uv() if existing: return existing if not _m()._is_termux_env(): return None # A Termux-packaged uv lands on PATH but not in the managed bin dir, so # resolve_uv() misses it. Use it before pip, which has no Android wheel and # would otherwise build uv from source on a low-memory device. system_uv = shutil.which("uv") if system_uv: return system_uv try: print(" → Termux detected: trying to install uv for faster dependency updates...") result = subprocess.run( pip_cmd + ["install", "uv", "--only-binary", ":all:"], cwd=_m().PROJECT_ROOT, check=False, ) if result.returncode != 0: return None except Exception: pass # After pip install, check managed path first, then PATH return resolve_uv() or shutil.which("uv") def _npm_manifest_paths() -> tuple[Path, ...]: """Manifests whose changes must defeat the update-skip. The lockfile alone is NOT a sufficient key: on a local checkout a dev can edit package.json (root or a workspace) without running npm — the lockfile is then unchanged but `hermes update` is exactly the step expected to sync node_modules (via the `npm install` fallback in _run_npm_install_deterministic). The workspace list is pulled from the root package.json's `workspaces` globs (npm's own source of truth) rather than hardcoded, so adding a workspace can never silently escape the skip key. Every workspace manifest belongs in the key — desktop included, even though the install only names ui-tui and web — because the single lockfile spans the whole workspace graph, so any manifest edit can put the lockfile out of sync and change what the install must do. Falls back to hashing just root manifests if package.json is unreadable (never skips more than main would have installed). """ root_pkg = _m().PROJECT_ROOT / "package.json" paths = [_m().PROJECT_ROOT / "package-lock.json", root_pkg] try: workspaces = json.loads(root_pkg.read_text(encoding="utf-8")).get( "workspaces", [] ) if isinstance(workspaces, dict): # legacy {"packages": [...]} form workspaces = workspaces.get("packages", []) for pattern in workspaces: for match in sorted(_m().PROJECT_ROOT.glob(str(pattern))): manifest = match / "package.json" if manifest.is_file(): paths.append(manifest) except (OSError, json.JSONDecodeError, TypeError): pass return tuple(paths) def _npm_manifests_digest() -> str | None: """Combined sha256 over the lockfile + all workspace package.json files. Returns None when the lockfile is missing (never skip then). """ if not (_m().PROJECT_ROOT / "package-lock.json").exists(): return None h = hashlib.sha256() for p in _npm_manifest_paths(): h.update(str(p.relative_to(_m().PROJECT_ROOT)).encode()) try: h.update(p.read_bytes()) except OSError: h.update(b"") return h.hexdigest() def _npm_lockfile_changed(hermes_root: Path) -> bool: current = _npm_manifests_digest() if current is None: return True # Also check that node_modules exists; a matching hash with missing # node_modules means the cache was recorded by another checkout. if not (_m().PROJECT_ROOT / "node_modules").is_dir(): return True # A matching lockfile hash over a tree whose web build toolchain never # landed must NOT skip the reinstall — otherwise every later `hermes # update` keeps rebuilding against a half-installed tree and serving a # stale dist. web_dir = _m().PROJECT_ROOT / "web" if (web_dir / "package.json").is_file() and not _web_build_toolchain_ready( *_web_toolchain_roots(web_dir) ): return True try: # Key the cache by PROJECT_ROOT so parallel worktrees don't collide. cache_key = hashlib.sha256(str(_m().PROJECT_ROOT).encode()).hexdigest()[:12] cache_file = hermes_root / f".npm_lock_hash_{cache_key}" if not cache_file.exists(): return True return cache_file.read_text(encoding="utf-8").strip() != current except OSError: return True def _record_npm_lockfile_hash(hermes_root: Path) -> None: digest = _npm_manifests_digest() if digest is None: return try: cache_key = hashlib.sha256(str(_m().PROJECT_ROOT).encode()).hexdigest()[:12] cache_file = hermes_root / f".npm_lock_hash_{cache_key}" cache_file.write_text(digest, encoding="utf-8") except OSError: logger.debug("Could not write npm lockfile hash cache") def _repair_node_deps_on_current_checkout( print_completion, *, assume_yes: bool = False, gateway_mode: bool = False, pre_update_snapshot_id: str | None = None, completion_message: str = "✓ Already up to date!", had_desktop_app_before_update: bool = False, ) -> bool: """Repair Node deps on the ``commit_count == 0`` path (#77211). A current checkout does not imply healthy Node deps: a previous npm install may have failed (EBADENGINE from a node/npm mismatch, network timeout, interrupted install) and its error message says to "re-run hermes update" — but the early return never reached the Node refresh, so that repair advice could never work. ``_update_node_dependencies`` self-gates on the lockfile hash, which is only recorded after a SUCCESSFUL npm install (and re-trips when node_modules is missing or the web toolchain never landed), so this is a cheap no-op on healthy installs and a real repair after a failed one. """ node_failures = _update_node_dependencies() if node_failures: print(f" ⚠ Node.js refresh failed for: {', '.join(node_failures)}") print(" Fix npm and re-run `hermes update`.") print_completion( "⚠ Checkout is current, but Node.js dependencies could not be repaired." ) return False # Pair the refresh with the web build like every other # _update_node_dependencies call site; it staleness-checks internally, # so this is a no-op when nothing changed. _m()._build_web_ui(_m().PROJECT_ROOT / "web") _check_and_apply_config_migration( assume_yes=assume_yes, gateway_mode=gateway_mode, pre_update_snapshot_id=pre_update_snapshot_id, ) # A current checkout can still owe a Desktop rebuild (#97343): the # packaged app is built from source the pull already landed — or, on the # Windows hand-off, by a child that never reaches the commits-pulled # rebuild. Skipping it leaves a stale desktop app behind a # successful-looking update. Self-gates on the build stamp, so this is a # no-op when nothing changed. if not _rebuild_desktop_after_update( _m().PROJECT_ROOT / "apps" / "desktop", had_desktop_app_before_update=had_desktop_app_before_update, ): # _rebuild_desktop_after_update already printed the retry hint; withhold # success rather than claiming the update finished (#88251). print_completion( "⚠ Update partially complete — the desktop app was not rebuilt " "and is still on the previous build." ) return False return bool(print_completion(completion_message)) def _update_node_dependencies() -> list[str]: """Refresh Node deps for the ui-tui and web workspaces. Returns the list of labels whose npm install failed (empty on success), so the caller can treat a Node refresh failure as a partial update rather than silently reporting ``Update complete!`` (#30271). """ if not (_m().PROJECT_ROOT / "package.json").exists(): return [] npm = _m()._resolve_node_runtime_npm() if not npm: # If the only npm reachable inside this WSL shell is the Windows one, # flag it loudly: silently skipping leaves ui-tui deps stale while the # rest of the update proceeds, and running it would corrupt the tree. from hermes_constants import is_wsl path_npm = shutil.which("npm") if is_wsl() and path_npm and _m()._is_windows_npm_path(path_npm): print("→ Updating Node.js dependencies...") print(" ⚠ Skipped: only a Windows npm is reachable from this WSL shell.") print(" Install Node.js inside the WSL distro (nvm, or your distro's") print(" package manager), then re-run `hermes update`.") failed = [] if any( (_m().PROJECT_ROOT / workspace / "package.json").exists() for workspace in ("ui-tui", "web") ): failed.append("ui-tui, web workspaces") return failed return [] from hermes_constants import get_default_hermes_root # This cache describes PROJECT_ROOT/node_modules, which is shared by every # Hermes profile using this checkout. Keep one per-checkout cache under the # shared Hermes root rather than rerunning npm once per named profile. shared_hermes_root = get_default_hermes_root() # Best-effort: warm npx's cache for agent-browser (#43564). Runs before # the lockfile-unchanged early return below since that's the common # `hermes update` case. Synchronous and can block ~11s on a true cold # cache (~0.4s once warm) — print first so that doesn't look like a hang. print("→ Warming npx cache for agent-browser...") try: from tools.browser_tool import warm_agent_browser_npx_cache warm_agent_browser_npx_cache() except Exception: pass if not _m()._npm_lockfile_changed(shared_hermes_root): logger.info("npm lockfile unchanged, skipping npm install") return [] # Root package.json has no dependencies of its own (agent-browser and # @streamdown/math were moved out — see #43564): agent-browser resolves # at runtime via `npx agent-browser` (tools/browser_tool.py), and # @streamdown/math is a desktop-only import now declared in # apps/desktop/package.json. That means a plain workspace-scoped install # can never prune anything root-only, so we only need to name the # workspaces the CLI/TUI/web build actually requires. apps/desktop pulls # in Electron as a devDependency with a ~200MB postinstall download, so # it's deliberately never named here — desktop deps install on demand # (see _desktop_build_needed). print("→ Updating Node.js dependencies...") def _partial_update_failure(*labels: str) -> list[str]: print() print(" ⚠ Node.js dependency refresh did not complete cleanly; the") print(" installation may be in a mixed state (updated code, stale Node") print(" deps). Fix npm and re-run `hermes update`.") return list(labels) install_args = [ "--no-fund", "--no-audit", "--prefer-offline", "--progress=false", "--workspace", "ui-tui", "--workspace", "web", # Root package.json's own devDependencies (the shared ESLint flat # config every workspace's eslint.config.mjs imports) are otherwise # pruned by this scoped install, same as agent-browser/@streamdown # math used to be before they moved out of root entirely (#43564). # Unlike those, root's devDependencies have nowhere else to live — # this flag still excludes apps/desktop, which is never named above. "--include-workspace-root", ] from hermes_constants import with_hermes_node_path nixos_env = with_hermes_node_path(_m()._nixos_build_env()) # NOTE: capture_output=False here is deliberate (#18840) — optional # postinstall scripts print download progress, and capturing it makes a # long download look hung. The chatty npm-deprecation noise during # `hermes update` comes from the *desktop* build, not this step; that # one is captured to update.log. result = _m()._run_npm_install_deterministic( npm, _m().PROJECT_ROOT, extra_args=tuple(install_args), capture_output=False, env=nixos_env, ) if result.returncode == 0: _record_npm_lockfile_hash(shared_hermes_root) print(" ✓ ui-tui, web workspaces installed (desktop skipped)") failures: list[str] = [] else: print(" ⚠ npm install failed") stderr = (result.stderr or "").strip() if result.stderr else "" if stderr: print(f" {stderr.splitlines()[-1]}") failures = _partial_update_failure("ui-tui, web workspaces") return failures def _log_only_write(text: str) -> None: """Write ``text`` to ``~/.hermes/logs/update.log`` only, never the terminal. During ``hermes update`` ``sys.stdout`` is an ``_UpdateOutputStream`` that mirrors to both the terminal and ``update.log``. Loud, low-signal subprocess output (npm installs, the Electron/vite build, the cua-driver installer's "Next steps" wall) should be captured and tucked into the log so failures stay debuggable, without flooding the user's terminal. This reaches past the mirroring stream straight to the underlying log handle. """ if not text: return stream = _m().sys.stdout log_file = getattr(stream, "_log", None) if log_file is None: return try: log_file.write(text if text.endswith("\n") else text + "\n") log_file.flush() except Exception: pass def _run_logged_subprocess(cmd, *, cwd=None, env=None): """Run ``cmd`` capturing combined output into update.log (not the terminal). Returns the ``CompletedProcess`` (with ``stdout`` populated) so the caller can decide whether to surface the captured output on failure. """ result = subprocess.run( cmd, cwd=cwd, env=env, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", ) _log_only_write(result.stdout or "") return result def _classify_fetch_failure(stderr: str) -> str: """Map git-fetch stderr to a one-line, user-facing diagnosis. Order matters: curl surfaces HTTP failures as ``fatal: unable to access '': The requested URL returned error: 429``, so the rate-limit/outage checks must run BEFORE the generic "unable to access" network check or a GitHub 429/5xx gets misreported as a local network problem. The caller always prints the first raw stderr line alongside this diagnosis — the friendly message adds guidance, it never replaces the wire error. """ def _has_http_code(*codes: str) -> bool: return any( f"HTTP {code}" in stderr or f"returned error: {code}" in stderr for code in codes ) if _has_http_code("429") or "rate limit" in stderr.lower(): return ( "✗ GitHub is rate limiting requests or having an outage (HTTP 429)" " — try again in 5 minutes." ) if _has_http_code("500", "502", "503", "504"): return ( "✗ GitHub appears to be having an outage — try again in a few" " minutes (https://www.githubstatus.com)." ) if "Could not resolve host" in stderr or "unable to access" in stderr: return "✗ Network error — cannot reach the remote repository." if "could not read Username" in stderr or "terminal prompts disabled" in stderr: # Anonymous fetch of a public repo got HTTP 401. GitHub does this # during outages (and for renamed/private repos) — it is not a # credentials problem on the user's side. return ( "✗ GitHub rejected the anonymous fetch (asked for a login) — this" " usually means a GitHub outage; try again in a few minutes" " (https://www.githubstatus.com). If it persists, check" " `git remote -v` points at a public repo." ) if "Authentication failed" in stderr: return "✗ Authentication failed — check your git credentials or SSH key." return "✗ Failed to fetch updates from origin." def _print_fetch_failure(stderr: str) -> None: """Print the classified diagnosis plus the first raw stderr line.""" stderr = (stderr or "").strip() print(_classify_fetch_failure(stderr)) if stderr: print(f" {stderr.splitlines()[0]}") def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): """Implement ``hermes update --check``: fetch and report without installing. ``branch`` selects which branch the check compares against. Default is "main"; callers can pass another branch to ask "are there new commits on origin/?" without performing the update. ``branch_explicit`` is True iff the caller passed --branch on the CLI. Installs that can't honor non-default branches (e.g. Docker) surface a one-line notice instead of silently dropping the flag. """ # Shared admission gate (#91277 Phase 3): same marker-first decision as # the apply path, so --check can never report git state for an install # whose real update mechanism is an image pull. from hermes_cli.update_contract import ( evaluate_update_admission, record_refusal_receipt, ) refusal = evaluate_update_admission(_m().PROJECT_ROOT) if refusal is not None: print(refusal.message) record_refusal_receipt(refusal) sys.exit(2) git_dir = _m().PROJECT_ROOT / ".git" if not git_dir.exists(): print("✗ Not a git repository — cannot check for updates.") sys.exit(1) git_cmd = ["git"] if sys.platform == "win32": git_cmd = ["git", "-c", "windows.appendAtomically=false"] # A crashed/interrupted fetch can leave .git/shallow.lock (or another git # lock file) behind; every later fetch then fails with "File exists" and # the check reports a hard failure (or, in the banner path, silently # compares stale refs). Self-heal abandoned locks before fetching. from hermes_cli.gitlock import clear_stale_git_locks, clear_stale_tmp_packs cleared = clear_stale_git_locks(_m().PROJECT_ROOT) for lock_path in cleared: print(f" (removed stale git lock: {lock_path})") # Aborted fetches on flaky lines also strand tmp_pack_* debris in # .git/objects/pack — unchecked it reached 6 GB and corrupted the pack # dir outright (#93732). Same age+process safety contract as the locks. swept = clear_stale_tmp_packs(_m().PROJECT_ROOT) if swept: print(f" (removed {len(swept)} aborted-fetch pack temp file(s))") # Fetch only the branch we compare against; prefer upstream as the canonical # reference. A bare `git fetch ` pulls every ref, and this repo has # thousands of auto-generated branches, so scope the fetch to . # Note: upstream/ may not exist for non-main branches (a fork's # bb/gui has no upstream counterpart), so when the caller picks a # non-default branch we skip the upstream probe and use origin directly. # Installer checkouts are shallow (`git clone --depth 1`). A plain # `git fetch` would unshallow the repo (dragging in the whole history — # the exact cost the shallow clone avoided) and the rev-list count below # would then report a huge bogus "behind" number. Detect shallow up front: # fetch with --depth 1 to preserve the boundary and report presence-only. is_shallow = ( subprocess.run( git_cmd + ["rev-parse", "--is-shallow-repository"], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ).stdout.strip() == "true" ) depth_args = ["--depth", "1"] if is_shallow else [] if branch == "main": # Probe locally (~6 ms) whether an 'upstream' remote exists at all # before spending a network fetch on it. Non-fork installs have no # 'upstream' remote, and the old flow burned a failed network attempt # (~0.3-1 s) on every --check before falling back to origin. has_upstream_remote = ( subprocess.run( git_cmd + ["remote", "get-url", "upstream"], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ).returncode == 0 ) fetch_result = None if has_upstream_remote: print("→ Fetching from upstream...") fetch_result = subprocess.run( git_cmd + ["fetch"] + depth_args + ["upstream", branch], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", **_no_prompt_git_kwargs(), ) if fetch_result is not None and fetch_result.returncode == 0: upstream_exists = True compare_branch = f"upstream/{branch}" else: # No upstream remote, or the upstream fetch failed — use origin. print("→ Fetching from origin...") fetch_result = subprocess.run( git_cmd + ["fetch"] + depth_args + ["origin", branch], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", **_no_prompt_git_kwargs(), ) upstream_exists = False compare_branch = f"origin/{branch}" else: # Non-default branch: compare against origin/ directly. print("→ Fetching from origin...") fetch_result = subprocess.run( git_cmd + ["fetch"] + depth_args + ["origin", branch], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", **_no_prompt_git_kwargs(), ) upstream_exists = False compare_branch = f"origin/{branch}" if fetch_result.returncode != 0: _print_fetch_failure(fetch_result.stderr) sys.exit(1) # Verify the compare ref actually exists before asking rev-list about it. # Without this, `git rev-list HEAD..origin/ --count` exits 128 and # (with check=True) raises CalledProcessError, surfacing a Python # traceback. Friendlier to detect-and-report. verify_result = subprocess.run( git_cmd + ["rev-parse", "--verify", "--quiet", compare_branch], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if verify_result.returncode != 0: print(f"✗ Branch '{branch}' not found on {compare_branch.split('/', 1)[0]}.") sys.exit(1) if is_shallow: # No history to count across the shallow boundary. Compare tip SHAs # (mirrors the banner's _check_via_local_git), then try to recover the # exact count via the GitHub compare API — the remote graph is complete # even when the local one is truncated. head_sha = subprocess.run( git_cmd + ["rev-parse", "HEAD"], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ).stdout.strip() target_sha = subprocess.run( git_cmd + ["rev-parse", compare_branch], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ).stdout.strip() if head_sha and target_sha and head_sha == target_sha: print("✓ Already up to date.") else: from hermes_cli.banner import _github_compare_behind from hermes_cli.config import recommended_update_command counted = _github_compare_behind(head_sha, target_sha) if counted == 0: # Local commits on top of the remote tip — not behind. print("✓ Already up to date.") return if counted is not None: commits_word = "commit" if counted == 1 else "commits" print(f"⚕ Update available: {counted} {commits_word} behind {compare_branch}.") else: print(f"⚕ Update available (behind {compare_branch}).") print(f" Run '{recommended_update_command()}' to install.") return rev_result = subprocess.run( git_cmd + ["rev-list", f"HEAD..{compare_branch}", "--count"], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", check=True, ) behind = int(rev_result.stdout.strip()) if behind == 0: print("✓ Already up to date.") else: commits_word = "commit" if behind == 1 else "commits" print(f"⚕ Update available: {behind} {commits_word} behind {compare_branch}.") from hermes_cli.config import recommended_update_command print(f" Run '{recommended_update_command()}' to install.") def _ensure_fhs_path_guard() -> None: """Ensure /usr/local/bin is on PATH for RHEL-family root non-login shells. Mirrors the post-symlink probe added to ``scripts/install.sh`` so that existing FHS-layout root installs on RHEL/CentOS/Rocky/Alma 8+ get repaired on ``hermes update`` without requiring a reinstall. The installer's assumption that ``/usr/local/bin`` is on PATH for every standard shell breaks on those distros in non-login interactive shells (su, sudo -s, tmux panes, some web terminals): /etc/bashrc doesn't add /usr/local/bin and /root/.bash_profile doesn't either. Symptom: ``hermes`` prints ``command not found`` even though the symlink lives at /usr/local/bin/hermes. Silent no-op on: non-Linux, non-root, non-FHS installs, and any system where ``bash -i -c 'command -v hermes'`` already resolves. Idempotent. """ if _m().sys.platform != "linux": return try: if os.geteuid() != 0: # windows-footgun: ok — Linux FHS helper, guarded by sys.platform == "linux" above + AttributeError catch return except AttributeError: return # Only act when this is actually an FHS-layout install (command link at # /usr/local/bin/hermes, code at /usr/local/lib/hermes-agent). fhs_link = Path("/usr/local/bin/hermes") if not fhs_link.is_symlink() and not fhs_link.exists(): return # Probe a fresh non-login interactive bash the way the user will use it. # ``bash -i -c`` sources ~/.bashrc but NOT ~/.bash_profile or /etc/profile, # which is the exact scenario where RHEL root loses /usr/local/bin. home = os.environ.get("HOME") or "/root" try: probe = subprocess.run( [ "env", "-i", f"HOME={home}", f"TERM={os.environ.get('TERM', 'dumb')}", "bash", "-i", "-c", "command -v hermes", ], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, ) except (FileNotFoundError, subprocess.TimeoutExpired): return # no bash or probe hung — don't block update on this if probe.returncode == 0: return # already on PATH, nothing to do path_line = 'export PATH="/usr/local/bin:$PATH"' path_comment = ( "# Hermes Agent — ensure /usr/local/bin is on PATH " "(RHEL non-login shells)" ) wrote_any = False for candidate in (".bashrc", ".bash_profile"): cfg = Path(home) / candidate if not cfg.is_file(): continue try: existing = cfg.read_text(errors="replace", encoding="utf-8") except OSError: continue # Idempotency: skip if any uncommented PATH= line already references # /usr/local/bin. Mirrors the grep pattern used by install.sh. already_guarded = any( "/usr/local/bin" in line and "PATH" in line and not line.lstrip().startswith("#") for line in existing.splitlines() ) if already_guarded: continue try: with cfg.open("a", encoding="utf-8") as f: f.write("\n" + path_comment + "\n" + path_line + "\n") except OSError as e: print(f" ⚠ Could not update {cfg}: {e}") continue print(f" ✓ Added /usr/local/bin to PATH in {cfg}") wrote_any = True if wrote_any: print(" (reload your shell or run 'source ~/.bashrc' to pick it up)") def _ensure_acp_launcher() -> None: r"""Self-heal: install a ``hermes-acp`` launcher next to the ``hermes`` one. Mirrors the launcher block in ``scripts/install.sh`` so existing installs gain the ACP command on ``hermes update`` without a reinstall. ACP hosts (Zed, JetBrains, Buzz Desktop) spawn the agent by resolving the ``hermes-acp`` command name against the login-shell PATH; the console script of that name lives inside the install's venv, which is not on that PATH, so those hosts report Hermes as not installed even when it is. The shim simply delegates to the sibling ``hermes`` launcher with the ``acp`` subcommand, which makes it correct for every install layout (venv wrapper, FHS symlink, pipx/pip console script) without having to reconstruct interpreter/entrypoint paths. No-op on Windows (install.ps1 stages the ``hermes`` / ``hermes-acp`` launchers into the managed binary dir ``$HermesHome\bin`` and puts THAT on the user PATH — never the whole ``venv\Scripts`` dir, which would shadow the user's ``python`` (#83797); when those launchers go missing, ``hermes_cli._install_repair.ensure_windows_bin_launchers`` re-stages them) and wherever a ``hermes-acp`` is already present next to the ``hermes`` command. Unwritable directories (e.g. ``/usr/local/bin`` as non-root) are skipped silently. Idempotent. """ if _m().sys.platform == "win32": # Windows launcher staging/repair lives in _install_repair # (ensure_windows_bin_launchers at process start, # migrate_windows_bin_path in this command's tail) — not here. return for bin_dir in (Path.home() / ".local" / "bin", Path("/usr/local/bin")): hermes_cmd = bin_dir / "hermes" acp_cmd = bin_dir / "hermes-acp" try: if not (hermes_cmd.is_file() or hermes_cmd.is_symlink()): continue # Already present — a console script (pip/pipx install), an # earlier shim, or a symlink. is_symlink() catches broken # symlinks that exists() would miss; never follow-and-overwrite # (the #21454 failure mode). if acp_cmd.exists() or acp_cmd.is_symlink(): continue shim = ( "#!/usr/bin/env bash\n" "# Hermes Agent — ACP launcher (written by `hermes update`).\n" "# ACP hosts (Zed, JetBrains, Buzz) resolve the agent by this\n" "# command name on the login-shell PATH.\n" f'exec "{hermes_cmd}" acp "$@"\n' ) acp_cmd.write_text(shim, encoding="utf-8") acp_cmd.chmod(acp_cmd.stat().st_mode | 0o755) except OSError: continue print(f" ✓ Installed hermes-acp launcher → {acp_cmd}") _PRE_UPDATE_SNAPSHOT_KEEP = 1 # Sibling-profile snapshot ids from the current run's pre-update backup # ({profile: snapshot_id}) — consumed by the post-update per-profile # cron-jobs safety net (#66140). Module-level because the snapshot and the # restore run in the same process but far apart in _cmd_update_impl. _LAST_SIBLING_SNAPSHOTS: dict = {} # Per-file size cap for the pre-update quick snapshot. Anything larger is # skipped with a warning: the snapshot exists to protect small, hard-to- # regenerate state (pairing JSONs, cron jobs, config, auth) — not to copy a # multi-GB state.db on every update (observed: a 24 GB state.db added ~60s # of wall time and silently ate 24 GB of disk per update). _PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE = 1 << 30 # 1 GiB def _resolve_pre_update_backup_mode(args) -> str: """Resolve the pre-update backup mode: ``"off"``, ``"quick"``, or ``"full"``. CLI flags win over config; ``--no-backup`` beats ``--backup`` when both are set. Config accepts the mode strings plus legacy booleans: ``true`` → ``full`` (the old zip behavior), ``false`` → ``off`` (an explicit opt-out now disables the quick snapshot too — previously it ran unconditionally, ignoring the user's setting). A missing key defaults to ``quick``. """ if getattr(args, "no_backup", False): return "off" if getattr(args, "backup", False): return "full" try: from hermes_cli.config import load_config cfg = load_config() except Exception as exc: logging.getLogger(__name__).debug( "Could not load config for pre-update backup: %s", exc ) cfg = {} updates_cfg = cfg.get("updates", {}) if isinstance(cfg, dict) else {} raw = updates_cfg.get("pre_update_backup", "quick") if raw is True: return "full" if raw is False: return "off" mode = str(raw).strip().lower() if mode in ("off", "false", "none", "disabled"): return "off" if mode in ("full", "zip", "true"): return "full" if mode == "quick": return "quick" logging.getLogger(__name__).warning( "Unknown updates.pre_update_backup value %r — using 'quick'", raw ) return "quick" def _run_pre_update_backup(args) -> Optional[str]: """Run the pre-update safety backup and return the quick-snapshot id. Single consolidated mechanism gated on ``updates.pre_update_backup``: - ``off`` — nothing runs. Explicit user opt-out is honored fully. - ``quick`` (default) — a state snapshot of critical small files (pairing JSONs, cron jobs, config, auth; see ``_QUICK_STATE_FILES``) under ``state-snapshots/``. Files over 1 GiB are skipped with a warning so a bloated state.db can never stall the update (issues #15733, #34600 are the reason this safety net exists). - ``full`` — the quick snapshot PLUS a full zip of HERMES_HOME under ``backups/`` (restorable via ``hermes import``; the #48200 wrong-path wipe is the reason this level exists). ``--backup`` forces ``full`` for one run; ``--no-backup`` forces ``off``. Never raises — a backup failure should not block the update itself. Returns the quick-snapshot id (used by the post-update cron-jobs restore safety net), or ``None`` when mode is ``off`` or the snapshot failed. """ mode = _resolve_pre_update_backup_mode(args) if mode == "off": if getattr(args, "no_backup", False): print("◆ Pre-update backup: skipped (--no-backup)") print() # Config-level off is silent — the user opted out; don't spam them # on every update. return None snapshot_id = None try: from hermes_cli.backup import ( _quick_snapshot_root, create_quick_snapshot, verify_sqlite_integrity, ) # NOTE: this function later does `from hermes_constants import # get_hermes_home`, which makes the name function-local — the # module-level import is shadowed and unbound here. Alias explicitly. from hermes_cli.config import get_hermes_home as _get_home snapshot_id = create_quick_snapshot( label="pre-update", keep=_PRE_UPDATE_SNAPSHOT_KEEP, max_file_size=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE, ) # After the snapshot, verify the source state.db is still intact. # The snapshot was taken via _safe_copy_db (read-only SQLite backup # API), but a concurrent process (antivirus, force-killed gateway # releasing file handles, Windows filter driver) can corrupt the live # file at any point. A silent zeroing at this point would proceed with # the update and exit code 0 — exactly the #68474 symptom. if snapshot_id: _src_path = _get_home() / "state.db" if _src_path.exists(): _integrity = verify_sqlite_integrity( _src_path, check_header=True, run_pragma=True, max_bytes=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE, ) if not _integrity.get("valid"): _msg = _integrity.get("message", "unknown error") print( f" ⚠ state.db integrity check FAILED after snapshot: {_msg}" ) # Check if the snapshot itself is valid. _snap_root = _quick_snapshot_root(_get_home()) _snap_state = _snap_root / snapshot_id / "state.db" if _snap_state.exists(): _snap_ok = verify_sqlite_integrity( _snap_state, check_header=True, run_pragma=True ) if _snap_ok.get("valid"): print( " ✓ Snapshot copy is valid — continuing update." ) print( " If state.db is lost after update it will be auto-restored." ) else: print( " ✗ Snapshot copy ALSO failed integrity — " "the source was already corrupted before the backup." ) else: print( " ⚠ Snapshot does not contain state.db (was skipped or too large)." ) print() if snapshot_id: print(f"◆ Pre-update snapshot: {snapshot_id}") # #66140: the code swap + fleet restart touch EVERY profile, so # every profile gets the same snapshot (same set, same 1GiB cap, # keep=1) under its own state-snapshots/. Best-effort per profile. try: from hermes_cli.backup import create_pre_update_snapshots_all_profiles _sibling_snaps = create_pre_update_snapshots_all_profiles( keep=_PRE_UPDATE_SNAPSHOT_KEEP, max_file_size=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE, ) if _sibling_snaps: print( f"◆ Sibling profile snapshot(s): " + ", ".join(sorted(_sibling_snaps)) ) try: from hermes_cli.update_receipt import record_step record_step( "sibling_profile_snapshots", True, ", ".join( f"{k}={v}" for k, v in sorted(_sibling_snaps.items()) ), ) except Exception: pass global _LAST_SIBLING_SNAPSHOTS _LAST_SIBLING_SNAPSHOTS = _sibling_snaps except Exception as _sib_exc: logging.getLogger(__name__).debug( "Sibling profile snapshots failed: %s", _sib_exc ) except Exception as exc: # Never let a snapshot failure block an update. logging.getLogger(__name__).debug("Pre-update snapshot failed: %s", exc) if mode != "full": if snapshot_id: print() return snapshot_id try: from hermes_cli.backup import create_pre_update_backup except Exception as exc: print( f"⚠ Pre-update backup: could not load backup module ({exc}); continuing update." ) print() return snapshot_id try: from hermes_cli.config import load_config _keep = (load_config() or {}).get("updates", {}).get("backup_keep", 5) except Exception: _keep = 5 print("◆ Creating pre-update backup...") t0 = _time.monotonic() try: out_path = create_pre_update_backup(keep=int(_keep)) except Exception as exc: # defensive — helper already swallows, but just in case print(f" ⚠ Backup failed: {exc}") print(" Continuing with update.") print() return snapshot_id elapsed = _time.monotonic() - t0 if out_path is None: print(" ⚠ Backup skipped (no files found or write failed); continuing update.") print() return snapshot_id try: size_bytes = out_path.stat().st_size except OSError: size_bytes = 0 # Human-readable size from hermes_cli.sizefmt import format_bytes size_str = format_bytes(size_bytes) # Render path using display_hermes_home so the user sees ~/.hermes/... try: from hermes_constants import get_hermes_home, display_hermes_home home = get_hermes_home() try: display_path = f"{display_hermes_home()}/{out_path.relative_to(home)}" except ValueError: display_path = str(out_path) except Exception: display_path = str(out_path) print(f" Saved: {display_path} ({size_str}, {elapsed:.1f}s)") print(f" Restore: hermes import {out_path}") print(" Disable: set updates.pre_update_backup: quick (or off) in config.yaml") print() return snapshot_id def _write_update_planned_stop_marker(profile_path: Path, pid: int) -> bool: """Write a planned-stop marker into a specific profile home.""" try: from datetime import timezone from gateway.status import _get_process_start_time from utils import atomic_json_write record = { "target_pid": pid, "target_start_time": _get_process_start_time(pid), "stopper_pid": os.getpid(), "written_at": datetime.now(timezone.utc).isoformat(), } atomic_json_write( Path(profile_path) / ".gateway-planned-stop.json", record, indent=None, separators=(",", ":"), ) return True except (OSError, PermissionError): return False def _wait_for_windows_update_gateway_exit( pids: list[int], *, timeout: float ) -> set[int]: """Wait for the given gateway PIDs to exit, returning survivors.""" if not pids: return set() from gateway.status import _pid_exists remaining = set(pids) deadline = _time.monotonic() + max(timeout, 0.0) while remaining and _time.monotonic() < deadline: for pid in list(remaining): try: if not _pid_exists(pid): remaining.discard(pid) except Exception: remaining.discard(pid) if remaining: _time.sleep(0.25) survivors: set[int] = set() for pid in remaining: try: if _pid_exists(pid): survivors.add(pid) except Exception: pass return survivors def _venv_core_imports_healthy() -> tuple[bool, str]: """Probe the project venv for the core imports the backend needs to boot. Runs a tiny import check inside the venv interpreter (NOT this process — ``hermes update`` may be driven by a different Python). Catches the half-updated-venv state: git checkout current but a dependency sync that failed or was killed partway (e.g. Windows access-denied on a loaded .pyd), leaving imports like ``fastapi``'s new transitive deps missing. Without this probe, ``hermes update`` on a current checkout prints "Already up to date!" and returns without ever re-syncing dependencies — the user's install stays broken no matter how many times they update (ryanc's incident, July 2026). Returns ``(healthy, detail)``. Never raises; unknown states report healthy so a probe failure can't force needless reinstalls. """ venv_dir = _m().PROJECT_ROOT / "venv" venv_python = venv_python_path(venv_dir, windows=_m()._is_windows()) if not venv_python.exists(): # No venv interpreter at all. In a dev checkout that's normal (the # dev may run hermes from any interpreter), so report healthy to # avoid forcing reinstalls. But on a MANAGED install (the Windows # installer / desktop bootstrap stamps `.hermes-bootstrap-complete`, # and an interrupted update leaves `.update-incomplete`), the venv # IS the install — its absence means a repair got interrupted after # the old venv was moved aside, and "Already up to date!" would # gaslight the user while nothing can run. managed_markers = ( _m().PROJECT_ROOT / ".hermes-bootstrap-complete", _m()._update_marker_path(), ) if any(m.exists() for m in managed_markers): return False, f"venv python missing ({venv_python})" return True, "" # Core web/serve imports plus their newest transitive deps. Import (not # just metadata) — a package can have intact dist-info but a missing # module after an interrupted uninstall/install cycle. check = ( "import importlib\n" "mods = ['fastapi', 'uvicorn', 'pydantic', 'openai', 'yaml']\n" "missing = []\n" "for m in mods:\n" " try: importlib.import_module(m)\n" " except Exception as e: missing.append(f'{m}: {e}')\n" "print('\\n'.join(missing))\n" ) try: result = subprocess.run( [str(venv_python), "-c", check], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=60, cwd=_m().PROJECT_ROOT, ) except Exception as exc: logger.debug("venv health probe failed to run: %s", exc) return True, "" missing = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()] if result.returncode != 0 and not missing: # Interpreter itself is broken (e.g. deleted stdlib) — that IS unhealthy. detail = (result.stderr or "").strip().splitlines() return False, detail[0] if detail else "venv python failed to run" if missing: return False, "; ".join(missing[:4]) return True, "" def _detect_venv_python_processes( *, exclude_pids: set[int] | None = None ) -> list[tuple[int, str, str]]: """Find live processes running from the project venv's interpreter. The hermes.exe shim guard misses the biggest lock-holder class on Windows: the Desktop app's backend (``python.exe -m hermes_cli.main serve``) and anything else running straight off ``venv\\Scripts\\python (w).exe``. Those processes keep native ``.pyd`` extensions mapped, so a dependency sync mid-update dies with access-denied and strands the venv half-updated (ryanc's brotlicffi/_sodium.pyd incidents, July 2026). Killing them from here is pointless — the Desktop app supervises its backend and respawns it within seconds — so the caller should refuse and tell the user to close the app instead. Returns ``(pid, name, cmdline)`` tuples; empty off-Windows / without psutil / when nothing matches. The calling process and its ancestors are always excluded (a CLI ``hermes update`` itself runs from the venv python). Never raises. """ if not _m()._is_windows(): return [] try: import psutil except Exception: return [] venv_dir = _m().PROJECT_ROOT / "venv" try: venv_prefix = str(venv_dir.resolve()).lower().rstrip(os.sep) + os.sep except OSError: venv_prefix = str(venv_dir).lower().rstrip(os.sep) + os.sep try: root_prefix = str(_m().PROJECT_ROOT.resolve()).lower().rstrip(os.sep) + os.sep except OSError: root_prefix = str(_m().PROJECT_ROOT).lower().rstrip(os.sep) + os.sep skip: set[int] = set(exclude_pids or set()) skip.add(os.getpid()) try: from gateway.status import looks_like_gateway_command_line as _is_gw except Exception: _is_gw = None try: for anc in psutil.Process().parents(): # #87594: do NOT blanket-exclude ancestors. When `/update` runs # from a messaging platform the updater is a CHILD of the gateway # — excluding all ancestors hides the gateway from the scan, so # the pause machinery downstream never sees the one process it # exists to stop, and the update dead-ends on `venv-blocked`. # A GATEWAY ancestor stays visible (the pause path stops it # gracefully; a detached child updater survives its parent's # stop on Windows). Every other ancestor (shells, terminals, # this CLI's own venv python chain) stays excluded — an updater # must never nominate its own interactive ancestry as blockers. try: anc_cmdline = " ".join(anc.cmdline() or []) except Exception: anc_cmdline = "" if _is_gw is not None and anc_cmdline and _is_gw(anc_cmdline): continue skip.add(int(anc.pid)) except Exception: pass matches: list[tuple[int, str, str]] = [] try: # On Windows, prefetching cmdline and cwd performs two expensive # per-process queries. A busy workstation can have 500+ processes, so # querying those fields for every unrelated process can exceed the # Desktop preflight watchdog. First collect only cheap identity fields; # fetch cmdline/cwd lazily for plausible Python/uv/Hermes candidates. proc_iter = psutil.process_iter(["pid", "exe", "name"]) except Exception: return [] for proc in proc_iter: try: info = proc.info except Exception: continue pid = info.get("pid") exe = info.get("exe") if not exe or pid is None or int(pid) in skip: continue try: exe_norm = str(Path(exe).resolve()).lower() except (OSError, ValueError): exe_norm = str(exe).lower() # Primary match: the executable itself lives under this venv # (venv\Scripts\python(w).exe — the desktop backend / gateway case). is_holder = exe_norm.startswith(venv_prefix) name = str(info.get("name") or Path(exe).name) name_low = name.lower() if not is_holder and not ( name_low.startswith(("python", "pypy")) or name_low in {"uv.exe", "uvx.exe", "hermes.exe"} ): continue try: cmdline_raw = " ".join(proc.cmdline() or []) except Exception: cmdline_raw = "" cmdline_low = cmdline_raw.lower() # Fallback: uv/base-interpreter trampolines run a python whose exe is # OUTSIDE the venv but which still imports from it and holds its .pyd # files. Catch those by what they're running: a cmdline that references # this venv's path, or a `-m hermes_cli.main ...` invocation tied to # this install (install root in the cmdline or as the working dir). if not is_holder and venv_prefix in cmdline_low: is_holder = True if not is_holder and "hermes_cli.main" in cmdline_low: try: cwd_low = str(proc.cwd() or "").lower().rstrip(os.sep) + os.sep except Exception: cwd_low = os.sep if root_prefix in cmdline_low or cwd_low.startswith(root_prefix): is_holder = True if not is_holder: continue name = info.get("name") or Path(exe).name # Return the FULL cmdline: callers match against it (the Desktop # preflight's pausable-gateway exemption parses for `gateway run`). # Truncating here cut long managed-runtime interpreter paths before # the `-m hermes_cli.main gateway run` argv, so autostarted gateways # were misreported as blockers and the update dead-ended. Truncate # only at display time. matches.append((int(pid), str(name), cmdline_raw)) return matches # Native-extension modules that pin files inside the venv once imported. If # the updater process itself has any of these loaded, the dependency sync # below cannot rewrite the backing ``.pyd``/``.dll`` — Windows blocks REPLACE # on a mapped image — and the update dies with ``os error 5`` between # uninstall and reinstall, stranding the venv half-updated (#83569). # ``cryptography`` is the canonical case: ``hermes_cli.main`` used to import # it at startup while resolving external secret sources; ``PyYAML``'s # ``_yaml`` C extension is loaded by every CLI process (config parsing). # Keep this guard as defence-in-depth against future eager imports (new # secret sources, plugins absorbed into core, refactors of the startup # order) — but the guard must be HONEST (#86735/#86780/#86781: a preflight # that fired on every run, before the fetch, re-bricked the exact flow it # was meant to protect). Two honesty gates: # # 1. It only fires when the dependency sync would actually REWRITE the # loaded distribution (``_dependency_sync_would_rewrite``): if the # installed version already satisfies the on-disk pyproject pins, uv/pip # will not touch the mapped ``.pyd``, so there is no lock to trip. # 2. It runs AFTER the code swap (git pull / ZIP commit), immediately # before the venv rewrite — so the on-disk pyproject is the NEW one # (gate 1 compares against the right target) and a deferral no longer # strands the user on the old checkout: the next launch's marker # recovery completes the dependency install against the already-updated # pyproject. # # Keys are module prefixes in ``sys.modules``; values are # ``(display name, PyPI distribution name)``. _SELF_LOCKING_NATIVE_MODULES: dict[str, tuple[str, str]] = { "cryptography.hazmat.bindings._rust": ("cryptography (_rust.pyd)", "cryptography"), "yaml._yaml": ("PyYAML (_yaml.pyd)", "pyyaml"), } def _dependency_sync_would_rewrite(dist_name: str) -> bool | None: """Whether ``uv pip install -e .[all]`` would replace *dist_name*'s files. Compares the installed distribution version against every applicable requirement for it in the on-disk ``pyproject.toml`` (base dependencies plus all optional extras). Returns: - ``False`` — installed version satisfies every pin: the resolver will leave the wheel alone, so a mapped extension is NOT at risk. - ``True`` — some pin is not satisfied (or the distribution is missing): the sync will rewrite it. - ``None`` — could not determine (parse failure, unparseable pins). Never raises. Callers treat ``None`` as fail-OPEN (no deferral): a module in the registry can be loaded by every process (PyYAML), so deferring on uncertainty would recreate the #86735 always-firing loop. """ try: from importlib import metadata as _ilmd installed = _ilmd.version(dist_name) except Exception: return True # not installed → the sync will definitely install it try: import tomllib from packaging.requirements import Requirement from packaging.utils import canonicalize_name from packaging.version import Version pyproject = _m().PROJECT_ROOT / "pyproject.toml" data = tomllib.loads(pyproject.read_text(encoding="utf-8")) project = data.get("project") or {} req_strings: list[str] = list(project.get("dependencies") or []) for extra_reqs in (project.get("optional-dependencies") or {}).values(): req_strings.extend(extra_reqs or []) target = canonicalize_name(dist_name) installed_v = Version(installed) saw_pin = False for req_str in req_strings: try: req = Requirement(req_str) except Exception: continue if canonicalize_name(req.name) != target: continue if req.marker is not None and not req.marker.evaluate(): continue saw_pin = True if installed_v not in req.specifier: return True if saw_pin: return False # Not pinned anywhere in pyproject: the resolver may still move it # as a transitive — we cannot cheaply predict that, so stay honest # about the uncertainty. return None except Exception: return None def _detect_self_loaded_native_modules() -> list[str]: """Native venv extensions loaded into THIS process that the sync would rewrite. Returns display names (empty off Windows — POSIX lets a running process keep using an unlinked inode, so self-locking is a Windows-only hazard). A loaded module whose installed version already satisfies the on-disk pyproject pins is NOT reported: the dependency sync will not touch its files, so there is no swap at risk (#86735 — the always-firing variant of this preflight bricked every Windows update). Never raises. """ if not _m()._is_windows(): return [] found = [] for prefix, (display, dist) in _SELF_LOCKING_NATIVE_MODULES.items(): if prefix not in sys.modules: continue # Defer ONLY on a CONFIRMED pending rewrite. An "unknown" result # (unreadable/unparseable pyproject, no pin found) must fail OPEN: # PyYAML is loaded in every CLI process, so treating unknown as # at-risk would re-create the exact always-firing loop this guard's # first version caused (#86735). The downside of a missed deferral # is the pre-existing failure mode — a mid-sync os error 5 that the # marker recovery already handles — which is strictly less harmful # than an update that can never run. if _m()._dependency_sync_would_rewrite(dist) is not True: continue found.append(display) return sorted(set(found)) def _abort_dependency_sync_if_self_locked(gateway_resume=None) -> None: """Defer the venv rewrite when THIS process holds something it must replace. Runs at the last moment before the venv rewrite — after the code swap — so the on-disk pyproject reflects the update target and a deferral leaves the user on NEW code with only the dependency install pending. No-op when nothing at-risk is held. Two hazards, both "this process holds a file the sync must replace", and they end differently because their recoveries differ: - A mapped native extension (``.pyd``). Exit 2 and let the next launch's marker recovery finish the install: that launch runs the install before importing anything heavy, so it maps nothing and the swap succeeds. - The ``hermes.exe`` console shim we were launched from (#88838, #89599). The marker cannot help here — every future ``hermes`` launch is also the shim, so deferring to the next launch defers forever. Hand the install to a child under the venv interpreter and exit, releasing the shim. """ locked = _m()._detect_self_loaded_native_modules() if locked: _m()._defer_update_for_self_lock(locked) if gateway_resume is not None: _m()._resume_windows_gateways_after_update(gateway_resume) sys.exit(2) if _m()._reexec_dependency_sync_off_windows_shim(): if gateway_resume is not None: _m()._resume_windows_gateways_after_update(gateway_resume) sys.exit(0) def _defer_update_for_self_lock(loaded: list[str]) -> None: """Bail out before the dependency sync when the updater holds a lock. The install cannot win this race from inside the locked process — even killing threads would not unmap the image — so defer it: drop the update-incomplete marker (next launch's fresh process completes the install before importing anything heavy), explain, and exit 2 like the other preflight refusals. """ print("✗ This updater process has already loaded native venv modules that") print(" the dependency sync must replace:") for name in loaded: print(f" {name}") print() print(" On Windows a mapped extension cannot be replaced by the process") print(" holding it. The code update has been applied; only the dependency") print(" sync has been deferred: the next `hermes` launch will complete it") print(" in a fresh process before anything imports these modules.") _m()._write_update_incomplete_marker() _HOLDER_VALUE_FLAGS_FALLBACK = frozenset( { "--profile", "-p", "--config", "--model", "-m", "--provider", "--reasoning", "--toolsets", "-t", "--skills", "-s", "--continue", "-c", "--resume", "-r", "--oneshot", "-z", "--in", "--usage-file", } ) _holder_value_flags_cache: frozenset | None = None def _holder_value_flags() -> frozenset: """Top-level CLI flags that consume a value — derived from the REAL parser. Introspects ``build_top_level_parser()`` (every option with nargs != 0) so the holder classifier can never drift from the argparse surface (#91869 review: a handwritten subset misparsed ``--reasoning high serve`` as subcommand ``high`` and ``-m dashboard serve`` as ``dashboard`` — recreating the wrong-hint class). The pre-argparse profile selectors (``--profile``/``-p``, ``--config``) are added explicitly since they are stripped before argparse sees argv. Falls back to a static snapshot when the parser cannot be imported (the updater must classify holders even mid-upgrade on a broken tree). Cached per process. """ global _holder_value_flags_cache if _holder_value_flags_cache is not None: return _holder_value_flags_cache flags: set[str] = {"--profile", "-p", "--config"} try: from hermes_cli._parser import build_top_level_parser parser = build_top_level_parser()[0] for action in parser._actions: if action.option_strings and action.nargs != 0: flags.update(action.option_strings) _holder_value_flags_cache = frozenset(flags) except Exception: _holder_value_flags_cache = _HOLDER_VALUE_FLAGS_FALLBACK return _holder_value_flags_cache def _hermes_holder_subcommand(cmdline: str) -> str | None: """The actual Hermes SUBCOMMAND a venv-holder argv runs, or None. Token-based, never substring (#90778: ``kanban --preserve-cache`` contained \"serve\" and got labeled as the Desktop backend). Finds the ``hermes_cli.main`` / ``hermes(.exe)`` entry token, then returns the first following token that is not a flag or a flag's value. Profile selectors (``--profile X``, ``-p X``) are skipped like the canonical gateway matcher does. Returns None when no subcommand can be determined — callers must NOT guess a label in that case. """ try: import shlex tokens = shlex.split(cmdline, posix=False) except Exception: tokens = cmdline.split() entry_idx: int | None = None for i, token in enumerate(tokens): low = token.lower().strip('"') if low.endswith("hermes_cli.main") and i > 0 and tokens[i - 1] == "-m": entry_idx = i break base = low.rsplit("\\", 1)[-1].rsplit("/", 1)[-1] if base in ("hermes", "hermes.exe"): entry_idx = i break if entry_idx is None: return None value_flags = _holder_value_flags() i = entry_idx + 1 while i < len(tokens): token = tokens[i] if token in value_flags or token.split("=", 1)[0] in value_flags: # --flag value consumes two tokens; --flag=value consumes one. i += 1 if "=" in token else 2 continue if token.startswith("-"): i += 1 continue return token.lower() return None def _format_venv_python_holders_message(matches: list[tuple[int, str, str]]) -> str: """Explain which venv processes block the update and how to clear them. Holder labels come from the parsed SUBCOMMAND, never substring matching (#90778): a standalone ``hermes dashboard`` must not be labeled as the Desktop backend (advice to close an app that isn't running), and flags like ``--preserve-cache`` must not match \"serve\". Unknown argv gets no hint rather than a wrong one. """ lines = [ "✗ Other Hermes processes are running from this install's venv:", ] hint_by_subcommand = { "serve": " ← Hermes backend (if the Desktop app is open, close it)", "dashboard": " ← hermes dashboard (stop it: hermes dashboard stop, or close that terminal)", "gateway": " ← gateway", } for pid, name, cmdline in matches[:6]: sub = _hermes_holder_subcommand(cmdline) hint = hint_by_subcommand.get(sub or "", "") lines.append(f" PID {pid} {name} {cmdline[:120]}{hint}") if len(matches) > 6: lines.append(f" ... and {len(matches) - 6} more") lines.append("") lines.append( " On Windows these keep native extension files (.pyd) locked, so the" ) lines.append( " dependency update would fail partway and leave a broken install." ) lines.append( " Close the Hermes desktop app / other Hermes terminals, then re-run:" ) lines.append(" hermes update") lines.append(" (or use `hermes update --force-venv` to proceed anyway at your own risk)") return "\n".join(lines) def _venv_launcher_ancestors(pids: list[int]) -> list[int]: """Return venv-interpreter ancestors of *pids* that hold the install open. On Windows a gateway started through the venv shim is a **two-process chain**: ``venv\\Scripts\\python.exe`` (the launcher, which keeps native ``.pyd`` files from the venv mapped) spawns the actual interpreter from uv's managed CPython directory (``AppData\\Roaming\\uv\\python\\...``). The gateway writes its PID file from the *child*, so ``find_gateway_pids()`` — and therefore this module's pause set — only ever sees the uv-side worker. ``_detect_venv_python_processes()`` matches on the venv path prefix, so the guard downstream of the pause sees the *launcher* instead. The two sets are disjoint, which meant a paused gateway still tripped the venv-holder guard and aborted the update every time (the Desktop "venv-blocked: N process(es) hold the install" dead-end, where the reported holder is a gateway the updater believes it already stopped). Walking one hop up from each mapped gateway PID and keeping ancestors that live under the project venv closes the gap. Only the venv-side parent is returned — unrelated ancestors (the Scheduled Task's ``cmd.exe``, an operator's shell) are ignored so we never widen the blast radius beyond the gateway's own launcher. Never raises. """ if not _m()._is_windows() or not pids: return [] try: import psutil except Exception: return [] venv_dir = _m().PROJECT_ROOT / "venv" try: venv_prefix = str(venv_dir.resolve()).lower().rstrip(os.sep) + os.sep except OSError: venv_prefix = str(venv_dir).lower().rstrip(os.sep) + os.sep # Never return ourselves or our own ancestry: a CLI ``hermes update`` # runs from the venv python and would otherwise nominate itself. # Same #87594 carve-out as _detect_venv_python_processes: a GATEWAY # ancestor is not "our own ancestry" in the interactive sense — it is # the process the pause machinery must see (the /update-from-gateway # topology makes the updater the gateway's child). try: from gateway.status import looks_like_gateway_command_line as _is_gw except Exception: _is_gw = None skip: set[int] = {os.getpid()} try: for anc in psutil.Process().parents(): try: anc_cmdline = " ".join(anc.cmdline() or []) except Exception: anc_cmdline = "" if _is_gw is not None and anc_cmdline and _is_gw(anc_cmdline): continue skip.add(int(anc.pid)) except Exception: pass found: list[int] = [] for pid in pids: try: parent = psutil.Process(int(pid)).parent() except Exception: continue if parent is None: continue ppid = int(parent.pid) if ppid in skip or ppid in found or ppid in set(pids): continue try: exe = (parent.exe() or "").lower() except Exception: continue if exe.startswith(venv_prefix): found.append(ppid) return found def _leftover_pausable_gateway_pids( matches: list[tuple[int, str, str]], ) -> list[int] | None: """PIDs from *matches* when every remaining venv holder is a pausable gateway. ``_pause_windows_gateways_for_update()`` stops every gateway its discovery finds, but the venv-holder guard downstream sees the process table as it is *now*: a gateway respawned by its supervisor (Scheduled Task, login watchdog) inside the pause→guard window, or one started through a spawn path the discovery does not map, still holds venv ``.pyd`` files and would dead-end the update — an abort pointed at exactly the kind of process the pause machinery exists to stop. Holders are classified with the same matcher the Desktop preflight uses to exempt them (``_is_pausable_gateway``), so the preflight's exemption and this guard's tolerance cannot drift apart — matcher drift between two views of the same process table is what produced the launcher/worker dead-end fixed above. The scan captures only a 120-char cmdline prefix, so the live argv is re-read where psutil allows; an unreadable argv falls back to the captured prefix. Returns ``None`` when any holder is not a pausable gateway — an operator REPL, a stray script, or the Desktop backend has no pause machinery downstream, and the guard must keep refusing exactly as before. """ from hermes_cli._scan_venv_blockers import _is_pausable_gateway try: import psutil # type: ignore except Exception: psutil = None pids: list[int] = [] for pid, _name, cmdline in matches: argv = cmdline if psutil is not None: try: argv = " ".join(psutil.Process(int(pid)).cmdline()) or cmdline except Exception: pass if not _is_pausable_gateway(argv): return None pids.append(int(pid)) return pids def _refuse_gateway_ancestor_tree_kill( pids: list[int], *, gateway_mode: bool ) -> bool: """Refuse a plain Windows update that would kill its own process tree. A chat agent can launch plain ``hermes update`` through its terminal tool. In that topology the updater is a child of the gateway. The leftover holder recovery below uses ``taskkill /T /F`` on Windows, so force-stopping that gateway also kills the updater before it can mutate the checkout (#98814). ``/update`` uses the supported ``--gateway`` hand-off and is deliberately exempt: it detaches the updater and provides file-based progress/result delivery. For every other invocation, refuse only when a nominated gateway is positively identified as this process's ancestor. If ancestry cannot be established, preserve the existing holder recovery behavior. """ if gateway_mode or not pids: return False try: from hermes_cli.gateway import _is_pid_ancestor_of_current_process ancestors = [ int(pid) for pid in pids if _is_pid_ancestor_of_current_process(int(pid)) ] except Exception as exc: logger.debug("Could not inspect gateway ancestry before tree-kill: %s", exc) return False if not ancestors: return False rendered = ", ".join(str(pid) for pid in ancestors) print( "✗ Refusing to stop the gateway process tree because this updater " f"is running inside it (gateway PID(s): {rendered})." ) print( " On Windows, taskkill /T would terminate the updater before the " "update can run." ) print(" From a chat platform, use `/update` instead.") print(" Otherwise, run `hermes update` from a separate terminal.") return True def _ledger_manual_serve_holders( matches: list[tuple[int, str, str]], ) -> list[dict]: """Ledger entries for venv holders that are MANUAL serve/dashboard backends. Positive identity only (#63206): the process self-registered in the spawn ledger with purpose serve/dashboard, its (pid, create_time) still matches a live process, and its recorded spawner is NOT alive (a Desktop-owned backend keeps its live Electron spawner and must keep the refusal — the app would respawn what we kill; a PowerShell-launched serve has no live Hermes spawner). Returns the full ledger entries so the relauncher can rebuild the launch command from structured host/port/profile instead of parsing argv. """ try: from hermes_cli.process_identity import ledger_entries, spawner_is_dead except Exception: return [] holder_pids = {int(pid) for pid, _name, _cmd in matches} out: list[dict] = [] for entry in ledger_entries(): if entry.get("purpose") not in ("serve", "dashboard"): continue pid = entry.get("pid") if not isinstance(pid, int) or pid not in holder_pids: continue if spawner_is_dead(entry) is False: continue # live Desktop supervisor owns it — keep refusing out.append(entry) return out def _serve_relaunch_commands(entries: list[dict]) -> list[list[str]]: """Rebuild launch commands for stopped serves from structured identity. Uses the ledger's host/port/profile fields — never argv parsing (a joined argv string cannot round-trip Windows paths with spaces). Entries without a recorded port are skipped; the caller prints the manual hint for those. """ commands: list[list[str]] = [] hermes = None try: scripts_dir = _m()._venv_scripts_dir() if scripts_dir is not None: for name in ("hermes.exe", "hermes"): candidate = scripts_dir / name if candidate.is_file(): hermes = str(candidate) break except Exception: hermes = None if hermes is None: hermes = "hermes" for entry in entries: port = entry.get("port") if not isinstance(port, int) or port <= 0: continue cmd = [hermes] profile = str(entry.get("profile") or "") if profile and profile != "default": cmd += ["--profile", profile] cmd.append(str(entry.get("purpose"))) host = str(entry.get("host") or "") if host: cmd += ["--host", host] cmd += ["--port", str(port)] commands.append(cmd) return commands def _relaunch_stopped_serves(token: dict) -> None: """Idempotent atexit relaunch of manual serves stopped by the venv guard. Mirrors the gateway resume token contract: `pending` flips False on the first invocation so the explicit call and the atexit registration cannot double-spawn (#63206). """ if not token.get("pending"): return token["pending"] = False entries = token.get("entries") or [] if not entries: return commands = _serve_relaunch_commands(entries) skipped = len(entries) - len(commands) failed: list = [] if commands: print(" ⟲ Relaunching stopped serve/dashboard backend(s)") failed = _m()._respawn_dashboard_processes(commands) if skipped or failed: print( " ⚠ Some stopped backends could not be relaunched automatically; " "restart them manually (hermes serve --host --port )." ) try: from hermes_cli.update_receipt import record_step record_step( "serve_relaunch", not failed and not skipped, f"relaunched={len(commands) - len(failed)} failed={len(failed)} skipped={skipped}", ) except Exception: pass def _orphaned_desktop_backend_pids( matches: list[tuple[int, str, str]], ) -> list[tuple[int, int]] | None: """PIDs from *matches* when every remaining holder is an ORPHANED backend. The venv-holder guard refuses on the Desktop app's ``serve`` backend by design: while the Desktop is open, killing its backend is futile (the app supervises and respawns it within seconds), so the user must close the app. But in the GUI-updater handoff path the Desktop has *already exited* — by contract it tree-kills its backends and waits for the venv shim before spawning hermes-setup, and the update-in-progress marker parks any relaunched Desktop from spawning a fresh backend (#50238). A ``serve`` backend still holding the venv at that point is a straggler whose supervisor is gone: SIGTERM raced its spawn, or it belongs to a crashed window. Nothing will respawn it, and refusing on it dead-ends the update with "Hermes is still running" while the user stares at zero open windows (ryanc's 2026-08-09 01:59/02:17 failures). A holder qualifies only when BOTH hold: - its cmdline is a Hermes backend (``hermes_cli.main`` + ``serve`` / ``dashboard``), and - its supervising parent is demonstrably gone: the parent PID no longer exists, or the PID was reused (parent created *after* the child). Tree-aware: the scanner can return an orphaned backend AND one of its managed-runtime descendants (the ``.hermes-runtime`` interpreter child) in the same holder set. That descendant has a live parent — the orphaned backend itself — and isn't a ``serve`` cmdline, so per-process rules would refuse a set that is entirely safe to reap. Holders that sit inside an accepted orphan root's tree are therefore folded into that root (only roots are returned; ``taskkill /T`` reaps the descendants). Any other live-parent backend (the Desktop is still open), non-backend holder outside an orphan tree, or unprovable case disqualifies the whole set — the guard must keep refusing exactly as before. Returns ``None`` in that case, or when psutil is unavailable (can't prove orphanhood → refuse). Never raises. """ try: import psutil # type: ignore except Exception: return None def _is_backend(argv_low: str) -> bool: return "hermes_cli.main" in argv_low and ( " serve" in argv_low or " dashboard" in argv_low ) # Pass 1: find orphaned backend ROOTS among the holders. roots: list[tuple[int, int]] = [] remaining: list[tuple[int, str]] = [] # (pid, argv_low) still to justify for pid, _name, cmdline in matches: argv = cmdline try: argv = " ".join(psutil.Process(int(pid)).cmdline()) or cmdline except psutil.NoSuchProcess: # Holder exited between scan and classification — nothing to # reap, nothing blocking. Skip it. continue except Exception: pass low = argv.lower() if not _is_backend(low): remaining.append((int(pid), low)) continue try: proc = psutil.Process(int(pid)) # Fingerprint from the SAME psutil handle used for classification # below, quantized to centiseconds — the exact scheme # gateway.status.get_process_start_time uses on Windows, so the # value round-trips through pid_is_hermes at kill time. (Reading # /proc//stat here instead would consult the HOST process # table and use different units.) process_start_time = int(round(proc.create_time() * 100)) except psutil.NoSuchProcess: # The candidate itself exited during classification; there is # nothing left to reap and no identity to pass to taskkill. continue except Exception: return None try: ppid = proc.ppid() parent = psutil.Process(ppid) if ppid else None if parent is not None and parent.is_running(): # PID-reuse check: a "parent" created after its child is a # recycled PID, not the real (dead) supervisor. if parent.create_time() <= proc.create_time(): # Live parent — NOT a root. But it may still be a # descendant of an orphan root: the venv python.exe is # a trampoline that re-execs the uv-managed interpreter # with the SAME backend argv, so the worker half of the # two-process chain lands here. Defer to pass 2 instead # of refusing outright. remaining.append((int(pid), low)) continue except psutil.NoSuchProcess: pass # parent gone → orphan except Exception: return None roots.append((int(pid), process_start_time)) # Pass 2: every non-backend holder must be a descendant of an accepted # orphan root — then it dies with the root's tree reap. Anything else # (operator REPL, stray script) keeps the refusal. root_set = {pid for pid, _start_time in roots} for pid, _low in remaining: if not root_set: return None try: ancestors = {int(a.pid) for a in psutil.Process(pid).parents()} except psutil.NoSuchProcess: continue # exited already except Exception: return None if not (ancestors & root_set): return None return roots def _ledger_reapable_backend_pids( matches: list[tuple[int, str, str]], ) -> list[int]: """PIDs positively identified by the spawn ledger as orphaned backends. The strongest rung: instead of inferring lineage from PPIDs or cmdline shape, look each venv holder up in the machine spawn ledger (``hermes_cli.process_identity``). A holder qualifies when ALL of: - its ``(pid, create_time)`` matches a live ledger entry (PID reuse cannot forge this pair); - the entry's purpose is a reapable backend kind (serve/dashboard/ gateway — never interactive processes); - the entry's recorded SPAWNER is provably dead (``spawner_is_dead``). Unlike the heuristic rungs, this is safe in ANY update context — no hand-off contract needed — because the ownership claim is explicit: the process itself declared who supervises it, and that supervisor is gone. Holders not in the ledger are simply not returned (they fall through to the later rungs); they never disqualify the identified ones. Never raises. """ try: from hermes_cli.process_identity import ( REAPABLE_PURPOSES, ledger_entries, spawner_is_dead, ) entries = ledger_entries() except Exception: return [] by_pid = {e.get("pid"): e for e in entries if isinstance(e.get("pid"), int)} roots: list[int] = [] for pid, _name, _cmdline in matches: entry = by_pid.get(int(pid)) if not entry: continue if entry.get("purpose") not in REAPABLE_PURPOSES: continue if spawner_is_dead(entry) is True: roots.append(int(pid)) return roots def _handoff_reapable_backend_pids( matches: list[tuple[int, str, str]], ) -> list[int] | None: """PIDs of Hermes ``serve``/``dashboard`` backends safe to reap during a GUI-updater hand-off, INCLUDING ones with a still-live parent. Complements ``_orphaned_desktop_backend_pids``, which only reaps backends whose supervisor is provably dead. That check returns ``None`` (keep refusing) the moment ANY holder still has a live parent — which is exactly the case that produced the field incident this fixes: a Windows Desktop update hand-off (``update --yes --gateway --force``) left a *swarm* of per-profile ``serve`` backends (mr-tester, probe-inherit, turqoise, …) holding ``cryptography\\_rust.pyd``. Several still had a lingering parent (the tearing-down Electron process, or the two-hop venv launcher→worker chain mid-exit), so the orphan check disqualified the WHOLE set and the update dead-ended — the user saw a 12-minute hang, then force-closed, and the half-done state stranded bot sessions. The hand-off is the safe signal: when the update-incomplete marker is present (the GUI updater claimed it) AND this is a ``--gateway`` hand-off run AND no live Desktop shim (``hermes.exe``) is open, NOTHING legitimate is supervising or respawning a ``serve`` backend from this venv — by the hand-off contract the Desktop tree-kills its backends and parks any relaunch behind the marker (#50238). Any ``serve`` backend still holding the venv here is therefore a leak, live parent or not, and reaping its tree is correct rather than a race. Guarded conservatively: - Only Hermes backends (``hermes_cli.main`` + ``serve``/``dashboard``) from THIS install's venv qualify; a non-backend holder (operator REPL, stray script) disqualifies the whole set → ``None`` (keep refusing), so we never widen the blast radius during a hand-off. - Only runs when the CALLER has confirmed the hand-off context (``args.gateway`` AND a claimed update-incomplete marker AND no live ``hermes.exe`` shim) — outside that gate this function is never called and the stricter orphan-only path stands. - psutil unavailable → ``None`` (can't re-read argv to classify → refuse). Returns the backend root PIDs to tree-reap, or ``None`` to leave the decision to the caller's existing rungs. Never raises. """ try: import psutil # type: ignore except Exception: return None def _is_backend(argv_low: str) -> bool: return "hermes_cli.main" in argv_low and ( " serve" in argv_low or " dashboard" in argv_low ) roots: list[int] = [] for pid, _name, cmdline in matches: argv = cmdline try: argv = " ".join(psutil.Process(int(pid)).cmdline()) or cmdline except psutil.NoSuchProcess: # Exited between scan and classification — nothing to reap. continue except Exception: pass if not _is_backend(argv.lower()): # A non-backend holder during a hand-off is unexpected; refuse the # whole set rather than reap something we cannot justify. return None roots.append(int(pid)) return roots or None def _stop_process_trees( pids: list[int] | list[tuple[int, int]], ) -> None: """Force-stop each PID with its full child tree (Windows). ``taskkill /T /F`` mirrors the Desktop's ``forceKillProcessTree`` and install.ps1's venv sweep: stopping only the parent can leave a managed ``.hermes-runtime`` interpreter child alive and holding the install open (#70026). Best effort; never raises. """ from gateway.status import get_process_start_time from hermes_cli._subprocess_compat import pid_is_hermes, windows_hide_flags for entry in pids: if isinstance(entry, tuple): pid, expected_start_time = entry else: pid = int(entry) expected_start_time = get_process_start_time(pid) try: if expected_start_time is None: logger.debug( "Skipping taskkill of PID %s: process identity unavailable", pid, ) continue if not pid_is_hermes( pid, expected_start_time=expected_start_time, ): logger.debug( "Skipping taskkill of non-Hermes or changed PID %s", pid, ) continue subprocess.run( ["taskkill", "/PID", str(pid), "/T", "/F"], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL, creationflags=windows_hide_flags(), ) except Exception as exc: logger.debug("Could not stop process tree %s: %s", pid, exc) def _looks_like_desktop_control_plane(cmdline: str) -> bool: """True for this-install ``hermes serve`` / ``hermes dashboard`` argv. That is the Desktop control plane, not the messaging gateway. Serve and dashboard do not host platform adapters (#92091); do not feed this into ``looks_like_gateway_command_line``. Token-based via the parser-derived subcommand classifier — never substring (#90778/#91869: ``kanban --preserve-cache`` contains "serve", ``-m dashboard chat`` contains " dashboard"; both are NOT control planes). A cmdline whose subcommand cannot be determined is NOT a control plane — callers must not guess ownership. """ if "hermes_cli.main" not in (cmdline or "").lower(): return False return _hermes_holder_subcommand(cmdline) in ("serve", "dashboard") def _desktop_owns_gateway_lifecycle() -> bool: """True when Desktop currently supervises this install's control plane. The updater must not steal gateway start in that case: Desktop owns start/stop via ``/api/gateway/*``. This is *not* proof messaging is already served — a live serve process is the control plane, and the gateway is a detached sibling (#76129 / #92091). Prefer the spawn ledger (owned identity). Fall back to the install-scoped venv-holder scan already used by the lock guard; an orphaned control-plane process (supervisor gone) does not count. """ try: from hermes_cli.process_identity import ledger_entries, spawner_is_dead for entry in ledger_entries(): if entry.get("purpose") not in ("serve", "dashboard"): continue if spawner_is_dead(entry) is False: return True except Exception as exc: logger.debug("Desktop-lifecycle ledger probe failed: %s", exc) try: import psutil except Exception: psutil = None try: holders = _m()._detect_venv_python_processes() except Exception as exc: logger.debug("Desktop-lifecycle holder scan failed: %s", exc) return False for pid, _name, cmdline in holders: if not _looks_like_desktop_control_plane(cmdline): continue if psutil is None: # Cannot prove orphanhood; a live this-install control plane is # enough to refuse stealing gateway start. return True try: proc = psutil.Process(int(pid)) parent = proc.parent() if parent is None or not parent.is_running(): continue if parent.create_time() > proc.create_time(): continue return True except Exception: continue return False def _stop_windows_gateway_service( name: str, *, expected_processes: tuple[tuple[int, float], ...] = (), expected_service_identity: tuple[int, float] | None = None, expected_gateway_identity: tuple[int, float] | None = None, timeout: float = 30.0, ) -> None: """Stop one verified Windows service and wait until SCM reports it down.""" import psutil # noqa: PLC0415 service = psutil.win_service_get(name) if expected_service_identity is not None: try: current_status = str(service.status()) current_service_pid = int(service.pid() or 0) except Exception as exc: raise RuntimeError( f"Windows service {name} SCM identity is unavailable before stop" ) from exc if current_status != "running": raise RuntimeError( f"Windows service {name} is not stably running before stop: {current_status}" ) if current_service_pid != int(expected_service_identity[0]): raise RuntimeError( f"Windows service {name} SCM process identity changed before stop" ) for label, identity in ( ("service", expected_service_identity), ("gateway", expected_gateway_identity), ): if identity is None: continue pid, create_time = identity try: current = float(psutil.Process(int(pid)).create_time()) except Exception as exc: raise RuntimeError( f"Windows {label} process identity is unavailable before stop" ) from exc if abs(current - float(create_time)) > 0.001: raise RuntimeError( f"Windows {label} process identity changed before stop" ) if expected_service_identity is not None and expected_gateway_identity is not None: service_pid = int(expected_service_identity[0]) gateway_pid = int(expected_gateway_identity[0]) try: ancestor_pids = { int(parent.pid) for parent in psutil.Process(gateway_pid).parents() } except Exception as exc: raise RuntimeError( "Windows gateway ancestry is unavailable before service stop" ) from exc if service_pid not in ancestor_pids: raise RuntimeError( f"Windows gateway is no longer owned by service {name}" ) result = subprocess.run( ["sc.exe", "stop", name], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, check=False, ) if result.returncode != 0 and service.status() != "stopped": detail = (result.stderr or result.stdout).strip() raise RuntimeError(detail or f"sc.exe stop failed with {result.returncode}") def _original_process_is_alive(pid: int, create_time: float) -> bool: try: current = float(psutil.Process(pid).create_time()) except (psutil.NoSuchProcess, psutil.ZombieProcess): # A vanished process is clear. return False except Exception: # AccessDenied or any unknown probe failure stays fail-closed # because the venv may still be locked. return True return abs(current - create_time) <= 0.001 alive = [ pid for pid, create_time in expected_processes if _original_process_is_alive(pid, create_time) ] deadline = _time.monotonic() + timeout while _time.monotonic() < deadline: service_stopped = service.status() == "stopped" alive = [ pid for pid, create_time in expected_processes if _original_process_is_alive(pid, create_time) ] if service_stopped and not alive: return _time.sleep(0.2) if service.status() == "stopped": # We only return if the original processes have also exited their identity. # A lingering process with a matching creation time means the venv mutation # must not proceed — fail closed. alive_after_stop = [ pid for pid, create_time in expected_processes if _original_process_is_alive(pid, create_time) ] if alive_after_stop: raise RuntimeError( f"Windows service {name} stopped but its process tree is still alive: " f"{alive_after_stop}" ) return # If we reach here, the timeout elapsed without the service reaching a stable stopped state # while its original descendants are still alive. Fail closed — venv mutation is unsafe. raise RuntimeError( f"Windows service {name} did not stop within {timeout:.0f}s; venv mutation unsafe." ) def _start_windows_gateway_service(name: str, *, timeout: float = 30.0) -> None: """Start one previously paused Windows service and verify it is running.""" import psutil # noqa: PLC0415 service = psutil.win_service_get(name) result = subprocess.run( ["sc.exe", "start", name], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, check=False, ) if result.returncode != 0 and service.status() != "running": detail = (result.stderr or result.stdout).strip() raise RuntimeError(detail or f"sc.exe start failed with {result.returncode}") deadline = _time.monotonic() + timeout while _time.monotonic() < deadline: if service.status() == "running": return _time.sleep(0.2) raise RuntimeError(f"Windows service {name} did not start within {timeout:.0f}s") def _restore_windows_gateway_service(name: str, *, timeout: float = 60.0) -> None: """Restore a service after an uncertain stop, including STOP_PENDING.""" import psutil # noqa: PLC0415 service = psutil.win_service_get(name) deadline = _time.monotonic() + timeout while _time.monotonic() < deadline: status = service.status() if status == "running": return if status == "stopped": _start_windows_gateway_service(name) return _time.sleep(0.2) raise RuntimeError( f"Windows service {name} did not reach a restorable state within {timeout:.0f}s" ) def _pause_windows_gateways_for_update() -> dict | None: """Stop running Windows gateways before mutating the checkout or venv. Windows scheduled/startup gateways run through pythonw.exe, so the generic hermes.exe concurrent-instance guard does not see them. They still import from the checkout and can keep files locked while ``git`` or ``uv`` updates the install. Stop only PIDs that the gateway discovery code identifies. """ if not _m()._is_windows(): return None try: from gateway.status import get_process_start_time, terminate_pid from hermes_cli.gateway import ( _capture_gateway_argv, _get_restart_drain_timeout, find_gateway_pids, find_profile_gateway_processes, find_windows_gateway_services, ) except Exception as exc: raise RuntimeError( f"Could not prepare Windows gateway pause for update: {exc}" ) from exc try: profile_process_list = find_profile_gateway_processes(strict=True) profile_processes = {proc.pid: proc for proc in profile_process_list} except Exception as exc: raise RuntimeError( f"Could not map Windows gateway PIDs to profiles: {exc}" ) from exc try: service_gateways = find_windows_gateway_services( profile_processes=profile_process_list ) except Exception as exc: raise RuntimeError( f"Could not determine Windows gateway service ownership: {exc}" ) from exc service_gateway_pids = {int(service.gateway_pid) for service in service_gateways} try: running_pids = list( dict.fromkeys( [ *find_gateway_pids(all_profiles=True), *sorted(profile_processes), *sorted(service_gateway_pids), ] ) ) except Exception as exc: raise RuntimeError( f"Could not discover Windows gateway PIDs before update: {exc}" ) from exc if not running_pids: # No gateway is running right now, but the user may have installed an # autostart entry (Scheduled Task or Startup-folder login item) — that # is an explicit "I want a gateway" signal. A gateway that died between # updates (e.g. the spawning terminal/TUI closed, taking its child with # it) would otherwise never come back: the autostart entry only fires on # the next login, and the update flow's resume path only relaunched # gateways that were running when the update began. Cold-start one after # the update so an installed gateway is actually up post-update. Users # who run gateway-less (no autostart entry) get nothing forced on them. # # Exception: Desktop currently owns this install's gateway lifecycle # (live supervised serve/dashboard). A vestigial Startup/Scheduled # Task is not the owner — spawning ``gateway run`` beside Desktop # races ports/state (#76129). Serve is the control plane, not proof # messaging is served; the skip is ownership, not liveness (#92091). try: if _desktop_owns_gateway_lifecycle(): logger.debug( "Skipping Windows gateway cold-start plan: " "Desktop owns gateway lifecycle" ) return None except Exception as exc: logger.debug( "Could not check Desktop gateway-lifecycle ownership before update: %s", exc, ) try: from hermes_cli import gateway_windows if gateway_windows.is_installed(): return { "resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": [], "cold_start_if_installed": True, } except Exception as exc: logger.debug( "Could not check Windows gateway autostart state before update: %s", exc, ) return None profiles: dict[str, int] = {} mapped_pids = [] socket_acks: list[dict] = [] for pid in running_pids: if pid in service_gateway_pids: continue proc = profile_processes.get(pid) if proc is None: continue profiles[str(proc.profile)] = int(pid) mapped_pids.append(int(pid)) _write_update_planned_stop_marker(Path(proc.path), int(pid)) # Socket-first pause (#92091 step 2): ask the gateway to drain and # exit itself instead of relying on the marker poll + force-kill # ladder. A positive ACK means the gateway is running its own # graceful restart path (same drain as SIGUSR1/service restarts) and # will release its venv handles on the way out. No answer (older # gateway, no socket) → the marker watcher / force-kill fallback # below behaves exactly as before this verb existed. try: from gateway.control_socket import pause_gateway_for_update ack = pause_gateway_for_update(Path(proc.path)) if ack and (ack.get("pausing") or ack.get("already_stopping")): socket_acks.append(ack) except Exception as exc: logger.debug( "Socket pause unavailable for gateway %s: %s", pid, exc ) # Resolve each mapped worker's venv-side launcher BEFORE draining: the # drain stops tracking a PID exactly when it dies, so a gracefully # drained worker is gone by the time the wait returns — and a dead pid's # parent cannot be recovered (psutil raises NoSuchProcess). The snapshot # is stopped after the drain alongside the survivors. # # Why launchers matter: the drain targets the PID that wrote the PID # file (the uv-side worker). On Windows that worker's parent is usually # the venv-side ``python.exe`` launcher, which keeps venv ``.pyd`` files # mapped and is what ``_detect_venv_python_processes()`` reports # downstream. Left alive, it trips the venv-holder guard and aborts the # update even though the gateway itself is stopped. launcher_pids = _m()._venv_launcher_ancestors(mapped_pids) print("→ Stopping Windows gateway process(es) before updating Hermes...") try: drain_timeout = max(float(_get_restart_drain_timeout()), 1.0) except Exception: drain_timeout = 10.0 if socket_acks: # A socket-paused gateway drains its ACTIVE TURN before exiting; give # it the budget it declared (plus teardown grace) rather than only # the local default, so a mid-turn gateway isn't force-killed at the # end of a too-short wait — the exact outcome the verb exists to # prevent. try: declared = max( float(a.get("drain_timeout") or 0.0) for a in socket_acks ) drain_timeout = max(drain_timeout, declared + 10.0) except Exception: pass print( f" → {len(socket_acks)} gateway(s) ACKed socket pause; " f"waiting up to {int(drain_timeout)}s for graceful exit" ) survivors = _m()._wait_for_windows_update_gateway_exit( mapped_pids, timeout=drain_timeout, ) unmapped_pids = [ pid for pid in running_pids if pid not in profile_processes and pid not in service_gateway_pids ] # Snapshot each unmapped gateway's command line *before* we force-kill it, # so ``_resume_windows_gateways_after_update`` can respawn it by replaying # its own argv. Unmapped gateways are ones with no profile→PID-file mapping # — e.g. a Windows Scheduled Task running ``pythonw.exe -m hermes_cli.main # gateway run``. Without this snapshot they were force-killed and never # restarted (the "Restart manually after update" dead-end from #50090). unmapped: list[dict] = [] for pid in unmapped_pids: argv = None try: argv = _capture_gateway_argv(int(pid)) except Exception as exc: logger.debug("Could not capture argv for unmapped gateway %s: %s", pid, exc) unmapped.append({"pid": int(pid), "argv": argv}) # Stop drain survivors, unmapped gateways, and the pre-drain launcher # snapshot. ``terminate_pid(force=True)`` is a tree kill, so a launcher # that outlived its worker takes any stragglers with it; a launcher that # already exited with its drained worker raises ProcessLookupError below # and is skipped. force_killed = [] for pid in sorted(set(survivors).union(unmapped_pids).union(launcher_pids)): try: pid_int = int(pid) terminate_pid( pid_int, force=True, expected_start_time=get_process_start_time(pid_int), ) force_killed.append(pid_int) except (ProcessLookupError, PermissionError, OSError): pass if profiles: print(f" ✓ Paused gateway profile(s): {', '.join(sorted(profiles))}") if force_killed: print(f" → Force-stopped {len(force_killed)} gateway process(es)") if unmapped_pids: respawnable = sum(1 for u in unmapped if u.get("argv")) print( f" → Stopped {len(unmapped_pids)} gateway process(es) without profile mapping" ) if respawnable < len(unmapped_pids): # Some had no recoverable command line (psutil missing, access # denied, already gone): those still need a manual restart. print(" Restart manually after update: hermes gateway run") token = { "resume_needed": True, "profiles": profiles, "unmapped_pids": unmapped_pids, "unmapped": unmapped, } # Stop SCM-supervised gateways only after every fallible preparation step # for ordinary gateways is complete. From this point to return, any error # restores both the attempted services and the already-paused ordinary # gateways before aborting the update. paused_services = [] current_service_name = None try: for service in service_gateways: current_service_name = str(service.name) _stop_windows_gateway_service( current_service_name, expected_processes=tuple( getattr(service, "descendant_identities", ()) ), expected_service_identity=( int(service.service_pid), float(service.service_create_time), ), expected_gateway_identity=( int(service.gateway_pid), float(service.gateway_create_time), ), ) paused_services.append(current_service_name) current_service_name = None if paused_services: token["services"] = paused_services token["expected_services"] = list(paused_services) token["restarted_services"] = [] token["service_profiles"] = { str(service.name): str(service.profile) for service in service_gateways if str(service.name) in paused_services } print( " ✓ Paused Windows gateway service(s): " + ", ".join(paused_services) ) return token except Exception as exc: restore_names = [] if current_service_name: restore_names.append(current_service_name) restore_names.extend(reversed(paused_services)) rollback_failures = [] for service_name in dict.fromkeys(restore_names): try: _restore_windows_gateway_service(service_name) except Exception as restore_exc: rollback_failures.append(f"{service_name}: {restore_exc}") if profiles or unmapped: try: _resume_windows_gateways_after_update(token) except Exception as restore_exc: rollback_failures.append(f"ordinary gateways: {restore_exc}") failed_service = current_service_name or "unknown" detail = f"Could not stop Windows gateway service {failed_service}: {exc}" if rollback_failures: detail += "; rollback failures: " + "; ".join(rollback_failures) raise RuntimeError(detail) from exc def _cold_start_windows_gateway_after_update() -> bool: """Start a fresh detached gateway after update when one is installed but down. Invoked from ``_resume_windows_gateways_after_update`` for the ``cold_start_if_installed`` case: no gateway was running when the update began, but an autostart entry (Scheduled Task / Startup-folder login item) is installed, signalling the user wants a gateway. Unlike the relaunch paths — which watch an old PID and respawn once it exits — this is a direct fresh spawn via the same hidden-console + breakaway path that ``hermes gateway start`` uses (``gateway_windows._spawn_detached``). Best-effort and idempotent: re-checks that nothing is running first so a concurrent start (e.g. the autostart entry firing) can't produce a duplicate gateway. A successful ``Popen`` only proves the process was created, not that it survived (e.g. a Windows job object denying breakaway kills it before it logs anything — #84185). So the success line is gated on the same post-spawn liveness poll every other ``_spawn_detached`` caller uses (``gateway_windows._report_gateway_start``), instead of being printed unconditionally from the returned PID. """ if not _m()._is_windows(): return True try: from hermes_cli import gateway_windows from hermes_cli.gateway import find_gateway_pids except Exception as exc: raise RuntimeError( f"Could not load Windows gateway cold-start helpers: {exc}" ) from exc # Re-check liveness right before spawning — between pause and resume the # autostart entry may have already brought a gateway up, or a leftover # process may have re-registered. Don't double-start. try: if list(find_gateway_pids(all_profiles=True)): return True except Exception as exc: raise RuntimeError( f"Could not re-check gateway liveness before cold-start: {exc}" ) from exc try: if _desktop_owns_gateway_lifecycle(): logger.debug( "Skipping Windows gateway cold-start: Desktop owns gateway lifecycle" ) return True except Exception as exc: raise RuntimeError( "Could not re-check Desktop gateway-lifecycle ownership before cold-start: " f"{exc}" ) from exc try: pid = gateway_windows._spawn_detached() except Exception as exc: raise RuntimeError(f"Could not cold-start Windows gateway after update: {exc}") from exc if not pid: raise RuntimeError("Windows gateway cold-start did not return a process ID") ready_pids = gateway_windows._wait_for_gateway_ready() if not ready_pids: raise RuntimeError( f"Windows gateway cold-start PID {pid} did not become ready" ) print() print( "✓ Gateway started via cold-start after update " f"(PID: {', '.join(map(str, ready_pids))})" ) # Persist the PIDs this ✓ vouched for so a death AFTER the updater exits # (parent Job Object teardown, #91675) is reported by the next CLI # invocation instead of staying silent. Best-effort. try: gateway_windows._write_start_attestation( ready_pids, "cold-start after update" ) except Exception: pass return True def _for_each_systemd_gateway_unit( list_units_stdout: str, *, process_unit, on_unit_timeout, ) -> None: """Process each ``hermes-gateway*.service``/``hermes-serve*.service`` unit from ``systemctl list-units``. ``subprocess.TimeoutExpired`` raised by ``process_unit`` is isolated to that unit via ``on_unit_timeout`` so one wedged systemctl call cannot abort the rest of the fleet (#68523). """ for line in (list_units_stdout or "").strip().splitlines(): parts = line.split() if not parts: continue unit = parts[0] if not unit.endswith(".service"): continue # list-units is already pattern-filtered, but keep the name gate so a # stray non-gateway/serve line cannot enter the restart path. # ``unit.startswith("hermes-serve")`` alone would also accept the # unrelated ``hermes-server.service`` — require the exact base unit # or the hyphenated profile family instead (review on #83595). if not ( unit == "hermes-gateway.service" or unit.startswith("hermes-gateway-") or unit == "hermes-serve.service" or unit.startswith("hermes-serve-") ): continue svc_name = unit.removesuffix(".service") try: process_unit(svc_name) except subprocess.TimeoutExpired as exc: on_unit_timeout(svc_name, exc) def _service_unit_supports_graceful_sigusr1_restart(svc_name: str) -> bool: """Whether *svc_name* wires SIGUSR1 to a graceful drain-then-restart. Only ``hermes-gateway*`` units run ``gateway/run.py``, which installs the SIGUSR1 handler. ``hermes-serve*`` units (#83438) don't, so sending them SIGUSR1 would just invoke the default terminate action and burn the full drain budget waiting for an exit that was never graceful — go straight to the blunt ``systemctl restart`` path for those instead. Uses the same strict exact/hyphenated shape as the unit-name gate in ``_for_each_systemd_gateway_unit`` so a hypothetical near-prefix unit (``hermes-gateway-helper`` is fine — profile units are ``hermes-gateway-`` — but ``hermes-gatewayd``-style names are not) can't be sent a SIGUSR1 it doesn't handle. """ return svc_name == "hermes-gateway" or svc_name.startswith("hermes-gateway-") def _warn_incomplete_gateway_fleet_restart(failed_units: list) -> None: """Print an explicit incomplete-update warning for unrestarted units.""" from hermes_cli.gateway import is_macos if not failed_units: return # Preserve discovery order while de-duplicating. seen = set() ordered = [] for name in failed_units: if name in seen: continue seen.add(name) ordered.append(name) print() print("⚠ Update incomplete — some units were not restarted:") for name in ordered: print(f" - {name}") if is_macos(): # A launchd label reaches this list when launchd was not supervising a # live process after the restart (#88848), so the unit is not merely # stale — it is very likely deregistered, and `launchctl kickstart` # cannot revive a job launchd no longer knows about. print(" Listed services may be deregistered from launchd, or still") print(" running pre-update code (mixed sys.modules). Recover with:") print(" hermes gateway status") print(" launchctl list | grep