yilsem/aiturk-hermes-ide
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

Browse Source
This commit is contained in:
yilsem
2026-09-05 13:26:46 +03:00
commit 03634b1ca3
11340 changed files with 3442369 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/memory/holographic/README.md
+36
View File
@@ -0,0 +1,36 @@
# Holographic Memory Provider
Local SQLite fact store with FTS5 search, trust scoring, entity resolution, and HRR-based compositional retrieval.
## Requirements
None — uses SQLite (always available). NumPy optional for HRR algebra.
## Setup
```bash
hermes memory setup # select "holographic"
```
Or manually:
```bash
hermes config set memory.provider holographic
```
## Config
Config in `config.yaml` under `plugins.hermes-memory-store`:
| Key | Default | Description |
|-----|---------|-------------|
| `db_path` | `$HERMES_HOME/memory_store.db` | SQLite database path |
| `auto_extract` | `false` | Auto-extract facts at session end |
| `default_trust` | `0.5` | Default trust score for new facts |
| `hrr_dim` | `1024` | HRR vector dimensions |
## Tools
| Tool | Description |
|------|-------------|
| `fact_store` | 9 actions: add, search, probe, related, reason, contradict, update, remove, list |
| `fact_feedback` | Rate facts as helpful/unhelpful (trains trust scores) |
plugins/memory/holographic/__init__.py
+462
View File
@@ -0,0 +1,462 @@
"""hermes-memory-store — holographic memory plugin using MemoryProvider interface.
Registers as a MemoryProvider plugin, giving the agent structured fact storage
with entity resolution, trust scoring, and HRR-based compositional retrieval.
Original plugin by dusterbloom (PR #2351), adapted to the MemoryProvider ABC.
Config in $HERMES_HOME/config.yaml (profile-scoped):
plugins:
hermes-memory-store:
db_path: $HERMES_HOME/memory_store.db # omit to use the default
auto_extract: false
default_trust: 0.5
min_trust_threshold: 0.3
temporal_decay_half_life: 0
"""
from __future__ import annotations
import json
import logging
import re
from typing import Any, Dict, List
from agent.memory_provider import MemoryProvider
from tools.registry import tool_error
from utils import is_truthy_value
from .store import MemoryStore
from .retrieval import FactRetriever
from hermes_cli.config import cfg_get
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Tool schemas (unchanged from original PR)
# ---------------------------------------------------------------------------
FACT_STORE_SCHEMA = {
"name": "fact_store",
"description": (
"Deep structured memory with algebraic reasoning. "
"Use alongside the memory tool — memory for always-on context, "
"fact_store for deep recall and compositional queries.\n\n"
"ACTIONS (simple → powerful):\n"
"• add — Store a fact the user would expect you to remember.\n"
"• search — Keyword lookup ('editor config', 'deploy process').\n"
"• probe — Entity recall: ALL facts about a person/thing.\n"
"• related — What connects to an entity? Structural adjacency.\n"
"• reason — Compositional: facts connected to MULTIPLE entities simultaneously.\n"
"• contradict — Memory hygiene: find facts making conflicting claims.\n"
"• update/remove/list — CRUD operations.\n\n"
"IMPORTANT: Before answering questions about the user, ALWAYS probe or reason first."
),
"parameters": {
"type": "object",
"properties": {
"action": {
"type": "string",
"enum": ["add", "search", "probe", "related", "reason", "contradict", "update", "remove", "list"],
},
"content": {"type": "string", "description": "Fact content (required for 'add')."},
"query": {"type": "string", "description": "Search query (required for 'search')."},
"entity": {"type": "string", "description": "Entity name for 'probe'/'related'."},
"entities": {"type": "array", "items": {"type": "string"}, "description": "Entity names for 'reason'."},
"fact_id": {"type": "integer", "description": "Fact ID for 'update'/'remove'."},
"category": {"type": "string", "enum": ["user_pref", "project", "tool", "general"]},
"tags": {"type": "string", "description": "Comma-separated tags."},
"trust_delta": {"type": "number", "description": "Trust adjustment for 'update'."},
"min_trust": {"type": "number", "description": "Minimum trust filter (default: 0.3)."},
"limit": {"type": "integer", "description": "Max results (default: 10)."},
},
"required": ["action"],
},
}
FACT_FEEDBACK_SCHEMA = {
"name": "fact_feedback",
"description": (
"Rate a fact after using it. Mark 'helpful' if accurate, 'unhelpful' if outdated. "
"This trains the memory — good facts rise, bad facts sink."
),
"parameters": {
"type": "object",
"properties": {
"action": {"type": "string", "enum": ["helpful", "unhelpful"]},
"fact_id": {"type": "integer", "description": "The fact ID to rate."},
},
"required": ["action", "fact_id"],
},
}
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
def _load_plugin_config() -> dict:
try:
# Canonical loader: behavioral read now honors the managed-scope
# overlay + ${VAR} expansion (e.g. an api key template) too.
from hermes_cli.config import load_config_readonly
all_config = load_config_readonly()
return cfg_get(all_config, "plugins", "hermes-memory-store", default={}) or {}
except Exception:
return {}
# ---------------------------------------------------------------------------
# MemoryProvider implementation
# ---------------------------------------------------------------------------
class HolographicMemoryProvider(MemoryProvider):
"""Holographic memory with structured facts, entity resolution, and HRR retrieval."""
def __init__(self, config: dict | None = None):
self._config = config or _load_plugin_config()
self._store = None
self._retriever = None
self._min_trust = float(self._config.get("min_trust_threshold", 0.3))
@property
def name(self) -> str:
return "holographic"
def is_available(self) -> bool:
return True # SQLite is always available, numpy is optional
def save_config(self, values, hermes_home):
"""Write config to config.yaml under plugins.hermes-memory-store."""
from pathlib import Path
config_path = Path(hermes_home) / "config.yaml"
try:
import yaml
# Write-back round-trip: raw read is correct (merged defaults
# must not be persisted back into the user's file).
from hermes_cli.config import read_user_config_raw
existing = read_user_config_raw(config_path)
existing.setdefault("plugins", {})
existing["plugins"]["hermes-memory-store"] = values
with open(config_path, "w", encoding="utf-8") as f:
yaml.dump(existing, f, default_flow_style=False)
except Exception:
pass
def get_config_schema(self):
from hermes_constants import display_hermes_home
_default_db = f"{display_hermes_home()}/memory_store.db"
return [
{"key": "db_path", "description": "SQLite database path", "default": _default_db},
{"key": "auto_extract", "description": "Auto-extract facts at session end", "default": "false", "choices": ["true", "false"]},
{"key": "default_trust", "description": "Default trust score for new facts", "default": "0.5"},
{"key": "hrr_dim", "description": "HRR vector dimensions", "default": "1024"},
]
def initialize(self, session_id: str, **kwargs) -> None:
from hermes_constants import get_hermes_home
_hermes_home = str(get_hermes_home())
_default_db = _hermes_home + "/memory_store.db"
db_path = self._config.get("db_path", _default_db)
# Expand $HERMES_HOME in user-supplied paths so config values like
# "$HERMES_HOME/memory_store.db" or "~/.hermes/memory_store.db" both
# resolve to the active profile's directory.
if isinstance(db_path, str):
db_path = db_path.replace("$HERMES_HOME", _hermes_home)
db_path = db_path.replace("${HERMES_HOME}", _hermes_home)
default_trust = float(self._config.get("default_trust", 0.5))
hrr_dim = int(self._config.get("hrr_dim", 1024))
hrr_weight = float(self._config.get("hrr_weight", 0.3))
temporal_decay = int(self._config.get("temporal_decay_half_life", 0))
self._store = MemoryStore(db_path=db_path, default_trust=default_trust, hrr_dim=hrr_dim)
self._retriever = FactRetriever(
store=self._store,
temporal_decay_half_life=temporal_decay,
hrr_weight=hrr_weight,
hrr_dim=hrr_dim,
)
self._session_id = session_id
def system_prompt_block(self) -> str:
if not self._store:
return ""
try:
total = self._store._conn.execute(
"SELECT COUNT(*) FROM facts"
).fetchone()[0]
except Exception:
total = 0
if total == 0:
return (
"# Holographic Memory\n"
"Active. Empty fact store — proactively add facts the user would expect you to remember.\n"
"Use fact_store(action='add') to store durable structured facts about people, projects, preferences, decisions.\n"
"Use fact_feedback to rate facts after using them (trains trust scores)."
)
return (
f"# Holographic Memory\n"
f"Active. {total} facts stored with entity resolution and trust scoring.\n"
f"Use fact_store to search, probe entities, reason across entities, or add facts.\n"
f"Use fact_feedback to rate facts after using them (trains trust scores)."
)
def prefetch(self, query: str, *, session_id: str = "") -> str:
if not self._retriever or not query:
return ""
try:
results = self._retriever.search(query, min_trust=self._min_trust, limit=5)
if not results:
return ""
lines = []
for r in results:
trust = r.get("trust_score", r.get("trust", 0))
lines.append(f"- [{trust:.1f}] {r.get('content', '')}")
return "## Holographic Memory\n" + "\n".join(lines)
except Exception as e:
logger.debug("Holographic prefetch failed: %s", e)
return ""
def sync_turn(self, user_content: str, assistant_content: str, *, session_id: str = "") -> None:
# Holographic memory stores explicit facts via tools, not auto-sync.
# The on_session_end hook handles auto-extraction if configured.
pass
def get_tool_schemas(self) -> List[Dict[str, Any]]:
return [FACT_STORE_SCHEMA, FACT_FEEDBACK_SCHEMA]
def handle_tool_call(self, tool_name: str, args: Dict[str, Any], **kwargs) -> str:
if tool_name == "fact_store":
return self._handle_fact_store(args)
elif tool_name == "fact_feedback":
return self._handle_fact_feedback(args)
return tool_error(f"Unknown tool: {tool_name}")
def on_session_end(self, messages: List[Dict[str, Any]]) -> None:
# is_truthy_value: the config schema declares auto_extract as a string
# enum ("false"/"true"), and a plain truthiness check treats the string
# "false" as enabled (#57682).
if not is_truthy_value(self._config.get("auto_extract", False)):
return
if not self._store or not messages:
return
self._auto_extract_facts(messages)
def on_memory_write(self, action: str, target: str, content: str) -> None:
"""Mirror built-in memory writes as facts."""
if action == "add" and self._store and content:
try:
category = "user_pref" if target == "user" else "general"
self._store.add_fact(content, category=category)
except Exception as e:
logger.debug("Holographic memory_write mirror failed: %s", e)
def shutdown(self) -> None:
# Release the shared SQLite connection deterministically on the
# caller's thread. Dropping the reference alone leaves fd finalization
# to GC, which keeps the connection (and its write lock) alive on a
# long-running gateway and prolongs the "database is locked" contention
# this store's shared-connection refcounting is meant to eliminate.
# close() is idempotent and refcount-guarded, so siblings stay safe.
if self._store is not None:
try:
self._store.close()
except Exception as e:
logger.debug("Holographic shutdown close() failed: %s", e)
self._store = None
self._retriever = None
# -- Tool handlers -------------------------------------------------------
def _handle_fact_store(self, args: dict) -> str:
try:
action = args["action"]
store = self._store
retriever = self._retriever
if action == "add":
fact_id = store.add_fact(
args["content"],
category=args.get("category", "general"),
tags=args.get("tags", ""),
)
return json.dumps({"fact_id": fact_id, "status": "added"})
elif action == "search":
results = retriever.search(
args["query"],
category=args.get("category"),
min_trust=float(args.get("min_trust", self._min_trust)),
limit=int(args.get("limit", 10)),
)
return json.dumps({"results": results, "count": len(results)})
elif action == "probe":
results = retriever.probe(
args["entity"],
category=args.get("category"),
limit=int(args.get("limit", 10)),
)
return json.dumps({"results": results, "count": len(results)})
elif action == "related":
results = retriever.related(
args["entity"],
category=args.get("category"),
limit=int(args.get("limit", 10)),
)
return json.dumps({"results": results, "count": len(results)})
elif action == "reason":
entities = args.get("entities", [])
if not entities:
return tool_error("reason requires 'entities' list")
results = retriever.reason(
entities,
category=args.get("category"),
limit=int(args.get("limit", 10)),
)
return json.dumps({"results": results, "count": len(results)})
elif action == "contradict":
results = retriever.contradict(
category=args.get("category"),
limit=int(args.get("limit", 10)),
)
return json.dumps({"results": results, "count": len(results)})
elif action == "update":
updated = store.update_fact(
int(args["fact_id"]),
content=args.get("content"),
trust_delta=float(args["trust_delta"]) if "trust_delta" in args else None,
tags=args.get("tags"),
category=args.get("category"),
)
return json.dumps({"updated": updated})
elif action == "remove":
removed = store.remove_fact(int(args["fact_id"]))
return json.dumps({"removed": removed})
elif action == "list":
facts = store.list_facts(
category=args.get("category"),
min_trust=float(args.get("min_trust", 0.0)),
limit=int(args.get("limit", 10)),
)
return json.dumps({"facts": facts, "count": len(facts)})
else:
return tool_error(f"Unknown action: {action}")
except KeyError as exc:
return tool_error(f"Missing required argument: {exc}")
except Exception as exc:
return tool_error(str(exc))
def _handle_fact_feedback(self, args: dict) -> str:
try:
fact_id = int(args["fact_id"])
helpful = args["action"] == "helpful"
result = self._store.record_feedback(fact_id, helpful=helpful)
return json.dumps(result)
except KeyError as exc:
return tool_error(f"Missing required argument: {exc}")
except Exception as exc:
return tool_error(str(exc))
# -- Auto-extraction (on_session_end) ------------------------------------
def _auto_extract_facts(self, messages: list) -> None:
# Local import (pattern used in initialize()): the compressor module is
# heavier than this plugin and is only needed when auto_extract is on.
from agent.context_compressor import (
_MERGED_PRIOR_CONTEXT_HEADER,
_MERGED_SUMMARY_DELIMITER,
is_compaction_summary_message,
)
def _pre_delimiter_user_segment(msg: dict):
"""Return the genuine user text preceding a merged-into-tail
compaction summary, or None when the whole message is a summary.
Merge-into-tail messages (agent/context_compressor.py ~3163-3190)
wrap real prior tail content BEFORE ``_MERGED_SUMMARY_DELIMITER``,
prefixed with ``_MERGED_PRIOR_CONTEXT_HEADER``, then append the
generated handoff summary AFTER the delimiter. Dropping the whole
row (as ``is_compaction_summary_message`` alone would suggest)
discards that genuine pre-delimiter content too (#57690 review).
Only the summary suffix must be excluded from harvesting.
"""
content = msg.get("content", "")
if not isinstance(content, str) or _MERGED_SUMMARY_DELIMITER not in content:
return None
pre = content.split(_MERGED_SUMMARY_DELIMITER, 1)[0]
if pre.startswith(_MERGED_PRIOR_CONTEXT_HEADER):
pre = pre[len(_MERGED_PRIOR_CONTEXT_HEADER):]
pre = pre.strip()
return pre or None
_PREF_PATTERNS = [
re.compile(r'\bI\s+(?:prefer|like|love|use|want|need)\s+(.+)', re.IGNORECASE),
re.compile(r'\bmy\s+(?:favorite|preferred|default)\s+\w+\s+is\s+(.+)', re.IGNORECASE),
re.compile(r'\bI\s+(?:always|never|usually)\s+(.+)', re.IGNORECASE),
]
_DECISION_PATTERNS = [
re.compile(r'\bwe\s+(?:decided|agreed|chose)\s+(?:to\s+)?(.+)', re.IGNORECASE),
re.compile(r'\bthe\s+project\s+(?:uses|needs|requires)\s+(.+)', re.IGNORECASE),
]
extracted = 0
for msg in messages:
if msg.get("role") != "user":
continue
# Compaction handoff summaries can be inserted as role="user"
# messages; their prose reliably matches the decision patterns, so
# without this guard the compactor's own output is stored as a
# durable "fact" on every rollover (#57682). A merge-into-tail
# summary also carries genuine pre-delimiter user content in the
# SAME row; harvest that segment instead of dropping the whole
# message (#57690 review).
pre_delimiter_segment = _pre_delimiter_user_segment(msg)
if pre_delimiter_segment is not None:
content = pre_delimiter_segment
elif is_compaction_summary_message(msg):
continue
else:
content = msg.get("content", "")
if not isinstance(content, str) or len(content) < 10:
continue
for pattern in _PREF_PATTERNS:
if pattern.search(content):
try:
self._store.add_fact(content[:400], category="user_pref")
extracted += 1
except Exception:
pass
break
for pattern in _DECISION_PATTERNS:
if pattern.search(content):
try:
self._store.add_fact(content[:400], category="project")
extracted += 1
except Exception:
pass
break
if extracted:
logger.info("Auto-extracted %d facts from conversation", extracted)
# ---------------------------------------------------------------------------
# Plugin entry point
# ---------------------------------------------------------------------------
def register(ctx) -> None:
"""Register the holographic memory provider with the plugin system."""
config = _load_plugin_config()
provider = HolographicMemoryProvider(config=config)
ctx.register_memory_provider(provider)
plugins/memory/holographic/holographic.py
+290
View File
@@ -0,0 +1,290 @@
"""Holographic Reduced Representations (HRR) with phase encoding.
HRRs are a vector symbolic architecture for encoding compositional structure
into fixed-width distributed representations. This module uses *phase vectors*:
each concept is a vector of angles in [0, 2π). The algebraic operations are:
bind — circular convolution (phase addition) — associates two concepts
unbind — circular correlation (phase subtraction) — retrieves a bound value
bundle — superposition (circular mean) — merges multiple concepts
Phase encoding is numerically stable, avoids the magnitude collapse of
traditional complex-number HRRs, and maps cleanly to cosine similarity.
Atoms are generated deterministically from SHA-256 so representations are
identical across processes, machines, and language versions.
References:
Plate (1995) — Holographic Reduced Representations
Gayler (2004) — Vector Symbolic Architectures answer Jackendoff's challenges
"""
import hashlib
import logging
import struct
import math
try:
import numpy as np
_HAS_NUMPY = True
except ImportError:
_HAS_NUMPY = False
logger = logging.getLogger(__name__)
_TWO_PI = 2.0 * math.pi
_FLOAT32_BLOB_PREFIX = b"HRR1"
def _require_numpy() -> None:
if not _HAS_NUMPY:
raise RuntimeError("numpy is required for holographic operations")
def _np():
"""Return the numpy module after the runtime availability guard."""
_require_numpy()
return np # type: ignore[name-defined]
def encode_atom(word: str, dim: int = 1024) -> "np.ndarray":
"""Deterministic phase vector via SHA-256 counter blocks.
Uses hashlib (not numpy RNG) for cross-platform reproducibility.
Algorithm:
- Generate enough SHA-256 blocks by hashing f"{word}:{i}" for i=0,1,2,...
- Concatenate digests, interpret as uint16 values via struct.unpack
- Scale to [0, 2π): phases = values * (2π / 65536)
- Truncate to dim elements
- Returns np.float64 array of shape (dim,)
"""
_require_numpy()
# Each SHA-256 digest is 32 bytes = 16 uint16 values.
values_per_block = 16
blocks_needed = math.ceil(dim / values_per_block)
uint16_values: list[int] = []
for i in range(blocks_needed):
digest = hashlib.sha256(f"{word}:{i}".encode()).digest()
uint16_values.extend(struct.unpack("<16H", digest))
phases = np.array(uint16_values[:dim], dtype=np.float64) * (_TWO_PI / 65536.0)
return phases
def bind(a: "np.ndarray", b: "np.ndarray") -> "np.ndarray":
"""Circular convolution = element-wise phase addition.
Binding associates two concepts into a single composite vector.
The result is dissimilar to both inputs (quasi-orthogonal).
"""
_require_numpy()
return (a + b) % _TWO_PI
def unbind(memory: "np.ndarray", key: "np.ndarray") -> "np.ndarray":
"""Circular correlation = element-wise phase subtraction.
Unbinding retrieves the value associated with a key from a memory vector.
unbind(bind(a, b), a) ≈ b (up to superposition noise)
"""
_require_numpy()
return (memory - key) % _TWO_PI
def bundle(*vectors: "np.ndarray") -> "np.ndarray":
"""Superposition via circular mean of complex exponentials.
Bundling merges multiple vectors into one that is similar to each input.
The result can hold O(sqrt(dim)) items before similarity degrades.
"""
_require_numpy()
complex_sum = np.sum([np.exp(1j * v) for v in vectors], axis=0)
return np.angle(complex_sum) % _TWO_PI
def similarity(a: "np.ndarray", b: "np.ndarray") -> float:
"""Phase cosine similarity. Range [-1, 1].
Returns 1.0 for identical vectors, near 0.0 for random (unrelated) vectors,
and -1.0 for perfectly anti-correlated vectors.
"""
_require_numpy()
return float(np.mean(np.cos(a - b)))
def encode_text(text: str, dim: int = 1024) -> "np.ndarray":
"""Bag-of-words: bundle of atom vectors for each token.
Tokenizes by lowercasing, splitting on whitespace, and stripping
leading/trailing punctuation from each token.
Returns bundle of all token atom vectors.
If text is empty or produces no tokens, returns encode_atom("__hrr_empty__", dim).
"""
_require_numpy()
tokens = [
token.strip(".,!?;:\"'()[]{}")
for token in text.lower().split()
]
tokens = [t for t in tokens if t]
if not tokens:
return encode_atom("__hrr_empty__", dim)
atom_vectors = [encode_atom(token, dim) for token in tokens]
return bundle(*atom_vectors)
def encode_fact(content: str, entities: list[str], dim: int = 1024) -> "np.ndarray":
"""Structured encoding: content bound to ROLE_CONTENT, each entity bound to ROLE_ENTITY, all bundled.
Role vectors are reserved atoms: "__hrr_role_content__", "__hrr_role_entity__"
Components:
1. bind(encode_text(content, dim), encode_atom("__hrr_role_content__", dim))
2. For each entity: bind(encode_atom(entity.lower(), dim), encode_atom("__hrr_role_entity__", dim))
3. bundle all components together
This enables algebraic extraction:
unbind(fact, bind(entity, ROLE_ENTITY)) ≈ content_vector
"""
_require_numpy()
role_content = encode_atom("__hrr_role_content__", dim)
role_entity = encode_atom("__hrr_role_entity__", dim)
components: list[np.ndarray] = [
bind(encode_text(content, dim), role_content)
]
for entity in entities:
components.append(bind(encode_atom(entity.lower(), dim), role_entity))
return bundle(*components)
def phases_to_bytes(phases: "np.ndarray", dim: int | None = None) -> bytes:
"""Serialize phase vectors as float32 blobs.
float32 halves SQLite BLOB storage versus the legacy float64 format
(4 KB + a 4-byte format prefix instead of 8 KB at dim=1024) while
preserving enough precision for phase-similarity retrieval.
``bytes_to_phases`` keeps reading legacy float64 blobs for backward
compatibility.
When ``dim`` is 1 the prefixed float32 blob (8 bytes) collides in size
with a raw float64 blob (8 bytes), making the format ambiguous. In
that case we fall back to writing raw float64 so that ``bytes_to_phases``
can never misinterpret the blob.
"""
numpy = _np()
if dim is None:
dim = int(phases.shape[0])
float32_blob_bytes = len(_FLOAT32_BLOB_PREFIX) + dim * numpy.dtype(numpy.float32).itemsize
float64_bytes = dim * numpy.dtype(numpy.float64).itemsize
if float32_blob_bytes == float64_bytes:
# dim=1: sizes collide, write legacy float64 to stay unambiguous
return numpy.asarray(phases, dtype=numpy.float64).tobytes()
payload = numpy.asarray(phases, dtype=numpy.float32).tobytes()
return _FLOAT32_BLOB_PREFIX + payload
def bytes_to_phases(data: bytes, dim: int | None = None) -> "np.ndarray":
"""Deserialize a phase vector from new float32 or legacy float64 storage.
New float32 blobs carry a small prefix so callers can round-trip without
knowing ``dim``. Legacy float64 blobs are raw NumPy bytes and remain
readable for backward compatibility. The returned array is copied and
promoted to float64 so downstream HRR math keeps the existing numerical
behavior.
When ``dim`` is 1 the prefixed float32 blob and the raw float64 blob are
both 8 bytes, so size alone cannot disambiguate. ``phases_to_bytes``
avoids writing prefixed blobs in that case; here we guard the remaining
collision window (a legacy float64 blob that happens to start with the
``HRR1`` prefix) by preferring the legacy interpretation when sizes
match and the caller supplied ``dim``.
"""
numpy = _np()
if dim is not None:
float32_payload_bytes = dim * numpy.dtype(numpy.float32).itemsize
float32_blob_bytes = len(_FLOAT32_BLOB_PREFIX) + float32_payload_bytes
float64_bytes = dim * numpy.dtype(numpy.float64).itemsize
# When sizes collide (dim=1), prefer legacy float64 for a blob that
# starts with the prefix, because phases_to_bytes never writes a
# prefixed float32 blob at dim=1 — any such blob must be legacy.
if float32_blob_bytes == float64_bytes:
if len(data) == float64_bytes:
return numpy.frombuffer(data, dtype=numpy.float64).copy()
if data.startswith(_FLOAT32_BLOB_PREFIX):
payload_len = len(data) - len(_FLOAT32_BLOB_PREFIX)
raise ValueError(
f"HRR vector blob has {len(data)} bytes ({payload_len} payload bytes after "
f"the float32 prefix); expected {float64_bytes} (legacy float64) for dim={dim}"
)
raise ValueError(
f"HRR legacy vector blob has {len(data)} bytes; expected "
f"{float64_bytes} (float64) for dim={dim}"
)
if data.startswith(_FLOAT32_BLOB_PREFIX) and len(data) == float32_blob_bytes:
payload = data[len(_FLOAT32_BLOB_PREFIX):]
return numpy.frombuffer(payload, dtype=numpy.float32).astype(numpy.float64)
if len(data) == float64_bytes:
return numpy.frombuffer(data, dtype=numpy.float64).copy()
if data.startswith(_FLOAT32_BLOB_PREFIX):
payload_len = len(data) - len(_FLOAT32_BLOB_PREFIX)
raise ValueError(
f"HRR vector blob has {len(data)} bytes ({payload_len} payload bytes after "
f"the float32 prefix); expected {float32_blob_bytes} (prefixed float32) "
f"or {float64_bytes} (legacy float64) for dim={dim}"
)
raise ValueError(
f"HRR legacy vector blob has {len(data)} bytes; expected "
f"{float64_bytes} (float64) for dim={dim}"
)
if data.startswith(_FLOAT32_BLOB_PREFIX):
payload = data[len(_FLOAT32_BLOB_PREFIX):]
if len(payload) % numpy.dtype(numpy.float32).itemsize != 0:
raise ValueError(
f"HRR float32 vector blob has invalid payload byte length: {len(payload)}"
)
return numpy.frombuffer(payload, dtype=numpy.float32).astype(numpy.float64)
if len(data) % numpy.dtype(numpy.float64).itemsize != 0:
raise ValueError(f"HRR legacy vector blob has invalid byte length: {len(data)}")
return numpy.frombuffer(data, dtype=numpy.float64).copy()
def snr_estimate(dim: int, n_items: int) -> float:
"""Signal-to-noise ratio estimate for holographic storage.
SNR = sqrt(dim / n_items) when n_items > 0, else inf.
The SNR falls below 2.0 when n_items > dim / 4, meaning retrieval
errors become likely. Logs a warning when this threshold is crossed.
"""
_require_numpy()
if n_items <= 0:
return float("inf")
snr = math.sqrt(dim / n_items)
if snr < 2.0:
logger.warning(
"HRR storage near capacity: SNR=%.2f (dim=%d, n_items=%d). "
"Retrieval accuracy may degrade. Consider increasing dim or reducing stored items.",
snr,
dim,
n_items,
)
return snr
plugins/memory/holographic/plugin.yaml
+5
View File
@@ -0,0 +1,5 @@
name: holographic
version: 0.1.0
description: "Holographic memory — local SQLite fact store with FTS5 search, trust scoring, and HRR-based compositional retrieval."
hooks:
- on_session_end
plugins/memory/holographic/retrieval.py
+668
View File
@@ -0,0 +1,668 @@
"""Hybrid keyword/BM25 retrieval for the memory store.
Ported from KIK memory_agent.py — combines FTS5 full-text search with
Jaccard similarity reranking and trust-weighted scoring.
"""
from __future__ import annotations
import math
from datetime import datetime, timezone
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .store import MemoryStore
try:
from . import holographic as hrr
except ImportError:
import holographic as hrr # type: ignore[no-redef]
class FactRetriever:
"""Multi-strategy fact retrieval with trust-weighted scoring."""
def __init__(
self,
store: MemoryStore,
temporal_decay_half_life: int = 0, # days, 0 = disabled
fts_weight: float = 0.4,
jaccard_weight: float = 0.3,
hrr_weight: float = 0.3,
hrr_dim: int = 1024,
):
self.store = store
self.half_life = temporal_decay_half_life
self.hrr_dim = hrr_dim
# Auto-redistribute weights if numpy unavailable
if hrr_weight > 0 and not hrr._HAS_NUMPY:
fts_weight = 0.6
jaccard_weight = 0.4
hrr_weight = 0.0
self.fts_weight = fts_weight
self.jaccard_weight = jaccard_weight
self.hrr_weight = hrr_weight
def search(
self,
query: str,
category: str | None = None,
min_trust: float = 0.3,
limit: int = 10,
) -> list[dict]:
"""Hybrid search: FTS5 candidates → Jaccard rerank → trust weighting.
Pipeline:
1. FTS5 search: Get limit*3 candidates from SQLite full-text search
2. Jaccard boost: Token overlap between query and fact content
3. Trust weighting: final_score = relevance * trust_score
4. Temporal decay (optional): decay = 0.5^(age_days / half_life)
Returns list of dicts with fact data + 'score' field, sorted by score desc.
"""
# Stage 1: Get FTS5 candidates (more than limit for reranking headroom)
candidates = self._fts_candidates(query, category, min_trust, limit * 3)
if not candidates:
return []
# Stage 2: Rerank with Jaccard + trust + optional decay
query_tokens = self._tokenize(query)
# The query vector is loop-invariant — encode it at most once, on
# the first candidate that actually carries an HRR vector. Lazy on
# purpose: migrated stores can have FTS candidates whose hrr_vector
# was never backfilled (MemoryStore._init_db adds the column
# without backfilling), and those must not pay for an encode
# nothing will use. encode_text is deterministic (SHA-256 counter
# blocks), so the hoisted vector is bit-identical to what the
# per-candidate calls produced.
query_vec = None
scored = []
for fact in candidates:
content_tokens = self._tokenize(fact["content"])
tag_tokens = self._tokenize(fact.get("tags", ""))
all_tokens = content_tokens | tag_tokens
jaccard = self._jaccard_similarity(query_tokens, all_tokens)
fts_score = fact.get("fts_rank", 0.0)
# HRR similarity
if self.hrr_weight > 0 and fact.get("hrr_vector"):
fact_vec = hrr.bytes_to_phases(fact["hrr_vector"], dim=self.hrr_dim)
if query_vec is None:
query_vec = hrr.encode_text(query, self.hrr_dim)
hrr_sim = (hrr.similarity(query_vec, fact_vec) + 1.0) / 2.0 # shift to [0,1]
else:
hrr_sim = 0.5 # neutral
# Combine FTS5 + Jaccard + HRR
relevance = (self.fts_weight * fts_score
+ self.jaccard_weight * jaccard
+ self.hrr_weight * hrr_sim)
# Trust weighting
score = relevance * fact["trust_score"]
# Optional temporal decay
if self.half_life > 0:
score *= self._temporal_decay(fact.get("updated_at") or fact.get("created_at"))
fact["score"] = score
scored.append(fact)
# Sort by score descending, return top limit
scored.sort(key=lambda x: x["score"], reverse=True)
results = scored[:limit]
# Strip raw HRR bytes — callers expect JSON-serializable dicts
for fact in results:
fact.pop("hrr_vector", None)
return results
def probe(
self,
entity: str,
category: str | None = None,
limit: int = 10,
) -> list[dict]:
"""Compositional entity query using HRR algebra.
Unbinds entity from memory bank to extract associated content.
This is NOT keyword search — it uses algebraic structure to find facts
where the entity plays a structural role.
Falls back to FTS5 search if numpy unavailable.
"""
if not hrr._HAS_NUMPY:
# Fallback to keyword search on entity name
return self.search(entity, category=category, limit=limit)
conn = self.store._conn
# Encode entity as role-bound vector
role_entity = hrr.encode_atom("__hrr_role_entity__", self.hrr_dim)
entity_vec = hrr.encode_atom(entity.lower(), self.hrr_dim)
probe_key = hrr.bind(entity_vec, role_entity)
# Try category-specific bank first, then all facts
if category:
bank_name = f"cat:{category}"
bank_row = conn.execute(
"SELECT vector FROM memory_banks WHERE bank_name = ?",
(bank_name,),
).fetchone()
if bank_row:
bank_vec = hrr.bytes_to_phases(bank_row["vector"], dim=self.hrr_dim)
extracted = hrr.unbind(bank_vec, probe_key)
# Use extracted signal to score individual facts
return self._score_facts_by_vector(
extracted, category=category, limit=limit
)
# Score against individual fact vectors directly
where = "WHERE hrr_vector IS NOT NULL"
params: list = []
if category:
where += " AND category = ?"
params.append(category)
rows = conn.execute(
f"""
SELECT fact_id, content, category, tags, trust_score,
retrieval_count, helpful_count, created_at, updated_at,
hrr_vector
FROM facts
{where}
""",
params,
).fetchall()
if not rows:
# Final fallback: keyword search
return self.search(entity, category=category, limit=limit)
# role_content is loop-invariant — encode it once (deterministic
# SHA-256-based atom) instead of once per fact row.
role_content = hrr.encode_atom("__hrr_role_content__", self.hrr_dim)
scored = []
for row in rows:
fact = dict(row)
fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"), dim=self.hrr_dim)
# Unbind probe key from fact to see if entity is structurally present
residual = hrr.unbind(fact_vec, probe_key)
# Compare residual against content signal
content_vec = hrr.bind(hrr.encode_text(fact["content"], self.hrr_dim), role_content)
sim = hrr.similarity(residual, content_vec)
fact["score"] = (sim + 1.0) / 2.0 * fact["trust_score"]
scored.append(fact)
scored.sort(key=lambda x: x["score"], reverse=True)
return scored[:limit]
def related(
self,
entity: str,
category: str | None = None,
limit: int = 10,
) -> list[dict]:
"""Discover facts that share structural connections with an entity.
Unlike probe (which finds facts *about* an entity), related finds
facts that are connected through shared context — e.g., other entities
mentioned alongside this one, or content that overlaps structurally.
Falls back to FTS5 search if numpy unavailable.
"""
if not hrr._HAS_NUMPY:
return self.search(entity, category=category, limit=limit)
conn = self.store._conn
# Encode entity as a bare atom (not role-bound — we want ANY structural match)
entity_vec = hrr.encode_atom(entity.lower(), self.hrr_dim)
# Get all facts with vectors
where = "WHERE hrr_vector IS NOT NULL"
params: list = []
if category:
where += " AND category = ?"
params.append(category)
rows = conn.execute(
f"""
SELECT fact_id, content, category, tags, trust_score,
retrieval_count, helpful_count, created_at, updated_at,
hrr_vector
FROM facts
{where}
""",
params,
).fetchall()
if not rows:
return self.search(entity, category=category, limit=limit)
# Score each fact by how much the entity's atom appears in its vector
# This catches both role-bound entity matches AND content word matches
# Both role atoms are loop-invariant — encode them once here
# (deterministic SHA-256-based atoms) instead of twice per fact row.
role_entity = hrr.encode_atom("__hrr_role_entity__", self.hrr_dim)
role_content = hrr.encode_atom("__hrr_role_content__", self.hrr_dim)
scored = []
for row in rows:
fact = dict(row)
fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"), dim=self.hrr_dim)
# Check structural similarity: unbind entity from fact
residual = hrr.unbind(fact_vec, entity_vec)
# A high-similarity residual to ANY known role vector means this entity
# plays a structural role in the fact
entity_role_sim = hrr.similarity(residual, role_entity)
content_role_sim = hrr.similarity(residual, role_content)
# Take the max — entity could appear in either role
best_sim = max(entity_role_sim, content_role_sim)
fact["score"] = (best_sim + 1.0) / 2.0 * fact["trust_score"]
scored.append(fact)
scored.sort(key=lambda x: x["score"], reverse=True)
return scored[:limit]
def reason(
self,
entities: list[str],
category: str | None = None,
limit: int = 10,
) -> list[dict]:
"""Multi-entity compositional query — vector-space JOIN.
Given multiple entities, algebraically intersects their structural
connections to find facts related to ALL of them simultaneously.
This is compositional reasoning that no embedding DB can do.
Example: reason(["peppi", "backend"]) finds facts where peppi AND
backend both play structural roles — without keyword matching.
Falls back to FTS5 search if numpy unavailable.
"""
if not hrr._HAS_NUMPY or not entities:
# Fallback: search with all entities as keywords
query = " ".join(entities)
return self.search(query, category=category, limit=limit)
conn = self.store._conn
role_entity = hrr.encode_atom("__hrr_role_entity__", self.hrr_dim)
# For each entity, compute what the bank "remembers" about it
# by unbinding entity+role from each fact vector
entity_residuals = []
for entity in entities:
entity_vec = hrr.encode_atom(entity.lower(), self.hrr_dim)
probe_key = hrr.bind(entity_vec, role_entity)
entity_residuals.append(probe_key)
# Get all facts with vectors
where = "WHERE hrr_vector IS NOT NULL"
params: list = []
if category:
where += " AND category = ?"
params.append(category)
rows = conn.execute(
f"""
SELECT fact_id, content, category, tags, trust_score,
retrieval_count, helpful_count, created_at, updated_at,
hrr_vector
FROM facts
{where}
""",
params,
).fetchall()
if not rows:
query = " ".join(entities)
return self.search(query, category=category, limit=limit)
# Score each fact by how much EACH entity is structurally present.
# A fact scores high only if ALL entities have structural presence
# (AND semantics via min, vs OR which would use mean/max).
role_content = hrr.encode_atom("__hrr_role_content__", self.hrr_dim)
scored = []
for row in rows:
fact = dict(row)
fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"), dim=self.hrr_dim)
entity_scores = []
for probe_key in entity_residuals:
residual = hrr.unbind(fact_vec, probe_key)
sim = hrr.similarity(residual, role_content)
entity_scores.append(sim)
min_sim = min(entity_scores)
fact["score"] = (min_sim + 1.0) / 2.0 * fact["trust_score"]
scored.append(fact)
scored.sort(key=lambda x: x["score"], reverse=True)
return scored[:limit]
def contradict(
self,
category: str | None = None,
threshold: float = 0.3,
limit: int = 10,
) -> list[dict]:
"""Find potentially contradictory facts via entity overlap + content divergence.
Two facts contradict when they share entities (same subject) but have
low content-vector similarity (different claims). This is automated
memory hygiene — no other memory system does this.
Returns pairs of facts with a contradiction score.
Falls back to empty list if numpy unavailable.
"""
if not hrr._HAS_NUMPY:
return []
conn = self.store._conn
# Get all facts with vectors and their linked entities
where = "WHERE f.hrr_vector IS NOT NULL"
params: list = []
if category:
where += " AND f.category = ?"
params.append(category)
rows = conn.execute(
f"""
SELECT f.fact_id, f.content, f.category, f.tags, f.trust_score,
f.created_at, f.updated_at, f.hrr_vector
FROM facts f
{where}
""",
params,
).fetchall()
if len(rows) < 2:
return []
# Guard against O(n²) explosion on large fact stores.
# At 500 facts, that's ~125K comparisons — acceptable.
# Above that, only check the most recently updated facts.
_MAX_CONTRADICT_FACTS = 500
if len(rows) > _MAX_CONTRADICT_FACTS:
rows = sorted(rows, key=lambda r: r["updated_at"] or r["created_at"], reverse=True)
rows = rows[:_MAX_CONTRADICT_FACTS]
# Build entity sets per fact
fact_entities: dict[int, set[str]] = {}
for row in rows:
fid = row["fact_id"]
entity_rows = conn.execute(
"""
SELECT e.name FROM entities e
JOIN fact_entities fe ON fe.entity_id = e.entity_id
WHERE fe.fact_id = ?
""",
(fid,),
).fetchall()
fact_entities[fid] = {r["name"].lower() for r in entity_rows}
# Compare all pairs: high entity overlap + low content similarity = contradiction
facts = [dict(r) for r in rows]
contradictions = []
for i in range(len(facts)):
for j in range(i + 1, len(facts)):
f1, f2 = facts[i], facts[j]
ents1 = fact_entities.get(f1["fact_id"], set())
ents2 = fact_entities.get(f2["fact_id"], set())
if not ents1 or not ents2:
continue
# Entity overlap (Jaccard)
entity_overlap = len(ents1 & ents2) / len(ents1 | ents2) if (ents1 | ents2) else 0.0
if entity_overlap < 0.3:
continue # Not enough entity overlap to be contradictory
# Content similarity via HRR vectors
v1 = hrr.bytes_to_phases(f1["hrr_vector"], dim=self.hrr_dim)
v2 = hrr.bytes_to_phases(f2["hrr_vector"], dim=self.hrr_dim)
content_sim = hrr.similarity(v1, v2)
# High entity overlap + low content similarity = potential contradiction
# contradiction_score: higher = more contradictory
contradiction_score = entity_overlap * (1.0 - (content_sim + 1.0) / 2.0)
if contradiction_score >= threshold:
# Strip hrr_vector from output (not JSON serializable)
f1_clean = {k: v for k, v in f1.items() if k != "hrr_vector"}
f2_clean = {k: v for k, v in f2.items() if k != "hrr_vector"}
contradictions.append({
"fact_a": f1_clean,
"fact_b": f2_clean,
"entity_overlap": round(entity_overlap, 3),
"content_similarity": round(content_sim, 3),
"contradiction_score": round(contradiction_score, 3),
"shared_entities": sorted(ents1 & ents2),
})
contradictions.sort(key=lambda x: x["contradiction_score"], reverse=True)
return contradictions[:limit]
def _score_facts_by_vector(
self,
target_vec: "np.ndarray",
category: str | None = None,
limit: int = 10,
) -> list[dict]:
"""Score facts by similarity to a target vector."""
conn = self.store._conn
where = "WHERE hrr_vector IS NOT NULL"
params: list = []
if category:
where += " AND category = ?"
params.append(category)
rows = conn.execute(
f"""
SELECT fact_id, content, category, tags, trust_score,
retrieval_count, helpful_count, created_at, updated_at,
hrr_vector
FROM facts
{where}
""",
params,
).fetchall()
scored = []
for row in rows:
fact = dict(row)
fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"), dim=self.hrr_dim)
sim = hrr.similarity(target_vec, fact_vec)
fact["score"] = (sim + 1.0) / 2.0 * fact["trust_score"]
scored.append(fact)
scored.sort(key=lambda x: x["score"], reverse=True)
return scored[:limit]
def _fts_candidates(
self,
query: str,
category: str | None,
min_trust: float,
limit: int,
) -> list[dict]:
"""Get raw FTS5 candidates from the store.
Uses the store's database connection directly for FTS5 MATCH
with rank scoring. Normalizes FTS5 rank to [0, 1] range.
"""
conn = self.store._conn
# Build query - FTS5 rank is negative (lower = better match)
# We need to join facts_fts with facts to get all columns
params: list = []
where_clauses = ["facts_fts MATCH ?"]
# FTS5 defaults to AND-between-tokens, which kills recall on
# natural-language queries ("what happened with the deployment
# rollback"). Sanitize: drop stopwords, OR-join content tokens, so
# any significant term can match.
params.append(self._sanitize_fts_query(query))
if category:
where_clauses.append("f.category = ?")
params.append(category)
where_clauses.append("f.trust_score >= ?")
params.append(min_trust)
where_sql = " AND ".join(where_clauses)
sql = f"""
SELECT f.*, facts_fts.rank as fts_rank_raw
FROM facts_fts
JOIN facts f ON f.fact_id = facts_fts.rowid
WHERE {where_sql}
ORDER BY facts_fts.rank
LIMIT ?
"""
params.append(limit)
try:
rows = conn.execute(sql, params).fetchall()
except Exception:
# FTS5 MATCH can fail on malformed queries — fall back to empty
return []
if not rows:
return []
# Normalize FTS5 rank: rank is negative, lower = better
# Convert to positive score in [0, 1] range
raw_ranks = [abs(row["fts_rank_raw"]) for row in rows]
max_rank = max(raw_ranks) if raw_ranks else 1.0
max_rank = max(max_rank, 1e-6) # avoid div by zero
results = []
for row, raw_rank in zip(rows, raw_ranks):
fact = dict(row)
fact.pop("fts_rank_raw", None)
fact["fts_rank"] = raw_rank / max_rank # normalize to [0, 1]
results.append(fact)
return results
@staticmethod
def _tokenize(text: str) -> set[str]:
"""Simple whitespace tokenization with lowercasing.
Strips common punctuation. No stemming/lemmatization (Phase 1).
"""
if not text:
return set()
# Split on whitespace, lowercase, strip punctuation
tokens = set()
for word in text.lower().split():
cleaned = word.strip(".,;:!?\"'()[]{}#@<>")
if cleaned:
tokens.add(cleaned)
return tokens
# Stopwords dropped before FTS5 OR-expansion. Short English function
# words that carry no retrieval signal and force false-negative AND
# matches when left in the query.
_FTS_STOPWORDS = frozenset({
"a", "about", "above", "after", "again", "all", "am", "an", "and",
"any", "are", "as", "at", "be", "because", "been", "before", "being",
"between", "both", "but", "by", "can", "could", "did", "do", "does",
"doing", "don", "down", "during", "each", "few", "for", "from",
"further", "had", "has", "have", "having", "he", "her", "here",
"hers", "herself", "him", "himself", "his", "how", "i", "if", "in",
"into", "is", "it", "its", "itself", "just", "me", "more", "most",
"my", "myself", "no", "nor", "not", "now", "of", "off", "on", "once",
"only", "or", "other", "our", "ours", "ourselves", "out", "over",
"own", "same", "she", "should", "so", "some", "such", "than", "that",
"the", "their", "theirs", "them", "themselves", "then", "there",
"these", "they", "this", "those", "through", "to", "too", "under",
"until", "up", "very", "was", "we", "were", "what", "when", "where",
"which", "while", "who", "whom", "why", "will", "with", "would",
"you", "your", "yours", "yourself", "yourselves",
})
@classmethod
def _sanitize_fts_query(cls, query: str) -> str:
"""Convert a natural-language query to an FTS5-safe OR expression.
FTS5 treats a multi-word MATCH argument as AND-joined by default,
which tanks recall on prose queries. This helper:
- tokenizes the query
- drops stopwords and short (<2 char) tokens
- strips FTS5 special characters from each token
- OR-joins the survivors
If nothing remains (pathological query), falls back to the raw
query so the caller sees zero results instead of a SQL error.
"""
if not query:
return ""
# Strip FTS5 operator characters from EACH token to avoid
# accidentally creating a malformed query.
_FTS_SPECIAL = '"()*^:-+'
tokens: list[str] = []
for raw in query.lower().split():
cleaned = raw.strip(".,;:!?\"'()[]{}#@<>") .translate(
str.maketrans("", "", _FTS_SPECIAL)
)
if len(cleaned) < 2:
continue
if cleaned in cls._FTS_STOPWORDS:
continue
# FTS5 phrase-literal each token to ensure no special chars
# sneak through as operators.
tokens.append(f'"{cleaned}"')
if not tokens:
# Fallback: raw query (likely returns 0, but never crashes)
return query
return " OR ".join(tokens)
@staticmethod
def _jaccard_similarity(set_a: set, set_b: set) -> float:
"""Jaccard similarity coefficient: |A ∩ B| / |A ∪ B|."""
if not set_a or not set_b:
return 0.0
intersection = len(set_a & set_b)
union = len(set_a | set_b)
return intersection / union if union > 0 else 0.0
def _temporal_decay(self, timestamp_str: str | None) -> float:
"""Exponential decay: 0.5^(age_days / half_life_days).
Returns 1.0 if decay is disabled or timestamp is missing.
"""
if not self.half_life or not timestamp_str:
return 1.0
try:
if isinstance(timestamp_str, str):
# Parse ISO format timestamp from SQLite
ts = datetime.fromisoformat(timestamp_str.replace("Z", "+00:00"))
else:
ts = timestamp_str
if ts.tzinfo is None:
ts = ts.replace(tzinfo=timezone.utc)
age_days = (datetime.now(timezone.utc) - ts).total_seconds() / 86400
if age_days < 0:
return 1.0
return math.pow(0.5, age_days / self.half_life)
except (ValueError, TypeError):
return 1.0
plugins/memory/holographic/store.py
+688
View File
@@ -0,0 +1,688 @@
"""
SQLite-backed fact store with entity resolution and trust scoring.
Single-user Hermes memory store plugin.
"""
import os
import re
import sqlite3
import threading
from pathlib import Path
try:
from . import holographic as hrr
except ImportError:
import holographic as hrr # type: ignore[no-redef]
_SCHEMA = """
CREATE TABLE IF NOT EXISTS facts (
fact_id INTEGER PRIMARY KEY AUTOINCREMENT,
content TEXT NOT NULL UNIQUE,
category TEXT DEFAULT 'general',
tags TEXT DEFAULT '',
trust_score REAL DEFAULT 0.5,
retrieval_count INTEGER DEFAULT 0,
helpful_count INTEGER DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
hrr_vector BLOB
);
CREATE TABLE IF NOT EXISTS entities (
entity_id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
entity_type TEXT DEFAULT 'unknown',
aliases TEXT DEFAULT '',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS fact_entities (
fact_id INTEGER REFERENCES facts(fact_id),
entity_id INTEGER REFERENCES entities(entity_id),
PRIMARY KEY (fact_id, entity_id)
);
CREATE INDEX IF NOT EXISTS idx_facts_trust ON facts(trust_score DESC);
CREATE INDEX IF NOT EXISTS idx_facts_category ON facts(category);
CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
CREATE VIRTUAL TABLE IF NOT EXISTS facts_fts
USING fts5(content, tags, content=facts, content_rowid=fact_id);
CREATE TRIGGER IF NOT EXISTS facts_ai AFTER INSERT ON facts BEGIN
INSERT INTO facts_fts(rowid, content, tags)
VALUES (new.fact_id, new.content, new.tags);
END;
CREATE TRIGGER IF NOT EXISTS facts_ad AFTER DELETE ON facts BEGIN
INSERT INTO facts_fts(facts_fts, rowid, content, tags)
VALUES ('delete', old.fact_id, old.content, old.tags);
END;
CREATE TRIGGER IF NOT EXISTS facts_au AFTER UPDATE ON facts BEGIN
INSERT INTO facts_fts(facts_fts, rowid, content, tags)
VALUES ('delete', old.fact_id, old.content, old.tags);
INSERT INTO facts_fts(rowid, content, tags)
VALUES (new.fact_id, new.content, new.tags);
END;
CREATE TABLE IF NOT EXISTS memory_banks (
bank_id INTEGER PRIMARY KEY AUTOINCREMENT,
bank_name TEXT NOT NULL UNIQUE,
vector BLOB NOT NULL,
dim INTEGER NOT NULL,
fact_count INTEGER DEFAULT 0,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
"""
# Trust adjustment constants
_HELPFUL_DELTA = 0.05
_UNHELPFUL_DELTA = -0.10
_TRUST_MIN = 0.0
_TRUST_MAX = 1.0
# Entity extraction patterns
_RE_CAPITALIZED = re.compile(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b')
_RE_DOUBLE_QUOTE = re.compile(r'"([^"]+)"')
_RE_SINGLE_QUOTE = re.compile(r"'([^']+)'")
_RE_AKA = re.compile(
r'(\w+(?:\s+\w+)*)\s+(?:aka|also known as)\s+(\w+(?:\s+\w+)*)',
re.IGNORECASE,
)
def _clamp_trust(value: float) -> float:
return max(_TRUST_MIN, min(_TRUST_MAX, value))
class MemoryStore:
"""SQLite-backed fact store with entity resolution and trust scoring."""
# --- Process-wide shared connection registry -------------------------
# SQLite permits only one writer at a time. Each MemoryStore instance used
# to open its own connection guarded by its own RLock, so the several
# providers that coexist in one process (the main agent plus every
# delegate_task subagent) raced as independent WAL writers. Combined with
# writes that were not rolled back on error, one connection could leave an
# open write transaction that pinned the write lock and made every other
# connection's write fail with "database is locked" for the full busy
# timeout. All instances for the same database now share ONE connection and
# ONE re-entrant lock, so access is fully serialized and cross-connection
# contention is impossible. The shared connection is refcounted, so closing
# one instance never tears the connection out from under a live sibling.
_shared: dict = {}
_shared_guard = threading.Lock()
def __init__(
self,
db_path: "str | Path | None" = None,
default_trust: float = 0.5,
hrr_dim: int = 1024,
) -> None:
if db_path is None:
from hermes_constants import get_hermes_home
db_path = str(get_hermes_home() / "memory_store.db")
self.db_path = Path(db_path).expanduser()
self.db_path.parent.mkdir(parents=True, exist_ok=True)
self.default_trust = _clamp_trust(default_trust)
self.hrr_dim = hrr_dim
self._hrr_available = hrr._HAS_NUMPY
# Acquire (or open) the process-wide shared connection for this DB.
# resolve() (not just expanduser) so symlinked/relative paths to the
# same file share ONE connection instead of silently reintroducing
# the multi-writer contention this registry exists to prevent.
try:
self._key = str(self.db_path.resolve())
except OSError:
self._key = str(self.db_path)
with MemoryStore._shared_guard:
entry = MemoryStore._shared.get(self._key)
if entry is None:
conn = sqlite3.connect(
self._key,
check_same_thread=False,
timeout=10.0,
# Autocommit: every statement is its own transaction, so a
# write that raises mid-method can never leave a dangling
# transaction (and its write lock) open. The explicit
# commit() calls below become harmless no-ops.
isolation_level=None,
)
conn.row_factory = sqlite3.Row
entry = {"conn": conn, "lock": threading.RLock(), "refs": 0, "ready": False}
MemoryStore._shared[self._key] = entry
entry["refs"] += 1
self._entry = entry
self._conn = entry["conn"]
self._lock = entry["lock"]
# Initialise the schema once per shared connection.
with self._lock:
if not self._entry["ready"]:
self._init_db()
self._entry["ready"] = True
# ------------------------------------------------------------------
# Initialisation
# ------------------------------------------------------------------
def _init_db(self) -> None:
"""Create tables, indexes, and triggers if they do not exist. Enable WAL mode."""
# Use the shared WAL-fallback helper so memory_store.db degrades
# gracefully on NFS/SMB/FUSE-mounted HERMES_HOME (same issue as
# state.db / kanban.db — see hermes_state._WAL_INCOMPAT_MARKERS).
from hermes_state import apply_wal_with_fallback
apply_wal_with_fallback(self._conn, db_label="memory_store.db (holographic)")
self._conn.executescript(_SCHEMA)
# Migrate: add hrr_vector column if missing (safe for existing databases)
columns = {row[1] for row in self._conn.execute("PRAGMA table_info(facts)").fetchall()}
if "hrr_vector" not in columns:
self._conn.execute("ALTER TABLE facts ADD COLUMN hrr_vector BLOB")
self._conn.commit()
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def add_fact(
self,
content: str,
category: str = "general",
tags: str = "",
) -> int:
"""Insert a fact and return its fact_id.
Deduplicates by content (UNIQUE constraint). On duplicate, returns
the existing fact_id without modifying the row. Extracts entities from
the content and links them to the fact.
"""
with self._lock:
content = content.strip()
if not content:
raise ValueError("content must not be empty")
try:
cur = self._conn.execute(
"""
INSERT INTO facts (content, category, tags, trust_score)
VALUES (?, ?, ?, ?)
""",
(content, category, tags, self.default_trust),
)
self._conn.commit()
fact_id: int = cur.lastrowid # type: ignore[assignment]
except sqlite3.IntegrityError:
# Duplicate content — return existing id
row = self._conn.execute(
"SELECT fact_id FROM facts WHERE content = ?", (content,)
).fetchone()
return int(row["fact_id"])
# Entity extraction and linking
for name in self._extract_entities(content):
entity_id = self._resolve_entity(name)
self._link_fact_entity(fact_id, entity_id)
# Compute HRR vector after entity linking
self._compute_hrr_vector(fact_id, content)
self._rebuild_bank(category)
return fact_id
def search_facts(
self,
query: str,
category: str | None = None,
min_trust: float = 0.3,
limit: int = 10,
) -> list[dict]:
"""Full-text search over facts using FTS5.
Returns a list of fact dicts ordered by FTS5 rank, then trust_score
descending. Also increments retrieval_count for matched facts.
"""
with self._lock:
query = query.strip()
if not query:
return []
# FTS5 AND-joins tokens by default, which zeroes out recall on
# natural-language queries. Reuse the retriever's sanitizer
# (stopword drop + OR-join content tokens). Imported lazily to
# avoid a store->retrieval import cycle.
from plugins.memory.holographic.retrieval import FactRetriever
match_query = FactRetriever._sanitize_fts_query(query)
params: list = [match_query, min_trust]
category_clause = ""
if category is not None:
category_clause = "AND f.category = ?"
params.append(category)
params.append(limit)
sql = f"""
SELECT f.fact_id, f.content, f.category, f.tags,
f.trust_score, f.retrieval_count, f.helpful_count,
f.created_at, f.updated_at
FROM facts f
JOIN facts_fts fts ON fts.rowid = f.fact_id
WHERE facts_fts MATCH ?
AND f.trust_score >= ?
{category_clause}
ORDER BY fts.rank, f.trust_score DESC
LIMIT ?
"""
rows = self._conn.execute(sql, params).fetchall()
results = [self._row_to_dict(r) for r in rows]
if results:
ids = [r["fact_id"] for r in results]
placeholders = ",".join("?" * len(ids))
self._conn.execute(
f"UPDATE facts SET retrieval_count = retrieval_count + 1 WHERE fact_id IN ({placeholders})",
ids,
)
self._conn.commit()
return results
def update_fact(
self,
fact_id: int,
content: str | None = None,
trust_delta: float | None = None,
tags: str | None = None,
category: str | None = None,
) -> bool:
"""Partially update a fact. Trust is clamped to [0, 1].
Returns True if the row existed, False otherwise.
"""
with self._lock:
row = self._conn.execute(
"SELECT fact_id, trust_score FROM facts WHERE fact_id = ?", (fact_id,)
).fetchone()
if row is None:
return False
assignments: list[str] = ["updated_at = CURRENT_TIMESTAMP"]
params: list = []
if content is not None:
assignments.append("content = ?")
params.append(content.strip())
if tags is not None:
assignments.append("tags = ?")
params.append(tags)
if category is not None:
assignments.append("category = ?")
params.append(category)
if trust_delta is not None:
new_trust = _clamp_trust(row["trust_score"] + trust_delta)
assignments.append("trust_score = ?")
params.append(new_trust)
params.append(fact_id)
self._conn.execute(
f"UPDATE facts SET {', '.join(assignments)} WHERE fact_id = ?",
params,
)
self._conn.commit()
# If content changed, re-extract entities
if content is not None:
self._conn.execute(
"DELETE FROM fact_entities WHERE fact_id = ?", (fact_id,)
)
for name in self._extract_entities(content):
entity_id = self._resolve_entity(name)
self._link_fact_entity(fact_id, entity_id)
self._conn.commit()
# Recompute HRR vector if content changed
if content is not None:
self._compute_hrr_vector(fact_id, content)
# Rebuild bank for relevant category
cat = category or self._conn.execute(
"SELECT category FROM facts WHERE fact_id = ?", (fact_id,)
).fetchone()["category"]
self._rebuild_bank(cat)
return True
def remove_fact(self, fact_id: int) -> bool:
"""Delete a fact and its entity links. Returns True if the row existed."""
with self._lock:
row = self._conn.execute(
"SELECT fact_id, category FROM facts WHERE fact_id = ?", (fact_id,)
).fetchone()
if row is None:
return False
self._conn.execute(
"DELETE FROM fact_entities WHERE fact_id = ?", (fact_id,)
)
self._conn.execute("DELETE FROM facts WHERE fact_id = ?", (fact_id,))
self._conn.commit()
self._rebuild_bank(row["category"])
return True
def list_facts(
self,
category: str | None = None,
min_trust: float = 0.0,
limit: int = 50,
) -> list[dict]:
"""Browse facts ordered by trust_score descending.
Optionally filter by category and minimum trust score.
"""
with self._lock:
params: list = [min_trust]
category_clause = ""
if category is not None:
category_clause = "AND category = ?"
params.append(category)
params.append(limit)
sql = f"""
SELECT fact_id, content, category, tags, trust_score,
retrieval_count, helpful_count, created_at, updated_at
FROM facts
WHERE trust_score >= ?
{category_clause}
ORDER BY trust_score DESC
LIMIT ?
"""
rows = self._conn.execute(sql, params).fetchall()
return [self._row_to_dict(r) for r in rows]
def record_feedback(self, fact_id: int, helpful: bool) -> dict:
"""Record user feedback and adjust trust asymmetrically.
helpful=True -> trust += 0.05, helpful_count += 1
helpful=False -> trust -= 0.10
Returns a dict with fact_id, old_trust, new_trust, helpful_count.
Raises KeyError if fact_id does not exist.
"""
with self._lock:
row = self._conn.execute(
"SELECT fact_id, trust_score, helpful_count FROM facts WHERE fact_id = ?",
(fact_id,),
).fetchone()
if row is None:
raise KeyError(f"fact_id {fact_id} not found")
old_trust: float = row["trust_score"]
delta = _HELPFUL_DELTA if helpful else _UNHELPFUL_DELTA
new_trust = _clamp_trust(old_trust + delta)
helpful_increment = 1 if helpful else 0
self._conn.execute(
"""
UPDATE facts
SET trust_score = ?,
helpful_count = helpful_count + ?,
updated_at = CURRENT_TIMESTAMP
WHERE fact_id = ?
""",
(new_trust, helpful_increment, fact_id),
)
self._conn.commit()
return {
"fact_id": fact_id,
"old_trust": old_trust,
"new_trust": new_trust,
"helpful_count": row["helpful_count"] + helpful_increment,
}
# ------------------------------------------------------------------
# Entity helpers
# ------------------------------------------------------------------
def _extract_entities(self, text: str) -> list[str]:
"""Extract entity candidates from text using simple regex rules.
Rules applied (in order):
1. Capitalized multi-word phrases e.g. "John Doe"
2. Double-quoted terms e.g. "Python"
3. Single-quoted terms e.g. 'pytest'
4. AKA patterns e.g. "Guido aka BDFL" -> two entities
Returns a deduplicated list preserving first-seen order.
"""
seen: set[str] = set()
candidates: list[str] = []
def _add(name: str) -> None:
stripped = name.strip()
if stripped and stripped.lower() not in seen:
seen.add(stripped.lower())
candidates.append(stripped)
for m in _RE_CAPITALIZED.finditer(text):
_add(m.group(1))
for m in _RE_DOUBLE_QUOTE.finditer(text):
_add(m.group(1))
for m in _RE_SINGLE_QUOTE.finditer(text):
_add(m.group(1))
for m in _RE_AKA.finditer(text):
_add(m.group(1))
_add(m.group(2))
return candidates
def _resolve_entity(self, name: str) -> int:
"""Find an existing entity by name or alias (case-insensitive) or create one.
Returns the entity_id.
"""
# Exact name match
row = self._conn.execute(
"SELECT entity_id FROM entities WHERE name LIKE ?", (name,)
).fetchone()
if row is not None:
return int(row["entity_id"])
# Search aliases — aliases stored as comma-separated; use LIKE with % boundaries
alias_row = self._conn.execute(
"""
SELECT entity_id FROM entities
WHERE ',' || aliases || ',' LIKE '%,' || ? || ',%'
""",
(name,),
).fetchone()
if alias_row is not None:
return int(alias_row["entity_id"])
# Create new entity
cur = self._conn.execute(
"INSERT INTO entities (name) VALUES (?)", (name,)
)
self._conn.commit()
return int(cur.lastrowid) # type: ignore[return-value]
def _link_fact_entity(self, fact_id: int, entity_id: int) -> None:
"""Insert into fact_entities, silently ignore if the link already exists."""
self._conn.execute(
"""
INSERT OR IGNORE INTO fact_entities (fact_id, entity_id)
VALUES (?, ?)
""",
(fact_id, entity_id),
)
self._conn.commit()
def _compute_hrr_vector(self, fact_id: int, content: str) -> None:
"""Compute and store HRR vector for a fact. No-op if numpy unavailable."""
with self._lock:
if not self._hrr_available:
return
# Get entities linked to this fact
rows = self._conn.execute(
"""
SELECT e.name FROM entities e
JOIN fact_entities fe ON fe.entity_id = e.entity_id
WHERE fe.fact_id = ?
""",
(fact_id,),
).fetchall()
entities = [row["name"] for row in rows]
vector = hrr.encode_fact(content, entities, self.hrr_dim)
self._conn.execute(
"UPDATE facts SET hrr_vector = ? WHERE fact_id = ?",
(hrr.phases_to_bytes(vector), fact_id),
)
self._conn.commit()
def _rebuild_bank(self, category: str) -> None:
"""Full rebuild of a category's memory bank from all its fact vectors."""
with self._lock:
if not self._hrr_available:
return
bank_name = f"cat:{category}"
rows = self._conn.execute(
"SELECT hrr_vector FROM facts WHERE category = ? AND hrr_vector IS NOT NULL",
(category,),
).fetchall()
if not rows:
self._conn.execute("DELETE FROM memory_banks WHERE bank_name = ?", (bank_name,))
self._conn.commit()
return
vectors = [hrr.bytes_to_phases(row["hrr_vector"], dim=self.hrr_dim) for row in rows]
bank_vector = hrr.bundle(*vectors)
fact_count = len(vectors)
# Check SNR
hrr.snr_estimate(self.hrr_dim, fact_count)
self._conn.execute(
"""
INSERT INTO memory_banks (bank_name, vector, dim, fact_count, updated_at)
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(bank_name) DO UPDATE SET
vector = excluded.vector,
dim = excluded.dim,
fact_count = excluded.fact_count,
updated_at = excluded.updated_at
""",
(bank_name, hrr.phases_to_bytes(bank_vector), self.hrr_dim, fact_count),
)
self._conn.commit()
def rebuild_all_vectors(self, dim: int | None = None) -> int:
"""Recompute all HRR vectors + banks from text. For recovery/migration.
Returns the number of facts processed.
"""
with self._lock:
if not self._hrr_available:
return 0
if dim is not None:
self.hrr_dim = dim
rows = self._conn.execute(
"SELECT fact_id, content, category FROM facts"
).fetchall()
categories: set[str] = set()
for row in rows:
self._compute_hrr_vector(row["fact_id"], row["content"])
categories.add(row["category"])
for category in categories:
self._rebuild_bank(category)
return len(rows)
# ------------------------------------------------------------------
# Utilities
# ------------------------------------------------------------------
def _row_to_dict(self, row: sqlite3.Row) -> dict:
"""Convert a sqlite3.Row to a plain dict."""
return dict(row)
@classmethod
def release_all_under(cls, directory: "str | Path") -> int:
"""Force-close every shared connection whose database lives under ``directory``.
``close()`` is refcount-driven, so a live holder (e.g. an agent's
memory provider) keeps a profile's SQLite handle open indefinitely.
That is exactly what a profile delete must break on Windows: the
desktop's main ``serve`` process opens ``memory_store.db`` for every
known profile, and ``rmtree`` of the profile directory fails with
``WinError 32`` while any of those handles is open (#88347). This
closes the matching connections unconditionally — the directory is
going away, so later use by a stale holder is expected to fail — and
returns how many were closed. In a process that holds none (e.g. the
CLI deleting from outside serve) this is a harmless no-op returning 0.
"""
root = os.path.normcase(str(Path(directory).expanduser().resolve())) + os.sep
with cls._shared_guard:
# Snapshot the keys first so the registry stays stable while
# connections are closed inside their per-database locks (closing
# can run no user code, but this keeps the invariant obvious).
doomed = [
key
for key in cls._shared
if os.path.normcase(key).startswith(root)
]
for key in doomed:
entry = cls._shared.pop(key)
try:
with entry["lock"]:
entry["conn"].close()
except Exception:
# A connection that is already closed or broken must not
# abort releasing its siblings.
pass
return len(doomed)
def close(self) -> None:
"""Release this instance's reference to the shared connection.
The underlying connection is closed only when the last MemoryStore
referencing the same database is closed, so closing one instance can
never break sibling instances that still hold it. Idempotent.
"""
if getattr(self, "_entry", None) is None:
return
with MemoryStore._shared_guard:
entry = self._entry
if entry is None:
return
entry["refs"] -= 1
if entry["refs"] <= 0:
try:
entry["conn"].close()
finally:
# Pop only OUR entry. After release_all_under() force-
# closed this entry (profile delete, #88347) a same-path
# store may have re-registered a FRESH entry under the
# same key; a stale holder's late close() must not evict
# it — that would silently reintroduce the multi-writer
# contention this registry exists to prevent.
if MemoryStore._shared.get(self._key) is entry:
MemoryStore._shared.pop(self._key, None)
self._entry = None
def __enter__(self) -> "MemoryStore":
return self
def __exit__(self, *_: object) -> None:
self.close()