Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,108 @@
|
||||
---
|
||||
name: pinecone-research
|
||||
description: Agent RAG and long-term memory with Pinecone.
|
||||
version: 1.0.0
|
||||
author: immuhammadfurqan
|
||||
license: MIT
|
||||
dependencies: [pinecone-client, langchain-pinecone]
|
||||
platforms: [linux, macos, windows]
|
||||
metadata:
|
||||
hermes:
|
||||
tags: [RAG, Pinecone, Memory, Research, Vector Database, Agent, Retrieval]
|
||||
|
||||
---
|
||||
|
||||
# Pinecone Research — Agent RAG & Long-Term Memory
|
||||
|
||||
Use Pinecone as a retrieval-augmented generation (RAG) backend for agent
|
||||
conversations: persist embeddings, retrieve relevant context from past
|
||||
sessions, and build long-term memory.
|
||||
|
||||
## When to use this skill
|
||||
|
||||
**Use when:**
|
||||
- Building agent RAG pipelines with Pinecone as the vector store
|
||||
- Need persistent long-term memory across agent sessions
|
||||
- Combining retrieval with agent tool use
|
||||
- Researching or prototyping semantic search workflows
|
||||
|
||||
**Use the mlops/pinecone skill instead when:**
|
||||
- Need a general Pinecone reference (index management, CRUD, hybrid search)
|
||||
- Working on production infrastructure without agent integration
|
||||
|
||||
## Quick start
|
||||
|
||||
### Setup
|
||||
|
||||
```bash
|
||||
pip install pinecone-client langchain-pinecone langchain-openai
|
||||
```
|
||||
|
||||
Set your API key:
|
||||
```bash
|
||||
export PINECONE_API_KEY="your-api-key"
|
||||
```
|
||||
|
||||
### Basic RAG pipeline
|
||||
|
||||
```python
|
||||
from pinecone import Pinecone, ServerlessSpec
|
||||
from langchain_pinecone import PineconeVectorStore
|
||||
from langchain_openai import OpenAIEmbeddings
|
||||
|
||||
# Initialize Pinecone
|
||||
pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
|
||||
|
||||
# Create or connect to index
|
||||
index_name = "agent-memory"
|
||||
if index_name not in [i.name for i in pc.list_indexes()]:
|
||||
pc.create_index(
|
||||
name=index_name,
|
||||
dimension=1536,
|
||||
metric="cosine",
|
||||
spec=ServerlessSpec(cloud="aws", region="us-east-1"),
|
||||
)
|
||||
|
||||
# Build vector store
|
||||
vectorstore = PineconeVectorStore.from_documents(
|
||||
documents=docs,
|
||||
embedding=OpenAIEmbeddings(),
|
||||
index_name=index_name,
|
||||
)
|
||||
|
||||
# Retrieve relevant context
|
||||
retriever = vectorstore.as_retriever(search_kwargs={"k": 5})
|
||||
results = retriever.invoke("What did the agent discuss yesterday?")
|
||||
```
|
||||
|
||||
### Namespace-based session memory
|
||||
|
||||
```python
|
||||
# Store per-session memory
|
||||
vectorstore = PineconeVectorStore(
|
||||
index=pc.Index(index_name),
|
||||
embedding=OpenAIEmbeddings(),
|
||||
namespace=f"session-{session_id}",
|
||||
)
|
||||
|
||||
# Query across all sessions (no namespace filter)
|
||||
all_memory = PineconeVectorStore(
|
||||
index=pc.Index(index_name),
|
||||
embedding=OpenAIEmbeddings(),
|
||||
)
|
||||
results = all_memory.similarity_search("relevant query", k=10)
|
||||
```
|
||||
|
||||
## Best practices
|
||||
|
||||
1. **Namespace by session or user** — isolate data for multi-tenant agents
|
||||
2. **Batch upserts** — 100–200 vectors per batch for efficiency
|
||||
3. **Metadata filtering** — tag vectors with session ID, timestamp, topic
|
||||
4. **Prune old memory** — delete stale namespaces to control costs
|
||||
5. **Use serverless** — auto-scaling, pay-per-use pricing
|
||||
|
||||
## Resources
|
||||
|
||||
- **Pinecone Docs**: https://docs.pinecone.io
|
||||
- **LangChain Integration**: https://python.langchain.com/docs/integrations/vectorstores/pinecone
|
||||
- **Free Tier**: 1 index, 100K vectors (1536 dimensions)
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Pinecone memory manager — namespace-based session memory for agents.
|
||||
|
||||
Provides helpers for storing and retrieving agent conversation memory
|
||||
using Pinecone namespaces. Each session gets its own namespace for isolation,
|
||||
with cross-session search available via the global namespace.
|
||||
|
||||
Usage:
|
||||
export PINECONE_API_KEY="your-key"
|
||||
export OPENAI_API_KEY="your-key"
|
||||
python memory_manager.py --index-name agent-memory --action store \
|
||||
--session-id sess-001 --text "User discussed project architecture"
|
||||
python memory_manager.py --index-name agent-memory --action recall \
|
||||
--query "architecture decisions"
|
||||
python memory_manager.py --index-name agent-memory --action cleanup \
|
||||
--session-id sess-001
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def get_pinecone_client():
|
||||
"""Initialize Pinecone client from environment."""
|
||||
try:
|
||||
from pinecone import Pinecone
|
||||
except ImportError:
|
||||
print("Error: pinecone-client not installed. Run: pip install pinecone-client", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
api_key = os.environ.get("PINECONE_API_KEY")
|
||||
if not api_key:
|
||||
print("Error: PINECONE_API_KEY environment variable not set.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
return Pinecone(api_key=api_key)
|
||||
|
||||
|
||||
def get_embeddings():
|
||||
"""Get the embedding model."""
|
||||
try:
|
||||
from langchain_openai import OpenAIEmbeddings
|
||||
except ImportError:
|
||||
print("Error: langchain-openai not installed. Run: pip install langchain-openai", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return OpenAIEmbeddings()
|
||||
|
||||
|
||||
def store_memory(index, session_id: str, text: str, metadata: dict | None = None):
|
||||
"""Store a memory entry in the session namespace."""
|
||||
embeddings = get_embeddings()
|
||||
vector = embeddings.embed_query(text)
|
||||
|
||||
doc_id = hashlib.sha256(f"{session_id}:{text}:{time.time()}".encode()).hexdigest()[:16]
|
||||
entry_metadata = {
|
||||
"text": text[:1000],
|
||||
"session_id": session_id,
|
||||
"timestamp": int(time.time()),
|
||||
}
|
||||
if metadata:
|
||||
entry_metadata.update(metadata)
|
||||
|
||||
index.upsert(
|
||||
vectors=[{"id": doc_id, "values": vector, "metadata": entry_metadata}],
|
||||
namespace=session_id,
|
||||
)
|
||||
print(f"Stored memory [{doc_id}] in namespace '{session_id}'")
|
||||
return doc_id
|
||||
|
||||
|
||||
def recall_memories(index, query: str, session_id: str | None = None, top_k: int = 5):
|
||||
"""Recall memories matching a query, optionally scoped to a session."""
|
||||
embeddings = get_embeddings()
|
||||
query_vector = embeddings.embed_query(query)
|
||||
|
||||
kwargs = {"vector": query_vector, "top_k": top_k, "include_metadata": True}
|
||||
if session_id:
|
||||
kwargs["namespace"] = session_id
|
||||
|
||||
results = index.query(**kwargs)
|
||||
|
||||
print(f"\nRecalling memories for: {query!r}")
|
||||
if session_id:
|
||||
print(f"Scoped to session: {session_id}")
|
||||
print(f"Found {len(results['matches'])} results:\n")
|
||||
|
||||
for match in results["matches"]:
|
||||
score = match["score"]
|
||||
text = match["metadata"].get("text", "")[:200]
|
||||
sess = match["metadata"].get("session_id", "unknown")
|
||||
ts = match["metadata"].get("timestamp", 0)
|
||||
print(f" [{score:.4f}] session={sess} time={ts}")
|
||||
print(f" {text}")
|
||||
print()
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def cleanup_session(index, session_id: str):
|
||||
"""Delete all vectors in a session namespace."""
|
||||
index.delete(delete_all=True, namespace=session_id)
|
||||
print(f"Cleaned up namespace '{session_id}'")
|
||||
|
||||
|
||||
def show_stats(index):
|
||||
"""Show index statistics."""
|
||||
stats = index.describe_index_stats()
|
||||
print(f"Total vectors: {stats['total_vector_count']}")
|
||||
namespaces = stats.get("namespaces", {})
|
||||
if namespaces:
|
||||
print(f"Namespaces ({len(namespaces)}):")
|
||||
for ns, info in sorted(namespaces.items()):
|
||||
print(f" '{ns}': {info['vector_count']} vectors")
|
||||
else:
|
||||
print("No namespaces found.")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Pinecone agent memory manager")
|
||||
parser.add_argument("--index-name", required=True, help="Pinecone index name")
|
||||
parser.add_argument(
|
||||
"--action",
|
||||
choices=["store", "recall", "cleanup", "stats"],
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument("--session-id", help="Session namespace ID")
|
||||
parser.add_argument("--text", help="Text to store as memory")
|
||||
parser.add_argument("--query", help="Query for recall")
|
||||
parser.add_argument("--top-k", type=int, default=5, help="Number of results")
|
||||
args = parser.parse_args()
|
||||
|
||||
pc = get_pinecone_client()
|
||||
index = pc.Index(args.index_name)
|
||||
|
||||
if args.action == "store":
|
||||
if not args.session_id or not args.text:
|
||||
parser.error("--session-id and --text required for store action")
|
||||
store_memory(index, args.session_id, args.text)
|
||||
elif args.action == "recall":
|
||||
if not args.query:
|
||||
parser.error("--query required for recall action")
|
||||
recall_memories(index, args.query, session_id=args.session_id, top_k=args.top_k)
|
||||
elif args.action == "cleanup":
|
||||
if not args.session_id:
|
||||
parser.error("--session-id required for cleanup action")
|
||||
cleanup_session(index, args.session_id)
|
||||
elif args.action == "stats":
|
||||
show_stats(index)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Pinecone RAG pipeline — index documents and query with retrieval-augmented generation.
|
||||
|
||||
Usage:
|
||||
export PINECONE_API_KEY="your-key"
|
||||
export OPENAI_API_KEY="your-key"
|
||||
python rag_pipeline.py --index-name agent-memory --action index --docs-dir ./docs
|
||||
python rag_pipeline.py --index-name agent-memory --action query --query "How does X work?"
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_pinecone_client():
|
||||
"""Initialize Pinecone client from environment."""
|
||||
try:
|
||||
from pinecone import Pinecone
|
||||
except ImportError:
|
||||
print("Error: pinecone-client not installed. Run: pip install pinecone-client", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
api_key = os.environ.get("PINECONE_API_KEY")
|
||||
if not api_key:
|
||||
print("Error: PINECONE_API_KEY environment variable not set.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
return Pinecone(api_key=api_key)
|
||||
|
||||
|
||||
def ensure_index(pc, index_name: str, dimension: int = 1536):
|
||||
"""Create the index if it doesn't exist."""
|
||||
from pinecone import ServerlessSpec
|
||||
|
||||
existing = [idx.name for idx in pc.list_indexes()]
|
||||
if index_name not in existing:
|
||||
pc.create_index(
|
||||
name=index_name,
|
||||
dimension=dimension,
|
||||
metric="cosine",
|
||||
spec=ServerlessSpec(cloud="aws", region="us-east-1"),
|
||||
)
|
||||
print(f"Created index: {index_name}")
|
||||
else:
|
||||
print(f"Index already exists: {index_name}")
|
||||
return pc.Index(index_name)
|
||||
|
||||
|
||||
def load_documents(docs_dir: str) -> list[dict]:
|
||||
"""Load text files from a directory as documents."""
|
||||
docs = []
|
||||
docs_path = Path(docs_dir)
|
||||
if not docs_path.is_dir():
|
||||
print(f"Error: {docs_dir} is not a directory.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
for filepath in sorted(docs_path.rglob("*.txt")):
|
||||
text = filepath.read_text(encoding="utf-8").strip()
|
||||
if text:
|
||||
docs.append({
|
||||
"id": str(filepath.relative_to(docs_path)),
|
||||
"text": text,
|
||||
"metadata": {"source": str(filepath.name)},
|
||||
})
|
||||
return docs
|
||||
|
||||
|
||||
def index_documents(index, docs: list[dict], batch_size: int = 100):
|
||||
"""Embed and upsert documents into Pinecone."""
|
||||
try:
|
||||
from langchain_openai import OpenAIEmbeddings
|
||||
except ImportError:
|
||||
print("Error: langchain-openai not installed. Run: pip install langchain-openai", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
embeddings = OpenAIEmbeddings()
|
||||
vectors = []
|
||||
|
||||
for doc in docs:
|
||||
embedding = embeddings.embed_query(doc["text"])
|
||||
vectors.append({
|
||||
"id": doc["id"],
|
||||
"values": embedding,
|
||||
"metadata": {**doc["metadata"], "text": doc["text"][:1000]},
|
||||
})
|
||||
|
||||
# Batch upsert
|
||||
for i in range(0, len(vectors), batch_size):
|
||||
batch = vectors[i : i + batch_size]
|
||||
index.upsert(vectors=batch)
|
||||
print(f"Upserted batch {i // batch_size + 1} ({len(batch)} vectors)")
|
||||
|
||||
print(f"Total vectors indexed: {len(vectors)}")
|
||||
|
||||
|
||||
def query_index(index, query: str, top_k: int = 5):
|
||||
"""Embed a query and retrieve similar documents from Pinecone."""
|
||||
try:
|
||||
from langchain_openai import OpenAIEmbeddings
|
||||
except ImportError:
|
||||
print("Error: langchain-openai not installed. Run: pip install langchain-openai", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
embeddings = OpenAIEmbeddings()
|
||||
query_vector = embeddings.embed_query(query)
|
||||
|
||||
results = index.query(vector=query_vector, top_k=top_k, include_metadata=True)
|
||||
|
||||
print(f"\nQuery: {query}")
|
||||
print(f"Top {top_k} results:\n")
|
||||
for match in results["matches"]:
|
||||
score = match["score"]
|
||||
source = match["metadata"].get("source", "unknown")
|
||||
text_preview = match["metadata"].get("text", "")[:200]
|
||||
print(f" [{score:.4f}] {source}")
|
||||
print(f" {text_preview}...")
|
||||
print()
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Pinecone RAG pipeline")
|
||||
parser.add_argument("--index-name", required=True, help="Pinecone index name")
|
||||
parser.add_argument("--action", choices=["index", "query", "stats"], required=True)
|
||||
parser.add_argument("--docs-dir", help="Directory of .txt files to index")
|
||||
parser.add_argument("--query", help="Query string for retrieval")
|
||||
parser.add_argument("--top-k", type=int, default=5, help="Number of results to return")
|
||||
args = parser.parse_args()
|
||||
|
||||
pc = get_pinecone_client()
|
||||
index = ensure_index(pc, args.index_name)
|
||||
|
||||
if args.action == "index":
|
||||
if not args.docs_dir:
|
||||
parser.error("--docs-dir required for index action")
|
||||
docs = load_documents(args.docs_dir)
|
||||
if not docs:
|
||||
print("No .txt documents found.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
index_documents(index, docs)
|
||||
elif args.action == "query":
|
||||
if not args.query:
|
||||
parser.error("--query required for query action")
|
||||
query_index(index, args.query, top_k=args.top_k)
|
||||
elif args.action == "stats":
|
||||
stats = index.describe_index_stats()
|
||||
print(f"Total vectors: {stats['total_vector_count']}")
|
||||
for ns, info in stats.get("namespaces", {}).items():
|
||||
print(f" Namespace '{ns}': {info['vector_count']} vectors")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user