Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license
This commit is contained in:
@@ -0,0 +1,417 @@
|
||||
"""Autonomous action queue: what should the agent do RIGHT NOW for this subject?
|
||||
|
||||
`next_actions` turns (dossier, broker DB, config, ledger) into an ordered queue of
|
||||
concrete agent actions plus a human digest. The agent's whole run becomes a loop:
|
||||
|
||||
while True:
|
||||
q = pdd.py next <subject>
|
||||
if not q["actions"]: break
|
||||
execute each action, record outcomes
|
||||
present q["human_digest"] once; schedule cron at q["next_wake_at"]
|
||||
|
||||
Policy (cfg["autonomy"]):
|
||||
full - intake consent is standing authorization; T0-T2 agent actions are
|
||||
executed without pausing. Humans appear only in the digest.
|
||||
assisted - same queue, but every submission action carries confirm_first=True.
|
||||
|
||||
The queue is deterministic and side-effect free: it never mutates the ledger, it
|
||||
only reads. Executing + recording stays with the agent (and the record command).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import brokers as brokers_mod
|
||||
import emailer
|
||||
import ledger as ledger_mod
|
||||
import paths
|
||||
import registry
|
||||
import tiers
|
||||
|
||||
CACHE_STALE_DAYS = 7 # refresh the live broker list after this
|
||||
FANOUT_THRESHOLD = 8 # above this many unscanned brokers, use delegate_task fan-out
|
||||
|
||||
# States with nothing left to do (absent a due recheck).
|
||||
_TERMINAL = {"not_found", "confirmed_removed"}
|
||||
_IN_FLIGHT = {"submitted", "verification_pending", "awaiting_processing"}
|
||||
|
||||
|
||||
def cache_age_days(now: float | None = None) -> float | None:
|
||||
"""Age of the live BADBOOL cache in days, or None if never pulled."""
|
||||
p: Path = paths.brokers_cache_path()
|
||||
if not p.exists():
|
||||
return None
|
||||
now = now if now is not None else _dt.datetime.now().timestamp()
|
||||
return max(0.0, (now - p.stat().st_mtime) / 86400.0)
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _min_future_recheck(ledger: dict, at: str) -> str | None:
|
||||
future = [c.get("next_recheck_at") for c in ledger.values()
|
||||
if c.get("next_recheck_at") and c["next_recheck_at"] > at]
|
||||
return min(future) if future else None
|
||||
|
||||
|
||||
def _digest(broker_row: dict, reason: str, steps: list[str], prep: list[str] | None = None) -> dict:
|
||||
return {
|
||||
"broker_id": broker_row.get("broker_id"),
|
||||
"broker_name": broker_row.get("broker_name"),
|
||||
"reason": reason,
|
||||
"agent_prep": prep or [], # commands the agent runs BEFORE handing this to the human
|
||||
"steps": steps, # what the human actually does
|
||||
"withhold": ["SSN", "full driver's-license / passport numbers"],
|
||||
}
|
||||
|
||||
|
||||
def request_kind(dossier: dict, allowed: list[str] | None = None) -> str:
|
||||
"""Pick the honest legal basis for a deletion request from the subject's residency.
|
||||
|
||||
ccpa only for California residents, gdpr only for EU/UK residents, generic otherwise.
|
||||
`allowed` (from the broker's deletion.kinds) can restrict DOWN to generic but never
|
||||
upgrades to a law the subject can't truthfully claim.
|
||||
"""
|
||||
res = (dossier.get("residency_jurisdiction") or "US").upper()
|
||||
if res.startswith("US-CA"):
|
||||
kind = "ccpa"
|
||||
elif res.startswith(("EU", "UK", "GB")):
|
||||
kind = "gdpr"
|
||||
else:
|
||||
kind = "generic"
|
||||
if allowed and kind not in allowed and "generic" in allowed:
|
||||
kind = "generic"
|
||||
return kind
|
||||
|
||||
|
||||
_HUMAN_GATES = ("gov_id", "fax", "mail", "phone_voice", "phone_callback", "account")
|
||||
|
||||
|
||||
def _email_lane(row: dict) -> tuple[str | None, str]:
|
||||
"""(address, why) for the autonomous email lane of this broker, if one exists.
|
||||
|
||||
Lane rules:
|
||||
1. the broker's primary opt-out method IS email;
|
||||
2. the record marks its deletion lane email-preferred (deletion.via == "email");
|
||||
3. RESCUE: the primary flow is human-gated (gov ID / fax / phone / account) but a
|
||||
right-to-delete email exists - the email lane restores full autonomy (this is the
|
||||
verified Whitepages pattern: privacyrequest@ accepts requests precisely so people
|
||||
don't have to do the phone-callback tool).
|
||||
"""
|
||||
deletion = row.get("deletion") or {}
|
||||
req = row.get("optout_requires") or {}
|
||||
if row.get("method") == "email":
|
||||
addr = row.get("optout_email") or deletion.get("email")
|
||||
return (addr, "primary opt-out method is email") if addr else (None, "")
|
||||
if deletion.get("via") == "email" and deletion.get("email"):
|
||||
return deletion["email"], "record prefers the right-to-delete email lane"
|
||||
if (row.get("tier") == "T3" or any(req.get(k) for k in _HUMAN_GATES)) and deletion.get("email"):
|
||||
return deletion["email"], "rescue: primary flow is human-gated; deletion email restores autonomy"
|
||||
return None, ""
|
||||
|
||||
|
||||
def _optout_action(row: dict, playbook: dict[str, dict], subject_id: str, dossier: dict,
|
||||
email_mode: str, smtp_ok: bool, confirm_first: bool) -> tuple[dict | None, dict | None]:
|
||||
"""Map one actionable `found` row to (agent_action, human_digest_entry).
|
||||
|
||||
Routing order maximizes autonomy: (1) the email lane (primary email method, preferred
|
||||
right-to-delete email, or rescue from a human-gated form) beats everything when SMTP is
|
||||
up; (2) genuinely human-only flows go to the digest; (3) web forms are driven with the
|
||||
record's own field-verified playbook steps.
|
||||
"""
|
||||
bid = row["broker_id"]
|
||||
req = row.get("optout_requires") or {}
|
||||
tier = row.get("tier")
|
||||
deletion = row.get("deletion") or {}
|
||||
|
||||
# 1) The autonomous EMAIL LANE (right-to-delete by email + confirm the reply).
|
||||
# Autonomous when SMTP is configured (programmatic/alias) OR in browser mode (agent sends via
|
||||
# the operator's logged-in webmail; no password needed).
|
||||
email_addr, lane_why = _email_lane(row)
|
||||
can_email = (email_mode in ("programmatic", "alias") and smtp_ok) or email_mode == "browser"
|
||||
if email_addr and can_email:
|
||||
kind = request_kind(dossier, deletion.get("kinds"))
|
||||
via = "browser" if email_mode == "browser" else "smtp"
|
||||
then = ("send-email records it + returns a recipient-locked payload; compose and send it in "
|
||||
"the operator's webmail via browser_*, then `verify-link` on the reply and open the link"
|
||||
if via == "browser" else
|
||||
"state auto-records as submitted; poll-verification picks up their verification reply, "
|
||||
"open its link, then record")
|
||||
return {
|
||||
"type": "optout_email_send",
|
||||
"broker_id": bid, "broker_name": row.get("broker_name"), "tier": tier,
|
||||
"confirm_first": confirm_first, "send_via": via,
|
||||
"to": email_addr, "kind": kind, "why": lane_why,
|
||||
"command": f"python3 scripts/pdd.py send-email {subject_id} {bid} --kind {kind} "
|
||||
f"--to {email_addr} --listing <confirmed-url>",
|
||||
"then": then,
|
||||
}, None
|
||||
if row.get("method") == "email":
|
||||
return None, _digest(row, "email opt-out (draft mode: a human must hit send)",
|
||||
["Send the rendered draft from your own mail client",
|
||||
f"Then: python3 scripts/pdd.py record {subject_id} {bid} submitted "
|
||||
f"--disclosed contact_email --channel email"],
|
||||
prep=[f"python3 scripts/pdd.py render-email {subject_id} {bid} --listing <confirmed-url>"])
|
||||
|
||||
# 2) Genuinely human-only work goes to the digest (no email lane could rescue it).
|
||||
if tier == "T3":
|
||||
return None, _digest(row, "human-only opt-out (gov ID / fax / mail / voice phone)",
|
||||
[f"Follow the broker's process at {row.get('optout_url') or row.get('optout_email')}",
|
||||
"Provide only the fields the listing already shows; cross out ID numbers on any document"])
|
||||
if req.get("phone_callback"):
|
||||
return None, _digest(row, "phone-callback verification (operator must be on the phone)",
|
||||
[f"Open {row.get('optout_url')} and submit with only the planned fields",
|
||||
"Answer the automated call and enter the 4-digit code to finish"],
|
||||
prep=[f"python3 scripts/pdd.py plan {subject_id} --batch # confirm fields first"])
|
||||
if req.get("account"):
|
||||
return None, _digest(row, "requires creating/holding an account with the broker",
|
||||
[f"Create/log in at {row.get('optout_url')} and submit the opt-out",
|
||||
"Use the subject's contact email; no extra PII beyond the planned fields"])
|
||||
|
||||
# 3) web_form: drive the browser with the record's own playbook steps.
|
||||
steps = (playbook.get(bid) or {}).get("steps") or list(row.get("optout_playbook") or []) \
|
||||
or tiers.synthesize_steps(row)
|
||||
action = {
|
||||
"type": "optout_web_form",
|
||||
"broker_id": bid, "broker_name": row.get("broker_name"), "tier": tier,
|
||||
"confirm_first": confirm_first,
|
||||
"optout_url": row.get("optout_url"),
|
||||
"clears_children": row.get("clears_children") or [],
|
||||
"steps": steps,
|
||||
"after": f"python3 scripts/pdd.py record {subject_id} {bid} submitted "
|
||||
f"--disclosed <field>... --channel web_form",
|
||||
}
|
||||
if deletion:
|
||||
if deletion.get("prefer", True):
|
||||
action["prefer_deletion"] = ("this record has a right-to-delete lane -- complete the "
|
||||
"DELETION flow, not just suppression"
|
||||
+ (f" ({deletion.get('notes')})" if deletion.get("notes") else ""))
|
||||
else:
|
||||
# Some brokers invert the usual rule: deleting the account removes suppressions and
|
||||
# does not stop public-records re-listing (e.g. PeopleConnect). Suppress and maintain.
|
||||
action["prefer_suppression"] = (deletion.get("notes")
|
||||
or "suppression (maintained) is what removes you here; "
|
||||
"deleting undoes it and does not stop re-listing")
|
||||
if req.get("captcha"):
|
||||
action["note"] = ("CAPTCHA-gated: attempt with the configured browser backend once; if it "
|
||||
"does not clear, record blocked (do NOT retry-loop or bypass)")
|
||||
return action, None
|
||||
|
||||
|
||||
def next_actions(dossier: dict, brokers_list: list[dict], cfg: dict,
|
||||
ledger: dict | None = None, env: dict | None = None) -> dict:
|
||||
env = os.environ if env is None else env
|
||||
ledger = ledger or {}
|
||||
subject_id = dossier.get("subject_id", "")
|
||||
autonomy = cfg.get("autonomy", "full")
|
||||
confirm_first = autonomy == "assisted"
|
||||
email_mode = cfg.get("email_mode", "draft_only")
|
||||
mail = emailer.available(env)
|
||||
at = _now_iso()
|
||||
|
||||
batch = tiers.batch_plan(dossier, brokers_list, cfg, ledger,
|
||||
browser_clears_captcha=cfg.get("browser_backend") == "browserbase"
|
||||
or bool(env.get("BROWSERBASE_API_KEY")))
|
||||
groups = batch["groups"]
|
||||
playbook = {p["broker_id"]: p for p in batch.get("parent_playbook") or []}
|
||||
by_id = {b.get("id"): b for b in brokers_list}
|
||||
|
||||
actions: list[dict] = []
|
||||
digest: list[dict] = []
|
||||
|
||||
# 0) keep the broker DB fresh (autonomously)
|
||||
age = cache_age_days()
|
||||
if age is None or age > CACHE_STALE_DAYS:
|
||||
actions.append({
|
||||
"type": "refresh_brokers",
|
||||
"why": "live broker cache missing" if age is None else f"cache is {age:.0f} days old",
|
||||
"command": "python3 scripts/pdd.py refresh-brokers",
|
||||
})
|
||||
|
||||
# 0b) DROP one-shot: for a CA resident, ONE request deletes from every registered
|
||||
# broker (the whole CA Data Broker Registry) -- the highest-leverage removal there is.
|
||||
registry_recs = brokers_mod.load_registry_cache()
|
||||
residency = (dossier.get("residency_jurisdiction") or "US").upper()
|
||||
drop_filed = bool((dossier.get("preferences") or {}).get("drop_filed_at"))
|
||||
if registry_recs and residency.startswith("US-CA") and not drop_filed:
|
||||
actions.append({
|
||||
"type": "drop_submit",
|
||||
"one_shot": True,
|
||||
"registry_count": len(registry_recs),
|
||||
"url": registry.DROP_URL,
|
||||
"command": f"python3 scripts/pdd.py drop {subject_id}",
|
||||
"why": f"CA resident: one DROP request deletes from all {len(registry_recs)} registered "
|
||||
"data brokers at once (superset of what commercial services cover).",
|
||||
"after": f"python3 scripts/pdd.py drop {subject_id} --filed",
|
||||
})
|
||||
|
||||
# 1) Phase 1 crawl: everything unscanned (read-only, parallel-safe)
|
||||
unscanned = groups.get("unscanned") or []
|
||||
if unscanned:
|
||||
ids = [r["broker_id"] for r in unscanned]
|
||||
if len(ids) > FANOUT_THRESHOLD:
|
||||
actions.append({
|
||||
"type": "fanout_scan",
|
||||
"broker_ids": ids,
|
||||
"command": f"python3 scripts/pdd.py fanout {subject_id}",
|
||||
"how": "spawn ONE delegate_task subagent per batch IN PARALLEL with each batch's brief; "
|
||||
"parent re-verifies key `found` claims before trusting them",
|
||||
})
|
||||
else:
|
||||
actions.append({
|
||||
"type": "scan_inline",
|
||||
"broker_ids": ids,
|
||||
"command": f"python3 scripts/pdd.py plan {subject_id}",
|
||||
"how": "run every search_vector per broker via the methods.md ladder "
|
||||
"(web_extract -> site: probe -> browser), record a verdict per broker",
|
||||
})
|
||||
|
||||
# 2) in-flight email verifications: poll the inbox (or hand to the human in draft mode)
|
||||
for st in ("submitted", "verification_pending"):
|
||||
for bid, case in sorted(ledger.items()):
|
||||
if case.get("state") != st:
|
||||
continue
|
||||
broker = by_id.get(bid) or {}
|
||||
if not ((broker.get("optout") or {}).get("requires") or {}).get("email_verification"):
|
||||
continue
|
||||
if mail["imap"]:
|
||||
actions.append({
|
||||
"type": "poll_verification", "via": "imap",
|
||||
"broker_id": bid,
|
||||
"command": f"python3 scripts/pdd.py poll-verification {subject_id} --broker {bid}",
|
||||
"then": "browser_navigate the returned link IN THE SAME AGENT BROWSER (sessions are "
|
||||
"browser-bound), complete the flow, then record: awaiting_processing",
|
||||
})
|
||||
elif email_mode == "browser":
|
||||
actions.append({
|
||||
"type": "poll_verification", "via": "browser", "broker_id": bid,
|
||||
"how": "open the broker's confirmation email in the operator's logged-in webmail "
|
||||
f"(browser_*), then `python3 scripts/pdd.py verify-link {subject_id} {bid} "
|
||||
"--text '<email body>'` to score the link, browser_navigate it in the SAME "
|
||||
"browser, then record awaiting_processing",
|
||||
})
|
||||
else:
|
||||
digest.append(_digest(
|
||||
{"broker_id": bid, "broker_name": (broker.get("name") or bid)},
|
||||
"verification email must be opened by a human (draft mode, no inbox access)",
|
||||
["Open the broker's verification email in the subject's inbox and click the link",
|
||||
f"Then: python3 scripts/pdd.py record {subject_id} {bid} awaiting_processing"]))
|
||||
|
||||
# 3) due rechecks: processing windows elapsed / reappearance sweeps
|
||||
for case in ledger_mod.due(subject_id, at=at, ledger=ledger):
|
||||
bid = case.get("broker_id")
|
||||
st = case.get("state")
|
||||
if st in ("awaiting_processing", "confirmed_removed"):
|
||||
actions.append({
|
||||
"type": "verify_removal",
|
||||
"broker_id": bid,
|
||||
"why": "processing window elapsed" if st == "awaiting_processing" else "periodic reappearance re-scan",
|
||||
"how": "re-run this broker's search_vectors; if gone record confirmed_removed; "
|
||||
"if still listed record reappeared and requeue the opt-out",
|
||||
})
|
||||
elif st in ("submitted", "verification_pending") and not mail["imap"]:
|
||||
pass # already covered by the digest entry above
|
||||
|
||||
# 4) Phase 2 opt-outs: parents first (batch_plan already ordered them)
|
||||
for row in groups.get("found") or []:
|
||||
action, task = _optout_action(row, playbook, subject_id, dossier,
|
||||
email_mode, mail["smtp"], confirm_first)
|
||||
if action:
|
||||
actions.append(action)
|
||||
if task:
|
||||
digest.append(task)
|
||||
|
||||
# 5) indirect exposure: targeted delete-my-PII requests
|
||||
for row in groups.get("indirect_exposure") or []:
|
||||
bid = row["broker_id"]
|
||||
has_email = bool(row.get("optout_email") or (row.get("deletion") or {}).get("email"))
|
||||
if not has_email and row.get("optout_url"):
|
||||
# No email lane (e.g. ThatsThem is web-form-only): drive the opt-out FORM, submitting
|
||||
# ONLY the subject's own identifiers to scrub from the third party's record.
|
||||
actions.append({
|
||||
"type": "indirect_web_form",
|
||||
"broker_id": bid, "confirm_first": confirm_first,
|
||||
"optout_url": row.get("optout_url"),
|
||||
"steps": [f"browser_navigate {row.get('optout_url')}",
|
||||
"submit ONLY the subject's own identifiers (the fields the form requires) to "
|
||||
"remove them from the third party's record; disclose nothing extra",
|
||||
"confirm the success state, screenshot into evidence/"],
|
||||
"after": f"python3 scripts/pdd.py record {subject_id} {bid} submitted --channel web_form",
|
||||
})
|
||||
elif (email_mode in ("programmatic", "alias") and mail["smtp"]) or email_mode == "browser":
|
||||
actions.append({
|
||||
"type": "indirect_email_send",
|
||||
"broker_id": bid, "confirm_first": confirm_first,
|
||||
"send_via": "browser" if email_mode == "browser" else "smtp",
|
||||
"command": f"python3 scripts/pdd.py send-email {subject_id} {bid} --kind ccpa_indirect "
|
||||
f"--listing <third-party-listing-url>",
|
||||
})
|
||||
else:
|
||||
digest.append(_digest(row, "indirect-exposure request (draft mode: a human must hit send)",
|
||||
["Send the rendered ccpa_indirect draft",
|
||||
f"Then: python3 scripts/pdd.py record {subject_id} {bid} submitted "
|
||||
f"--disclosed contact_email --channel email"],
|
||||
prep=[f"python3 scripts/pdd.py render-email {subject_id} {bid} "
|
||||
f"--kind ccpa_indirect --listing <url>"]))
|
||||
|
||||
# 6) blocked sites: stealth pass if we have one, else the operator-browser path
|
||||
blocked = groups.get("blocked") or []
|
||||
if blocked:
|
||||
ids = [r["broker_id"] for r in blocked]
|
||||
if bool(env.get("BROWSERBASE_API_KEY")):
|
||||
actions.append({
|
||||
"type": "stealth_rescan",
|
||||
"broker_ids": ids,
|
||||
"how": "retry these with the cloud/stealth browser backend, then record real verdicts",
|
||||
})
|
||||
else:
|
||||
for r in blocked:
|
||||
digest.append(_digest(r, "site blocks automated access (anti-bot); a human browser gets through",
|
||||
["Open the paste-ready search URL from `plan` in your everyday browser",
|
||||
"Report the verdict (or a screenshot) back to the agent",
|
||||
f"Agent records: python3 scripts/pdd.py record {subject_id} "
|
||||
f"{r['broker_id']} <found|not_found|indirect_exposure>"]))
|
||||
|
||||
# 7) anything already parked as a human task
|
||||
for bid, case in sorted(ledger.items()):
|
||||
if case.get("state") == "human_task_queued":
|
||||
broker = by_id.get(bid) or {}
|
||||
digest.append(_digest({"broker_id": bid, "broker_name": broker.get("name") or bid},
|
||||
case.get("human_task_reason") or "queued manual step",
|
||||
["See `pdd.py tasks` for the exact steps recorded with this case"]))
|
||||
|
||||
# registry coverage summary (breadth beyond the scannable people-search sites)
|
||||
coverage = None
|
||||
if registry_recs:
|
||||
coverage = {
|
||||
"people_search_sites": len(brokers_list),
|
||||
"registered_data_brokers": len(registry_recs),
|
||||
"worked_via": "CA DROP one-shot" if residency.startswith("US-CA") else "targeted CCPA/GDPR email",
|
||||
}
|
||||
if not residency.startswith("US-CA"):
|
||||
coverage["note"] = ("DROP is CA-only; for this subject the registry is covered by targeted "
|
||||
"CCPA/GDPR deletion emails (`registry --search` then `send-email`), "
|
||||
"not a single portal request.")
|
||||
elif drop_filed:
|
||||
coverage["note"] = "DROP already filed; registry deletions are in the brokers' hands."
|
||||
|
||||
next_wake = _min_future_recheck(ledger, at)
|
||||
return {
|
||||
"subject": subject_id,
|
||||
"autonomy": autonomy,
|
||||
"phase": batch.get("phase"),
|
||||
"counts": batch.get("counts"),
|
||||
"actions": actions,
|
||||
"human_digest": digest,
|
||||
"coverage": coverage,
|
||||
"done_for_now": not actions,
|
||||
"fully_done": not actions and not digest and not next_wake,
|
||||
"next_wake_at": next_wake,
|
||||
"note": ("assisted mode: pause for operator confirmation on every action with confirm_first=true"
|
||||
if confirm_first else
|
||||
"full autonomy: recorded intake consent authorizes these submissions; do not pause. "
|
||||
"Present human_digest ONCE at the end of the run, not per item."),
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Pull and parse the Big-Ass Data Broker Opt-Out List (BADBOOL) into broker records.
|
||||
|
||||
BADBOOL (https://github.com/yaelwrites/Big-Ass-Data-Broker-Opt-Out-List) is a
|
||||
maintained, frequently-updated markdown list. `refresh` fetches it and parses the
|
||||
"People Search Sites" section into records that merge UNDER the curated DB (curated
|
||||
records always win). Auto-parsed records carry source="BADBOOL-auto" and
|
||||
confidence="auto" so the agent treats their URLs as best guesses to verify first.
|
||||
|
||||
`parse()` is pure (markdown in, records out) so it is tested offline; `fetch()` is
|
||||
the only network call and can be bypassed by passing markdown directly to refresh().
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
import storage
|
||||
|
||||
DEFAULT_URL = (
|
||||
"https://raw.githubusercontent.com/yaelwrites/"
|
||||
"Big-Ass-Data-Broker-Opt-Out-List/master/README.md"
|
||||
)
|
||||
USER_AGENT = "Mozilla/5.0 (compatible; unbroker/1.0; data opt-out)"
|
||||
|
||||
# BADBOOL legend symbols.
|
||||
SYMBOLS = {
|
||||
"crucial": "\U0001F490", # 💐
|
||||
"high": "\u2620", # ☠
|
||||
"gov_id": "\U0001F3AB", # 🎫
|
||||
"phone": "\U0001F4DE", # 📞
|
||||
"payment": "\U0001F4B0", # 💰
|
||||
}
|
||||
|
||||
_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
|
||||
_OPTOUT_HINT = re.compile(
|
||||
r"opt[\- ]?out|optout|removal|remove|suppress|control-privacy|delete", re.I
|
||||
)
|
||||
_FIND_HINT = re.compile(r"find|your information|search|look ?up|look for", re.I)
|
||||
|
||||
|
||||
def slug(name: str) -> str:
|
||||
# Drop a trailing .com/.org/.info on the displayed name so "FastPeopleSearch.com"
|
||||
# matches the curated id "fastpeoplesearch"; keep .net/.id so distinct sites differ.
|
||||
n = re.sub(r"\.(com|org|info)\b", "", name.strip(), flags=re.I)
|
||||
return re.sub(r"[^a-z0-9]+", "", n.lower())
|
||||
|
||||
|
||||
def _heading_flags(heading: str) -> tuple[str, dict]:
|
||||
flags = {key: (sym in heading) for key, sym in SYMBOLS.items()}
|
||||
name = heading
|
||||
for sym in SYMBOLS.values():
|
||||
name = name.replace(sym, "")
|
||||
name = name.replace("\ufe0f", "").strip()
|
||||
return name, flags
|
||||
|
||||
|
||||
def _priority(flags: dict) -> str:
|
||||
if flags["crucial"]:
|
||||
return "crucial"
|
||||
if flags["high"]:
|
||||
return "high"
|
||||
return "standard"
|
||||
|
||||
|
||||
def _pick(links: list[tuple[str, str]], hint: re.Pattern) -> str | None:
|
||||
for _text, url in links:
|
||||
if hint.search(url):
|
||||
return url
|
||||
for text, url in links:
|
||||
if hint.search(text):
|
||||
return url
|
||||
return None
|
||||
|
||||
|
||||
def _clean(text: str) -> str:
|
||||
return re.sub(r"\s+", " ", text).strip()[:600]
|
||||
|
||||
|
||||
def _build(name: str, flags: dict, body: str) -> dict:
|
||||
links = _LINK_RE.findall(body)
|
||||
web = [(t, u) for t, u in links if u.lower().startswith("http")]
|
||||
mailtos = [u[7:] for _t, u in links if u.lower().startswith("mailto:")]
|
||||
optout_url = _pick(web, _OPTOUT_HINT)
|
||||
search_url = _pick(web, _FIND_HINT) or (web[0][1] if web else None)
|
||||
|
||||
if flags["phone"]:
|
||||
method = "phone"
|
||||
elif optout_url:
|
||||
method = "web_form"
|
||||
elif mailtos:
|
||||
method = "email"
|
||||
else:
|
||||
method = "manual"
|
||||
|
||||
return {
|
||||
"id": slug(name),
|
||||
"name": name,
|
||||
"category": "people_search",
|
||||
"priority": _priority(flags),
|
||||
"jurisdictions": ["US"],
|
||||
"search": {"method": "url_pattern", "url": search_url, "fetch": "browser",
|
||||
"match_signal": "result", "by": ["name", "phone", "address"]},
|
||||
"optout": {
|
||||
"method": method,
|
||||
"url": optout_url,
|
||||
"email": mailtos[0] if mailtos else None,
|
||||
"requires": {
|
||||
"gov_id": flags["gov_id"],
|
||||
"phone_voice": flags["phone"],
|
||||
"payment": flags["payment"],
|
||||
"email_verification": False,
|
||||
"captcha": False,
|
||||
"account": False,
|
||||
"phone_callback": False,
|
||||
},
|
||||
"inputs": ["full_name", "contact_email"],
|
||||
"notes": _clean(body),
|
||||
"links": [{"text": t, "url": u} for t, u in links],
|
||||
"est_processing_days": 14, # unknown for auto records; drives next_recheck_at
|
||||
},
|
||||
"source": "BADBOOL-auto",
|
||||
"confidence": "auto",
|
||||
"last_verified": None,
|
||||
}
|
||||
|
||||
|
||||
def parse(markdown: str) -> list[dict]:
|
||||
"""Parse the 'People Search Sites' section of BADBOOL into broker records."""
|
||||
records: list[dict] = []
|
||||
in_people = False
|
||||
heading: str | None = None
|
||||
body: list[str] = []
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal heading, body
|
||||
if heading is not None:
|
||||
name, flags = _heading_flags(heading)
|
||||
if name:
|
||||
records.append(_build(name, flags, "\n".join(body).strip()))
|
||||
heading, body = None, []
|
||||
|
||||
for line in markdown.splitlines():
|
||||
if line.startswith("## "):
|
||||
flush()
|
||||
in_people = line[3:].strip().lower().startswith("people search")
|
||||
continue
|
||||
if not in_people:
|
||||
continue
|
||||
if line.startswith("### "):
|
||||
flush()
|
||||
heading = line[4:].strip()
|
||||
elif heading is not None:
|
||||
body.append(line)
|
||||
flush()
|
||||
return records
|
||||
|
||||
|
||||
def fetch(url: str = DEFAULT_URL, timeout: int = 30) -> str:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
MIN_EXPECTED = 20 # BADBOOL's People Search section lists ~47; far fewer => upstream reorg, warn
|
||||
|
||||
|
||||
def refresh(cache_path: Path, url: str = DEFAULT_URL, markdown: str | None = None) -> dict:
|
||||
"""Fetch (or accept) BADBOOL markdown, parse it, and write the snapshot cache."""
|
||||
md = markdown if markdown is not None else fetch(url)
|
||||
records = parse(md)
|
||||
storage.write_json(cache_path, records)
|
||||
out = {"parsed": len(records), "cache_path": str(cache_path), "source_url": url}
|
||||
if len(records) < MIN_EXPECTED:
|
||||
out["warning"] = (f"only {len(records)} parsed (expected >{MIN_EXPECTED}); BADBOOL's "
|
||||
"'People Search Sites' section may have moved/reorganized - check the parser")
|
||||
return out
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Load and query the broker database (references/brokers/*.json).
|
||||
|
||||
Each broker is one JSON file for clean diffs/PRs. Files beginning with `_` are
|
||||
ignored (reserved for notes/scratch).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import paths
|
||||
import storage
|
||||
|
||||
PRIORITY_ORDER = {"crucial": 0, "high": 1, "standard": 2, "long_tail": 3}
|
||||
|
||||
|
||||
def _load_curated(directory: Path | None = None) -> list[dict]:
|
||||
directory = directory or paths.brokers_dir()
|
||||
out: list[dict] = []
|
||||
if not directory.exists():
|
||||
return out
|
||||
for fp in sorted(directory.glob("*.json")):
|
||||
if fp.name.startswith("_"):
|
||||
continue
|
||||
out.append(json.loads(fp.read_text(encoding="utf-8")))
|
||||
return out
|
||||
|
||||
|
||||
def load_live_cache() -> list[dict]:
|
||||
"""Records pulled from BADBOOL via `refresh-brokers` (empty until refreshed)."""
|
||||
return storage.read_json(paths.brokers_cache_path(), []) or []
|
||||
|
||||
|
||||
def load_registry_cache() -> list[dict]:
|
||||
"""CA Data Broker Registry records (separate coverage lane; empty until refreshed).
|
||||
|
||||
Kept OUT of load_all() by default: these are not people-search sites to scan, they
|
||||
are worked via the CA DROP one-shot + CCPA email. Consumers of the scan/plan/fanout
|
||||
pipeline must not receive them; use this directly for coverage counts and the DROP/
|
||||
email lanes.
|
||||
"""
|
||||
return storage.read_json(paths.registry_cache_path(), []) or []
|
||||
|
||||
|
||||
def load_all(directory: Path | None = None, include_live: bool = True) -> list[dict]:
|
||||
"""Curated records, with live BADBOOL records merged underneath (curated wins)."""
|
||||
merged: dict[str, dict] = {b["id"]: b for b in _load_curated(directory)}
|
||||
if include_live:
|
||||
for b in load_live_cache():
|
||||
bid = b.get("id")
|
||||
if bid and bid not in merged:
|
||||
merged[bid] = b
|
||||
out = list(merged.values())
|
||||
out.sort(key=lambda b: (PRIORITY_ORDER.get(b.get("priority", "standard"), 9), b.get("id", "")))
|
||||
return out
|
||||
|
||||
|
||||
def get(broker_id: str, directory: Path | None = None) -> dict | None:
|
||||
for b in load_all(directory):
|
||||
if b.get("id") == broker_id:
|
||||
return b
|
||||
return None
|
||||
|
||||
|
||||
def by_priority(*levels: str, directory: Path | None = None) -> list[dict]:
|
||||
wanted = set(levels) if levels else None
|
||||
return [b for b in load_all(directory) if wanted is None or b.get("priority") in wanted]
|
||||
|
||||
|
||||
def clusters(directory: Path | None = None) -> dict[str, list[str]]:
|
||||
"""Map a parent broker id -> child site ids it can clear (force-multipliers)."""
|
||||
out: dict[str, list[str]] = {}
|
||||
for b in load_all(directory):
|
||||
owns = b.get("owns") or []
|
||||
if owns:
|
||||
out[b["id"]] = list(owns)
|
||||
return out
|
||||
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Launch (or detect) the operator's local Chrome/Chromium over the DevTools Protocol (CDP).
|
||||
|
||||
Phase-2 work -- sending opt-out/CCPA email through the operator's logged-in webmail, and driving
|
||||
session-bound multi-step opt-out gates (e.g. PeopleConnect guided-mode) -- must run in the
|
||||
operator's OWN browser: real fingerprint, residential IP, and the operator's signed-in sessions.
|
||||
A headless cloud browser (Browserbase) is the wrong tool there (it has no webmail session and is
|
||||
itself anti-bot-gated on those exact flows). This module launches the operator's real Chrome with
|
||||
remote debugging on a DEDICATED profile so Hermes's browser tools can attach at 127.0.0.1:<port>.
|
||||
|
||||
Stdlib only; cross-platform (macOS / Linux / Windows). Nothing here touches a password or PII.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
import paths
|
||||
|
||||
DEFAULT_PORT = 9222
|
||||
|
||||
# Chromium-family binaries we know how to drive, in preference order. Names first (works on any OS
|
||||
# where one is on PATH), then per-OS absolute-path fallbacks below.
|
||||
_PATH_NAMES = (
|
||||
"google-chrome", "google-chrome-stable", "chromium", "chromium-browser",
|
||||
"brave-browser", "microsoft-edge", "microsoft-edge-stable", "chrome",
|
||||
)
|
||||
|
||||
|
||||
def default_profile() -> Path:
|
||||
"""Dedicated debug profile dir, NOT the operator's Default Chrome profile.
|
||||
|
||||
Chrome refuses remote-debugging on a profile that is already open in another Chrome instance,
|
||||
so we isolate the debug session in its own user-data-dir under HERMES_HOME.
|
||||
"""
|
||||
return paths.hermes_home() / "chrome-debug"
|
||||
|
||||
|
||||
def _mac_candidates() -> list[str]:
|
||||
return [
|
||||
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
||||
"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
|
||||
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
||||
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
|
||||
]
|
||||
|
||||
|
||||
def _windows_candidates() -> list[str]:
|
||||
bases = [
|
||||
os.environ.get("ProgramFiles", r"C:\Program Files"),
|
||||
os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"),
|
||||
os.environ.get("LOCALAPPDATA", ""),
|
||||
]
|
||||
rels = [
|
||||
r"Google\Chrome\Application\chrome.exe",
|
||||
r"Chromium\Application\chrome.exe",
|
||||
r"BraveSoftware\Brave-Browser\Application\brave.exe",
|
||||
r"Microsoft\Edge\Application\msedge.exe",
|
||||
]
|
||||
out: list[str] = []
|
||||
for base in bases:
|
||||
if not base:
|
||||
continue
|
||||
for rel in rels:
|
||||
out.append(str(Path(base) / rel))
|
||||
return out
|
||||
|
||||
|
||||
def find_browser(override: str | None = None) -> str | None:
|
||||
"""Return the first usable Chromium-family browser path/command, or None.
|
||||
|
||||
`override` (an explicit path, or a command on PATH) wins when it resolves.
|
||||
"""
|
||||
if override:
|
||||
if Path(override).exists():
|
||||
return override
|
||||
return shutil.which(override) # may be None -> caller reports "not found"
|
||||
for name in _PATH_NAMES:
|
||||
found = shutil.which(name)
|
||||
if found:
|
||||
return found
|
||||
if sys.platform == "darwin":
|
||||
candidates = _mac_candidates()
|
||||
elif sys.platform == "win32":
|
||||
candidates = _windows_candidates()
|
||||
else:
|
||||
candidates = []
|
||||
for cand in candidates:
|
||||
if Path(cand).exists():
|
||||
return cand
|
||||
return None
|
||||
|
||||
|
||||
def launch_command(browser: str, port: int = DEFAULT_PORT, profile: Path | None = None) -> list[str]:
|
||||
"""The exact argv used to start the debug browser (also handy for `--print`)."""
|
||||
profile = profile or default_profile()
|
||||
return [
|
||||
browser,
|
||||
f"--remote-debugging-port={int(port)}",
|
||||
f"--user-data-dir={profile}",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
]
|
||||
|
||||
|
||||
def _http_get(url: str, timeout: float) -> bytes:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "unbroker-cdp/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 (localhost only)
|
||||
return resp.read()
|
||||
|
||||
|
||||
def endpoint_status(port: int = DEFAULT_PORT, host: str = "127.0.0.1",
|
||||
timeout: float = 1.0) -> dict | None:
|
||||
"""Return the CDP `/json/version` dict if a debuggable browser is live at host:port, else None.
|
||||
|
||||
(Chrome restricts this endpoint to localhost/IP Host headers, so we always hit 127.0.0.1.)
|
||||
"""
|
||||
url = f"http://{host}:{int(port)}/json/version"
|
||||
try:
|
||||
raw = _http_get(url, timeout)
|
||||
except (urllib.error.URLError, TimeoutError, ConnectionError, OSError, ValueError):
|
||||
return None
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8", errors="replace"))
|
||||
except (ValueError, AttributeError):
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def launch(browser: str, port: int = DEFAULT_PORT, profile: Path | None = None) -> int:
|
||||
"""Start the browser detached with remote debugging; return the child PID.
|
||||
|
||||
Detach so the browser outlives this short-lived CLI call. POSIX uses start_new_session (which
|
||||
avoids referencing os.setsid, so there is no Windows import-time footgun); Windows uses
|
||||
DETACHED_PROCESS + a new process group.
|
||||
"""
|
||||
profile = profile or default_profile()
|
||||
profile.mkdir(parents=True, exist_ok=True)
|
||||
cmd = launch_command(browser, port, profile)
|
||||
kwargs: dict = {
|
||||
"stdin": subprocess.DEVNULL,
|
||||
"stdout": subprocess.DEVNULL,
|
||||
"stderr": subprocess.DEVNULL,
|
||||
}
|
||||
if sys.platform == "win32":
|
||||
kwargs["creationflags"] = (
|
||||
subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP # windows-footgun: ok
|
||||
)
|
||||
else:
|
||||
kwargs["start_new_session"] = True
|
||||
proc = subprocess.Popen(cmd, **kwargs)
|
||||
return proc.pid
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Install-wide configuration with easiest-first defaults.
|
||||
|
||||
Everything works zero-config. `setup --auto` (the autonomous path) detects what
|
||||
this environment can do and picks the MOST AUTONOMOUS valid configuration without
|
||||
asking anyone; plain `setup` keeps the easiest-first defaults and only upgrades a
|
||||
setting when a flag opts in.
|
||||
|
||||
`autonomy` is policy, orthogonal to capability:
|
||||
full - intake consent is standing authorization; the agent submits T0-T2
|
||||
opt-outs without pausing per submission (default).
|
||||
assisted - the agent pauses for operator confirmation before each submission.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from shutil import which
|
||||
|
||||
import emailer
|
||||
import paths
|
||||
import storage
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
"autonomy": "full", # hands-off after intake+consent
|
||||
"email_mode": "draft_only", # zero credentials
|
||||
"browser_backend": "auto", # auto = Browserbase when BROWSERBASE_API_KEY is set
|
||||
# (recommended default; clears soft CAPTCHAs), else plain browser
|
||||
"tracker_backend": "local-json", # no external dependency
|
||||
"encryption": "none", # files still written 0600
|
||||
"default_rescan_interval_days": 120,
|
||||
"email_min_interval_seconds": 20, # pace SMTP sends so a run can't torch the account
|
||||
}
|
||||
|
||||
VALID = {
|
||||
"autonomy": {"full", "assisted"},
|
||||
# email_mode:
|
||||
# draft_only - render drafts; the operator sends + clicks verify links (zero setup)
|
||||
# browser - the agent sends + opens verify links through the operator's logged-in
|
||||
# webmail via browser_* tools (NO password stored; needs a browser the
|
||||
# operator's inbox is signed into)
|
||||
# programmatic - CLI sends via SMTP + reads verify links via IMAP (needs EMAIL_* creds)
|
||||
# alias - AgentMail agent-owned inboxes / per-broker aliases
|
||||
"email_mode": {"draft_only", "browser", "programmatic", "alias"},
|
||||
"browser_backend": {"auto", "browserbase", "agent-browser", "camofox"},
|
||||
"tracker_backend": {"local-json", "google-sheets"},
|
||||
"encryption": {"none", "age"},
|
||||
}
|
||||
|
||||
|
||||
def load_config() -> dict:
|
||||
cfg = dict(DEFAULT_CONFIG)
|
||||
cfg.update(storage.read_json(paths.config_path(), {}) or {})
|
||||
return cfg
|
||||
|
||||
|
||||
def save_config(cfg: dict) -> Path:
|
||||
merged = dict(DEFAULT_CONFIG)
|
||||
merged.update(cfg)
|
||||
for key, allowed in VALID.items():
|
||||
if merged.get(key) not in allowed:
|
||||
raise ValueError(f"invalid {key!r}: {merged.get(key)!r} (allowed: {sorted(allowed)})")
|
||||
return storage.write_json(paths.config_path(), merged)
|
||||
|
||||
|
||||
def dotenv_env() -> dict:
|
||||
"""Shell env overlaid on `$HERMES_HOME/.env`, so capability detection sees the creds Hermes
|
||||
loads for its own tools (BROWSERBASE_API_KEY, EMAIL_*, AGENTMAIL_API_KEY, ...) even though the
|
||||
terminal-tool shell doesn't export them. Shell env wins; the .env only fills gaps."""
|
||||
merged: dict = {}
|
||||
p = paths.hermes_home() / ".env"
|
||||
if p.exists():
|
||||
try:
|
||||
for line in p.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
merged[k.strip()] = v.strip().strip('"').strip("'")
|
||||
except OSError:
|
||||
pass
|
||||
merged.update(os.environ)
|
||||
return merged
|
||||
|
||||
|
||||
def detect_capabilities(env: dict | None = None) -> dict:
|
||||
"""Report which opt-in upgrades are available without extra setup."""
|
||||
env = os.environ if env is None else env
|
||||
home = paths.hermes_home()
|
||||
google = (
|
||||
(home / "google_token.json").exists()
|
||||
or (home / "skills" / "productivity" / "google-workspace").exists()
|
||||
or (home / "skills" / "google-workspace").exists()
|
||||
)
|
||||
mail = emailer.available(env)
|
||||
return {
|
||||
"browserbase": bool(env.get("BROWSERBASE_API_KEY")),
|
||||
"agentmail": bool(env.get("AGENTMAIL_API_KEY")),
|
||||
"email_imap_smtp": bool(env.get("EMAIL_ADDRESS") and env.get("EMAIL_PASSWORD")),
|
||||
"smtp_send": mail["smtp"], # CLI can SEND opt-out emails itself
|
||||
"imap_read": mail["imap"], # CLI can POLL verification links itself
|
||||
"google_workspace": google,
|
||||
"age": which("age") is not None,
|
||||
}
|
||||
|
||||
|
||||
def auto_configure(env: dict | None = None) -> dict:
|
||||
"""Pick the most autonomous configuration this environment supports (no questions).
|
||||
|
||||
- email: programmatic when SMTP creds exist (CLI sends + IMAP-verifies itself);
|
||||
alias mode when only AgentMail exists; draft_only as the capability floor.
|
||||
- browser: browserbase when the key exists (clears soft CAPTCHAs -> more T1).
|
||||
- encryption: age when the binary is installed (free privacy, zero human cost).
|
||||
- tracker: stays local-json (google-sheets needs a sheet id -> a human choice).
|
||||
"""
|
||||
caps = detect_capabilities(env)
|
||||
cfg = load_config()
|
||||
cfg["autonomy"] = "full"
|
||||
if caps["smtp_send"]:
|
||||
cfg["email_mode"] = "programmatic"
|
||||
elif caps["agentmail"]:
|
||||
cfg["email_mode"] = "alias"
|
||||
else:
|
||||
cfg["email_mode"] = "draft_only"
|
||||
cfg["browser_backend"] = "browserbase" if caps["browserbase"] else "auto"
|
||||
if caps["age"]:
|
||||
cfg["encryption"] = "age"
|
||||
return cfg
|
||||
|
||||
|
||||
def browser_clears_captcha(cfg: dict, env: dict | None = None) -> bool:
|
||||
"""True if the chosen browser backend can clear soft CAPTCHAs (shifts T2 -> T1).
|
||||
|
||||
Browserbase is the recommended default: a real residential-IP cloud browser passes
|
||||
soft/managed challenges (Turnstile, hCaptcha/reCAPTCHA checkbox) as normal operation.
|
||||
This is NOT solving/spoofing - hard interactive challenges still escalate to a human.
|
||||
`auto` inherits this whenever BROWSERBASE_API_KEY is present.
|
||||
"""
|
||||
backend = cfg.get("browser_backend", "auto")
|
||||
if backend == "browserbase":
|
||||
return True
|
||||
if backend == "auto":
|
||||
env = os.environ if env is None else env
|
||||
return bool(env.get("BROWSERBASE_API_KEY"))
|
||||
return False
|
||||
@@ -0,0 +1,88 @@
|
||||
"""At-rest encryption for sensitive files via the `age` binary (optional).
|
||||
|
||||
Engaged ONLY when config `encryption: age` AND an age identity key exists AND the
|
||||
`age`/`age-keygen` binaries are available. When engaged, JSON docs under
|
||||
`subjects/` (dossier, ledger) are written as `<file>.age` ciphertext; the audit
|
||||
log (field NAMES + states only, no raw PII values), `config.json`, and the broker
|
||||
cache stay plaintext so the engine can read them.
|
||||
|
||||
Threat model (be honest): this protects against casual disk inspection, accidental
|
||||
`git add`/commits, screen-shares, and backup/cloud-sync leakage. The identity key
|
||||
defaults to living beside the data at `$PDD_DATA_DIR/age-identity.txt` (0600); set
|
||||
`PDD_AGE_IDENTITY` to a separate volume/token for true key separation. It does NOT
|
||||
protect against an attacker who can already read your whole HERMES_HOME (they get
|
||||
key + data together).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from shutil import which
|
||||
|
||||
import paths
|
||||
|
||||
|
||||
def age_available() -> bool:
|
||||
return which("age") is not None and which("age-keygen") is not None
|
||||
|
||||
|
||||
def encryption_setting() -> str:
|
||||
"""Read `encryption` straight from config.json (no config/storage import => no cycle)."""
|
||||
cfg = paths.config_path()
|
||||
if not cfg.exists():
|
||||
return "none"
|
||||
try:
|
||||
return (json.loads(cfg.read_text(encoding="utf-8")) or {}).get("encryption", "none")
|
||||
except (ValueError, OSError):
|
||||
return "none"
|
||||
|
||||
|
||||
def identity_path() -> Path:
|
||||
return paths.age_identity_path()
|
||||
|
||||
|
||||
def ensure_identity() -> Path:
|
||||
"""Generate an age identity (X25519 keypair) if missing; return its path."""
|
||||
if not age_available():
|
||||
raise RuntimeError("`age`/`age-keygen` not found; cannot enable encryption")
|
||||
p = identity_path()
|
||||
if not p.exists():
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
p.parent.chmod(0o700)
|
||||
except OSError:
|
||||
pass
|
||||
subprocess.run(["age-keygen", "-o", str(p)], check=True, capture_output=True)
|
||||
try:
|
||||
p.chmod(0o600)
|
||||
except OSError:
|
||||
pass
|
||||
return p
|
||||
|
||||
|
||||
def recipient() -> str:
|
||||
"""The age public key (recipient) for the identity, parsed from its header."""
|
||||
p = ensure_identity()
|
||||
for line in p.read_text(encoding="utf-8").splitlines():
|
||||
s = line.strip()
|
||||
if s.lower().startswith("# public key:"):
|
||||
return s.split(":", 1)[1].strip()
|
||||
if s.startswith("age1"):
|
||||
return s
|
||||
raise RuntimeError(f"no public key found in {p}")
|
||||
|
||||
|
||||
def is_engaged() -> bool:
|
||||
"""True only when encryption is actually active (configured + available + key present)."""
|
||||
return encryption_setting() == "age" and age_available() and identity_path().exists()
|
||||
|
||||
|
||||
def encrypt(data: bytes) -> bytes:
|
||||
out = subprocess.run(["age", "-r", recipient()], input=data, capture_output=True, check=True)
|
||||
return out.stdout
|
||||
|
||||
|
||||
def decrypt(data: bytes) -> bytes:
|
||||
out = subprocess.run(["age", "-d", "-i", str(identity_path())], input=data, capture_output=True, check=True)
|
||||
return out.stdout
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Subject dossier management + consent gate + least-disclosure field selection."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import paths
|
||||
import storage
|
||||
|
||||
# Identifiers we never volunteer in an opt-out (would expand exposure, not reduce it).
|
||||
NEVER_VOLUNTEER = {"ssn", "social_security_number", "passport", "drivers_license"}
|
||||
|
||||
VALID_CONSENT_METHODS = {"self", "written_authorization", "poa"}
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def new_subject_id(full_name: str = "") -> str:
|
||||
# Opaque id: derives NOTHING from the name, so PII never leaks into directory names,
|
||||
# case ids, drafts, or the audit log. full_name kept only for call compatibility.
|
||||
return "sub_" + hashlib.sha1(os.urandom(8)).hexdigest()[:10]
|
||||
|
||||
|
||||
def create(identity: dict, consent: dict, residency: str = "US", prefs: dict | None = None) -> dict:
|
||||
dossier = {
|
||||
"subject_id": new_subject_id(identity.get("full_name", "subject")),
|
||||
"consent": consent,
|
||||
"identity": identity,
|
||||
"residency_jurisdiction": residency,
|
||||
"preferences": prefs or {"email_mode": "draft_only", "rescan_interval_days": 120},
|
||||
"created_at": now(),
|
||||
}
|
||||
save(dossier)
|
||||
return dossier
|
||||
|
||||
|
||||
def load(subject_id: str) -> dict | None:
|
||||
return storage.read_json(paths.dossier_path(subject_id), None)
|
||||
|
||||
|
||||
def save(dossier: dict) -> Path:
|
||||
return storage.write_json(paths.dossier_path(dossier["subject_id"]), dossier)
|
||||
|
||||
|
||||
def is_authorized(dossier: dict) -> bool:
|
||||
c = dossier.get("consent") or {}
|
||||
return bool(c.get("authorized")) and c.get("method") in VALID_CONSENT_METHODS
|
||||
|
||||
|
||||
def require_authorized(dossier: dict) -> None:
|
||||
if not is_authorized(dossier):
|
||||
raise PermissionError(
|
||||
f"subject {dossier.get('subject_id')!r} has no recorded authorization; refusing to act"
|
||||
)
|
||||
|
||||
|
||||
def all_names(dossier: dict) -> list[str]:
|
||||
"""Primary name + aliases (maiden/married/nicknames), deduped, in priority order."""
|
||||
ident = dossier.get("identity", {})
|
||||
out: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for n in [ident.get("full_name"), *(ident.get("also_known_as") or [])]:
|
||||
if n and n.lower() not in seen:
|
||||
seen.add(n.lower())
|
||||
out.append(n)
|
||||
return out
|
||||
|
||||
|
||||
def all_addresses(dossier: dict) -> list[dict]:
|
||||
"""Current + prior addresses, each tagged with `kind` (current|prior)."""
|
||||
ident = dossier.get("identity", {})
|
||||
out: list[dict] = []
|
||||
cur = ident.get("current_address")
|
||||
if cur:
|
||||
out.append({**cur, "kind": cur.get("kind", "current")})
|
||||
for a in ident.get("prior_addresses") or []:
|
||||
out.append({**a, "kind": a.get("kind", "prior")})
|
||||
return out
|
||||
|
||||
|
||||
def all_locations(dossier: dict) -> list[dict]:
|
||||
"""Distinct city/state pairs across all addresses (the vectors for name searches)."""
|
||||
out: list[dict] = []
|
||||
seen: set[tuple] = set()
|
||||
for a in all_addresses(dossier):
|
||||
city = a.get("city")
|
||||
key = ((city or "").lower(), (a.get("state") or "").lower())
|
||||
if city and key not in seen:
|
||||
seen.add(key)
|
||||
out.append({"city": city, "state": a.get("state")})
|
||||
return out
|
||||
|
||||
|
||||
def contact_email(dossier: dict) -> str | None:
|
||||
"""The single email used for opt-out correspondence (designated, else the first)."""
|
||||
ident = dossier.get("identity", {})
|
||||
prefs = dossier.get("preferences", {})
|
||||
emails = ident.get("emails") or []
|
||||
return prefs.get("contact_email_for_optouts") or (emails[0] if emails else None)
|
||||
|
||||
|
||||
def select_disclosure(dossier: dict, inputs: list[str], override_email: str | None = None) -> dict:
|
||||
"""Return ONLY the dossier fields a broker's opt-out actually requires.
|
||||
|
||||
Enforces least-disclosure: skips anything in NEVER_VOLUNTEER, and skips
|
||||
`profile_url` (that is captured per-listing at submit time, not from the dossier).
|
||||
A single contact email is used for correspondence even when the subject has several
|
||||
(see all_names / all_addresses / search vectors for using every alternate to *find* listings).
|
||||
"""
|
||||
ident = dossier.get("identity", {})
|
||||
addr = ident.get("current_address") or {}
|
||||
phones = ident.get("phones") or []
|
||||
available = {
|
||||
"full_name": ident.get("full_name"),
|
||||
"first_name": (ident.get("full_name") or "").split(" ")[0] or None,
|
||||
"contact_email": override_email or contact_email(dossier),
|
||||
"current_address": addr or None,
|
||||
"street": addr.get("line1"),
|
||||
"city": addr.get("city"),
|
||||
"state": addr.get("state"),
|
||||
"postal": addr.get("postal"),
|
||||
"date_of_birth": ident.get("date_of_birth"),
|
||||
"phone": phones[0] if phones else None,
|
||||
}
|
||||
out: dict = {}
|
||||
for key in inputs:
|
||||
if key in NEVER_VOLUNTEER or key == "profile_url":
|
||||
continue
|
||||
if available.get(key) is not None:
|
||||
out[key] = available[key]
|
||||
return out
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Email modes A/B/C helpers + anti-phishing verification-link extraction.
|
||||
|
||||
Mode A (default): render a ready-to-send draft to disk; the operator sends it.
|
||||
Mode B/C: the agent SENDS via a Hermes email mechanism (IMAP/SMTP gateway,
|
||||
`himalaya`, AgentMail, or Gmail via `google-workspace`) and READS the reply to
|
||||
resolve the verification link with `extract_verification_link`. Those transports
|
||||
are driven by the agent through native tools; this module stays network-free so
|
||||
the hermetic tests pass.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import legal
|
||||
import paths
|
||||
|
||||
_LINK_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.IGNORECASE)
|
||||
_VERIFY_HINTS = ("opt", "remov", "verif", "confirm", "unsubscrib", "suppress", "delete", "privacy")
|
||||
|
||||
|
||||
def render_draft(broker: dict, fields: dict, out_dir: Path | None = None) -> Path:
|
||||
"""Mode A: write a ready-to-send opt-out email for the operator to send."""
|
||||
body = legal.render_optout_email(broker, fields)
|
||||
out_dir = out_dir or (paths.data_dir() / "drafts")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
fp = out_dir / f"{broker.get('id', 'broker')}.txt"
|
||||
fp.write_text(body, encoding="utf-8")
|
||||
return fp
|
||||
|
||||
|
||||
def render_request_draft(broker: dict, fields: dict, kind: str = "generic",
|
||||
out_dir: Path | None = None) -> Path:
|
||||
"""Mode A: write a ready-to-send request of a specific KIND.
|
||||
|
||||
kind: generic | ccpa | ccpa_agent | ccpa_indirect | gdpr. Used for indirect-exposure
|
||||
(ccpa_indirect) and explicit legal requests, where the generic opt-out wording is wrong.
|
||||
The filename is suffixed with the kind so an indirect request does not overwrite an opt-out draft.
|
||||
"""
|
||||
body = legal.render_request(kind, broker, fields)
|
||||
out_dir = out_dir or (paths.data_dir() / "drafts")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
suffix = "" if kind == "generic" else f"-{kind}"
|
||||
fp = out_dir / f"{broker.get('id', 'broker')}{suffix}.txt"
|
||||
fp.write_text(body, encoding="utf-8")
|
||||
return fp
|
||||
|
||||
|
||||
def extract_verification_link(email_body: str, broker: dict | None = None) -> str | None:
|
||||
"""Return the most likely opt-out/verification link from an email body.
|
||||
|
||||
Anti-phishing: a link is only returned if its URL matches an opt-out hint
|
||||
and/or the broker's own domain; arbitrary links score 0 and are ignored.
|
||||
"""
|
||||
candidates = _LINK_RE.findall(email_body or "")
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
domain = ""
|
||||
if broker:
|
||||
url = (broker.get("optout") or {}).get("url") or (broker.get("search") or {}).get("url") or ""
|
||||
m = re.search(r"https?://([^/]+)", url)
|
||||
if m:
|
||||
domain = m.group(1).replace("www.", "")
|
||||
|
||||
best_score, best_link = 0, None
|
||||
for link in candidates:
|
||||
low = link.lower()
|
||||
score = 0
|
||||
if any(h in low for h in _VERIFY_HINTS):
|
||||
score += 2
|
||||
if domain and domain in low:
|
||||
score += 3
|
||||
if score > best_score:
|
||||
best_score, best_link = score, link
|
||||
return best_link
|
||||
@@ -0,0 +1,342 @@
|
||||
"""Programmatic email (Mode B) via stdlib smtplib/imaplib - no human in the loop.
|
||||
|
||||
This is what turns email opt-outs autonomous: `send()` delivers the rendered
|
||||
request straight to the broker's known opt-out address, and `find_verification_link()`
|
||||
polls the inbox for the broker's confirmation email and extracts the link (scored
|
||||
by email_modes.extract_verification_link, so arbitrary/phishing links are ignored).
|
||||
The agent still OPENS the link with its own browser - several brokers bind the
|
||||
verification session to the browser that opens it (see the intelius record).
|
||||
|
||||
Configuration comes from the same env vars the Hermes email gateway uses:
|
||||
EMAIL_ADDRESS / EMAIL_PASSWORD (required for Mode B)
|
||||
EMAIL_SMTP_HOST / EMAIL_SMTP_PORT (optional; inferred for common providers)
|
||||
EMAIL_IMAP_HOST / EMAIL_IMAP_PORT (optional; inferred for common providers)
|
||||
|
||||
Anti-misuse: `send()` refuses a recipient that is not the broker record's own
|
||||
opt-out/privacy address - this module cannot be repurposed to email arbitrary people.
|
||||
All network calls live behind small functions that the hermetic tests monkeypatch.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import email as _email
|
||||
import email.utils
|
||||
import imaplib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import smtplib
|
||||
import time
|
||||
from email.message import EmailMessage
|
||||
from pathlib import Path
|
||||
|
||||
import email_modes
|
||||
import paths
|
||||
|
||||
# provider domain -> (smtp_host, smtp_port, imap_host, imap_port)
|
||||
PROVIDERS = {
|
||||
"gmail.com": ("smtp.gmail.com", 587, "imap.gmail.com", 993),
|
||||
"googlemail.com": ("smtp.gmail.com", 587, "imap.gmail.com", 993),
|
||||
"outlook.com": ("smtp-mail.outlook.com", 587, "outlook.office365.com", 993),
|
||||
"hotmail.com": ("smtp-mail.outlook.com", 587, "outlook.office365.com", 993),
|
||||
"live.com": ("smtp-mail.outlook.com", 587, "outlook.office365.com", 993),
|
||||
"yahoo.com": ("smtp.mail.yahoo.com", 587, "imap.mail.yahoo.com", 993),
|
||||
"icloud.com": ("smtp.mail.me.com", 587, "imap.mail.me.com", 993),
|
||||
"me.com": ("smtp.mail.me.com", 587, "imap.mail.me.com", 993),
|
||||
"fastmail.com": ("smtp.fastmail.com", 587, "imap.fastmail.com", 993),
|
||||
}
|
||||
|
||||
|
||||
def _domain(address: str) -> str:
|
||||
return address.rsplit("@", 1)[-1].lower() if "@" in address else ""
|
||||
|
||||
|
||||
def smtp_settings(env: dict | None = None) -> dict | None:
|
||||
"""SMTP connection settings, or None when sending is not configured."""
|
||||
env = os.environ if env is None else env
|
||||
address, password = env.get("EMAIL_ADDRESS"), env.get("EMAIL_PASSWORD")
|
||||
if not (address and password):
|
||||
return None
|
||||
inferred = PROVIDERS.get(_domain(address))
|
||||
host = env.get("EMAIL_SMTP_HOST") or (inferred[0] if inferred else None)
|
||||
if not host:
|
||||
return None # unknown provider and no explicit host
|
||||
port = int(env.get("EMAIL_SMTP_PORT") or (inferred[1] if inferred else 587))
|
||||
return {"host": host, "port": port, "address": address, "password": password}
|
||||
|
||||
|
||||
def imap_settings(env: dict | None = None) -> dict | None:
|
||||
"""IMAP connection settings, or None when inbox reading is not configured."""
|
||||
env = os.environ if env is None else env
|
||||
address, password = env.get("EMAIL_ADDRESS"), env.get("EMAIL_PASSWORD")
|
||||
if not (address and password):
|
||||
return None
|
||||
inferred = PROVIDERS.get(_domain(address))
|
||||
host = env.get("EMAIL_IMAP_HOST") or (inferred[2] if inferred else None)
|
||||
if not host:
|
||||
return None
|
||||
port = int(env.get("EMAIL_IMAP_PORT") or (inferred[3] if inferred else 993))
|
||||
return {"host": host, "port": port, "address": address, "password": password}
|
||||
|
||||
|
||||
def available(env: dict | None = None) -> dict:
|
||||
return {"smtp": smtp_settings(env) is not None, "imap": imap_settings(env) is not None}
|
||||
|
||||
|
||||
# --- sending ------------------------------------------------------------------
|
||||
|
||||
def broker_addresses(broker: dict) -> list[str]:
|
||||
"""Every address the broker record itself declares (the ONLY valid recipients).
|
||||
|
||||
Includes the primary opt-out email, the right-to-delete lane's email
|
||||
(optout.deletion.email), and any mailto: links parsed from BADBOOL.
|
||||
"""
|
||||
opt = broker.get("optout") or {}
|
||||
out = [a for a in [opt.get("email"), (opt.get("deletion") or {}).get("email")] if a]
|
||||
for link in opt.get("links") or []:
|
||||
url = (link.get("url") or "")
|
||||
if url.lower().startswith("mailto:"):
|
||||
out.append(url[7:].split("?")[0])
|
||||
seen: set[str] = set()
|
||||
deduped = []
|
||||
for a in out:
|
||||
if a.lower() not in seen:
|
||||
seen.add(a.lower())
|
||||
deduped.append(a)
|
||||
return deduped
|
||||
|
||||
|
||||
def _split_subject_body(text: str) -> tuple[str, str]:
|
||||
"""Templates start with a 'Subject: ...' line; split it out for the MIME header."""
|
||||
lines = text.splitlines()
|
||||
if lines and lines[0].lower().startswith("subject:"):
|
||||
return lines[0].split(":", 1)[1].strip(), "\n".join(lines[1:]).lstrip("\n")
|
||||
return "Data removal request", text
|
||||
|
||||
|
||||
def browser_send_payload(broker: dict, body_text: str, to: str | None = None) -> dict:
|
||||
"""Build a recipient-locked {to, subject, body} for the agent to send via browser webmail.
|
||||
|
||||
No network and no credentials: the deterministic part (recipient-lock to the broker's own
|
||||
declared address, subject/body split) happens here; the agent then composes and sends it in
|
||||
the operator's logged-in webmail with browser_* tools. Same recipient guard as `send()`, so
|
||||
the browser lane cannot be pointed at an arbitrary person either.
|
||||
"""
|
||||
allowed = broker_addresses(broker)
|
||||
if not allowed:
|
||||
raise RuntimeError(f"broker {broker.get('id')!r} declares no opt-out email address")
|
||||
recipient = to or allowed[0]
|
||||
if recipient.lower() not in {a.lower() for a in allowed}:
|
||||
raise PermissionError(
|
||||
f"refusing to target {recipient!r}: not an address the broker record declares "
|
||||
f"(allowed: {allowed})"
|
||||
)
|
||||
subject, body = _split_subject_body(body_text)
|
||||
return {"to": recipient, "subject": subject, "body": body}
|
||||
|
||||
|
||||
def _rate_limit_path() -> Path:
|
||||
return paths.data_dir() / "email-rate.json"
|
||||
|
||||
|
||||
def _respect_rate_limit(min_interval: float, sleep, now, state_path=None) -> None:
|
||||
"""Pace sends across CLI invocations so a run can't torch the sending account.
|
||||
|
||||
Persists the last-send wall-clock time; if the next send is too soon, sleep the
|
||||
remainder. Cross-process because each `send-email` is a separate invocation.
|
||||
"""
|
||||
if min_interval <= 0:
|
||||
return
|
||||
p = state_path or _rate_limit_path()
|
||||
last = 0.0
|
||||
try:
|
||||
last = float(json.loads(p.read_text(encoding="utf-8")).get("last", 0.0))
|
||||
except (OSError, ValueError, TypeError):
|
||||
last = 0.0
|
||||
wait = min_interval - (now() - last)
|
||||
if wait > 0:
|
||||
sleep(min(wait, min_interval))
|
||||
try:
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps({"last": now()}), encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# SMTP errors that are permanent (don't retry) vs transient (retry with backoff).
|
||||
_SMTP_PERMANENT = (smtplib.SMTPAuthenticationError, smtplib.SMTPRecipientsRefused,
|
||||
smtplib.SMTPSenderRefused, smtplib.SMTPDataError)
|
||||
|
||||
|
||||
def send(broker: dict, body_text: str, to: str | None = None,
|
||||
env: dict | None = None, _smtp_factory=None,
|
||||
min_interval: float = 0.0, max_retries: int = 3,
|
||||
_sleep=time.sleep, _now=time.time, _rate_state=None) -> dict:
|
||||
"""Send an opt-out/legal request to the broker's own opt-out address.
|
||||
|
||||
Recipient is locked to an address the broker record declares (PermissionError
|
||||
otherwise). `min_interval` paces sends across invocations (deliverability /
|
||||
account-safety); transient SMTP/socket failures retry with exponential backoff,
|
||||
permanent ones (auth, recipient refused) raise immediately. NOTE: a successful
|
||||
SMTP handoff is NOT proof of delivery - real bounces arrive later as inbound mail;
|
||||
in programmatic mode `poll-verification`/inbox review surfaces them, and the
|
||||
due-queue re-scan is the true confirmation. Returns send metadata.
|
||||
"""
|
||||
settings = smtp_settings(env)
|
||||
if not settings:
|
||||
raise RuntimeError(
|
||||
"programmatic email not configured (need EMAIL_ADDRESS + EMAIL_PASSWORD, and "
|
||||
"EMAIL_SMTP_HOST for non-mainstream providers); fall back to `render-email` drafts"
|
||||
)
|
||||
allowed = broker_addresses(broker)
|
||||
if not allowed:
|
||||
raise RuntimeError(f"broker {broker.get('id')!r} declares no opt-out email address")
|
||||
recipient = to or allowed[0]
|
||||
if recipient.lower() not in {a.lower() for a in allowed}:
|
||||
raise PermissionError(
|
||||
f"refusing to send to {recipient!r}: not an address the broker record declares "
|
||||
f"(allowed: {allowed})"
|
||||
)
|
||||
|
||||
subject, body = _split_subject_body(body_text)
|
||||
msg = EmailMessage()
|
||||
msg["From"] = settings["address"]
|
||||
msg["To"] = recipient
|
||||
msg["Subject"] = subject
|
||||
msg["Date"] = email.utils.formatdate(localtime=True)
|
||||
msg["Message-ID"] = email.utils.make_msgid()
|
||||
msg.set_content(body)
|
||||
|
||||
_respect_rate_limit(min_interval, _sleep, _now, _rate_state)
|
||||
|
||||
factory = _smtp_factory or smtplib.SMTP
|
||||
attempts = 0
|
||||
while True:
|
||||
attempts += 1
|
||||
try:
|
||||
with factory(settings["host"], settings["port"], timeout=30) as smtp:
|
||||
smtp.ehlo()
|
||||
try:
|
||||
smtp.starttls()
|
||||
smtp.ehlo()
|
||||
except smtplib.SMTPNotSupportedError:
|
||||
pass # already-TLS ports / test doubles
|
||||
smtp.login(settings["address"], settings["password"])
|
||||
smtp.send_message(msg)
|
||||
break
|
||||
except _SMTP_PERMANENT:
|
||||
raise # auth / recipient refused: retrying won't help
|
||||
except (smtplib.SMTPException, OSError) as exc:
|
||||
if attempts > max_retries:
|
||||
raise RuntimeError(f"SMTP send failed after {attempts} attempts: {exc}") from exc
|
||||
_sleep(min(2 ** (attempts - 1), 30)) # 1s, 2s, 4s... capped
|
||||
return {"to": recipient, "subject": subject, "message_id": msg["Message-ID"],
|
||||
"from": settings["address"], "attempts": attempts,
|
||||
"delivery_note": "SMTP accepted; not proof of delivery - a bounce would arrive as "
|
||||
"inbound mail. The due-queue re-scan is the real confirmation."}
|
||||
|
||||
|
||||
# --- inbox polling ------------------------------------------------------------
|
||||
|
||||
def _decode_part(part) -> str:
|
||||
try:
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload is None:
|
||||
return ""
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
return payload.decode(charset, errors="replace")
|
||||
except Exception: # noqa: BLE001 - malformed MIME must not kill the poll
|
||||
return ""
|
||||
|
||||
|
||||
def message_text(msg) -> str:
|
||||
"""All text/plain + text/html content of a parsed email message."""
|
||||
chunks: list[str] = []
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
if part.get_content_type() in ("text/plain", "text/html"):
|
||||
chunks.append(_decode_part(part))
|
||||
else:
|
||||
chunks.append(_decode_part(msg))
|
||||
return "\n".join(c for c in chunks if c)
|
||||
|
||||
|
||||
def _broker_domains(broker: dict) -> list[str]:
|
||||
"""Domains this broker legitimately mails from (site domains + optout email domain)."""
|
||||
domains: list[str] = []
|
||||
for section in ("optout", "search"):
|
||||
url = ((broker.get(section) or {}).get("url")) or ""
|
||||
m = re.search(r"https?://([^/]+)", url)
|
||||
if m:
|
||||
domains.append(m.group(1).lower().removeprefix("www."))
|
||||
opt_email = (broker.get("optout") or {}).get("email")
|
||||
if opt_email and "@" in opt_email:
|
||||
domains.append(_domain(opt_email))
|
||||
# strip subdomains to the registrable-ish tail (mailer.intelius.com -> intelius.com)
|
||||
tails = {".".join(d.split(".")[-2:]) for d in domains if d}
|
||||
return sorted(tails)
|
||||
|
||||
|
||||
def fetch_recent(env: dict | None = None, since_days: int = 3, limit: int = 30,
|
||||
_imap_factory=None) -> list[dict]:
|
||||
"""Fetch recent inbox messages: [{from, subject, date, text}], newest first."""
|
||||
settings = imap_settings(env)
|
||||
if not settings:
|
||||
raise RuntimeError("IMAP not configured (need EMAIL_ADDRESS + EMAIL_PASSWORD, and "
|
||||
"EMAIL_IMAP_HOST for non-mainstream providers)")
|
||||
import datetime as _dt
|
||||
since = (_dt.date.today() - _dt.timedelta(days=max(0, since_days))).strftime("%d-%b-%Y")
|
||||
|
||||
factory = _imap_factory or imaplib.IMAP4_SSL
|
||||
conn = factory(settings["host"], settings["port"])
|
||||
try:
|
||||
conn.login(settings["address"], settings["password"])
|
||||
conn.select("INBOX", readonly=True)
|
||||
_typ, data = conn.search(None, "SINCE", since)
|
||||
ids = (data[0].split() if data and data[0] else [])[-limit:]
|
||||
out: list[dict] = []
|
||||
for mid in reversed(ids): # newest first
|
||||
_typ, msg_data = conn.fetch(mid, "(RFC822)")
|
||||
raw = next((p[1] for p in msg_data or [] if isinstance(p, tuple)), None)
|
||||
if not raw:
|
||||
continue
|
||||
msg = _email.message_from_bytes(raw)
|
||||
out.append({
|
||||
"from": msg.get("From", ""),
|
||||
"subject": msg.get("Subject", ""),
|
||||
"date": msg.get("Date", ""),
|
||||
"text": message_text(msg),
|
||||
})
|
||||
return out
|
||||
finally:
|
||||
try:
|
||||
conn.logout()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
def link_from_messages(messages: list[dict], broker: dict) -> dict | None:
|
||||
"""Pure: find the broker's verification link in already-fetched messages.
|
||||
|
||||
A message is only considered if its From domain OR any contained link matches
|
||||
the broker's own domains; the link itself must pass the anti-phishing scorer.
|
||||
"""
|
||||
domains = _broker_domains(broker)
|
||||
for m in messages:
|
||||
sender = (m.get("from") or "").lower()
|
||||
text = m.get("text") or ""
|
||||
sender_match = any(d in sender for d in domains)
|
||||
body_match = any(d in text.lower() for d in domains)
|
||||
if not (sender_match or body_match):
|
||||
continue
|
||||
link = email_modes.extract_verification_link(text, broker)
|
||||
if link:
|
||||
return {"link": link, "from": m.get("from"), "subject": m.get("subject"),
|
||||
"date": m.get("date")}
|
||||
return None
|
||||
|
||||
|
||||
def find_verification_link(broker: dict, env: dict | None = None, since_days: int = 3,
|
||||
_imap_factory=None) -> dict | None:
|
||||
"""Poll the inbox and return the broker's verification link (or None yet)."""
|
||||
messages = fetch_recent(env, since_days=since_days, _imap_factory=_imap_factory)
|
||||
return link_from_messages(messages, broker)
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Case ledger: opt-out state machine + append-only audit log.
|
||||
|
||||
A "case" is one (subject x broker) record. State changes are validated against
|
||||
TRANSITIONS and mirrored into audit.jsonl so every action is auditable.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
from pathlib import Path
|
||||
|
||||
import paths
|
||||
import storage
|
||||
|
||||
STATES = [
|
||||
"new", "searching", "not_found", "found", "indirect_exposure", "action_selected", "submitted",
|
||||
"verification_pending", "awaiting_processing", "confirmed_removed", "reappeared",
|
||||
"human_task_queued", "blocked",
|
||||
]
|
||||
|
||||
TRANSITIONS: dict[str, set[str]] = {
|
||||
"new": {"searching", "found", "not_found", "indirect_exposure", "blocked"},
|
||||
"searching": {"not_found", "found", "indirect_exposure", "blocked"},
|
||||
"not_found": {"searching", "found", "indirect_exposure", "blocked"},
|
||||
# found -> not_found: a parent re-verification (or re-scan) found the "found" was a false
|
||||
# positive (namesake, or an address-only property-record match) -- retract it with evidence.
|
||||
"found": {"action_selected", "submitted", "human_task_queued", "indirect_exposure", "blocked",
|
||||
"not_found"},
|
||||
# indirect_exposure: subject's PII (email/phone/name) sits on a THIRD PARTY's record. The
|
||||
# self-service opt-out form does not apply; the lever is a targeted CCPA/GDPR delete-my-PII
|
||||
# request (-> submitted) or a human task. Re-scan can clear it (-> not_found) or upgrade it to a
|
||||
# direct listing (-> found).
|
||||
"indirect_exposure": {"submitted", "human_task_queued", "not_found", "found", "blocked"},
|
||||
"action_selected": {"submitted", "human_task_queued", "blocked"},
|
||||
"submitted": {"verification_pending", "awaiting_processing", "human_task_queued", "blocked"},
|
||||
# verification_pending -> awaiting_processing: the verify link was opened/acknowledged and the
|
||||
# broker is now processing the removal (their stated window). confirmed_removed still requires a
|
||||
# verifying re-scan, never the submission flow's own say-so.
|
||||
"verification_pending": {"awaiting_processing", "confirmed_removed", "human_task_queued", "blocked"},
|
||||
"awaiting_processing": {"confirmed_removed", "human_task_queued", "blocked"},
|
||||
"confirmed_removed": {"reappeared", "confirmed_removed"},
|
||||
"reappeared": {"found", "indirect_exposure"},
|
||||
"human_task_queued": {
|
||||
"found", "indirect_exposure", "action_selected", "submitted", "verification_pending",
|
||||
"awaiting_processing", "confirmed_removed", "blocked",
|
||||
},
|
||||
# blocked: automated tools (web_extract/proxyless browser) couldn't read the site. A later pass
|
||||
# -- a stealth/cloud browser OR guiding the operator's own (residential) browser -- can resolve it
|
||||
# to any real scan verdict, so blocked reaches not_found / indirect_exposure too, not just found.
|
||||
# blocked -> human_task_queued: some blocked sites need an operator step to proceed at all
|
||||
# (face-recognition sites needing a selfie/gov-ID, etc.), so route them to the digest.
|
||||
"blocked": {"searching", "found", "not_found", "indirect_exposure", "action_selected",
|
||||
"human_task_queued"},
|
||||
}
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def load(subject_id: str) -> dict:
|
||||
return storage.read_json(paths.ledger_path(subject_id), {}) or {}
|
||||
|
||||
|
||||
def save(subject_id: str, ledger: dict) -> Path:
|
||||
return storage.write_json(paths.ledger_path(subject_id), ledger)
|
||||
|
||||
|
||||
def new_case(subject_id: str, broker_id: str) -> dict:
|
||||
return {
|
||||
"case_id": f"case_{subject_id}_{broker_id}",
|
||||
"subject_id": subject_id,
|
||||
"broker_id": broker_id,
|
||||
"state": "new",
|
||||
"found": None,
|
||||
"evidence": {},
|
||||
"disclosure_log": [],
|
||||
"history": [],
|
||||
}
|
||||
|
||||
|
||||
def get_case(subject_id: str, broker_id: str) -> dict:
|
||||
return load(subject_id).get(broker_id) or new_case(subject_id, broker_id)
|
||||
|
||||
|
||||
def can_transition(old: str, new: str) -> bool:
|
||||
return new == old or new in TRANSITIONS.get(old, set())
|
||||
|
||||
|
||||
def transition(subject_id: str, broker_id: str, new_state: str, **fields) -> dict:
|
||||
if new_state not in STATES:
|
||||
raise ValueError(f"unknown state {new_state!r}")
|
||||
# Lock the whole load-modify-save so a concurrent cron re-scan / other tenant
|
||||
# can't read a stale ledger and clobber this transition.
|
||||
with storage.locked(paths.ledger_path(subject_id)):
|
||||
ledger = load(subject_id)
|
||||
case = ledger.get(broker_id) or new_case(subject_id, broker_id)
|
||||
old = case.get("state", "new")
|
||||
if not can_transition(old, new_state):
|
||||
raise ValueError(f"illegal transition {old!r} -> {new_state!r} for broker {broker_id!r}")
|
||||
case["state"] = new_state
|
||||
for key, value in fields.items():
|
||||
case[key] = value
|
||||
stamp = now()
|
||||
case.setdefault("history", []).append({"at": stamp, "from": old, "to": new_state})
|
||||
ledger[broker_id] = case
|
||||
save(subject_id, ledger)
|
||||
storage.append_jsonl(
|
||||
paths.audit_path(subject_id),
|
||||
{"at": stamp, "broker_id": broker_id, "event": "transition", "from": old, "to": new_state},
|
||||
)
|
||||
return case
|
||||
|
||||
|
||||
DEFAULT_PROCESSING_DAYS = 14 # when a broker record doesn't state est_processing_days
|
||||
VERIFICATION_POLL_DAYS = 1 # how soon to re-poll for an unarrived verification email
|
||||
|
||||
|
||||
def _plus_days(days: int, start: str | None = None) -> str:
|
||||
base = _dt.datetime.strptime(start, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=_dt.timezone.utc) \
|
||||
if start else _dt.datetime.now(_dt.timezone.utc)
|
||||
return (base + _dt.timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def followup_fields(new_state: str, broker: dict | None = None,
|
||||
dossier: dict | None = None) -> dict:
|
||||
"""Auto-scheduling stamps for a transition, so nobody has to remember follow-ups.
|
||||
|
||||
submitted / awaiting_processing -> recheck after the broker's stated processing window;
|
||||
verification_pending -> re-poll the inbox quickly;
|
||||
confirmed_removed -> periodic reappearance re-scan per subject preference.
|
||||
"""
|
||||
if new_state in ("submitted", "awaiting_processing"):
|
||||
days = ((broker or {}).get("optout") or {}).get("est_processing_days") or DEFAULT_PROCESSING_DAYS
|
||||
return {"next_recheck_at": _plus_days(int(days))}
|
||||
if new_state == "verification_pending":
|
||||
return {"next_recheck_at": _plus_days(VERIFICATION_POLL_DAYS)}
|
||||
if new_state == "confirmed_removed":
|
||||
interval = ((dossier or {}).get("preferences") or {}).get("rescan_interval_days") or 120
|
||||
return {"removal_confirmed_at": now(), "next_recheck_at": _plus_days(int(interval))}
|
||||
return {}
|
||||
|
||||
|
||||
def due(subject_id: str, at: str | None = None, ledger: dict | None = None) -> list[dict]:
|
||||
"""Cases whose next_recheck_at has arrived - the autonomous follow-up queue."""
|
||||
stamp = at or now()
|
||||
out = []
|
||||
for case in (ledger if ledger is not None else load(subject_id)).values():
|
||||
when = case.get("next_recheck_at")
|
||||
if when and when <= stamp:
|
||||
out.append(case)
|
||||
out.sort(key=lambda c: c.get("next_recheck_at") or "")
|
||||
return out
|
||||
|
||||
|
||||
def log_disclosure(subject_id: str, broker_id: str, fields: list[str], channel: str) -> dict:
|
||||
"""Record exactly which PII field *names* were disclosed to a broker."""
|
||||
with storage.locked(paths.ledger_path(subject_id)):
|
||||
ledger = load(subject_id)
|
||||
case = ledger.get(broker_id) or new_case(subject_id, broker_id)
|
||||
stamp = now()
|
||||
record = {"at": stamp, "fields": sorted(fields), "channel": channel}
|
||||
case.setdefault("disclosure_log", []).append(record)
|
||||
ledger[broker_id] = case
|
||||
save(subject_id, ledger)
|
||||
storage.append_jsonl(
|
||||
paths.audit_path(subject_id),
|
||||
{"at": stamp, "broker_id": broker_id, "event": "disclosure",
|
||||
"fields": record["fields"], "channel": channel},
|
||||
)
|
||||
return record
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Render opt-out / legal request text from templates/ with safe substitution.
|
||||
|
||||
Templates use {field} placeholders. Missing fields are left literal (never crash,
|
||||
never inject blanks that look like real data). Field values come from the
|
||||
least-disclosure selection in dossier.select_disclosure.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import paths
|
||||
|
||||
|
||||
class _SafeDict(dict):
|
||||
def __missing__(self, key): # leave unknown placeholders untouched
|
||||
return "{" + key + "}"
|
||||
|
||||
|
||||
def template_path(name: str) -> Path:
|
||||
return paths.templates_dir() / name
|
||||
|
||||
|
||||
def render(template_name: str, fields: dict) -> str:
|
||||
text = template_path(template_name).read_text(encoding="utf-8")
|
||||
return text.format_map(_SafeDict(fields))
|
||||
|
||||
|
||||
def _join_listings(value) -> str:
|
||||
if isinstance(value, (list, tuple)):
|
||||
return "\n".join(str(v) for v in value)
|
||||
return str(value or "")
|
||||
|
||||
|
||||
def _join_identifiers(value) -> str:
|
||||
"""Render the subject's OWN identifiers as a bullet list for an indirect-exposure request."""
|
||||
if isinstance(value, (list, tuple)):
|
||||
return "\n".join(f" - {v}" for v in value if v)
|
||||
return f" - {value}" if value else ""
|
||||
|
||||
|
||||
def render_optout_email(broker: dict, fields: dict) -> str:
|
||||
ctx = dict(fields)
|
||||
ctx.setdefault("broker_name", broker.get("name", "the data broker"))
|
||||
ctx["listing_urls"] = _join_listings(fields.get("listing_urls"))
|
||||
ctx.setdefault("full_name", fields.get("full_name", "[your name]"))
|
||||
ctx.setdefault("contact_email", fields.get("contact_email", "[your email]"))
|
||||
return render("emails/generic-optout.txt", ctx)
|
||||
|
||||
|
||||
def render_request(kind: str, broker: dict, fields: dict) -> str:
|
||||
"""kind: generic | ccpa | ccpa_agent | ccpa_indirect | gdpr"""
|
||||
template = {
|
||||
"generic": "emails/generic-optout.txt",
|
||||
"ccpa": "emails/ccpa-deletion.txt",
|
||||
"ccpa_agent": "emails/ccpa-authorized-agent.txt",
|
||||
"ccpa_indirect": "emails/ccpa-indirect-deletion.txt",
|
||||
"gdpr": "emails/gdpr-erasure.txt",
|
||||
}.get(kind, "emails/generic-optout.txt")
|
||||
ctx = dict(fields)
|
||||
ctx.setdefault("broker_name", broker.get("name", "the data broker"))
|
||||
ctx["listing_urls"] = _join_listings(fields.get("listing_urls"))
|
||||
ctx["my_identifiers"] = _join_identifiers(fields.get("my_identifiers"))
|
||||
return render(template, ctx)
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Filesystem paths for the unbroker skill (stdlib only).
|
||||
|
||||
All per-subject data lives under PDD_DATA_DIR (default: $HERMES_HOME/unbroker),
|
||||
which is the same trust boundary Hermes uses for .env and OAuth tokens.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def hermes_home() -> Path:
|
||||
return Path(os.environ.get("HERMES_HOME") or (Path.home() / ".hermes"))
|
||||
|
||||
|
||||
def data_dir() -> Path:
|
||||
override = os.environ.get("PDD_DATA_DIR")
|
||||
return Path(override) if override else hermes_home() / "unbroker"
|
||||
|
||||
|
||||
def config_path() -> Path:
|
||||
return data_dir() / "config.json"
|
||||
|
||||
|
||||
def subjects_dir() -> Path:
|
||||
return data_dir() / "subjects"
|
||||
|
||||
|
||||
def subject_dir(subject_id: str) -> Path:
|
||||
return subjects_dir() / subject_id
|
||||
|
||||
|
||||
def dossier_path(subject_id: str) -> Path:
|
||||
return subject_dir(subject_id) / "dossier.json"
|
||||
|
||||
|
||||
def ledger_path(subject_id: str) -> Path:
|
||||
return subject_dir(subject_id) / "ledger.json"
|
||||
|
||||
|
||||
def audit_path(subject_id: str) -> Path:
|
||||
return subject_dir(subject_id) / "audit.jsonl"
|
||||
|
||||
|
||||
def evidence_dir(subject_id: str) -> Path:
|
||||
return subject_dir(subject_id) / "evidence"
|
||||
|
||||
|
||||
def skill_root() -> Path:
|
||||
"""The skill directory (parent of scripts/)."""
|
||||
return Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def brokers_dir() -> Path:
|
||||
return skill_root() / "references" / "brokers"
|
||||
|
||||
|
||||
def brokers_cache_path() -> Path:
|
||||
"""Live broker snapshot pulled from BADBOOL (merged under the curated DB)."""
|
||||
return data_dir() / "brokers-cache" / "badbool.json"
|
||||
|
||||
|
||||
def registry_cache_path() -> Path:
|
||||
"""CA Data Broker Registry snapshot (separate coverage lane; DROP/email, not scanned)."""
|
||||
return data_dir() / "brokers-cache" / "ca-registry.json"
|
||||
|
||||
|
||||
def age_identity_path() -> Path:
|
||||
"""age identity (private key) used for at-rest encryption when enabled.
|
||||
|
||||
Defaults beside the data; point PDD_AGE_IDENTITY at a separate volume/token
|
||||
for real key separation from the encrypted data.
|
||||
"""
|
||||
override = os.environ.get("PDD_AGE_IDENTITY")
|
||||
return Path(override) if override else data_dir() / "age-identity.txt"
|
||||
|
||||
|
||||
def templates_dir() -> Path:
|
||||
return skill_root() / "templates"
|
||||
@@ -0,0 +1,914 @@
|
||||
#!/usr/bin/env python3
|
||||
"""unbroker - deterministic CLI helper.
|
||||
|
||||
The Hermes agent orchestrates scanning and opt-out submission with native tools
|
||||
(`web_extract`, `browser_navigate`, email mechanisms). THIS CLI owns the
|
||||
deterministic state: config, dossiers + consent, the broker DB, tier planning,
|
||||
the ledger + audit log, draft/template rendering, and reports.
|
||||
|
||||
Run it through the `terminal` tool (it can read PII files under HERMES_HOME);
|
||||
do NOT run it through `execute_code` (that sandbox scrubs env and redacts output).
|
||||
|
||||
Examples:
|
||||
python pdd.py setup
|
||||
python pdd.py intake --full-name "Jane Q. Public" --email jane@example.com \
|
||||
--city Oakland --state CA --residency US-CA --consent --consent-method self
|
||||
python pdd.py plan sub_xxxx --priority crucial
|
||||
python pdd.py record sub_xxxx spokeo found --found true \
|
||||
--evidence '{"listing_urls":["https://www.spokeo.com/..."]}'
|
||||
python pdd.py render-email sub_xxxx spokeo --listing https://www.spokeo.com/...
|
||||
python pdd.py status sub_xxxx
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
import autopilot # noqa: E402
|
||||
import badbool # noqa: E402
|
||||
import cdp # noqa: E402
|
||||
import brokers as brokers_mod # noqa: E402
|
||||
import config as config_mod # noqa: E402
|
||||
import crypto # noqa: E402
|
||||
import dossier as dossier_mod # noqa: E402
|
||||
import email_modes # noqa: E402
|
||||
import emailer # noqa: E402
|
||||
import ledger as ledger_mod # noqa: E402
|
||||
import legal # noqa: E402
|
||||
import paths as paths_mod # noqa: E402
|
||||
import registry # noqa: E402
|
||||
import report as report_mod # noqa: E402
|
||||
import tiers # noqa: E402
|
||||
|
||||
|
||||
def _out(obj) -> None:
|
||||
print(json.dumps(obj, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
def _require_subject(subject_id: str) -> dict:
|
||||
d = dossier_mod.load(subject_id)
|
||||
if not d:
|
||||
sys.exit(f"error: unknown subject {subject_id!r} (run `intake` first)")
|
||||
return d
|
||||
|
||||
|
||||
def cmd_setup(args) -> None:
|
||||
if getattr(args, "auto", False):
|
||||
# Autonomous path: detect capabilities and pick the most autonomous valid config without
|
||||
# asking anyone. Read creds from $HERMES_HOME/.env too (the terminal shell doesn't export
|
||||
# them). Explicit flags still win below.
|
||||
cfg = config_mod.auto_configure(env=config_mod.dotenv_env())
|
||||
else:
|
||||
cfg = config_mod.load_config()
|
||||
for key in ("autonomy", "email_mode", "browser_backend", "tracker_backend", "encryption"):
|
||||
val = getattr(args, key)
|
||||
if val:
|
||||
cfg[key] = val
|
||||
if cfg.get("encryption") == "age":
|
||||
if not crypto.age_available():
|
||||
sys.exit("error: encryption=age requested but `age`/`age-keygen` not found. "
|
||||
"Install age (e.g. `brew install age`) or use `--encryption none`.")
|
||||
crypto.ensure_identity() # generate the key now so encryption is actually engaged
|
||||
path = config_mod.save_config(cfg)
|
||||
migrated = _migrate_subjects() # rewrite existing dossiers/ledgers into the new at-rest format
|
||||
out = {
|
||||
"config_path": str(path),
|
||||
"config": cfg,
|
||||
"encryption_engaged": crypto.is_engaged(),
|
||||
"detected_upgrades": config_mod.detect_capabilities(),
|
||||
"migrated_subjects": migrated,
|
||||
"note": "Defaults are easiest-first (draft email, auto browser, local tracker, no encryption). "
|
||||
"Pass flags to opt into upgrades, then run `doctor` for a readiness summary.",
|
||||
}
|
||||
if cfg.get("encryption") == "age":
|
||||
out["age_identity"] = str(crypto.identity_path())
|
||||
_out(out)
|
||||
|
||||
|
||||
def _migrate_subjects() -> int:
|
||||
"""Re-save each subject's dossier + ledger so they match the current at-rest format."""
|
||||
sd = paths_mod.subjects_dir()
|
||||
if not sd.exists():
|
||||
return 0
|
||||
n = 0
|
||||
for child in sorted(sd.iterdir()):
|
||||
if not child.is_dir():
|
||||
continue
|
||||
sid = child.name
|
||||
d = dossier_mod.load(sid)
|
||||
if d is not None:
|
||||
dossier_mod.save(d)
|
||||
n += 1
|
||||
led = ledger_mod.load(sid)
|
||||
if led:
|
||||
ledger_mod.save(sid, led)
|
||||
return n
|
||||
|
||||
|
||||
def _check_writable(path) -> bool:
|
||||
try:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
probe = path / ".write_test"
|
||||
probe.write_text("x", encoding="utf-8")
|
||||
probe.unlink()
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def cmd_doctor(args) -> None:
|
||||
import platform
|
||||
|
||||
cfg = config_mod.load_config()
|
||||
caps = config_mod.detect_capabilities(config_mod.dotenv_env()) # see creds in $HERMES_HOME/.env too
|
||||
data = paths_mod.data_dir()
|
||||
writable = _check_writable(data)
|
||||
curated = len(brokers_mod._load_curated())
|
||||
live = len(brokers_mod.load_live_cache())
|
||||
total = len(brokers_mod.load_all())
|
||||
|
||||
L = ["unbroker - readiness check", "=" * 42,
|
||||
f"Python : {platform.python_version()}",
|
||||
f"Data dir : {data} ({'writable' if writable else 'NOT writable'})",
|
||||
f"Config : autonomy={cfg.get('autonomy', 'full')} email={cfg['email_mode']} "
|
||||
f"browser={cfg['browser_backend']} "
|
||||
f"tracker={cfg['tracker_backend']} encryption={cfg['encryption']}",
|
||||
f"Brokers : {total} available ({curated} curated + {live} live"
|
||||
+ ("" if live else ", run `refresh-brokers` to expand to ~50") + ")",
|
||||
"", "Opt-in upgrades:"]
|
||||
rows = [
|
||||
("Cloud browser (Browserbase) *RECOMMENDED*", caps["browserbase"],
|
||||
"default backend: clears soft CAPTCHAs (Turnstile/hCaptcha) -> more T1", "set BROWSERBASE_API_KEY"),
|
||||
("Email auto (AgentMail)", caps["agentmail"],
|
||||
"send + auto-verify, per-broker aliases (Mode B/C)", "install agentmail skill / set AGENTMAIL_API_KEY"),
|
||||
("Email send (CLI SMTP)", caps["smtp_send"],
|
||||
"`send-email` delivers opt-outs itself (Mode B)", "set EMAIL_ADDRESS / EMAIL_PASSWORD (+ EMAIL_SMTP_HOST)"),
|
||||
("Verify-link poll (CLI IMAP)", caps["imap_read"],
|
||||
"`poll-verification` reads confirmation links itself", "set EMAIL_ADDRESS / EMAIL_PASSWORD (+ EMAIL_IMAP_HOST)"),
|
||||
("Google Sheets tracker", caps["google_workspace"],
|
||||
"shared status dashboard", "set up the google-workspace skill"),
|
||||
]
|
||||
for name, ok, enables, how in rows:
|
||||
L.append(f" [{'ON ' if ok else 'off'}] {name:<28} {enables}")
|
||||
if not ok:
|
||||
L.append(f" enable: {how}")
|
||||
|
||||
# At-rest encryption: report TRUE engagement (configured + key present), not just binary presence.
|
||||
engaged = crypto.is_engaged()
|
||||
L.append(f" [{'ON ' if engaged else 'off'}] {'At-rest encryption (age)':<28} "
|
||||
"encrypts dossiers + ledgers on disk")
|
||||
if engaged:
|
||||
L.append(f" key: {crypto.identity_path()} (0600) - guards casual/backup/commit "
|
||||
"exposure, NOT a full-HERMES_HOME read")
|
||||
elif cfg["encryption"] == "age":
|
||||
L.append(" WARNING: encryption=age is SET but NOT engaged (age binary or key missing);"
|
||||
" dossiers would be PLAINTEXT")
|
||||
elif caps["age"]:
|
||||
L.append(" off - dossiers are plaintext (0600). enable: `setup --encryption age`")
|
||||
else:
|
||||
L.append(" off - dossiers are plaintext (0600). install `age` first to enable")
|
||||
|
||||
L += ["", "Verdict:", " Ready now in DRAFT mode (no setup needed): scan brokers, draft opt-out",
|
||||
" emails for you to send, and track everything in the ledger."]
|
||||
if caps["browserbase"]:
|
||||
L.append(" Cloud browser ON (recommended default): soft/managed CAPTCHAs "
|
||||
"(Turnstile/hCaptcha) clear automatically -> those brokers stay T1.")
|
||||
else:
|
||||
L.append(" No cloud browser: set BROWSERBASE_API_KEY (the recommended default) so soft "
|
||||
"CAPTCHAs clear automatically; without it those brokers drop to T2 (human tasks).")
|
||||
if cfg["email_mode"] == "draft_only":
|
||||
L.append(" Email is draft-only: you send drafts + click verify links. For hands-off email "
|
||||
"WITHOUT storing a password, run `setup --email-mode browser` (agent sends + opens "
|
||||
"verify links via your logged-in webmail); or set EMAIL_* for SMTP/IMAP.")
|
||||
elif cfg["email_mode"] == "browser":
|
||||
L.append(" Email mode: browser (no password) - the agent sends opt-outs and opens verify "
|
||||
"links via the operator's logged-in webmail. This needs Hermes pointed at the "
|
||||
"operator's OWN Chrome over CDP (launch with --remote-debugging-port=9222 "
|
||||
"--user-data-dir=~/.hermes/chrome-debug, signed into the webmail once); else it falls "
|
||||
"back to drafts. Run `pdd.py cdp` to launch it (or `pdd.py cdp --print` for the command). "
|
||||
"See methods.md 'Browser backends'.")
|
||||
cloud_scan = cfg.get("browser_backend") == "browserbase" or (
|
||||
cfg.get("browser_backend") == "auto" and caps.get("browserbase"))
|
||||
if cloud_scan:
|
||||
L.append(" NOTE: your scan backend is a cloud browser (Browserbase). It is great for "
|
||||
"Phase-1 scanning but CANNOT be the browser that sends webmail (no inbox session) "
|
||||
"and is itself Cloudflare/DataDome-gated on session-bound gates (e.g. PeopleConnect). "
|
||||
"For Phase-2 email/verify, launch the operator's Chrome over CDP: `pdd.py cdp`.")
|
||||
if not crypto.is_engaged():
|
||||
L.append(" Storage: dossiers are PLAINTEXT JSON (0600 under HERMES_HOME). "
|
||||
"Run `setup --encryption age` for at-rest encryption.")
|
||||
if not live:
|
||||
L.append(" Next: run `refresh-brokers` to load the full broker list.")
|
||||
|
||||
# Freshness: warn when cached lists / curated mechanics are going stale (silent broker rot).
|
||||
import time as _time
|
||||
STALE_CACHE_DAYS, STALE_VERIFY_DAYS = 30, 180
|
||||
|
||||
def _age_days(p) -> float | None:
|
||||
try:
|
||||
return (_time.time() - p.stat().st_mtime) / 86400.0
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
fresh = []
|
||||
for label, p in [("BADBOOL", paths_mod.brokers_cache_path()),
|
||||
("CA registry", paths_mod.registry_cache_path())]:
|
||||
age = _age_days(p)
|
||||
if age is None:
|
||||
fresh.append(f"{label}: not pulled")
|
||||
elif age > STALE_CACHE_DAYS:
|
||||
fresh.append(f"{label}: {age:.0f}d old (stale, re-pull)")
|
||||
stale_curated = documented = 0
|
||||
for b in brokers_mod._load_curated():
|
||||
conf = b.get("confidence")
|
||||
lv = b.get("last_verified")
|
||||
if conf == "documented" or not lv:
|
||||
documented += 1
|
||||
continue
|
||||
try:
|
||||
if (_time.time() - _time.mktime(_time.strptime(lv, "%Y-%m-%d"))) / 86400.0 > STALE_VERIFY_DAYS:
|
||||
stale_curated += 1
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if fresh:
|
||||
L.append(" Freshness: " + "; ".join(fresh) + " (run `refresh-brokers`).")
|
||||
if stale_curated or documented:
|
||||
L.append(f" Freshness: {stale_curated} curated broker(s) last-verified >{STALE_VERIFY_DAYS}d ago; "
|
||||
f"{documented} documented broker(s) awaiting first-use verification.")
|
||||
print("\n".join(L))
|
||||
|
||||
|
||||
def cmd_cdp(args) -> None:
|
||||
"""Launch (or detect) the operator's Chrome over CDP for Phase-2 browser + webmail work.
|
||||
|
||||
A cloud browser cannot send the operator's webmail or clear session-bound gates; this points
|
||||
Hermes at the operator's real Chrome on a dedicated debug profile (see methods.md).
|
||||
"""
|
||||
import shlex
|
||||
import time
|
||||
|
||||
port = args.port
|
||||
profile = Path(args.profile).expanduser() if args.profile else cdp.default_profile()
|
||||
|
||||
live = cdp.endpoint_status(port)
|
||||
if live:
|
||||
_out({"running": True, "endpoint": f"127.0.0.1:{port}",
|
||||
"browser": live.get("Browser"),
|
||||
"webSocketDebuggerUrl": live.get("webSocketDebuggerUrl"),
|
||||
"note": "a debuggable browser is already listening; point Hermes's browser tools at "
|
||||
f"127.0.0.1:{port} and make sure the operator's webmail is signed in in THAT browser."})
|
||||
return
|
||||
|
||||
if getattr(args, "check", False):
|
||||
_out({"running": False, "endpoint": f"127.0.0.1:{port}",
|
||||
"note": f"no debuggable browser here yet; run `pdd.py cdp --port {port}` (no --check) to launch one."})
|
||||
return
|
||||
|
||||
browser = cdp.find_browser(args.browser)
|
||||
if not browser:
|
||||
_out({"running": False, "error": "no Chrome/Chromium-family browser found",
|
||||
"fix": "install Google Chrome, or pass --browser /path/to/chrome (or a command on PATH)"})
|
||||
return
|
||||
|
||||
cmd = cdp.launch_command(browser, port, profile)
|
||||
if getattr(args, "print_only", False):
|
||||
_out({"running": False, "browser": browser, "profile": str(profile), "command": cmd,
|
||||
"shell": " ".join(shlex.quote(c) for c in cmd),
|
||||
"note": "run this yourself to launch the debug browser, then sign into your webmail once."})
|
||||
return
|
||||
|
||||
pid = cdp.launch(browser, port, profile)
|
||||
live = None
|
||||
for _ in range(20): # give Chrome a few seconds to open the debug port
|
||||
live = cdp.endpoint_status(port)
|
||||
if live:
|
||||
break
|
||||
time.sleep(0.5)
|
||||
_out({"running": bool(live), "launched_pid": pid, "browser": browser,
|
||||
"profile": str(profile), "endpoint": f"127.0.0.1:{port}",
|
||||
"webSocketDebuggerUrl": (live or {}).get("webSocketDebuggerUrl"),
|
||||
"next": ([f"point Hermes's browser tools at 127.0.0.1:{port} (CDP)",
|
||||
"in the launched browser, sign into the operator's webmail ONCE (dedicated debug profile)",
|
||||
"then run email/verify flows in browser mode -- they use this logged-in session"]
|
||||
if live else
|
||||
["browser launched but the debug port has not answered yet; give it a few seconds, then "
|
||||
f"re-run `pdd.py cdp --check --port {port}`"])})
|
||||
|
||||
|
||||
def cmd_intake(args) -> None:
|
||||
if args.json:
|
||||
data = json.loads(Path(args.json).read_text(encoding="utf-8"))
|
||||
identity = data["identity"]
|
||||
consent = data.get("consent", {})
|
||||
residency = data.get("residency_jurisdiction", "US")
|
||||
prefs = data.get("preferences")
|
||||
else:
|
||||
if not args.full_name:
|
||||
sys.exit("error: --full-name (or --json) is required")
|
||||
identity = {"full_name": args.full_name, "emails": args.email or [], "phones": args.phone or []}
|
||||
if args.alias:
|
||||
identity["also_known_as"] = args.alias
|
||||
if args.dob:
|
||||
identity["date_of_birth"] = args.dob
|
||||
addr = {k: v for k, v in {"line1": args.street, "city": args.city,
|
||||
"state": args.state, "postal": args.postal}.items() if v}
|
||||
if addr:
|
||||
identity["current_address"] = addr
|
||||
priors = []
|
||||
for loc in args.prior_location or []:
|
||||
parts = [p.strip() for p in loc.split(",") if p.strip()]
|
||||
if not parts:
|
||||
continue
|
||||
entry = {"city": parts[0]}
|
||||
if len(parts) > 1:
|
||||
entry["state"] = parts[1]
|
||||
if len(parts) > 2:
|
||||
entry["postal"] = parts[2]
|
||||
priors.append(entry)
|
||||
if priors:
|
||||
identity["prior_addresses"] = priors
|
||||
cfg = config_mod.load_config()
|
||||
consent = {"authorized": bool(args.consent), "method": args.consent_method, "recorded_at": dossier_mod.now()}
|
||||
residency = args.residency or "US"
|
||||
prefs = {
|
||||
"email_mode": args.email_mode or cfg["email_mode"],
|
||||
"rescan_interval_days": cfg["default_rescan_interval_days"],
|
||||
}
|
||||
if args.contact_email:
|
||||
prefs["contact_email_for_optouts"] = args.contact_email
|
||||
d = dossier_mod.create(identity, consent, residency, prefs)
|
||||
_out({"subject_id": d["subject_id"], "authorized": dossier_mod.is_authorized(d),
|
||||
"residency": residency, "email_mode": (prefs or {}).get("email_mode"),
|
||||
"names": dossier_mod.all_names(d),
|
||||
"emails": len(d["identity"].get("emails") or []),
|
||||
"phones": len(d["identity"].get("phones") or []),
|
||||
"addresses": len(dossier_mod.all_addresses(d))})
|
||||
|
||||
|
||||
def cmd_brokers(args) -> None:
|
||||
bl = brokers_mod.by_priority(*(args.priority or [])) if args.priority else brokers_mod.load_all()
|
||||
_out([
|
||||
{"id": b.get("id"), "name": b.get("name"), "priority": b.get("priority"),
|
||||
"method": (b.get("optout") or {}).get("method"), "owns": b.get("owns") or [],
|
||||
"source": b.get("source"), "confidence": b.get("confidence", "curated")}
|
||||
for b in bl
|
||||
])
|
||||
|
||||
|
||||
def cmd_refresh_brokers(args) -> None:
|
||||
res = badbool.refresh(paths_mod.brokers_cache_path())
|
||||
curated_ids = {b["id"] for b in brokers_mod._load_curated()}
|
||||
new = [b["id"] for b in brokers_mod.load_live_cache() if b["id"] not in curated_ids]
|
||||
out = {**res, "curated": len(curated_ids), "new_from_live": len(new),
|
||||
"people_search_total": len(brokers_mod.load_all()),
|
||||
"note": "Live records have confidence=auto; verify their opt-out URL before acting."}
|
||||
if not getattr(args, "no_registry", False):
|
||||
try:
|
||||
reg = registry.refresh_all(paths_mod.registry_cache_path())
|
||||
out["registry"] = {"total": reg["total"], "sources": reg["sources"],
|
||||
"portals": reg["portals"],
|
||||
"note": "Coverage lane worked via the CA DROP one-shot + CCPA email, "
|
||||
"not the people-search scan. VT/OR/TX are search portals (no "
|
||||
"bulk export); CA is the superset. See `drop` and `registry`."}
|
||||
except Exception as exc: # noqa: BLE001 - registry pull is best-effort
|
||||
out["registry_error"] = str(exc)
|
||||
_out(out)
|
||||
|
||||
|
||||
def cmd_registry(args) -> None:
|
||||
recs = brokers_mod.load_registry_cache()
|
||||
if not recs:
|
||||
_out({"registered_brokers": 0,
|
||||
"note": "registry empty - run `refresh-brokers` (pulls the CA Data Broker Registry)"})
|
||||
return
|
||||
fcra = sum(1 for r in recs if (r.get("optout") or {}).get("fcra"))
|
||||
out = {"registered_brokers": len(recs), "fcra_regulated": fcra,
|
||||
"source": "CA Data Broker Registry (CPPA, 2025)", "drop_url": registry.DROP_URL,
|
||||
"other_state_portals": registry.portals()}
|
||||
if args.search:
|
||||
q = args.search.lower()
|
||||
hits = [r for r in recs if q in (r.get("name") or "").lower()
|
||||
or q in (r.get("id") or "") or q in ((r.get("optout") or {}).get("email") or "").lower()]
|
||||
out["matches"] = [{"id": r["id"], "name": r["name"],
|
||||
"email": (r.get("optout") or {}).get("email"),
|
||||
"url": (r.get("optout") or {}).get("url"),
|
||||
"fcra": (r.get("optout") or {}).get("fcra")} for r in hits[:args.limit]]
|
||||
out["match_count"] = len(hits)
|
||||
_out(out)
|
||||
|
||||
|
||||
def cmd_drop(args) -> None:
|
||||
"""The one-shot legal lever: CA DROP deletes from ALL registered brokers at once."""
|
||||
d = _require_subject(args.subject)
|
||||
dossier_mod.require_authorized(d)
|
||||
reg = brokers_mod.load_registry_cache()
|
||||
res = (d.get("residency_jurisdiction") or "US").upper()
|
||||
eligible = res.startswith("US-CA")
|
||||
if args.filed:
|
||||
prefs = d.setdefault("preferences", {})
|
||||
prefs["drop_filed_at"] = dossier_mod.now()
|
||||
dossier_mod.save(d)
|
||||
_out({"subject": args.subject, "drop_filed_at": prefs["drop_filed_at"],
|
||||
"note": "recorded; `next` will stop surfacing the DROP one-shot"})
|
||||
return
|
||||
_out({
|
||||
"subject": args.subject,
|
||||
"eligible": eligible,
|
||||
"residency": res,
|
||||
"drop_url": registry.DROP_URL,
|
||||
"covers_registered_brokers": len(reg),
|
||||
"steps": ([
|
||||
"Go to privacy.ca.gov/drop and create/verify a DROP account (CA resident).",
|
||||
"Submit ONE deletion request; it applies to EVERY registered data broker "
|
||||
f"({len(reg)} in the current registry). Brokers must process starting 2026-08-01.",
|
||||
"After filing, run `drop <subject> --filed` so the loop stops re-surfacing it.",
|
||||
] if eligible else [
|
||||
"DROP is a California mechanism; this subject's residency is not US-CA.",
|
||||
"Parity path for non-CA: work the people-search sites via `next`, and send targeted "
|
||||
"CCPA/GDPR deletion emails to registry brokers that hold this person's data "
|
||||
"(`registry --search`, then `send-email`).",
|
||||
]),
|
||||
"note": "DROP is the highest-leverage removal: one request covers the whole registry.",
|
||||
})
|
||||
|
||||
|
||||
def cmd_plan(args) -> None:
|
||||
d = _require_subject(args.subject)
|
||||
dossier_mod.require_authorized(d)
|
||||
cfg = config_mod.load_config()
|
||||
bl = brokers_mod.by_priority(*(args.priority or [])) if args.priority else brokers_mod.load_all()
|
||||
bcc = config_mod.browser_clears_captcha(cfg)
|
||||
if getattr(args, "batch", False):
|
||||
_out(tiers.batch_plan(d, bl, cfg, ledger_mod.load(args.subject), bcc))
|
||||
else:
|
||||
_out(tiers.plan(d, bl, cfg, bcc))
|
||||
|
||||
|
||||
def cmd_fanout(args) -> None:
|
||||
d = _require_subject(args.subject)
|
||||
dossier_mod.require_authorized(d)
|
||||
bl = brokers_mod.by_priority(*(args.priority or [])) if args.priority else brokers_mod.load_all()
|
||||
grouping = tiers.fanout(bl, batch_size=args.size)
|
||||
mode = "scan AND opt-out (operator authorized submissions)" if args.optout \
|
||||
else "READ-ONLY scan (submit nothing; reconnaissance only)"
|
||||
batches = []
|
||||
for i, ids in enumerate(grouping["batches"], 1):
|
||||
brief = (
|
||||
f"You are scan worker {i} of {len(grouping['batches'])} for the `unbroker` skill. First "
|
||||
f"load the `unbroker` skill and read its references/methods.md. Use the `web` toolset "
|
||||
f"(web_search `site:` + web_extract), NOT `browser` (browser navigation is heavy and times "
|
||||
f"out). Subject id: {args.subject}. Handle ONLY these brokers: {', '.join(ids)}. "
|
||||
f"For EACH broker: read references/brokers/<id>.json; run EVERY search vector from "
|
||||
f"`pdd.py plan {args.subject}` (filtered to your brokers); build URLs from search.url_patterns "
|
||||
f"and heed url_format_quirks; a 404 is INCONCLUSIVE (rebuild/try the on-site search box), not "
|
||||
f"not_found. ECONOMY: at most ~3 web calls per broker; the moment a page shows antibot "
|
||||
f"(Cloudflare 'just a moment'/DataDome) or hangs, record `blocked` and move on -- do NOT "
|
||||
f"retry-loop. Confirm the SUBJECT vs namesakes/relatives by ADDRESS/DOB before recording "
|
||||
f"`found` (ignore SEO-templated page titles/intro that just echo the query -- require a real "
|
||||
f"result card; a public property/address record with no displayed personal NAME is "
|
||||
f"not_found, not found). Record each outcome via `pdd.py record {args.subject} <broker> "
|
||||
f"<found|not_found|indirect_exposure|blocked> --found <bool> --evidence '{{\"listing_urls\":[...]}}'`. "
|
||||
f"Mode: {mode}. Broker JSON files are READ-ONLY for you -- do NOT edit them; if you discover "
|
||||
f"a URL/quirk, put it in your report for the parent to fold in. Return a concise structured "
|
||||
f"per-broker report."
|
||||
)
|
||||
batches.append({"batch": i, "brokers": ids, "brief": brief})
|
||||
_out({
|
||||
"subject": args.subject,
|
||||
"broker_count": grouping["broker_count"],
|
||||
"batch_size": grouping["batch_size"],
|
||||
"should_fanout": grouping["should_fanout"],
|
||||
"batch_count": len(batches),
|
||||
"batches": batches,
|
||||
"instruction": (
|
||||
"If should_fanout is true you MUST spawn ONE delegate_task subagent per batch IN PARALLEL, "
|
||||
"passing each batch's `brief`; do not scan all brokers yourself sequentially. Wait for every "
|
||||
"report, consolidate, then proceed to opt-outs. If false, just scan the brokers inline."
|
||||
),
|
||||
})
|
||||
|
||||
|
||||
def cmd_record(args) -> None:
|
||||
d = _require_subject(args.subject)
|
||||
dossier_mod.require_authorized(d)
|
||||
broker = brokers_mod.get(args.broker)
|
||||
# Auto-stamp follow-up scheduling (next_recheck_at / removal_confirmed_at) so the
|
||||
# autonomous loop knows when to come back without anyone remembering to set it.
|
||||
fields = ledger_mod.followup_fields(args.state, broker, d)
|
||||
if args.found is not None:
|
||||
fields["found"] = args.found
|
||||
if args.evidence:
|
||||
fields["evidence"] = json.loads(args.evidence)
|
||||
if args.reason:
|
||||
fields["human_task_reason"] = args.reason
|
||||
case = ledger_mod.transition(args.subject, args.broker, args.state, **fields)
|
||||
if args.disclosed:
|
||||
ledger_mod.log_disclosure(args.subject, args.broker, args.disclosed, args.channel or "unknown")
|
||||
_out({"broker": args.broker, "state": case["state"],
|
||||
"next_recheck_at": case.get("next_recheck_at")})
|
||||
|
||||
|
||||
def _email_request(d: dict, b: dict, kind: str, listings, identifiers) -> tuple[dict, list[str]]:
|
||||
"""Least-disclosure (fields, disclosed_names) for an opt-out/legal email of KIND.
|
||||
|
||||
A removal letter must self-identify. Name + a contact email are already known to the
|
||||
broker (the name is displayed on the very listing being removed), so not extra exposure.
|
||||
"""
|
||||
fields = dossier_mod.select_disclosure(d, (b.get("optout") or {}).get("inputs", []))
|
||||
ident = d.get("identity", {})
|
||||
if ident.get("full_name"):
|
||||
fields.setdefault("full_name", ident["full_name"])
|
||||
fields.setdefault("contact_email", dossier_mod.contact_email(d) or "")
|
||||
if listings:
|
||||
fields["listing_urls"] = listings
|
||||
if kind == "ccpa_indirect":
|
||||
# Indirect exposure: name ONLY the subject's own identifiers to scrub from a third party's
|
||||
# record. Default to the contact email + the subject's name-as-relative if none specified.
|
||||
# The indirect template renders ONLY these placeholders; do not over-report disclosure with
|
||||
# unrelated dossier fields (phone/street/postal) that select_disclosure happened to populate.
|
||||
ids = list(identifiers or [])
|
||||
if not ids:
|
||||
ids = [contact for contact in [dossier_mod.contact_email(d)] if contact]
|
||||
ids.append(f'the name "{ident.get("full_name")}" where it appears as a relative/associated person')
|
||||
fields = {
|
||||
"full_name": fields.get("full_name"),
|
||||
"contact_email": fields.get("contact_email"),
|
||||
"listing_urls": fields.get("listing_urls"),
|
||||
"my_identifiers": ids,
|
||||
}
|
||||
return fields, ["contact_email", "full_name", "my_identifiers"]
|
||||
return fields, sorted(fields.keys())
|
||||
|
||||
|
||||
def cmd_render_email(args) -> None:
|
||||
d = _require_subject(args.subject)
|
||||
dossier_mod.require_authorized(d)
|
||||
b = brokers_mod.get(args.broker)
|
||||
if not b:
|
||||
sys.exit(f"error: unknown broker {args.broker!r}")
|
||||
kind = getattr(args, "kind", "generic") or "generic"
|
||||
fields, disclosed = _email_request(d, b, kind, args.listing, getattr(args, "identifier", None))
|
||||
if kind == "generic":
|
||||
draft = email_modes.render_draft(b, fields)
|
||||
else:
|
||||
draft = email_modes.render_request_draft(b, fields, kind=kind)
|
||||
ledger_mod.log_disclosure(args.subject, args.broker, list(disclosed), f"email_draft:{kind}")
|
||||
_out({"draft": str(draft), "kind": kind, "disclosed_fields": disclosed})
|
||||
|
||||
|
||||
def cmd_send_email(args) -> None:
|
||||
"""Mode B: render AND deliver the opt-out/legal request - no human in the loop.
|
||||
|
||||
Sends ONLY to an address the broker record itself declares (emailer enforces it),
|
||||
then records the ledger transition + disclosure and auto-stamps the recheck date.
|
||||
"""
|
||||
d = _require_subject(args.subject)
|
||||
dossier_mod.require_authorized(d)
|
||||
b = brokers_mod.get(args.broker)
|
||||
if not b:
|
||||
sys.exit(f"error: unknown broker {args.broker!r}")
|
||||
cfg = config_mod.load_config()
|
||||
mode = cfg.get("email_mode")
|
||||
if mode not in ("programmatic", "alias", "browser"):
|
||||
sys.exit("error: email_mode is draft_only; run `setup --email-mode browser` (no password; "
|
||||
"sends via your logged-in webmail) or `--email-mode programmatic`, or use "
|
||||
"`render-email` and send it yourself")
|
||||
if not args.listing:
|
||||
sys.exit("error: --listing <confirmed-url> is required (verify-before-disclose: never "
|
||||
"email a broker about an unconfirmed listing)")
|
||||
# Idempotency: don't re-send if this case is already submitted/beyond (prevents duplicate
|
||||
# requests when an action is retried). --force overrides.
|
||||
_POST_SUBMIT = {"submitted", "verification_pending", "awaiting_processing", "confirmed_removed"}
|
||||
current = ledger_mod.get_case(args.subject, args.broker).get("state")
|
||||
if current in _POST_SUBMIT and not getattr(args, "force", False):
|
||||
_out({"skipped": True, "broker": args.broker, "state": current,
|
||||
"note": "already submitted; not re-sending (idempotent). Use --force to re-send."})
|
||||
return
|
||||
kind = getattr(args, "kind", "generic") or "generic"
|
||||
fields, disclosed = _email_request(d, b, kind, args.listing, getattr(args, "identifier", None))
|
||||
body = legal.render_optout_email(b, fields) if kind == "generic" else legal.render_request(kind, b, fields)
|
||||
|
||||
if mode == "browser":
|
||||
# No network / no credentials: hand the agent a recipient-locked payload to send in the
|
||||
# operator's webmail via browser_* tools. State still records deterministically here.
|
||||
payload = emailer.browser_send_payload(b, body, to=args.to)
|
||||
ledger_mod.log_disclosure(args.subject, args.broker, list(disclosed), f"email_browser:{kind}")
|
||||
case = ledger_mod.transition(args.subject, args.broker, "submitted",
|
||||
**ledger_mod.followup_fields("submitted", b, d))
|
||||
_out({"send_via": "browser", "compose": payload, "kind": kind, "disclosed_fields": disclosed,
|
||||
"state": case["state"], "next_recheck_at": case.get("next_recheck_at"),
|
||||
"instruction": "In the operator's logged-in webmail, compose a NEW email to compose.to "
|
||||
"with compose.subject/body EXACTLY (disclose nothing beyond it) and send "
|
||||
"it via browser_* tools. Then use `verify-link` on any confirmation reply.",
|
||||
"note": "recipient is locked to the broker's declared address"})
|
||||
return
|
||||
|
||||
result = emailer.send(b, body, to=args.to,
|
||||
min_interval=float(cfg.get("email_min_interval_seconds", 0) or 0))
|
||||
ledger_mod.log_disclosure(args.subject, args.broker, list(disclosed), f"email_sent:{kind}")
|
||||
case = ledger_mod.transition(args.subject, args.broker, "submitted",
|
||||
**ledger_mod.followup_fields("submitted", b, d))
|
||||
_out({"sent": result, "send_via": "smtp", "kind": kind, "disclosed_fields": disclosed,
|
||||
"state": case["state"], "next_recheck_at": case.get("next_recheck_at"),
|
||||
"note": "if this broker verifies by email, `poll-verification` will pick up the link"})
|
||||
|
||||
|
||||
def cmd_verify_link(args) -> None:
|
||||
"""Extract a broker's verification link from email text the agent read in webmail (browser mode).
|
||||
|
||||
IMAP-free counterpart to `poll-verification`: the agent opens the broker's confirmation email
|
||||
in the operator's webmail, pastes the body here, and gets the anti-phishing-scored link back.
|
||||
"""
|
||||
_require_subject(args.subject)
|
||||
b = brokers_mod.get(args.broker)
|
||||
if not b:
|
||||
sys.exit(f"error: unknown broker {args.broker!r}")
|
||||
text = args.text
|
||||
if args.file:
|
||||
text = Path(args.file).read_text(encoding="utf-8", errors="replace")
|
||||
if not text:
|
||||
sys.exit("error: provide --text '<email body>' (or --file) from the broker's confirmation email")
|
||||
link = email_modes.extract_verification_link(text, b)
|
||||
_out({"broker": args.broker, "verification_link": link,
|
||||
"next": ("browser_navigate the link IN THE SAME browser (sessions are browser-bound), "
|
||||
f"complete the flow, then `record {args.subject} {args.broker} awaiting_processing`"
|
||||
if link else
|
||||
"no broker/opt-out-scoped link found in that text; confirm you opened the right email")})
|
||||
|
||||
|
||||
def cmd_poll_verification(args) -> None:
|
||||
"""Poll the inbox for brokers' verification links (Mode B) - replaces the human click-chase.
|
||||
|
||||
For each in-flight case (submitted / verification_pending with email_verification),
|
||||
extract the broker's link (anti-phishing scored). A found link auto-advances
|
||||
submitted -> verification_pending (the email HAS arrived); the agent must then OPEN
|
||||
the link in its own browser (sessions are browser-bound) and record the next state.
|
||||
"""
|
||||
d = _require_subject(args.subject)
|
||||
dossier_mod.require_authorized(d)
|
||||
led = ledger_mod.load(args.subject)
|
||||
targets = []
|
||||
for bid, case in sorted(led.items()):
|
||||
if args.broker and bid != args.broker:
|
||||
continue
|
||||
if case.get("state") not in ("submitted", "verification_pending"):
|
||||
continue
|
||||
b = brokers_mod.get(bid)
|
||||
if b and (((b.get("optout") or {}).get("requires")) or {}).get("email_verification"):
|
||||
targets.append((bid, case, b))
|
||||
if not targets:
|
||||
_out({"subject": args.subject, "results": [],
|
||||
"note": "no in-flight cases awaiting email verification"})
|
||||
return
|
||||
results = []
|
||||
for bid, case, b in targets:
|
||||
hit = emailer.find_verification_link(b, since_days=args.since_days)
|
||||
if hit:
|
||||
if case.get("state") == "submitted":
|
||||
ledger_mod.transition(args.subject, bid, "verification_pending",
|
||||
**ledger_mod.followup_fields("verification_pending", b, d))
|
||||
results.append({"broker": bid, "verification_link": hit["link"],
|
||||
"email_from": hit.get("from"), "email_subject": hit.get("subject"),
|
||||
"next": f"browser_navigate the link IN THE AGENT'S OWN BROWSER, complete "
|
||||
f"the flow, then `record {args.subject} {bid} awaiting_processing` "
|
||||
f"(or confirmed_removed only after a verifying re-scan)"})
|
||||
else:
|
||||
results.append({"broker": bid, "verification_link": None,
|
||||
"next": "no matching email yet; poll again later (next_recheck_at is set)"})
|
||||
_out({"subject": args.subject, "results": results})
|
||||
|
||||
|
||||
def cmd_next(args) -> None:
|
||||
d = _require_subject(args.subject)
|
||||
dossier_mod.require_authorized(d)
|
||||
cfg = config_mod.load_config()
|
||||
bl = brokers_mod.by_priority(*(args.priority or [])) if args.priority else brokers_mod.load_all()
|
||||
_out(autopilot.next_actions(d, bl, cfg, ledger_mod.load(args.subject)))
|
||||
|
||||
|
||||
def cmd_tasks(args) -> None:
|
||||
_require_subject(args.subject)
|
||||
print(report_mod.human_tasks_markdown(args.subject))
|
||||
|
||||
|
||||
def cmd_due(args) -> None:
|
||||
_require_subject(args.subject)
|
||||
cases = ledger_mod.due(args.subject)
|
||||
_out({"subject": args.subject, "due_count": len(cases),
|
||||
"cases": [{"broker_id": c.get("broker_id"), "state": c.get("state"),
|
||||
"next_recheck_at": c.get("next_recheck_at")} for c in cases],
|
||||
"note": "run `next` for the concrete follow-up action per case"})
|
||||
|
||||
|
||||
def cmd_show(args) -> None:
|
||||
"""Read a case's recorded state + evidence (so the parent can re-verify a subagent's `found`
|
||||
without re-deriving listing URLs)."""
|
||||
_require_subject(args.subject)
|
||||
case = ledger_mod.get_case(args.subject, args.broker)
|
||||
_out({"broker": args.broker, "state": case.get("state"), "found": case.get("found"),
|
||||
"evidence": case.get("evidence") or {},
|
||||
"disclosure_log": case.get("disclosure_log") or [],
|
||||
"next_recheck_at": case.get("next_recheck_at"),
|
||||
"human_task_reason": case.get("human_task_reason"),
|
||||
"history": case.get("history") or []})
|
||||
|
||||
|
||||
def cmd_status(args) -> None:
|
||||
_require_subject(args.subject)
|
||||
print(report_mod.render_markdown(args.subject))
|
||||
|
||||
|
||||
def cmd_report(args) -> None:
|
||||
_require_subject(args.subject)
|
||||
if args.sheets:
|
||||
_out(report_mod.sheets_rows(args.subject))
|
||||
else:
|
||||
print(report_mod.render_markdown(args.subject))
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(prog="pdd", description="unbroker helper CLI")
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
s = sub.add_parser("setup", help="write install config (easiest-first defaults; --auto = most autonomous)")
|
||||
s.add_argument("--auto", action="store_true",
|
||||
help="detect capabilities and pick the most autonomous valid config (no questions)")
|
||||
s.add_argument("--autonomy", dest="autonomy", choices=sorted(config_mod.VALID["autonomy"]))
|
||||
s.add_argument("--email-mode", dest="email_mode", choices=sorted(config_mod.VALID["email_mode"]))
|
||||
s.add_argument("--browser-backend", dest="browser_backend", choices=sorted(config_mod.VALID["browser_backend"]))
|
||||
s.add_argument("--tracker-backend", dest="tracker_backend", choices=sorted(config_mod.VALID["tracker_backend"]))
|
||||
s.add_argument("--encryption", dest="encryption", choices=sorted(config_mod.VALID["encryption"]))
|
||||
s.set_defaults(func=cmd_setup)
|
||||
|
||||
s = sub.add_parser("doctor", help="readiness check: config, brokers, available upgrades")
|
||||
s.set_defaults(func=cmd_doctor)
|
||||
|
||||
s = sub.add_parser("cdp",
|
||||
help="launch/detect the operator's Chrome over CDP (Phase-2 browser + webmail)")
|
||||
s.add_argument("--port", type=int, default=cdp.DEFAULT_PORT, help="remote debugging port (default 9222)")
|
||||
s.add_argument("--profile",
|
||||
help="user-data-dir (default: $HERMES_HOME/chrome-debug, a dedicated debug profile)")
|
||||
s.add_argument("--browser", help="path to (or PATH name of) a Chrome/Chromium/Brave/Edge binary")
|
||||
s.add_argument("--check", action="store_true",
|
||||
help="only report whether a debug browser is live; do not launch")
|
||||
s.add_argument("--print", dest="print_only", action="store_true",
|
||||
help="print the launch command instead of launching it (run it yourself)")
|
||||
s.set_defaults(func=cmd_cdp)
|
||||
|
||||
s = sub.add_parser("intake", help="create a subject dossier (records consent)")
|
||||
s.add_argument("--json", help="path to a dossier JSON file (overrides flags)")
|
||||
s.add_argument("--full-name")
|
||||
s.add_argument("--alias", action="append", metavar="NAME",
|
||||
help="other name the subject is listed under (maiden/married/nickname); repeatable")
|
||||
s.add_argument("--email", action="append", metavar="EMAIL", help="repeatable")
|
||||
s.add_argument("--phone", action="append", metavar="PHONE", help="repeatable")
|
||||
s.add_argument("--street", help="current street line1 (enables reverse-address search)")
|
||||
s.add_argument("--city")
|
||||
s.add_argument("--state")
|
||||
s.add_argument("--postal")
|
||||
s.add_argument("--prior-location", dest="prior_location", action="append", metavar="City,ST",
|
||||
help="a past city/state (or City,ST,ZIP); repeatable")
|
||||
s.add_argument("--dob", help="date of birth YYYY-MM-DD (only used if a broker requires it)")
|
||||
s.add_argument("--contact-email", dest="contact_email",
|
||||
help="which email to use for opt-out correspondence (default: first)")
|
||||
s.add_argument("--residency", help="e.g. US, US-CA")
|
||||
s.add_argument("--consent", action="store_true", help="subject authorizes removal on their behalf")
|
||||
s.add_argument("--consent-method", default="self", choices=["self", "written_authorization", "poa"])
|
||||
s.add_argument("--email-mode", dest="email_mode", choices=sorted(config_mod.VALID["email_mode"]))
|
||||
s.set_defaults(func=cmd_intake)
|
||||
|
||||
s = sub.add_parser("brokers", help="list the broker database (curated + live)")
|
||||
s.add_argument("--priority", action="append", choices=["crucial", "high", "standard", "long_tail"])
|
||||
s.set_defaults(func=cmd_brokers)
|
||||
|
||||
s = sub.add_parser("refresh-brokers",
|
||||
help="pull the latest BADBOOL people-search list + the CA data broker registry")
|
||||
s.add_argument("--no-registry", dest="no_registry", action="store_true",
|
||||
help="skip the CA registry pull (BADBOOL people-search only)")
|
||||
s.set_defaults(func=cmd_refresh_brokers)
|
||||
|
||||
s = sub.add_parser("registry",
|
||||
help="CA Data Broker Registry coverage (hundreds of brokers; DROP/email lane)")
|
||||
s.add_argument("--search", help="find registered brokers by name / id / email substring")
|
||||
s.add_argument("--limit", type=int, default=25, help="max matches to print (default 25)")
|
||||
s.set_defaults(func=cmd_registry)
|
||||
|
||||
s = sub.add_parser("drop",
|
||||
help="CA DROP one-shot: delete from ALL registered brokers in one request")
|
||||
s.add_argument("subject")
|
||||
s.add_argument("--filed", action="store_true", help="mark DROP as filed (stops `next` surfacing it)")
|
||||
s.set_defaults(func=cmd_drop)
|
||||
|
||||
s = sub.add_parser("plan", help="compute per-broker tier + next action for a subject")
|
||||
s.add_argument("subject")
|
||||
s.add_argument("--priority", action="append", choices=["crucial", "high", "standard", "long_tail"])
|
||||
s.add_argument("--batch", action="store_true",
|
||||
help="phase-oriented batch view: overlays ledger state, groups by next action "
|
||||
"(unscanned/found/indirect/blocked/in_progress/done), collapses ownership clusters")
|
||||
s.set_defaults(func=cmd_plan)
|
||||
|
||||
s = sub.add_parser("fanout", help="batch brokers into parallel delegate_task subagents (large runs)")
|
||||
s.add_argument("subject")
|
||||
s.add_argument("--priority", action="append", choices=["crucial", "high", "standard", "long_tail"])
|
||||
s.add_argument("--size", type=int, default=5, help="brokers per subagent batch (default 5; 8+ times out)")
|
||||
s.add_argument("--optout", action="store_true",
|
||||
help="brief authorizes opt-out submission (default: read-only scan)")
|
||||
s.set_defaults(func=cmd_fanout)
|
||||
|
||||
s = sub.add_parser("record", help="record a ledger state transition after an agent action")
|
||||
s.add_argument("subject")
|
||||
s.add_argument("broker")
|
||||
s.add_argument("state", choices=ledger_mod.STATES)
|
||||
s.add_argument("--found", type=lambda v: v.strip().lower() in ("1", "true", "yes", "y"))
|
||||
s.add_argument("--evidence", help="JSON object stored as case.evidence")
|
||||
s.add_argument("--disclosed", action="append", metavar="FIELD", help="field name disclosed")
|
||||
s.add_argument("--channel", help="disclosure channel, e.g. web_form / email")
|
||||
s.add_argument("--reason", help="for human_task_queued: why a human is needed (shown in `tasks`)")
|
||||
s.set_defaults(func=cmd_record)
|
||||
|
||||
s = sub.add_parser("next", help="autonomous action queue: exactly what to do right now")
|
||||
s.add_argument("subject")
|
||||
s.add_argument("--priority", action="append", choices=["crucial", "high", "standard", "long_tail"])
|
||||
s.set_defaults(func=cmd_next)
|
||||
|
||||
s = sub.add_parser("send-email", help="Mode B: render AND send the opt-out/legal request (records it)")
|
||||
s.add_argument("subject")
|
||||
s.add_argument("broker")
|
||||
s.add_argument("--listing", action="append", metavar="URL", required=False,
|
||||
help="confirmed listing URL (required: verify-before-disclose)")
|
||||
s.add_argument("--kind", choices=["generic", "ccpa", "ccpa_agent", "ccpa_indirect", "gdpr"],
|
||||
default="generic")
|
||||
s.add_argument("--identifier", action="append", metavar="ID",
|
||||
help="(ccpa_indirect only) a specific own-identifier to remove; repeatable")
|
||||
s.add_argument("--to", help="override recipient (must be an address the broker record declares)")
|
||||
s.add_argument("--force", action="store_true", help="re-send even if already submitted (default: idempotent skip)")
|
||||
s.set_defaults(func=cmd_send_email)
|
||||
|
||||
s = sub.add_parser("poll-verification",
|
||||
help="Mode B (IMAP): poll the inbox for brokers' verification links (anti-phishing scored)")
|
||||
s.add_argument("subject")
|
||||
s.add_argument("--broker", help="only this broker (default: every in-flight verification case)")
|
||||
s.add_argument("--since-days", dest="since_days", type=int, default=3)
|
||||
s.set_defaults(func=cmd_poll_verification)
|
||||
|
||||
s = sub.add_parser("verify-link",
|
||||
help="browser mode: extract a broker's verification link from pasted webmail text")
|
||||
s.add_argument("subject")
|
||||
s.add_argument("broker")
|
||||
s.add_argument("--text", help="the confirmation email body (read from the operator's webmail)")
|
||||
s.add_argument("--file", help="path to a file with the email body (alternative to --text)")
|
||||
s.set_defaults(func=cmd_verify_link)
|
||||
|
||||
s = sub.add_parser("tasks", help="ONE consolidated human-task digest (present at end of run)")
|
||||
s.add_argument("subject")
|
||||
s.set_defaults(func=cmd_tasks)
|
||||
|
||||
s = sub.add_parser("show", help="read a case's state + evidence (for parent re-verification)")
|
||||
s.add_argument("subject")
|
||||
s.add_argument("broker")
|
||||
s.set_defaults(func=cmd_show)
|
||||
|
||||
s = sub.add_parser("due", help="cases whose recheck window has arrived (cron re-scan queue)")
|
||||
s.add_argument("subject")
|
||||
s.set_defaults(func=cmd_due)
|
||||
|
||||
s = sub.add_parser("render-email", help="render a Mode-A opt-out / legal-request draft (least-disclosure)")
|
||||
s.add_argument("subject")
|
||||
s.add_argument("broker")
|
||||
s.add_argument("--listing", action="append", metavar="URL", help="confirmed listing URL")
|
||||
s.add_argument("--kind", choices=["generic", "ccpa", "ccpa_agent", "ccpa_indirect", "gdpr"],
|
||||
default="generic",
|
||||
help="request type. 'ccpa_indirect' = delete MY identifiers from a third party's "
|
||||
"record (indirect exposure); default 'generic' opt-out.")
|
||||
s.add_argument("--identifier", action="append", metavar="ID",
|
||||
help="(ccpa_indirect only) a specific own-identifier to request removal of "
|
||||
"(e.g. an email or phone). Repeatable. Defaults to the contact email + "
|
||||
"name-as-relative if omitted.")
|
||||
s.set_defaults(func=cmd_render_email)
|
||||
|
||||
s = sub.add_parser("status", help="print a Markdown status report")
|
||||
s.add_argument("subject")
|
||||
s.set_defaults(func=cmd_status)
|
||||
|
||||
s = sub.add_parser("report", help="status report (default) or --sheets rows")
|
||||
s.add_argument("subject")
|
||||
s.add_argument("--sheets", action="store_true", help="emit Google Sheets rows as JSON")
|
||||
s.set_defaults(func=cmd_report)
|
||||
return p
|
||||
|
||||
|
||||
def main(argv=None) -> None:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
args.func(args)
|
||||
except (PermissionError, ValueError, RuntimeError, FileNotFoundError) as exc:
|
||||
sys.exit(f"error: {exc}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,293 @@
|
||||
"""Ingest the California Data Broker Registry into broker records (coverage breadth).
|
||||
|
||||
The CA registry (CPPA, under the Delete Act) is the authoritative universe of data
|
||||
brokers doing business with California residents -- ~545 businesses in 2025, each
|
||||
required to publish a name, website, contact email, and a CCPA-rights/deletion URL.
|
||||
This is the same universe commercial services (DeleteMe/Incogni/Optery) draw from,
|
||||
plus the FCRA/GLBA-regulated and marketing/risk brokers most lists omit.
|
||||
|
||||
These are NOT people-search sites you scan with a name -- most have no per-person
|
||||
lookup UI. They are worked through the LEGAL lane: the CA DROP portal
|
||||
(privacy.ca.gov/drop) is a single request that deletes from ALL registered brokers
|
||||
at once (CA residents), and per-broker CCPA deletion emails to the contact address
|
||||
are the fallback / non-CA path. So registry records are kept in their own lane
|
||||
(loaded only when asked) and never dumped into the people-search scan pipeline.
|
||||
|
||||
`parse()` is pure (CSV text in, records out) so it is tested offline; `fetch()` is
|
||||
the only network call and can be bypassed by passing csv_text directly to refresh().
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import datetime
|
||||
import io
|
||||
import re
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
import storage
|
||||
|
||||
# CA CPPA registry CSVs are published per year (registry2024.csv, registry2025.csv, ...).
|
||||
# 2025 is the latest COMPLETE dataset; the current year's file is empty until the Jan
|
||||
# registration window closes. DEFAULT_URL is the known-good fallback; `ca_candidate_urls`
|
||||
# probes newer years first so coverage auto-advances when the next year is published.
|
||||
_CA_CSV = "https://cppa.ca.gov/data_broker_registry/registry{year}.csv"
|
||||
_CA_FLOOR_YEAR = 2025
|
||||
DEFAULT_URL = _CA_CSV.format(year=_CA_FLOOR_YEAR)
|
||||
DROP_URL = "https://privacy.ca.gov/drop"
|
||||
USER_AGENT = "Mozilla/5.0 (compatible; unbroker/1.0; data opt-out)"
|
||||
|
||||
|
||||
def ca_candidate_urls(today: datetime.date | None = None) -> list[str]:
|
||||
"""Newest-year-first CA registry URLs to try (auto-advances; never below the 2025 floor)."""
|
||||
year = (today or datetime.date.today()).year
|
||||
years = list(range(max(year, _CA_FLOOR_YEAR), _CA_FLOOR_YEAR - 1, -1))
|
||||
return [_CA_CSV.format(year=y) for y in years]
|
||||
|
||||
# Multi-source registry lane. Only California publishes a clean bulk CSV (with contact email +
|
||||
# CCPA-rights URL per broker) AND offers a one-shot deletion portal (DROP). Vermont, Oregon, and
|
||||
# Texas maintain registries too, but only as searchable PORTALS (no reliable bulk export) and with
|
||||
# no DROP-equivalent -- and they overlap CA heavily (CA is effectively the superset). So they are
|
||||
# wired as first-class portal sources (official URL surfaced to the operator) rather than scraped.
|
||||
# Adding any state that later publishes a CSV is a one-line "format: csv" entry (the parser is
|
||||
# column-detection based, not CA-specific).
|
||||
SOURCES = {
|
||||
"ca": {"jurisdiction": "US-CA", "format": "csv", "url": DEFAULT_URL, "has_drop": True,
|
||||
"name": "California Data Broker Registry (CPPA)"},
|
||||
"vt": {"jurisdiction": "US-VT", "format": "portal", "has_drop": False,
|
||||
"url": "https://bizfilings.vermont.gov/online/DatabrokerInquire/",
|
||||
"name": "Vermont Data Broker Registry (Secretary of State)"},
|
||||
"or": {"jurisdiction": "US-OR", "format": "portal", "has_drop": False,
|
||||
"url": "https://dfr.oregon.gov/business/licensing/data-broker-registry/Pages/index.aspx",
|
||||
"name": "Oregon Data Broker Registry (DCBS)"},
|
||||
"tx": {"jurisdiction": "US-TX", "format": "portal", "has_drop": False,
|
||||
"url": "https://texas-sos.appianportalsgov.com/data-broker-registry",
|
||||
"name": "Texas Data Broker Registry (Secretary of State)"},
|
||||
}
|
||||
|
||||
|
||||
def portals() -> list[dict]:
|
||||
"""Registry sources that are searchable portals (no bulk export) -- surfaced to the operator."""
|
||||
return [{"key": k, "jurisdiction": s["jurisdiction"], "name": s["name"], "url": s["url"]}
|
||||
for k, s in SOURCES.items() if s["format"] == "portal"]
|
||||
|
||||
# Field label -> substring to locate its column on the header row (robust to
|
||||
# year-to-year column shifts; the registry re-orders/adds columns between years).
|
||||
_LABELS = {
|
||||
"name": "data broker name:",
|
||||
"dba": "doing business as",
|
||||
"website": "data broker primary website:",
|
||||
"email": "primary contact email",
|
||||
"rights_url": "exercise their ca consumer privacy act rights",
|
||||
"fcra": "regulated by the federal fair credit reporting act (fcra):",
|
||||
}
|
||||
|
||||
|
||||
def _norm(s: str) -> str:
|
||||
"""Registry CSVs use NBSPs and a BOM; normalize for matching + clean values."""
|
||||
return re.sub(r"\s+", " ", (s or "").replace("\ufeff", "").replace("\xa0", " ")).strip()
|
||||
|
||||
|
||||
def slug(name: str, website: str = "") -> str:
|
||||
base = re.sub(r"\.(com|org|net|io|ai|inc|co|us|info|llc)\b", "", (name or "").strip(), flags=re.I)
|
||||
s = re.sub(r"[^a-z0-9]+", "", base.lower())
|
||||
if s:
|
||||
return s
|
||||
dom = re.sub(r"^https?://(www\.)?", "", (website or "").lower())
|
||||
return re.sub(r"[^a-z0-9]+", "", dom.split("/")[0]) or "broker"
|
||||
|
||||
|
||||
def _domain(website: str) -> str:
|
||||
dom = re.sub(r"^https?://(www\.)?", "", (website or "").strip().lower())
|
||||
return dom.split("/")[0]
|
||||
|
||||
|
||||
def _find_colmap(rows: list[list[str]]) -> tuple[int, dict[str, int]]:
|
||||
"""Locate the label row (col0 == 'Data broker name:') and map fields to columns."""
|
||||
for i, row in enumerate(rows[:5]):
|
||||
if row and _norm(row[0]).lower().startswith("data broker name:"):
|
||||
colmap: dict[str, int] = {}
|
||||
for field, needle in _LABELS.items():
|
||||
for j, cell in enumerate(row):
|
||||
c = _norm(cell).lower()
|
||||
if needle in c and not c.startswith("if the data broker"):
|
||||
colmap[field] = j
|
||||
break
|
||||
return i, colmap
|
||||
raise ValueError("CA registry: could not locate the header row")
|
||||
|
||||
|
||||
def _get(row: list[str], idx: int | None) -> str:
|
||||
return _norm(row[idx]) if idx is not None and idx < len(row) else ""
|
||||
|
||||
|
||||
def _build(row: list[str], cm: dict[str, int], jurisdiction: str = "US-CA",
|
||||
has_drop: bool = True) -> dict | None:
|
||||
name = _get(row, cm.get("name"))
|
||||
website = _get(row, cm.get("website"))
|
||||
if not (name or website):
|
||||
return None
|
||||
email = _get(row, cm.get("email"))
|
||||
rights = _get(row, cm.get("rights_url"))
|
||||
dba = _get(row, cm.get("dba"))
|
||||
fcra = _get(row, cm.get("fcra")).lower().startswith("y")
|
||||
state = jurisdiction.split("-")[-1]
|
||||
|
||||
method = "email" if email else ("web_form" if rights else "drop")
|
||||
if has_drop:
|
||||
notes = ("Registered CA data broker. One CA DROP request (privacy.ca.gov/drop) deletes from "
|
||||
"this and every registered broker at once; or send a CCPA deletion request to the "
|
||||
"contact email.")
|
||||
else:
|
||||
notes = (f"Registered {state} data broker (no one-shot delete portal in {state}). Send a "
|
||||
"CCPA/state-law deletion request to the contact email.")
|
||||
if fcra:
|
||||
notes += (" FCRA-regulated: some data is credit-reporting data with separate rules -- deletion "
|
||||
"may be limited; a consumer report dispute/security-freeze may apply instead.")
|
||||
return {
|
||||
"id": slug(name, website),
|
||||
"name": name or _domain(website),
|
||||
"dba": dba or None,
|
||||
"category": "data_broker",
|
||||
"priority": "long_tail",
|
||||
"jurisdictions": [jurisdiction],
|
||||
"search": {"method": "none", "url": website, "fetch": "none", "by": ["registry"]},
|
||||
"optout": {
|
||||
"method": method,
|
||||
"url": rights or website or None,
|
||||
"email": email or None,
|
||||
"requires": {"profile_url": False, "email_verification": False, "captcha": False,
|
||||
"gov_id": False, "account": False, "phone_callback": False, "payment": False},
|
||||
"inputs": ["full_name", "contact_email"],
|
||||
"deletion": {
|
||||
"via": "drop" if has_drop else "email",
|
||||
"email": email or None,
|
||||
"url": rights or None,
|
||||
"kinds": ["ccpa", "generic"],
|
||||
"notes": ("Covered by the CA DROP one-shot (privacy.ca.gov/drop); CCPA email fallback."
|
||||
if has_drop else "CCPA/state-law deletion email (no one-shot portal)."),
|
||||
},
|
||||
"fcra": fcra,
|
||||
"est_processing_days": 45,
|
||||
"notes": notes,
|
||||
},
|
||||
"source": f"{state}-registry",
|
||||
"confidence": "registry",
|
||||
"last_verified": None,
|
||||
}
|
||||
|
||||
|
||||
def parse(csv_text: str, jurisdiction: str = "US-CA", has_drop: bool = True) -> list[dict]:
|
||||
"""Parse a data-broker-registry CSV into broker records (deduped by id).
|
||||
|
||||
Column detection is by header label, not fixed position, so any state that publishes a
|
||||
registry CSV with name/website/email/rights columns parses without new code.
|
||||
"""
|
||||
rows = list(csv.reader(io.StringIO(csv_text)))
|
||||
if not rows:
|
||||
return []
|
||||
header_i, cm = _find_colmap(rows)
|
||||
out: list[dict] = []
|
||||
seen: dict[str, int] = {}
|
||||
for row in rows[header_i + 1:]:
|
||||
if not any(c.strip() for c in row):
|
||||
continue
|
||||
rec = _build(row, cm, jurisdiction, has_drop)
|
||||
if not rec:
|
||||
continue
|
||||
bid = rec["id"]
|
||||
if bid in seen: # disambiguate id collisions by domain, then a counter
|
||||
dom = re.sub(r"[^a-z0-9]+", "", _domain(rec["search"]["url"]))
|
||||
cand = f"{bid}-{dom}" if dom and dom != bid else bid
|
||||
while cand in seen:
|
||||
seen[bid] += 1
|
||||
cand = f"{bid}-{seen[bid]}"
|
||||
rec["id"] = cand
|
||||
seen.setdefault(rec["id"], 0)
|
||||
seen.setdefault(bid, 0)
|
||||
out.append(rec)
|
||||
return out
|
||||
|
||||
|
||||
MIN_EXPECTED_CA = 100 # CA registry has ~500+; far fewer => wrong/empty file, warn
|
||||
|
||||
|
||||
def fetch(url: str = DEFAULT_URL, timeout: int = 60) -> str:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _fetch_ca_latest() -> tuple[str, list[dict]]:
|
||||
"""Try newest CA registry year first; return (url, records) for the first non-empty."""
|
||||
last: tuple[str, list[dict]] = (DEFAULT_URL, [])
|
||||
for url in ca_candidate_urls():
|
||||
try:
|
||||
recs = parse(fetch(url), jurisdiction="US-CA", has_drop=True)
|
||||
except Exception: # noqa: BLE001 - a missing year 404s; fall through to older years
|
||||
continue
|
||||
if recs:
|
||||
return url, recs
|
||||
last = (url, recs)
|
||||
return last
|
||||
|
||||
|
||||
def refresh(cache_path: Path, url: str = DEFAULT_URL, csv_text: str | None = None) -> dict:
|
||||
"""CA single-source refresh: fetch (or accept) the CA CSV and write the cache."""
|
||||
text = csv_text if csv_text is not None else fetch(url)
|
||||
records = parse(text)
|
||||
storage.write_json(cache_path, records)
|
||||
fcra = sum(1 for r in records if (r.get("optout") or {}).get("fcra"))
|
||||
return {"parsed": len(records), "fcra_regulated": fcra,
|
||||
"cache_path": str(cache_path), "source_url": url}
|
||||
|
||||
|
||||
def refresh_all(cache_path: Path, fetched: dict[str, str] | None = None) -> dict:
|
||||
"""Multi-source refresh: pull every CSV source, dedupe across states by domain, cache.
|
||||
|
||||
`fetched` optionally supplies {source_key: csv_text} to bypass the network (tests). CSV
|
||||
sources are ingested as broker records; portal sources contribute their URL for the operator
|
||||
(no bulk export exists) but no records. CA is processed first so it wins domain collisions.
|
||||
"""
|
||||
all_recs: list[dict] = []
|
||||
seen_domains: set[str] = set()
|
||||
per_source: dict[str, dict] = {}
|
||||
for key, src in SOURCES.items():
|
||||
if src["format"] != "csv":
|
||||
per_source[key] = {"jurisdiction": src["jurisdiction"], "format": "portal",
|
||||
"url": src["url"], "records": 0,
|
||||
"note": "searchable portal (no bulk export); operator/agent searches by name"}
|
||||
continue
|
||||
used_url = src["url"]
|
||||
try:
|
||||
if fetched is not None:
|
||||
text = fetched.get(key)
|
||||
if text is None:
|
||||
raise RuntimeError("no CSV text supplied")
|
||||
recs = parse(text, jurisdiction=src["jurisdiction"], has_drop=src["has_drop"])
|
||||
elif key == "ca":
|
||||
used_url, recs = _fetch_ca_latest() # newest-year-first with fallback
|
||||
else:
|
||||
recs = parse(fetch(src["url"]), jurisdiction=src["jurisdiction"], has_drop=src["has_drop"])
|
||||
except Exception as exc: # noqa: BLE001 - one source failing must not sink the rest
|
||||
per_source[key] = {"jurisdiction": src["jurisdiction"], "format": "csv", "error": str(exc)}
|
||||
continue
|
||||
added = 0
|
||||
for r in recs:
|
||||
dom = _domain(r["search"]["url"])
|
||||
if dom and dom in seen_domains:
|
||||
continue
|
||||
if dom:
|
||||
seen_domains.add(dom)
|
||||
all_recs.append(r)
|
||||
added += 1
|
||||
entry = {"jurisdiction": src["jurisdiction"], "format": "csv", "url": used_url,
|
||||
"parsed": len(recs), "added_after_dedupe": added,
|
||||
"fcra": sum(1 for r in recs if (r.get("optout") or {}).get("fcra"))}
|
||||
if key == "ca" and len(recs) < MIN_EXPECTED_CA:
|
||||
entry["warning"] = (f"only {len(recs)} parsed (expected >{MIN_EXPECTED_CA}); the CA "
|
||||
"registry file may be empty/moved - verify the source URL")
|
||||
per_source[key] = entry
|
||||
storage.write_json(cache_path, all_recs)
|
||||
return {"total": len(all_recs), "sources": per_source, "portals": portals(),
|
||||
"cache_path": str(cache_path)}
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Status dashboards, Markdown reports, human-task digest, and Google Sheets row export."""
|
||||
from __future__ import annotations
|
||||
|
||||
import brokers as brokers_mod
|
||||
import ledger as ledger_mod
|
||||
|
||||
STATE_LABELS = {
|
||||
"new": "Not started",
|
||||
"searching": "Searching",
|
||||
"not_found": "Not found",
|
||||
"found": "Found (action needed)",
|
||||
"indirect_exposure": "Indirect exposure (PII on a relative's record)",
|
||||
"action_selected": "Action selected",
|
||||
"submitted": "Submitted",
|
||||
"verification_pending": "Awaiting verification",
|
||||
"awaiting_processing": "Processing",
|
||||
"confirmed_removed": "Removed",
|
||||
"reappeared": "Reappeared",
|
||||
"human_task_queued": "Human task",
|
||||
"blocked": "Blocked",
|
||||
}
|
||||
|
||||
|
||||
def status_counts(subject_id: str) -> dict:
|
||||
counts: dict[str, int] = {}
|
||||
for case in ledger_mod.load(subject_id).values():
|
||||
state = case.get("state", "new")
|
||||
counts[state] = counts.get(state, 0) + 1
|
||||
return counts
|
||||
|
||||
|
||||
def metrics(subject_id: str) -> dict:
|
||||
"""Outcome metrics: what's actually confirmed vs merely claimed, and what's overdue.
|
||||
|
||||
removal_rate is confirmed_removed over cases we actually acted on (found/submitted/... ),
|
||||
NOT over the whole broker DB, so it reflects real progress on real exposure. `in_flight`
|
||||
is 'claimed' (submitted/verifying/processing) but not yet re-scan-confirmed. `overdue`
|
||||
counts cases whose recheck window has already passed (the cron backlog).
|
||||
"""
|
||||
c = status_counts(subject_id)
|
||||
removed = c.get("confirmed_removed", 0)
|
||||
in_flight = c.get("submitted", 0) + c.get("verification_pending", 0) + c.get("awaiting_processing", 0)
|
||||
open_found = c.get("found", 0) + c.get("reappeared", 0) + c.get("action_selected", 0) \
|
||||
+ c.get("indirect_exposure", 0)
|
||||
acted = removed + in_flight + open_found + c.get("human_task_queued", 0) + c.get("blocked", 0)
|
||||
return {
|
||||
"confirmed_removed": removed,
|
||||
"in_flight_claimed": in_flight, # submitted but NOT yet verified gone
|
||||
"open_needs_action": open_found,
|
||||
"blocked": c.get("blocked", 0),
|
||||
"human_tasks": c.get("human_task_queued", 0),
|
||||
"acted_total": acted,
|
||||
"removal_rate": round(removed / acted, 3) if acted else 0.0,
|
||||
"overdue_rechecks": len(ledger_mod.due(subject_id)),
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(subject_id: str) -> str:
|
||||
ledger = ledger_mod.load(subject_id)
|
||||
counts = status_counts(subject_id)
|
||||
total = sum(counts.values())
|
||||
removed = counts.get("confirmed_removed", 0)
|
||||
|
||||
m = metrics(subject_id)
|
||||
lines = [
|
||||
f"# unbroker - status for `{subject_id}`",
|
||||
"",
|
||||
f"**{removed} / {total} confirmed removed** · removal rate (of acted-on cases): "
|
||||
f"{int(m['removal_rate'] * 100)}%",
|
||||
"",
|
||||
f"- Confirmed removed: {m['confirmed_removed']}",
|
||||
f"- In flight (submitted, not yet re-scan-confirmed): {m['in_flight_claimed']}",
|
||||
f"- Open / needs action: {m['open_needs_action']}",
|
||||
f"- Blocked (anti-bot): {m['blocked']} · Human tasks: {m['human_tasks']}",
|
||||
f"- Overdue rechecks (cron backlog): {m['overdue_rechecks']}",
|
||||
"",
|
||||
"| State | Count |",
|
||||
"|---|---|",
|
||||
]
|
||||
for state in ledger_mod.STATES:
|
||||
if counts.get(state):
|
||||
lines.append(f"| {STATE_LABELS.get(state, state)} | {counts[state]} |")
|
||||
|
||||
tasks = [c for c in ledger.values() if c.get("state") == "human_task_queued"]
|
||||
if tasks:
|
||||
lines += ["", "## Outstanding human tasks"]
|
||||
for c in tasks:
|
||||
reason = c.get("human_task_reason", "manual step required")
|
||||
lines.append(f"- **{c.get('broker_id')}** - {reason}")
|
||||
|
||||
indirect = [c for c in ledger.values() if c.get("state") == "indirect_exposure"]
|
||||
if indirect:
|
||||
lines += ["", "## Indirect exposure (your PII on third-party records)",
|
||||
"Not removable via the broker's self-service opt-out (the record is about someone "
|
||||
"else). Lever: a targeted CCPA/GDPR delete-my-PII request naming only your own "
|
||||
"identifiers."]
|
||||
for c in indirect:
|
||||
ev = c.get("evidence") or {}
|
||||
note = ev.get("summary") or "subject's identifiers appear on another person's listing"
|
||||
lines.append(f"- **{c.get('broker_id')}** - {note}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def human_tasks_markdown(subject_id: str) -> str:
|
||||
"""ONE consolidated digest of everything that genuinely needs a human.
|
||||
|
||||
The autonomous run accumulates human-only work silently (never interrupting);
|
||||
this digest is presented once, at the end, so the operator clears it in a
|
||||
single sitting. Includes queued tasks and blocked-site operator-browser checks.
|
||||
"""
|
||||
ledger = ledger_mod.load(subject_id)
|
||||
tasks = [(bid, c) for bid, c in sorted(ledger.items()) if c.get("state") == "human_task_queued"]
|
||||
blocked = [(bid, c) for bid, c in sorted(ledger.items()) if c.get("state") == "blocked"]
|
||||
|
||||
lines = [f"# Human tasks for `{subject_id}`", ""]
|
||||
if not tasks and not blocked:
|
||||
lines.append("Nothing needs a human right now.")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
lines.append(f"{len(tasks)} manual step(s) + {len(blocked)} blocked site(s). "
|
||||
"Everything else ran (or will run) autonomously.")
|
||||
if tasks:
|
||||
lines += ["", "## Manual steps"]
|
||||
for bid, c in tasks:
|
||||
b = brokers_mod.get(bid) or {}
|
||||
opt = b.get("optout") or {}
|
||||
lines.append(f"### {b.get('name', bid)}")
|
||||
lines.append(f"- Why: {c.get('human_task_reason', 'manual step required')}")
|
||||
where = opt.get("url") or opt.get("email") or "(see broker record)"
|
||||
lines.append(f"- Where: {where}")
|
||||
for q in (opt.get("quirks") or [])[:2]:
|
||||
lines.append(f"- Note: {q}")
|
||||
lines.append("- Withhold: SSN and full ID numbers - always.")
|
||||
lines.append(f"- When done, tell the agent so it records the outcome for `{bid}`.")
|
||||
if blocked:
|
||||
lines += ["", "## Blocked sites (open in YOUR browser - it gets through where bots don't)"]
|
||||
for bid, c in blocked:
|
||||
b = brokers_mod.get(bid) or {}
|
||||
url = ((b.get("search") or {}).get("url")) or "(see broker record)"
|
||||
lines.append(f"- **{b.get('name', bid)}** - open {url}, search the subject, and report "
|
||||
"the verdict (or a screenshot) back to the agent.")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def sheets_rows(subject_id: str) -> list[list[str]]:
|
||||
"""Header + one row per case for the optional Google Sheets tracker.
|
||||
|
||||
The agent appends these via the `google-workspace` skill, e.g.:
|
||||
google_api.py sheets append <SHEET_ID> "Sheet1!A:F" --values <json-rows>
|
||||
"""
|
||||
rows = [["broker_id", "state", "found", "tier", "removed_at", "next_recheck"]]
|
||||
for bid, c in sorted(ledger_mod.load(subject_id).items()):
|
||||
rows.append([
|
||||
bid,
|
||||
c.get("state", ""),
|
||||
str(c.get("found", "")),
|
||||
(c.get("automation") or {}).get("tier_used", ""),
|
||||
c.get("removal_confirmed_at") or "",
|
||||
c.get("next_recheck_at") or "",
|
||||
])
|
||||
return rows
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Stdlib fetch helper for simple url_pattern brokers (osint-style).
|
||||
|
||||
For JS-rendered or anti-bot pages the agent should use the `web_extract` or
|
||||
`browser_navigate` tools (and the `scrapling` skill for stealth/Cloudflare).
|
||||
This helper only covers plain static pages and is intentionally network-light so
|
||||
it can be mocked in tests.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
USER_AGENT = "Mozilla/5.0 (compatible; unbroker/1.0; data opt-out)"
|
||||
|
||||
|
||||
def fetch(url: str, timeout: int = 20) -> tuple[int, str]:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 (https only by convention)
|
||||
charset = resp.headers.get_content_charset() or "utf-8"
|
||||
return getattr(resp, "status", 200), resp.read().decode(charset, errors="replace")
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, ""
|
||||
except (urllib.error.URLError, TimeoutError, ValueError):
|
||||
return 0, ""
|
||||
|
||||
|
||||
def looks_listed(html: str, match_signal: str | None) -> bool:
|
||||
"""Naive confirmation heuristic for static pages: does the match signal appear?"""
|
||||
if not html or not match_signal:
|
||||
return False
|
||||
return match_signal.lower() in html.lower()
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Storage helpers (stdlib only): atomic JSON, append-only JSONL, strict perms.
|
||||
|
||||
Default backend is local-json. The optional google-sheets tracker is handled in
|
||||
report.py by emitting rows for the `google-workspace` skill; this module stays
|
||||
dependency-free so the hermetic tests never touch the network.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import crypto
|
||||
import paths
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def locked(target: Path, timeout: float = 10.0, stale: float = 30.0):
|
||||
"""Portable advisory lock via an O_EXCL lockfile next to `target`.
|
||||
|
||||
Serializes read-modify-write on shared JSON (the ledger) across concurrent
|
||||
processes - a cron re-scan overlapping a manual run, or multiple tenants -
|
||||
so one writer can't clobber another's update. A lock older than `stale`
|
||||
seconds is treated as abandoned (crashed writer) and broken, so a dead
|
||||
process can never deadlock the queue. Works on macOS/Linux/Windows (O_EXCL).
|
||||
"""
|
||||
ensure_dir(target.parent)
|
||||
lock = target.with_name(target.name + ".lock")
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
try:
|
||||
fd = os.open(str(lock), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||
try:
|
||||
os.write(fd, str(os.getpid()).encode())
|
||||
finally:
|
||||
os.close(fd)
|
||||
break
|
||||
except FileExistsError:
|
||||
try:
|
||||
if time.time() - lock.stat().st_mtime > stale:
|
||||
lock.unlink(missing_ok=True)
|
||||
continue
|
||||
except OSError:
|
||||
pass
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError(f"could not acquire lock {lock} within {timeout}s")
|
||||
time.sleep(0.05)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with contextlib.suppress(OSError):
|
||||
lock.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _secure(path: Path, mode: int) -> None:
|
||||
try:
|
||||
os.chmod(path, mode)
|
||||
except OSError:
|
||||
pass # non-POSIX / unsupported FS; HERMES_HOME directory perms still apply
|
||||
|
||||
|
||||
def ensure_dir(path: Path) -> Path:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
_secure(path, 0o700)
|
||||
return path
|
||||
|
||||
|
||||
def _is_sensitive(path: Path) -> bool:
|
||||
"""Per-subject docs (dossier, ledger) are sensitive; config/cache are not."""
|
||||
try:
|
||||
Path(path).resolve().relative_to(paths.subjects_dir().resolve())
|
||||
return True
|
||||
except (ValueError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def _age_path(path: Path) -> Path:
|
||||
return path.with_name(path.name + ".age")
|
||||
|
||||
|
||||
def _atomic_write(path: Path, data: bytes) -> Path:
|
||||
tmp = path.with_name(path.name + ".tmp")
|
||||
tmp.write_bytes(data)
|
||||
_secure(tmp, 0o600)
|
||||
os.replace(tmp, path)
|
||||
_secure(path, 0o600)
|
||||
return path
|
||||
|
||||
|
||||
def write_json(path: Path, obj: Any) -> Path:
|
||||
ensure_dir(path.parent)
|
||||
data = (json.dumps(obj, indent=2, ensure_ascii=False) + "\n").encode("utf-8")
|
||||
if _is_sensitive(path) and crypto.encryption_setting() == "age":
|
||||
if not crypto.age_available():
|
||||
raise RuntimeError(
|
||||
"encryption=age is configured but `age` is not available; "
|
||||
"refusing to write PII as plaintext. Install age or run `setup --encryption none`."
|
||||
)
|
||||
target = _atomic_write(_age_path(path), crypto.encrypt(data))
|
||||
if path.exists():
|
||||
path.unlink() # migrate plaintext -> ciphertext
|
||||
return target
|
||||
target = _atomic_write(path, data)
|
||||
ap = _age_path(path)
|
||||
if ap.exists():
|
||||
ap.unlink() # encryption turned off -> drop stale ciphertext
|
||||
return target
|
||||
|
||||
|
||||
def read_json(path: Path, default: Any = None) -> Any:
|
||||
ap = _age_path(path)
|
||||
if ap.exists():
|
||||
return json.loads(crypto.decrypt(ap.read_bytes()).decode("utf-8"))
|
||||
if path.exists():
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
return default
|
||||
|
||||
|
||||
def append_jsonl(path: Path, record: dict) -> Path:
|
||||
ensure_dir(path.parent)
|
||||
with path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
_secure(path, 0o600)
|
||||
return path
|
||||
|
||||
|
||||
def read_jsonl(path: Path) -> list[dict]:
|
||||
if not path.exists():
|
||||
return []
|
||||
out: list[dict] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
out.append(json.loads(line))
|
||||
return out
|
||||
@@ -0,0 +1,283 @@
|
||||
"""Automation-tier selection and per-subject action planning.
|
||||
|
||||
Tiers:
|
||||
T0 fully automated, no verification loop
|
||||
T1 automated submit + automated verification (email mode B/C, or backend-cleared captcha)
|
||||
T2 automated submit, verification needs a human (hard captcha / phone callback / account)
|
||||
T3 human-required end-to-end (gov ID, fax, mail, voice-only phone)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import dossier as dossier_mod
|
||||
import vectors as vectors_mod
|
||||
|
||||
HARD_HUMAN = ("gov_id", "fax", "mail", "phone_voice")
|
||||
|
||||
|
||||
def select_tier(broker: dict, email_mode: str = "draft_only",
|
||||
browser_clears_captcha: bool = False) -> str:
|
||||
req = ((broker.get("optout") or {}).get("requires")) or {}
|
||||
if not isinstance(req, dict):
|
||||
req = {} # defensive: a malformed record (e.g. requires as a list) must not crash planning
|
||||
|
||||
if any(req.get(k) for k in HARD_HUMAN):
|
||||
return "T3"
|
||||
if req.get("account"):
|
||||
return "T2"
|
||||
|
||||
captcha = bool(req.get("captcha"))
|
||||
if (captcha and not browser_clears_captcha) or req.get("phone_callback"):
|
||||
return "T2"
|
||||
|
||||
if req.get("email_verification"):
|
||||
return "T1" if email_mode in ("programmatic", "alias") else "T2"
|
||||
|
||||
if captcha and browser_clears_captcha:
|
||||
return "T1"
|
||||
return "T0"
|
||||
|
||||
|
||||
def plan(subject_dossier: dict, brokers_list: list[dict], cfg: dict,
|
||||
browser_clears_captcha: bool = False) -> list[dict]:
|
||||
email_mode = (subject_dossier.get("preferences") or {}).get("email_mode") \
|
||||
or cfg.get("email_mode", "draft_only")
|
||||
actions: list[dict] = []
|
||||
for b in brokers_list:
|
||||
opt = b.get("optout") or {}
|
||||
search = b.get("search") or {}
|
||||
# Defensive shape coercion: a subagent may have written a malformed record (requires as a
|
||||
# list, quirks as a string). Normalize here so nothing downstream crashes on a bad broker file.
|
||||
req = opt.get("requires") if isinstance(opt.get("requires"), dict) else {}
|
||||
q = opt.get("quirks")
|
||||
quirks = q if isinstance(q, list) else ([q] if isinstance(q, str) and q else [])
|
||||
tier = select_tier(b, email_mode, browser_clears_captcha)
|
||||
disclosure = dossier_mod.select_disclosure(subject_dossier, opt.get("inputs", []))
|
||||
svectors = vectors_mod.search_vectors(subject_dossier, b)
|
||||
# Pre-warn (don't discover mid-flow): a broker whose identity gate hard-requires DOB will
|
||||
# force a human touchpoint if DOB was not collected at intake (§4.1). Surface it now.
|
||||
prewarn: list[str] = []
|
||||
if req.get("dob") and not (subject_dossier.get("identity") or {}).get("date_of_birth"):
|
||||
prewarn.append("date_of_birth: this broker's identity gate requires DOB to match records; "
|
||||
"collect it up front (intake --dob) or expect a mid-flow human pause")
|
||||
actions.append({
|
||||
"broker_id": b.get("id"),
|
||||
"broker_name": b.get("name"),
|
||||
"priority": b.get("priority"),
|
||||
"method": opt.get("method"),
|
||||
"tier": tier,
|
||||
"human_required": tier == "T3",
|
||||
"search_url": search.get("url"),
|
||||
"fetch": search.get("fetch", "web_extract"),
|
||||
"antibot": search.get("antibot"),
|
||||
"search_by": vectors_mod.supported_by(b),
|
||||
"search_vectors": svectors,
|
||||
"optout_url": opt.get("url"),
|
||||
"optout_email": opt.get("email"),
|
||||
"disclosure_fields": sorted(disclosure.keys()),
|
||||
"needs_operator_input": prewarn,
|
||||
"owns": b.get("owns") or [],
|
||||
"notes": opt.get("notes", ""),
|
||||
"optout_quirks": quirks,
|
||||
"optout_requires": req,
|
||||
# The DELETION lane (right-to-delete), distinct from listing suppression. Structured so
|
||||
# the autopilot can route to it: {via: email|in_flow|web_form, email?, url?, kinds?, notes?}
|
||||
"deletion": opt.get("deletion") or {},
|
||||
# Exact ordered opt-out steps maintained IN the broker record (field-verified knowledge
|
||||
# lives with the data, not in code).
|
||||
"optout_playbook": opt.get("playbook") or [],
|
||||
})
|
||||
return actions
|
||||
|
||||
|
||||
def fanout(brokers_list: list[dict], batch_size: int = 5) -> dict:
|
||||
"""Group brokers into batches for parallel `delegate_task` scan subagents.
|
||||
|
||||
Scanning many brokers serially is slow and burns context; above `batch_size`
|
||||
the agent is expected to spawn one subagent per batch (see SKILL.md).
|
||||
"""
|
||||
ids = [b.get("id") for b in brokers_list if b.get("id")]
|
||||
batches = [ids[i:i + batch_size] for i in range(0, len(ids), batch_size)]
|
||||
return {
|
||||
"broker_count": len(ids),
|
||||
"batch_size": batch_size,
|
||||
"should_fanout": len(ids) > batch_size,
|
||||
"batches": batches,
|
||||
}
|
||||
|
||||
|
||||
# States that mean "the crawl reached a verdict for this broker".
|
||||
_SCANNED_STATES = {"found", "not_found", "indirect_exposure", "blocked", "submitted",
|
||||
"verification_pending", "awaiting_processing", "confirmed_removed", "reappeared",
|
||||
"action_selected", "human_task_queued"}
|
||||
# States that still need a deletion action taken.
|
||||
_ACTIONABLE_STATES = {"found", "indirect_exposure", "reappeared", "action_selected"}
|
||||
|
||||
|
||||
def batch_plan(subject_dossier: dict, brokers_list: list[dict], cfg: dict,
|
||||
ledger: dict | None = None, browser_clears_captcha: bool = False) -> dict:
|
||||
"""Reduce the per-broker plan into a phase-oriented batch view.
|
||||
|
||||
Overlays the current ledger state on each broker, groups by what the operator
|
||||
should DO next, and collapses ownership clusters so a parent removal that clears
|
||||
children is ONE action, not N. Read-only: computes, never mutates the ledger.
|
||||
"""
|
||||
ledger = ledger or {}
|
||||
actions = plan(subject_dossier, brokers_list, cfg, browser_clears_captcha)
|
||||
|
||||
# child id -> parent id (only for parents present in this plan set)
|
||||
child_to_parent: dict[str, str] = {}
|
||||
for a in actions:
|
||||
for child in a.get("owns") or []:
|
||||
child_to_parent[child] = a["broker_id"]
|
||||
|
||||
def state_of(bid: str) -> str:
|
||||
return (ledger.get(bid) or {}).get("state", "new")
|
||||
|
||||
groups: dict[str, list[dict]] = {
|
||||
"unscanned": [], # no verdict yet -> Phase 1 crawl
|
||||
"found": [], # direct removable listing -> Phase 2 opt-out (incl. reappeared/action_selected)
|
||||
"indirect_exposure": [],# PII on a third party's record -> CCPA/GDPR delete email
|
||||
"blocked": [], # anti-bot / needs stealth browser -> requeue
|
||||
"in_progress": [], # submitted / verification_pending / awaiting_processing
|
||||
"human": [], # human_task_queued -> the end-of-run digest, NOT re-scanning
|
||||
"done": [], # confirmed_removed
|
||||
"not_found": [],
|
||||
}
|
||||
covered_by_parent: dict[str, list[str]] = {}
|
||||
|
||||
for a in actions:
|
||||
bid = a["broker_id"]
|
||||
st = state_of(bid)
|
||||
# cluster collapse: if a parent in this set is already actioned, the child is covered
|
||||
parent = child_to_parent.get(bid)
|
||||
if parent and state_of(parent) in ("found", "reappeared", "action_selected", "submitted",
|
||||
"verification_pending", "awaiting_processing",
|
||||
"confirmed_removed", "human_task_queued"):
|
||||
covered_by_parent.setdefault(parent, []).append(bid)
|
||||
continue
|
||||
|
||||
row = {"broker_id": bid, "broker_name": a["broker_name"], "priority": a["priority"],
|
||||
"tier": a["tier"], "method": a["method"], "state": st,
|
||||
"optout_url": a["optout_url"], "optout_email": a.get("optout_email"),
|
||||
"clears_children": a.get("owns") or [],
|
||||
"optout_requires": a.get("optout_requires") or {},
|
||||
"optout_quirks": a.get("optout_quirks") or [],
|
||||
"deletion": a.get("deletion") or {},
|
||||
"optout_playbook": a.get("optout_playbook") or [],
|
||||
"notes": a.get("notes", "")}
|
||||
if st in ("submitted", "verification_pending", "awaiting_processing"):
|
||||
groups["in_progress"].append(row)
|
||||
elif st == "confirmed_removed":
|
||||
groups["done"].append(row)
|
||||
elif st in ("reappeared", "action_selected"):
|
||||
groups["found"].append(row) # still needs the opt-out action
|
||||
elif st == "human_task_queued":
|
||||
groups["human"].append(row) # parked for the digest; never re-queued as work
|
||||
elif st in groups:
|
||||
groups[st].append(row)
|
||||
elif st not in _SCANNED_STATES:
|
||||
groups["unscanned"].append(row)
|
||||
else:
|
||||
groups.setdefault(st, []).append(row)
|
||||
|
||||
# PARENTS FIRST: within the actionable 'found' group, order cluster parents (a removal
|
||||
# that clears children) ahead of standalone listings, most-children first. Working a
|
||||
# parent before its children is what makes the cluster dedup real -- do them in this order.
|
||||
groups["found"].sort(key=lambda r: (-len(r.get("clears_children") or []),
|
||||
{"T0": 0, "T1": 1, "T2": 2, "T3": 3}.get(r.get("tier") or "", 9),
|
||||
r["broker_id"]))
|
||||
|
||||
return {
|
||||
"subject": subject_dossier.get("subject_id"),
|
||||
"phase": "discover" if groups["unscanned"] else "delete",
|
||||
"counts": {k: len(v) for k, v in groups.items()},
|
||||
"groups": groups,
|
||||
"cluster_savings": {p: kids for p, kids in covered_by_parent.items()},
|
||||
"parent_playbook": _parent_playbook(groups["found"]),
|
||||
"next_actions": _batch_next(groups, covered_by_parent),
|
||||
}
|
||||
|
||||
|
||||
def synthesize_steps(r: dict) -> list[str]:
|
||||
"""Generic ordered opt-out steps derived from an optout record's structured fields.
|
||||
|
||||
Used for any broker without a hand-verified `optout.playbook`. Bespoke, field-verified
|
||||
step lists live IN the broker JSON (`optout.playbook`) - single source of truth that
|
||||
accrues knowledge as live runs discover mechanics (see methods.md logging rule).
|
||||
"""
|
||||
steps = [f"Opt out at {r.get('optout_url') or r.get('optout_email') or '(see broker record)'}"
|
||||
+ (f" -- clears {', '.join(r['clears_children'])}." if r.get("clears_children") else ".")]
|
||||
req = r.get("optout_requires") or {}
|
||||
if req.get("profile_url"):
|
||||
steps.append("Needs the confirmed profile_url (paste the listing URL you recorded).")
|
||||
if req.get("email_verification"):
|
||||
steps.append("Email verification: the same browser/inbox must open the confirmation link.")
|
||||
if req.get("phone_callback"):
|
||||
steps.append("Phone-callback code required; queue a human task if no operator is available.")
|
||||
if req.get("gov_id"):
|
||||
steps.append("Government ID demanded (T3): human task; never send SSN or a full ID number.")
|
||||
d = r.get("deletion") or {}
|
||||
if d.get("email"):
|
||||
steps.append(f"DELETION lane: a right-to-delete request can be emailed to {d['email']}"
|
||||
+ (f" ({d['notes']})" if d.get("notes") else "")
|
||||
+ " -- prefer deletion over suppression.")
|
||||
if r.get("notes"):
|
||||
steps.append(str(r["notes"]))
|
||||
for q in (r.get("optout_quirks") or [])[:3]:
|
||||
steps.append(str(q))
|
||||
return steps
|
||||
|
||||
|
||||
def _parent_playbook(found_rows: list[dict]) -> list[dict]:
|
||||
"""Tailored, ordered opt-out instructions for each cluster PARENT in the found group.
|
||||
|
||||
Steps come from the broker record's own `optout.playbook` (field-verified, maintained with
|
||||
the data) with a synthesised fallback so the guidance is never empty. Standalone listings
|
||||
are intentionally omitted -- the playbook exists to make the parents-first order concrete.
|
||||
"""
|
||||
playbook: list[dict] = []
|
||||
for i, r in enumerate([x for x in found_rows if x.get("clears_children")], start=1):
|
||||
steps = list(r.get("optout_playbook") or []) or synthesize_steps(r)
|
||||
playbook.append({
|
||||
"order": i,
|
||||
"broker_id": r["broker_id"],
|
||||
"broker_name": r["broker_name"],
|
||||
"tier": r["tier"],
|
||||
"clears_children": r["clears_children"],
|
||||
"optout_url": r.get("optout_url"),
|
||||
"optout_email": r.get("optout_email"),
|
||||
"deletion": r.get("deletion") or {},
|
||||
"steps": steps,
|
||||
})
|
||||
return playbook
|
||||
|
||||
|
||||
def _batch_next(groups: dict, covered: dict) -> list[str]:
|
||||
tips: list[str] = []
|
||||
if groups["unscanned"]:
|
||||
tips.append(f"PHASE 1 (crawl): {len(groups['unscanned'])} broker(s) unscanned -- run `fanout` and "
|
||||
"scan read-only before any deletion.")
|
||||
if groups["found"]:
|
||||
parents = [r for r in groups["found"] if r.get("clears_children")]
|
||||
if parents:
|
||||
order = " -> ".join(r["broker_id"] for r in parents)
|
||||
tips.append(f"PHASE 2 (opt-out): {len(groups['found'])} direct listing(s). DO CLUSTER PARENTS "
|
||||
f"FIRST, in this order: {order} (see `parent_playbook` for tailored per-parent "
|
||||
"steps), then the standalone listings.")
|
||||
else:
|
||||
tips.append(f"PHASE 2 (opt-out): {len(groups['found'])} direct listing(s) to remove.")
|
||||
if groups["indirect_exposure"]:
|
||||
tips.append(f"{len(groups['indirect_exposure'])} indirect-exposure case(s): send a targeted "
|
||||
"CCPA/GDPR delete-my-PII email (render-email --kind ccpa_indirect), do NOT use the opt-out form.")
|
||||
if groups["blocked"]:
|
||||
tips.append(f"{len(groups['blocked'])} blocked (anti-bot): requeue for a stealth/cloud browser "
|
||||
"pass; don't burn subagent time fighting CAPTCHAs.")
|
||||
if covered:
|
||||
n = sum(len(v) for v in covered.values())
|
||||
tips.append(f"Cluster dedup: {n} child site(s) covered by parent removals -- skip separate opt-outs.")
|
||||
if groups["in_progress"]:
|
||||
tips.append(f"{len(groups['in_progress'])} in progress: resolve verification links, then confirm removal.")
|
||||
if groups.get("human"):
|
||||
tips.append(f"{len(groups['human'])} parked human task(s): present via `tasks` at end of run "
|
||||
"(do not re-scan or re-queue them).")
|
||||
return tips
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Enumerate the search queries to run per broker, across ALL of a subject's identifiers.
|
||||
|
||||
People-search sites index a person under every name, phone, email, and address they
|
||||
have. A subject with two names (maiden/married) and three past cities can have many
|
||||
distinct listings on one broker, each found via a different search. `search_vectors`
|
||||
expands the dossier into the concrete searches to run, filtered by what each broker
|
||||
supports (`broker.search.by`, default ["name"]).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import dossier as dossier_mod
|
||||
|
||||
# What a broker can be searched by; default if a record doesn't declare it.
|
||||
DEFAULT_BY = ["name"]
|
||||
|
||||
|
||||
def supported_by(broker: dict) -> list[str]:
|
||||
return list((broker.get("search") or {}).get("by") or DEFAULT_BY)
|
||||
|
||||
|
||||
def search_vectors(subject_dossier: dict, broker: dict) -> list[dict]:
|
||||
"""List of {by, query} searches to run for this subject on this broker."""
|
||||
by = set(supported_by(broker))
|
||||
ident = subject_dossier.get("identity", {})
|
||||
vectors: list[dict] = []
|
||||
|
||||
if "name" in by:
|
||||
names = dossier_mod.all_names(subject_dossier)
|
||||
locations = dossier_mod.all_locations(subject_dossier)
|
||||
if locations:
|
||||
for name in names:
|
||||
for loc in locations:
|
||||
vectors.append({"by": "name",
|
||||
"query": {"full_name": name, "city": loc.get("city"), "state": loc.get("state")}})
|
||||
else:
|
||||
for name in names:
|
||||
vectors.append({"by": "name", "query": {"full_name": name}})
|
||||
|
||||
if "phone" in by:
|
||||
for phone in ident.get("phones") or []:
|
||||
vectors.append({"by": "phone", "query": {"phone": phone}})
|
||||
|
||||
if "email" in by:
|
||||
for email in ident.get("emails") or []:
|
||||
vectors.append({"by": "email", "query": {"email": email}})
|
||||
|
||||
if "address" in by:
|
||||
for a in dossier_mod.all_addresses(subject_dossier):
|
||||
if a.get("line1"):
|
||||
vectors.append({"by": "address",
|
||||
"query": {k: a.get(k) for k in ("line1", "city", "state", "postal")}})
|
||||
|
||||
return vectors
|
||||
Reference in New Issue
Block a user