Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

tests/website/test_generate_llms_txt.py

@@ -0,0 +1,133 @@
+"""`llms.txt` is how an LLM learns what Hermes can do.
+
+It is the index every model reads when pointed at our docs — including Hermes
+itself, whose `hermes-agent` skill routes unknown-feature questions there.
+`website/` is never packaged, so there is no shipped copy to fall back on.
+
+The index used to be a hand-written list of page paths, and it rotted to 53%
+coverage: Bot Mode, the desktop app, computer use, web search, and 22 messaging
+platforms were all absent, which is why an agent asked how to make bots talk to
+each other answered that it couldn't. These tests hold the two directions of
+that contract — every page reachable, every link real — so the index tracks the
+docs tree instead of someone's memory of it.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import re
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+GENERATOR = REPO_ROOT / "website" / "scripts" / "generate-llms-txt.py"
+
+
+@pytest.fixture(scope="module")
+def gen():
+    spec = importlib.util.spec_from_file_location("generate_llms_txt", GENERATOR)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.fixture(scope="module")
+def index(gen) -> str:
+    return gen.emit_llms_index()
+
+
+def _linked(gen, index: str) -> set[str]:
+    return set(re.findall(rf"\]\({re.escape(gen.SITE_BASE)}/([^)]+)\)", index))
+
+
+def _pages_on_disk(gen) -> set[str]:
+    """Walk the docs tree directly, duplicating only the two documented
+    exclusions.
+
+    Deliberately does not call `iter_docs()`: the index is built from that
+    enumeration, so checking one against the other would pass even if the
+    enumerator went blind to a whole tree — which is the failure being guarded.
+    """
+    pages = set()
+    for path in (*gen.DOCS.rglob("*.md"), *gen.DOCS.rglob("*.mdx")):
+        rel = path.relative_to(gen.DOCS).with_suffix("")
+        slug = str(rel.parent) if rel.name == "index" else str(rel)
+        # The docs landing page is the index's subject; per-skill pages are
+        # summarized by the two catalog reference pages.
+        if slug == "." or slug.startswith(("user-guide/skills/bundled", "user-guide/skills/optional")):
+            continue
+        pages.add(slug)
+    return pages
+
+
+def test_every_docs_page_is_indexed(gen, index):
+    """The regression: a page the index omits is a feature the agent denies."""
+    pages = _pages_on_disk(gen)
+    assert len(pages) > 100, "docs root resolved wrong — the rest of this file proves nothing"
+
+    missing = pages - _linked(gen, index)
+    assert not missing, (
+        f"{len(missing)} docs pages missing from llms.txt: {sorted(missing)} — "
+        "they should have been absorbed into a section automatically"
+    )
+
+
+def test_the_enumerator_sees_the_whole_docs_tree(gen):
+    """Everything downstream trusts `iter_docs()`, so pin it to the filesystem."""
+    assert set(gen.iter_docs()) == _pages_on_disk(gen)
+
+
+def test_every_indexed_page_exists(gen, index):
+    """The other direction: a renamed page leaves the index pointing at a 404."""
+    for slug in sorted(_linked(gen, index)):
+        assert gen.doc_path(slug) is not None, (
+            f"llms.txt links {slug}, which is not in the docs tree — "
+            "drop the SECTIONS row and let the page be absorbed under its new path"
+        )
+
+
+def test_pages_are_listed_once(gen, index):
+    """Curating a page must promote it, not duplicate it."""
+    entries = re.findall(rf"^- \[.*?\]\({re.escape(gen.SITE_BASE)}/([^)]+)\)", index, re.MULTILINE)
+    duplicated = {slug for slug in entries if entries.count(slug) > 1}
+    assert not duplicated, f"listed more than once in llms.txt: {sorted(duplicated)}"
+
+
+def test_curation_orders_pages_without_gatekeeping_them(gen):
+    """SECTIONS decides what leads a section, never what the index contains."""
+    curated = {slug for _section, items in gen.SECTIONS for slug, _t, _d in items}
+    pages = set(gen.iter_docs())
+
+    assert curated < pages, "every page is curated — absorption is no longer exercised"
+    assert gen.section_for("user-guide/features/some-feature-shipped-tomorrow") in dict(gen.ABSORB)
+    assert gen.section_for("a-tree-nobody-anticipated/page") == gen.MISC_SECTION
+
+
+def test_section_landing_pages_resolve_to_their_directory(gen):
+    """`messaging/index.md` is served at `/messaging`; `/messaging/index` 404s."""
+    assert gen.slug_for(gen.DOCS / "user-guide" / "messaging" / "index.md") == "user-guide/messaging"
+    assert "user-guide/messaging/index" not in _linked(gen, gen.emit_llms_index())
+
+
+def test_mdx_pages_are_indexed_without_their_imports(gen):
+    """MDX docs are real pages; their component imports are not prose."""
+    mdx = [p for p in gen.DOCS.rglob("*.mdx") if gen.slug_for(p)]
+    assert mdx, "no .mdx docs — this test no longer guards anything"
+    assert {gen.slug_for(p) for p in mdx} <= set(gen.iter_docs())
+
+    _meta, body = gen.read_frontmatter(mdx[0])
+    assert not re.search(r"^import\s", body, re.MULTILINE)
+
+
+def test_per_skill_catalog_pages_stay_out(gen):
+    """~195 generated skill pages would bury the product docs in the index."""
+    assert not [slug for slug in gen.iter_docs() if slug.startswith(gen.SKILL_CATALOG)]
+    assert "reference/skills-catalog" in gen.iter_docs(), "the summary page must remain"
+
+
+def test_bot_mode_is_reachable(gen, index):
+    """The page behind the original complaint, and the answer it has to carry."""
+    assert "user-guide/bot-mode" in _linked(gen, index)
+    assert "hermes peer dm" in (gen.DOCS / "user-guide" / "bot-mode.md").read_text(encoding="utf-8")