Autogenerate comprehensive docs llms.txt

docs/app/scripts/check_html_routes.py

@@ -14,6 +14,7 @@
 INLINE_SVG_PREVIEW_ROUTES = {"/overview/gallery/"}
 XY_PAYLOAD_PATTERN = re.compile(r'["\'](?P<url>/docs/xy/xy/[a-f0-9]+\.xyf)["\']')
 XY_PAYLOAD_MAGIC = b"XYBF"
+LLMS_DIRECTIVE = "For AI agents: the complete XY documentation index is at"
 
 
 def route_html_paths(route: str) -> tuple[Path, ...]:
@@ -109,6 +110,10 @@ def main() -> None:
         if not any(path.is_file() for path in html_paths):
             msg = f"Missing prerendered documentation route: {html_paths!r}"
             raise RuntimeError(msg)
+        for html_path in html_paths:
+            if html_path.is_file() and LLMS_DIRECTIVE not in html_path.read_text(encoding="utf-8"):
+                msg = f"Prerendered route omits the llms.txt directive: {html_path}"
+                raise RuntimeError(msg)
 
     for route in DOCS_REDIRECTS:
         module_path = route_module_path(route)

docs/app/scripts/check_markdown_assets.py

@@ -4,7 +4,12 @@
 
 from reflex_site_shared.docs import discover_docs
 from xy_docs.config import DOCS_CONFIG
-from xy_docs.plugins import page_markdown_with_api_reference
+from xy_docs.plugins import (
+    _markdown_directive,
+    build_llms_full_txt,
+    build_llms_txt,
+    page_markdown_with_api_reference,
+)
 
 APP_ROOT = Path(__file__).resolve().parents[1]
 BUILD_ROOT = APP_ROOT / ".web" / "build" / "client" / "docs" / "xy"
@@ -47,6 +52,25 @@ def main() -> None:
                 msg = f"Markdown asset differs from generated content: {output_path}"
                 raise RuntimeError(msg)
 
+            if not output_path.read_text(encoding="utf-8").startswith(
+                f"{_markdown_directive()}\n\n"
+            ):
+                msg = f"Markdown asset omits the llms.txt directive: {output_path}"
+                raise RuntimeError(msg)
+
+    expected_agent_files = {
+        "llms.txt": build_llms_txt(DOCS_CONFIG),
+        "llms-full.txt": build_llms_full_txt(DOCS_CONFIG),
+    }
+    for filename, expected in expected_agent_files.items():
+        output_path = BUILD_ROOT / filename
+        if not output_path.is_file():
+            msg = f"Missing agent documentation asset: {output_path}"
+            raise RuntimeError(msg)
+        if output_path.read_text(encoding="utf-8") != expected:
+            msg = f"Agent documentation asset differs from generated content: {output_path}"
+            raise RuntimeError(msg)
+
 
 if __name__ == "__main__":
     main()

docs/app/tests/test_agent_files.py

@@ -11,13 +11,15 @@
 from xy_docs.constants import LLMS_FULL_TXT_PATH, PUBLIC_DOCS_URL
 from xy_docs.plugins import (
     XYDocsAgentFilesPlugin,
+    _markdown_directive,
     build_llms_full_txt,
     build_llms_txt,
     markdown_asset_path,
     page_markdown_with_api_reference,
 )
 from xy_docs.prerender import XyDocsMarkdownPlugin
 from xy_docs.sidebar import xy_docs_sidebar
+from xy_docs.xy_docs import _llms_txt_directive
 
 
 def _headings(content: str, level: int) -> list[str]:
@@ -48,8 +50,31 @@ def test_llms_txt_indexes_every_page_under_the_public_url() -> None:
     """The index links llms-full.txt and each page's public Markdown URL."""
     content = build_llms_txt(DOCS_CONFIG)
     assert f"({PUBLIC_DOCS_URL}{LLMS_FULL_TXT_PATH})" in content
+    assert len(content) < 50_000
+    assert content.startswith(
+        "# XY Documentation\n\n> XY is a high-performance plotting library for Python and Reflex."
+    )
+    assert "\n## Docs\n\n" in content
+    expected_sections = (
+        "Overview",
+        "Core Concepts",
+        "Styling",
+        "Chart Gallery",
+        "Components",
+        "Integrations",
+        "Guides",
+        "Advanced",
+        "API Reference",
+    )
+    positions = [content.index(f"### {section}\n") for section in expected_sections]
+    assert positions == sorted(positions)
+    chart_gallery = content.split("### Chart Gallery\n", maxsplit=1)[1].split(
+        "### Components\n", maxsplit=1
+    )[0]
+    assert f"({PUBLIC_DOCS_URL}/overview/gallery.md)" in chart_gallery
     for page in discover_docs(DOCS_CONFIG):
         assert f"({PUBLIC_DOCS_URL}/{markdown_asset_path(page)})" in content
+        assert content.count(f"({PUBLIC_DOCS_URL}/{markdown_asset_path(page)})") == 1
 
 
 def test_llms_full_txt_keeps_section_headers_above_page_content() -> None:
@@ -83,7 +108,24 @@ def test_component_api_is_present_in_every_agent_markdown_export() -> None:
         assert "### xy.y_axis" in content
         assert "| Prop | Type | Description |" in content
 
-    assert published_markdown.startswith("---\n")
+    assert published_markdown.startswith(f"{_markdown_directive()}\n\n---\n")
+
+
+def test_every_page_markdown_has_the_agent_discovery_directive() -> None:
+    """Every direct and trailing-slash Markdown asset advertises llms.txt."""
+    directive = _markdown_directive()
+    assets = XyDocsMarkdownPlugin(docs=DOCS_CONFIG).get_static_assets()
+    assert assets
+    assert all(content.startswith(f"{directive}\n\n") for _path, content in assets)
+
+
+def test_html_shell_has_the_agent_discovery_directive() -> None:
+    """The hidden HTML directive uses the canonical XY llms.txt URL."""
+    rendered = str(_llms_txt_directive())
+    assert "For AI agents: the complete XY documentation index is at" in rendered
+    assert f"{PUBLIC_DOCS_URL}/llms.txt" in rendered
+    assert "Markdown versions are available" in rendered
+    assert "sr-only" in rendered
 
 
 def test_agent_files_publish_under_the_frontend_path(

docs/app/xy_docs/plugins.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import dataclasses
+from collections import defaultdict
 from pathlib import Path
 from typing import Any
 
@@ -14,6 +15,25 @@
 from xy_docs.api_reference import append_component_api_markdown
 from xy_docs.constants import LLMS_FULL_TXT_PATH, LLMS_TXT_PATH, PUBLIC_DOCS_URL
 
+MARKDOWN_DIRECTIVE = (
+    "> For AI agents: the complete XY documentation index is at "
+    "[llms.txt]({llms_txt_url}). Markdown versions are available by appending "
+    "`.md` or sending `Accept: text/markdown`."
+)
+
+_SECTION_TITLES = {
+    "overview": "Overview",
+    "core-concepts": "Core Concepts",
+    "styling": "Styling",
+    "charts": "Chart Gallery",
+    "components": "Components",
+    "integrations": "Integrations",
+    "guides": "Guides",
+    "advanced": "Advanced",
+    "api-reference": "API Reference",
+}
+_SECTION_ORDER = tuple(_SECTION_TITLES.values())
+
 
 def _public_url(path: str) -> str:
     """Return an absolute URL for an XY docs asset path."""
@@ -33,6 +53,32 @@ def _page_markdown_url(page: DocsPage) -> str:
     return f"{PUBLIC_DOCS_URL}/{markdown_asset_path(page)}"
 
 
+def _markdown_directive() -> str:
+    """Return the standard agent discovery directive for published Markdown."""
+    return MARKDOWN_DIRECTIVE.format(
+        llms_txt_url=_public_url(LLMS_TXT_PATH),
+    )
+
+
+def _strip_markdown_directive(content: str) -> str:
+    """Remove the generated discovery directive from combined page content."""
+    directive = _markdown_directive()
+    if content.startswith(directive):
+        return content.removeprefix(directive).lstrip()
+    return content
+
+
+def _section_for_page(page: DocsPage) -> str:
+    """Return the public navigation section for a discovered page."""
+    if page.route == "/overview/gallery/":
+        return "Chart Gallery"
+    route_parts = tuple(part for part in page.route.split("/") if part)
+    if not route_parts:
+        return "Overview"
+    segment = route_parts[0]
+    return _SECTION_TITLES.get(segment, segment.replace("-", " ").title())
+
+
 def _page_body(content: str) -> str:
     """Drop a page's leading H1 so section headers stay the top heading level."""
     first_line, separator, rest = content.lstrip("\n").partition("\n")
@@ -57,11 +103,12 @@ def page_markdown_with_api_reference(
         Agent-readable Markdown matching the rendered page's API content.
     """
     content = page.source_path.read_text(encoding="utf-8") if include_frontmatter else page.content
-    return append_component_api_markdown(content, page.metadata)
+    body = append_component_api_markdown(content, page.metadata).lstrip()
+    return f"{_markdown_directive()}\n\n{body}"
 
 
 def build_llms_txt(config: DocsSiteConfig) -> str:
-    """Build the concise agent-readable index of public XY pages."""
+    """Build the comprehensive agent-readable index of public XY pages."""
     lines = [
         "# XY Documentation",
         "",
@@ -75,9 +122,22 @@ def build_llms_txt(config: DocsSiteConfig) -> str:
         "## Docs",
         "",
     ]
+
+    sections: dict[str, list[DocsPage]] = defaultdict(list)
     for page in discover_docs(config):
-        description = f": {page.description}" if page.description else ""
-        lines.append(f"- [{page.title}]({_page_markdown_url(page)}){description}")
+        sections[_section_for_page(page)].append(page)
+
+    ordered_sections = [
+        *[section for section in _SECTION_ORDER if section in sections],
+        *sorted(section for section in sections if section not in _SECTION_ORDER),
+    ]
+    for section in ordered_sections:
+        lines.extend((f"### {section}", ""))
+        for page in sections[section]:
+            description = f": {page.description}" if page.description else ""
+            lines.append(f"- [{page.title}]({_page_markdown_url(page)}){description}")
+        lines.append("")
+
     return "\n".join(lines).rstrip() + "\n"
 
 
@@ -105,7 +165,11 @@ def build_llms_full_txt(config: DocsSiteConfig) -> str:
                 "",
                 f"Source: {_page_markdown_url(page)}",
                 "",
-                _page_body(page_markdown_with_api_reference(page)),
+                _page_body(
+                    _strip_markdown_directive(
+                        page_markdown_with_api_reference(page),
+                    )
+                ),
                 "",
             )
         )

docs/app/xy_docs/xy_docs.py

@@ -12,7 +12,7 @@
 
 from xy_docs.breadcrumb import xy_docs_breadcrumb
 from xy_docs.config import DOCS_CONFIG, DOCS_REDIRECTS
-from xy_docs.constants import PUBLIC_DOCS_URL, SOCIAL_IMAGE_URL
+from xy_docs.constants import LLMS_TXT_PATH, PUBLIC_DOCS_URL, SOCIAL_IMAGE_URL
 from xy_docs.footer import xy_docs_footer
 from xy_docs.markdown import page_with_api_reference_toc, render_xy_markdown_page
 from xy_docs.navbar import xy_docs_navbar
@@ -28,6 +28,22 @@
     "--chart-focus": "var(--primary-9)",
 }
 
+
+def _llms_txt_directive() -> rx.Component:
+    """Return the hidden agent-facing documentation index directive."""
+    return rx.el.blockquote(
+        rx.el.span("For AI agents: the complete XY documentation index is at "),
+        rx.el.a(
+            "llms.txt",
+            href=f"{PUBLIC_DOCS_URL}{LLMS_TXT_PATH}",
+        ),
+        rx.el.span(
+            ". Markdown versions are available by appending .md or sending Accept: text/markdown."
+        ),
+        class_name="sr-only",
+    )
+
+
 app = rx.App(
     style={**styles.BASE_STYLE, **_CHART_STYLE},
     app_wraps={},
@@ -72,6 +88,7 @@ def _without_faq_in_toc(page):
 def xy_docs_layout(page, content, navigation) -> rx.Component:
     """Render the shared docs layout with Reflex's TOC scroll highlighter."""
     return rx.box(
+        _llms_txt_directive(),
         docs_layout(
             page_with_api_reference_toc(_without_faq_in_toc(page)),
             content,