diff --git a/docs/app/scripts/check_html_routes.py b/docs/app/scripts/check_html_routes.py
index 72225151..cab1ab7f 100644
--- a/docs/app/scripts/check_html_routes.py
+++ b/docs/app/scripts/check_html_routes.py
@@ -14,6 +14,7 @@
INLINE_SVG_PREVIEW_ROUTES = {"/overview/gallery/"}
XY_PAYLOAD_PATTERN = re.compile(r'["\'](?P/docs/xy/xy/[a-f0-9]+\.xyf)["\']')
XY_PAYLOAD_MAGIC = b"XYBF"
+LLMS_DIRECTIVE = "For AI agents: the complete XY documentation index is at"
def route_html_paths(route: str) -> tuple[Path, ...]:
@@ -109,6 +110,10 @@ def main() -> None:
if not any(path.is_file() for path in html_paths):
msg = f"Missing prerendered documentation route: {html_paths!r}"
raise RuntimeError(msg)
+ for html_path in html_paths:
+ if html_path.is_file() and LLMS_DIRECTIVE not in html_path.read_text(encoding="utf-8"):
+ msg = f"Prerendered route omits the llms.txt directive: {html_path}"
+ raise RuntimeError(msg)
for route in DOCS_REDIRECTS:
module_path = route_module_path(route)
diff --git a/docs/app/scripts/check_markdown_assets.py b/docs/app/scripts/check_markdown_assets.py
index e0e32565..0e6b0b79 100644
--- a/docs/app/scripts/check_markdown_assets.py
+++ b/docs/app/scripts/check_markdown_assets.py
@@ -4,7 +4,12 @@
from reflex_site_shared.docs import discover_docs
from xy_docs.config import DOCS_CONFIG
-from xy_docs.plugins import page_markdown_with_api_reference
+from xy_docs.plugins import (
+ _markdown_directive,
+ build_llms_full_txt,
+ build_llms_txt,
+ page_markdown_with_api_reference,
+)
APP_ROOT = Path(__file__).resolve().parents[1]
BUILD_ROOT = APP_ROOT / ".web" / "build" / "client" / "docs" / "xy"
@@ -47,6 +52,25 @@ def main() -> None:
msg = f"Markdown asset differs from generated content: {output_path}"
raise RuntimeError(msg)
+ if not output_path.read_text(encoding="utf-8").startswith(
+ f"{_markdown_directive()}\n\n"
+ ):
+ msg = f"Markdown asset omits the llms.txt directive: {output_path}"
+ raise RuntimeError(msg)
+
+ expected_agent_files = {
+ "llms.txt": build_llms_txt(DOCS_CONFIG),
+ "llms-full.txt": build_llms_full_txt(DOCS_CONFIG),
+ }
+ for filename, expected in expected_agent_files.items():
+ output_path = BUILD_ROOT / filename
+ if not output_path.is_file():
+ msg = f"Missing agent documentation asset: {output_path}"
+ raise RuntimeError(msg)
+ if output_path.read_text(encoding="utf-8") != expected:
+ msg = f"Agent documentation asset differs from generated content: {output_path}"
+ raise RuntimeError(msg)
+
if __name__ == "__main__":
main()
diff --git a/docs/app/tests/test_agent_files.py b/docs/app/tests/test_agent_files.py
index 5fac925d..3762b620 100644
--- a/docs/app/tests/test_agent_files.py
+++ b/docs/app/tests/test_agent_files.py
@@ -11,6 +11,7 @@
from xy_docs.constants import LLMS_FULL_TXT_PATH, PUBLIC_DOCS_URL
from xy_docs.plugins import (
XYDocsAgentFilesPlugin,
+ _markdown_directive,
build_llms_full_txt,
build_llms_txt,
markdown_asset_path,
@@ -18,6 +19,7 @@
)
from xy_docs.prerender import XyDocsMarkdownPlugin
from xy_docs.sidebar import xy_docs_sidebar
+from xy_docs.xy_docs import _llms_txt_directive
def _headings(content: str, level: int) -> list[str]:
@@ -48,8 +50,31 @@ def test_llms_txt_indexes_every_page_under_the_public_url() -> None:
"""The index links llms-full.txt and each page's public Markdown URL."""
content = build_llms_txt(DOCS_CONFIG)
assert f"({PUBLIC_DOCS_URL}{LLMS_FULL_TXT_PATH})" in content
+ assert len(content) < 50_000
+ assert content.startswith(
+ "# XY Documentation\n\n> XY is a high-performance plotting library for Python and Reflex."
+ )
+ assert "\n## Docs\n\n" in content
+ expected_sections = (
+ "Overview",
+ "Core Concepts",
+ "Styling",
+ "Chart Gallery",
+ "Components",
+ "Integrations",
+ "Guides",
+ "Advanced",
+ "API Reference",
+ )
+ positions = [content.index(f"### {section}\n") for section in expected_sections]
+ assert positions == sorted(positions)
+ chart_gallery = content.split("### Chart Gallery\n", maxsplit=1)[1].split(
+ "### Components\n", maxsplit=1
+ )[0]
+ assert f"({PUBLIC_DOCS_URL}/overview/gallery.md)" in chart_gallery
for page in discover_docs(DOCS_CONFIG):
assert f"({PUBLIC_DOCS_URL}/{markdown_asset_path(page)})" in content
+ assert content.count(f"({PUBLIC_DOCS_URL}/{markdown_asset_path(page)})") == 1
def test_llms_full_txt_keeps_section_headers_above_page_content() -> None:
@@ -83,7 +108,24 @@ def test_component_api_is_present_in_every_agent_markdown_export() -> None:
assert "### xy.y_axis" in content
assert "| Prop | Type | Description |" in content
- assert published_markdown.startswith("---\n")
+ assert published_markdown.startswith(f"{_markdown_directive()}\n\n---\n")
+
+
+def test_every_page_markdown_has_the_agent_discovery_directive() -> None:
+ """Every direct and trailing-slash Markdown asset advertises llms.txt."""
+ directive = _markdown_directive()
+ assets = XyDocsMarkdownPlugin(docs=DOCS_CONFIG).get_static_assets()
+ assert assets
+ assert all(content.startswith(f"{directive}\n\n") for _path, content in assets)
+
+
+def test_html_shell_has_the_agent_discovery_directive() -> None:
+ """The hidden HTML directive uses the canonical XY llms.txt URL."""
+ rendered = str(_llms_txt_directive())
+ assert "For AI agents: the complete XY documentation index is at" in rendered
+ assert f"{PUBLIC_DOCS_URL}/llms.txt" in rendered
+ assert "Markdown versions are available" in rendered
+ assert "sr-only" in rendered
def test_agent_files_publish_under_the_frontend_path(
diff --git a/docs/app/xy_docs/plugins.py b/docs/app/xy_docs/plugins.py
index c8e1720b..65ab6a1f 100644
--- a/docs/app/xy_docs/plugins.py
+++ b/docs/app/xy_docs/plugins.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import dataclasses
+from collections import defaultdict
from pathlib import Path
from typing import Any
@@ -14,6 +15,25 @@
from xy_docs.api_reference import append_component_api_markdown
from xy_docs.constants import LLMS_FULL_TXT_PATH, LLMS_TXT_PATH, PUBLIC_DOCS_URL
+MARKDOWN_DIRECTIVE = (
+ "> For AI agents: the complete XY documentation index is at "
+ "[llms.txt]({llms_txt_url}). Markdown versions are available by appending "
+ "`.md` or sending `Accept: text/markdown`."
+)
+
+_SECTION_TITLES = {
+ "overview": "Overview",
+ "core-concepts": "Core Concepts",
+ "styling": "Styling",
+ "charts": "Chart Gallery",
+ "components": "Components",
+ "integrations": "Integrations",
+ "guides": "Guides",
+ "advanced": "Advanced",
+ "api-reference": "API Reference",
+}
+_SECTION_ORDER = tuple(_SECTION_TITLES.values())
+
def _public_url(path: str) -> str:
"""Return an absolute URL for an XY docs asset path."""
@@ -33,6 +53,32 @@ def _page_markdown_url(page: DocsPage) -> str:
return f"{PUBLIC_DOCS_URL}/{markdown_asset_path(page)}"
+def _markdown_directive() -> str:
+ """Return the standard agent discovery directive for published Markdown."""
+ return MARKDOWN_DIRECTIVE.format(
+ llms_txt_url=_public_url(LLMS_TXT_PATH),
+ )
+
+
+def _strip_markdown_directive(content: str) -> str:
+ """Remove the generated discovery directive from combined page content."""
+ directive = _markdown_directive()
+ if content.startswith(directive):
+ return content.removeprefix(directive).lstrip()
+ return content
+
+
+def _section_for_page(page: DocsPage) -> str:
+ """Return the public navigation section for a discovered page."""
+ if page.route == "/overview/gallery/":
+ return "Chart Gallery"
+ route_parts = tuple(part for part in page.route.split("/") if part)
+ if not route_parts:
+ return "Overview"
+ segment = route_parts[0]
+ return _SECTION_TITLES.get(segment, segment.replace("-", " ").title())
+
+
def _page_body(content: str) -> str:
"""Drop a page's leading H1 so section headers stay the top heading level."""
first_line, separator, rest = content.lstrip("\n").partition("\n")
@@ -57,11 +103,12 @@ def page_markdown_with_api_reference(
Agent-readable Markdown matching the rendered page's API content.
"""
content = page.source_path.read_text(encoding="utf-8") if include_frontmatter else page.content
- return append_component_api_markdown(content, page.metadata)
+ body = append_component_api_markdown(content, page.metadata).lstrip()
+ return f"{_markdown_directive()}\n\n{body}"
def build_llms_txt(config: DocsSiteConfig) -> str:
- """Build the concise agent-readable index of public XY pages."""
+ """Build the comprehensive agent-readable index of public XY pages."""
lines = [
"# XY Documentation",
"",
@@ -75,9 +122,22 @@ def build_llms_txt(config: DocsSiteConfig) -> str:
"## Docs",
"",
]
+
+ sections: dict[str, list[DocsPage]] = defaultdict(list)
for page in discover_docs(config):
- description = f": {page.description}" if page.description else ""
- lines.append(f"- [{page.title}]({_page_markdown_url(page)}){description}")
+ sections[_section_for_page(page)].append(page)
+
+ ordered_sections = [
+ *[section for section in _SECTION_ORDER if section in sections],
+ *sorted(section for section in sections if section not in _SECTION_ORDER),
+ ]
+ for section in ordered_sections:
+ lines.extend((f"### {section}", ""))
+ for page in sections[section]:
+ description = f": {page.description}" if page.description else ""
+ lines.append(f"- [{page.title}]({_page_markdown_url(page)}){description}")
+ lines.append("")
+
return "\n".join(lines).rstrip() + "\n"
@@ -105,7 +165,11 @@ def build_llms_full_txt(config: DocsSiteConfig) -> str:
"",
f"Source: {_page_markdown_url(page)}",
"",
- _page_body(page_markdown_with_api_reference(page)),
+ _page_body(
+ _strip_markdown_directive(
+ page_markdown_with_api_reference(page),
+ )
+ ),
"",
)
)
diff --git a/docs/app/xy_docs/xy_docs.py b/docs/app/xy_docs/xy_docs.py
index f9d8a231..c9239a79 100644
--- a/docs/app/xy_docs/xy_docs.py
+++ b/docs/app/xy_docs/xy_docs.py
@@ -12,7 +12,7 @@
from xy_docs.breadcrumb import xy_docs_breadcrumb
from xy_docs.config import DOCS_CONFIG, DOCS_REDIRECTS
-from xy_docs.constants import PUBLIC_DOCS_URL, SOCIAL_IMAGE_URL
+from xy_docs.constants import LLMS_TXT_PATH, PUBLIC_DOCS_URL, SOCIAL_IMAGE_URL
from xy_docs.footer import xy_docs_footer
from xy_docs.markdown import page_with_api_reference_toc, render_xy_markdown_page
from xy_docs.navbar import xy_docs_navbar
@@ -28,6 +28,22 @@
"--chart-focus": "var(--primary-9)",
}
+
+def _llms_txt_directive() -> rx.Component:
+ """Return the hidden agent-facing documentation index directive."""
+ return rx.el.blockquote(
+ rx.el.span("For AI agents: the complete XY documentation index is at "),
+ rx.el.a(
+ "llms.txt",
+ href=f"{PUBLIC_DOCS_URL}{LLMS_TXT_PATH}",
+ ),
+ rx.el.span(
+ ". Markdown versions are available by appending .md or sending Accept: text/markdown."
+ ),
+ class_name="sr-only",
+ )
+
+
app = rx.App(
style={**styles.BASE_STYLE, **_CHART_STYLE},
app_wraps={},
@@ -72,6 +88,7 @@ def _without_faq_in_toc(page):
def xy_docs_layout(page, content, navigation) -> rx.Component:
"""Render the shared docs layout with Reflex's TOC scroll highlighter."""
return rx.box(
+ _llms_txt_directive(),
docs_layout(
page_with_api_reference_toc(_without_faq_in_toc(page)),
content,