diff --git a/docs/app/scripts/check_html_routes.py b/docs/app/scripts/check_html_routes.py index 72225151..cab1ab7f 100644 --- a/docs/app/scripts/check_html_routes.py +++ b/docs/app/scripts/check_html_routes.py @@ -14,6 +14,7 @@ INLINE_SVG_PREVIEW_ROUTES = {"/overview/gallery/"} XY_PAYLOAD_PATTERN = re.compile(r'["\'](?P/docs/xy/xy/[a-f0-9]+\.xyf)["\']') XY_PAYLOAD_MAGIC = b"XYBF" +LLMS_DIRECTIVE = "For AI agents: the complete XY documentation index is at" def route_html_paths(route: str) -> tuple[Path, ...]: @@ -109,6 +110,10 @@ def main() -> None: if not any(path.is_file() for path in html_paths): msg = f"Missing prerendered documentation route: {html_paths!r}" raise RuntimeError(msg) + for html_path in html_paths: + if html_path.is_file() and LLMS_DIRECTIVE not in html_path.read_text(encoding="utf-8"): + msg = f"Prerendered route omits the llms.txt directive: {html_path}" + raise RuntimeError(msg) for route in DOCS_REDIRECTS: module_path = route_module_path(route) diff --git a/docs/app/scripts/check_markdown_assets.py b/docs/app/scripts/check_markdown_assets.py index e0e32565..0e6b0b79 100644 --- a/docs/app/scripts/check_markdown_assets.py +++ b/docs/app/scripts/check_markdown_assets.py @@ -4,7 +4,12 @@ from reflex_site_shared.docs import discover_docs from xy_docs.config import DOCS_CONFIG -from xy_docs.plugins import page_markdown_with_api_reference +from xy_docs.plugins import ( + _markdown_directive, + build_llms_full_txt, + build_llms_txt, + page_markdown_with_api_reference, +) APP_ROOT = Path(__file__).resolve().parents[1] BUILD_ROOT = APP_ROOT / ".web" / "build" / "client" / "docs" / "xy" @@ -47,6 +52,25 @@ def main() -> None: msg = f"Markdown asset differs from generated content: {output_path}" raise RuntimeError(msg) + if not output_path.read_text(encoding="utf-8").startswith( + f"{_markdown_directive()}\n\n" + ): + msg = f"Markdown asset omits the llms.txt directive: {output_path}" + raise RuntimeError(msg) + + expected_agent_files = { + "llms.txt": build_llms_txt(DOCS_CONFIG), + "llms-full.txt": build_llms_full_txt(DOCS_CONFIG), + } + for filename, expected in expected_agent_files.items(): + output_path = BUILD_ROOT / filename + if not output_path.is_file(): + msg = f"Missing agent documentation asset: {output_path}" + raise RuntimeError(msg) + if output_path.read_text(encoding="utf-8") != expected: + msg = f"Agent documentation asset differs from generated content: {output_path}" + raise RuntimeError(msg) + if __name__ == "__main__": main() diff --git a/docs/app/tests/test_agent_files.py b/docs/app/tests/test_agent_files.py index 5fac925d..3762b620 100644 --- a/docs/app/tests/test_agent_files.py +++ b/docs/app/tests/test_agent_files.py @@ -11,6 +11,7 @@ from xy_docs.constants import LLMS_FULL_TXT_PATH, PUBLIC_DOCS_URL from xy_docs.plugins import ( XYDocsAgentFilesPlugin, + _markdown_directive, build_llms_full_txt, build_llms_txt, markdown_asset_path, @@ -18,6 +19,7 @@ ) from xy_docs.prerender import XyDocsMarkdownPlugin from xy_docs.sidebar import xy_docs_sidebar +from xy_docs.xy_docs import _llms_txt_directive def _headings(content: str, level: int) -> list[str]: @@ -48,8 +50,31 @@ def test_llms_txt_indexes_every_page_under_the_public_url() -> None: """The index links llms-full.txt and each page's public Markdown URL.""" content = build_llms_txt(DOCS_CONFIG) assert f"({PUBLIC_DOCS_URL}{LLMS_FULL_TXT_PATH})" in content + assert len(content) < 50_000 + assert content.startswith( + "# XY Documentation\n\n> XY is a high-performance plotting library for Python and Reflex." + ) + assert "\n## Docs\n\n" in content + expected_sections = ( + "Overview", + "Core Concepts", + "Styling", + "Chart Gallery", + "Components", + "Integrations", + "Guides", + "Advanced", + "API Reference", + ) + positions = [content.index(f"### {section}\n") for section in expected_sections] + assert positions == sorted(positions) + chart_gallery = content.split("### Chart Gallery\n", maxsplit=1)[1].split( + "### Components\n", maxsplit=1 + )[0] + assert f"({PUBLIC_DOCS_URL}/overview/gallery.md)" in chart_gallery for page in discover_docs(DOCS_CONFIG): assert f"({PUBLIC_DOCS_URL}/{markdown_asset_path(page)})" in content + assert content.count(f"({PUBLIC_DOCS_URL}/{markdown_asset_path(page)})") == 1 def test_llms_full_txt_keeps_section_headers_above_page_content() -> None: @@ -83,7 +108,24 @@ def test_component_api_is_present_in_every_agent_markdown_export() -> None: assert "### xy.y_axis" in content assert "| Prop | Type | Description |" in content - assert published_markdown.startswith("---\n") + assert published_markdown.startswith(f"{_markdown_directive()}\n\n---\n") + + +def test_every_page_markdown_has_the_agent_discovery_directive() -> None: + """Every direct and trailing-slash Markdown asset advertises llms.txt.""" + directive = _markdown_directive() + assets = XyDocsMarkdownPlugin(docs=DOCS_CONFIG).get_static_assets() + assert assets + assert all(content.startswith(f"{directive}\n\n") for _path, content in assets) + + +def test_html_shell_has_the_agent_discovery_directive() -> None: + """The hidden HTML directive uses the canonical XY llms.txt URL.""" + rendered = str(_llms_txt_directive()) + assert "For AI agents: the complete XY documentation index is at" in rendered + assert f"{PUBLIC_DOCS_URL}/llms.txt" in rendered + assert "Markdown versions are available" in rendered + assert "sr-only" in rendered def test_agent_files_publish_under_the_frontend_path( diff --git a/docs/app/xy_docs/plugins.py b/docs/app/xy_docs/plugins.py index c8e1720b..65ab6a1f 100644 --- a/docs/app/xy_docs/plugins.py +++ b/docs/app/xy_docs/plugins.py @@ -3,6 +3,7 @@ from __future__ import annotations import dataclasses +from collections import defaultdict from pathlib import Path from typing import Any @@ -14,6 +15,25 @@ from xy_docs.api_reference import append_component_api_markdown from xy_docs.constants import LLMS_FULL_TXT_PATH, LLMS_TXT_PATH, PUBLIC_DOCS_URL +MARKDOWN_DIRECTIVE = ( + "> For AI agents: the complete XY documentation index is at " + "[llms.txt]({llms_txt_url}). Markdown versions are available by appending " + "`.md` or sending `Accept: text/markdown`." +) + +_SECTION_TITLES = { + "overview": "Overview", + "core-concepts": "Core Concepts", + "styling": "Styling", + "charts": "Chart Gallery", + "components": "Components", + "integrations": "Integrations", + "guides": "Guides", + "advanced": "Advanced", + "api-reference": "API Reference", +} +_SECTION_ORDER = tuple(_SECTION_TITLES.values()) + def _public_url(path: str) -> str: """Return an absolute URL for an XY docs asset path.""" @@ -33,6 +53,32 @@ def _page_markdown_url(page: DocsPage) -> str: return f"{PUBLIC_DOCS_URL}/{markdown_asset_path(page)}" +def _markdown_directive() -> str: + """Return the standard agent discovery directive for published Markdown.""" + return MARKDOWN_DIRECTIVE.format( + llms_txt_url=_public_url(LLMS_TXT_PATH), + ) + + +def _strip_markdown_directive(content: str) -> str: + """Remove the generated discovery directive from combined page content.""" + directive = _markdown_directive() + if content.startswith(directive): + return content.removeprefix(directive).lstrip() + return content + + +def _section_for_page(page: DocsPage) -> str: + """Return the public navigation section for a discovered page.""" + if page.route == "/overview/gallery/": + return "Chart Gallery" + route_parts = tuple(part for part in page.route.split("/") if part) + if not route_parts: + return "Overview" + segment = route_parts[0] + return _SECTION_TITLES.get(segment, segment.replace("-", " ").title()) + + def _page_body(content: str) -> str: """Drop a page's leading H1 so section headers stay the top heading level.""" first_line, separator, rest = content.lstrip("\n").partition("\n") @@ -57,11 +103,12 @@ def page_markdown_with_api_reference( Agent-readable Markdown matching the rendered page's API content. """ content = page.source_path.read_text(encoding="utf-8") if include_frontmatter else page.content - return append_component_api_markdown(content, page.metadata) + body = append_component_api_markdown(content, page.metadata).lstrip() + return f"{_markdown_directive()}\n\n{body}" def build_llms_txt(config: DocsSiteConfig) -> str: - """Build the concise agent-readable index of public XY pages.""" + """Build the comprehensive agent-readable index of public XY pages.""" lines = [ "# XY Documentation", "", @@ -75,9 +122,22 @@ def build_llms_txt(config: DocsSiteConfig) -> str: "## Docs", "", ] + + sections: dict[str, list[DocsPage]] = defaultdict(list) for page in discover_docs(config): - description = f": {page.description}" if page.description else "" - lines.append(f"- [{page.title}]({_page_markdown_url(page)}){description}") + sections[_section_for_page(page)].append(page) + + ordered_sections = [ + *[section for section in _SECTION_ORDER if section in sections], + *sorted(section for section in sections if section not in _SECTION_ORDER), + ] + for section in ordered_sections: + lines.extend((f"### {section}", "")) + for page in sections[section]: + description = f": {page.description}" if page.description else "" + lines.append(f"- [{page.title}]({_page_markdown_url(page)}){description}") + lines.append("") + return "\n".join(lines).rstrip() + "\n" @@ -105,7 +165,11 @@ def build_llms_full_txt(config: DocsSiteConfig) -> str: "", f"Source: {_page_markdown_url(page)}", "", - _page_body(page_markdown_with_api_reference(page)), + _page_body( + _strip_markdown_directive( + page_markdown_with_api_reference(page), + ) + ), "", ) ) diff --git a/docs/app/xy_docs/xy_docs.py b/docs/app/xy_docs/xy_docs.py index f9d8a231..c9239a79 100644 --- a/docs/app/xy_docs/xy_docs.py +++ b/docs/app/xy_docs/xy_docs.py @@ -12,7 +12,7 @@ from xy_docs.breadcrumb import xy_docs_breadcrumb from xy_docs.config import DOCS_CONFIG, DOCS_REDIRECTS -from xy_docs.constants import PUBLIC_DOCS_URL, SOCIAL_IMAGE_URL +from xy_docs.constants import LLMS_TXT_PATH, PUBLIC_DOCS_URL, SOCIAL_IMAGE_URL from xy_docs.footer import xy_docs_footer from xy_docs.markdown import page_with_api_reference_toc, render_xy_markdown_page from xy_docs.navbar import xy_docs_navbar @@ -28,6 +28,22 @@ "--chart-focus": "var(--primary-9)", } + +def _llms_txt_directive() -> rx.Component: + """Return the hidden agent-facing documentation index directive.""" + return rx.el.blockquote( + rx.el.span("For AI agents: the complete XY documentation index is at "), + rx.el.a( + "llms.txt", + href=f"{PUBLIC_DOCS_URL}{LLMS_TXT_PATH}", + ), + rx.el.span( + ". Markdown versions are available by appending .md or sending Accept: text/markdown." + ), + class_name="sr-only", + ) + + app = rx.App( style={**styles.BASE_STYLE, **_CHART_STYLE}, app_wraps={}, @@ -72,6 +88,7 @@ def _without_faq_in_toc(page): def xy_docs_layout(page, content, navigation) -> rx.Component: """Render the shared docs layout with Reflex's TOC scroll highlighter.""" return rx.box( + _llms_txt_directive(), docs_layout( page_with_api_reference_toc(_without_faq_in_toc(page)), content,