diff --git a/docs/package.json b/docs/package.json index 3227b5c0..bb1615fc 100644 --- a/docs/package.json +++ b/docs/package.json @@ -6,6 +6,8 @@ "docusaurus": "docusaurus", "start": "docusaurus start", "build": "docusaurus build", + "postbuild": "node scripts/validate-ai-baseline.mjs", + "validate:ai-baseline": "node scripts/validate-ai-baseline.mjs", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", @@ -15,7 +17,7 @@ "ci": "npm ci --legacy-peer-deps && npm run build" }, "dependencies": { - "@conduction/docusaurus-preset": "^2.6.1", + "@conduction/docusaurus-preset": "^3.4.0", "@docusaurus/core": "^3.7.0", "@docusaurus/preset-classic": "^3.7.0", "@docusaurus/theme-mermaid": "^3.7.0", diff --git a/docs/scripts/validate-ai-baseline.mjs b/docs/scripts/validate-ai-baseline.mjs new file mode 100755 index 00000000..83a127ed --- /dev/null +++ b/docs/scripts/validate-ai-baseline.mjs @@ -0,0 +1,177 @@ +#!/usr/bin/env node +/** + * scripts/validate-ai-baseline.mjs + * + * Generic AI-crawler baseline validator. Runs as a postbuild step on + * every Conduction Docusaurus site that consumes + * @conduction/docusaurus-preset >= 3.4.0. Asserts the SSG output + * carries the contract AI crawlers (GPTBot, ClaudeBot, PerplexityBot, + * OAI-SearchBot, Claude-SearchBot, Google AI Overviews) expect. + * + * Universal checks only - no site-specific routes. Sites that want + * additional gates (per-app SoftwareApplication, FAQPage on specific + * pages, etc.) extend this script in place. See conduction-website's + * version for an example of additional checks. + * + * Exit codes: + * 0 all checks passed + * 1 one or more checks failed (CI should block) + * 2 build directory not found (script invoked before build) + */ + +import {readFileSync, existsSync, statSync} from 'node:fs'; +import {join, resolve} from 'node:path'; + +const buildDir = resolve(process.argv[2] || 'build'); + +if (!existsSync(buildDir)) { + console.error(`✗ build directory not found: ${buildDir}`); + console.error(` Run \`npx docusaurus build\` first.`); + process.exit(2); +} + +const results = []; + +function check(name, fn) { + try { + const r = fn(); + results.push({name, ok: r.ok, msg: r.msg}); + } catch (e) { + results.push({name, ok: false, msg: `threw: ${e.message}`}); + } +} + +function readBuild(p) { + return readFileSync(join(buildDir, p), 'utf8'); +} + +/* robots.txt - shipped by the preset's ai-crawling plugin (or the + site's own static/robots.txt). Either way, the file must exist + and name at least one AI search bot so a `grep` audit can confirm + the posture at a glance. */ +check('robots.txt exists and is non-empty', () => { + const path = join(buildDir, 'robots.txt'); + if (!existsSync(path)) return {ok: false, msg: 'missing'}; + const size = statSync(path).size; + if (size < 50) return {ok: false, msg: `too small (${size} bytes)`}; + return {ok: true, msg: `${size} bytes`}; +}); + +check('robots.txt names at least one AI search bot', () => { + const body = readBuild('robots.txt'); + const candidates = ['OAI-SearchBot', 'Claude-SearchBot', 'PerplexityBot', 'ChatGPT-User', 'Claude-User']; + const found = candidates.filter(ua => body.includes(`User-agent: ${ua}`)); + if (found.length === 0) { + return {ok: false, msg: `none of [${candidates.join(', ')}] referenced`}; + } + return {ok: true, msg: `${found.length} bot(s): ${found.join(', ')}`}; +}); + +check('robots.txt has a Sitemap line', () => { + const body = readBuild('robots.txt'); + const matches = body.match(/^Sitemap:\s+https?:\/\//gm) || []; + if (matches.length === 0) return {ok: false, msg: 'no Sitemap: line'}; + return {ok: true, msg: `${matches.length} sitemap line(s)`}; +}); + +/* sitemap.xml - emitted by @docusaurus/plugin-sitemap (loaded via + the classic preset). Locale-specific sitemaps (e.g. /nl/sitemap.xml) + are present for i18n builds; we only check the canonical one + because some sites are single-locale. */ +check('sitemap.xml exists and has at least 1 URL', () => { + const path = join(buildDir, 'sitemap.xml'); + if (!existsSync(path)) return {ok: false, msg: 'missing'}; + const body = readBuild('sitemap.xml'); + const n = (body.match(//g) || []).length; + if (n < 1) return {ok: false, msg: 'no entries'}; + return {ok: true, msg: `${n} URLs`}; +}); + +/* Helper for the JSON-LD checks below. Docusaurus emits ld+json + tags via two paths with different attribute ordering: top-level + headTags renders