diff --git a/env.d.ts b/env.d.ts index 5a86c157..55798d47 100644 --- a/env.d.ts +++ b/env.d.ts @@ -24,3 +24,10 @@ interface ImportMetaEnv extends EnvironmentVariables {} interface ImportMeta { readonly env: ImportMetaEnv } + +declare module 'virtual:graphite-related-docs' { + import type { RelatedDocsManifest } from './src/lib/graphite-related-docs' + + const manifest: RelatedDocsManifest + export default manifest +} diff --git a/scripts/graphite-related-docs-plugin.ts b/scripts/graphite-related-docs-plugin.ts new file mode 100644 index 00000000..98a57b03 --- /dev/null +++ b/scripts/graphite-related-docs-plugin.ts @@ -0,0 +1,355 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import type { Plugin } from 'vite' +import { + canonicalDocsUrl, + GRAPHITE_RELATED_DOCS_ENDPOINT, + PUBLIC_DOCS_PREFIX, + parseGraphiteRelatedDocs, + type RelatedDocsLink, + type RelatedDocsManifest, + TEMPO_ORIGIN, +} from '../src/lib/graphite-related-docs' + +const virtualModuleId = 'virtual:graphite-related-docs' +const resolvedVirtualModuleId = `\0${virtualModuleId}` +const productionSitemapUrl = `${TEMPO_ORIGIN}/developers/sitemap.xml` +const cacheMaxAgeMs = 24 * 60 * 60 * 1_000 +const cacheVersion = 1 +const maxConcurrency = 6 +const requestStartIntervalMs = 75 +const requestTimeoutMs = 4_000 +const maxResponseBytes = 250_000 +const circuitBreakerFailureThreshold = 3 + +type CacheFile = { + generatedAt: number + manifest: RelatedDocsManifest + sourceUrls: string[] + version: number +} + +type FetchResult = { + links: RelatedDocsLink[] + ok: boolean + retryableFailure: boolean +} + +const manifestPromises = new Map>() +const requestPromises = new Map>() + +export function graphiteRelatedDocsPlugin(): Plugin { + let rootDirectory = process.cwd() + + return { + name: 'tempo-graphite-related-docs', + configResolved(config) { + rootDirectory = config.root + }, + resolveId(id) { + if (id === virtualModuleId) return resolvedVirtualModuleId + }, + async load(id) { + if (id !== resolvedVirtualModuleId) return + + let promise = manifestPromises.get(rootDirectory) + if (!promise) { + promise = buildRelatedDocsManifest(rootDirectory) + manifestPromises.set(rootDirectory, promise) + } + + const manifest = await promise.catch(() => ({})) + return `export default ${JSON.stringify(manifest)}` + }, + } +} + +export function docsPageRouteFromFile(filePath: string): string | undefined { + const normalized = filePath.replaceAll(path.sep, '/').replace(/^\.?\//, '') + const prefix = 'src/pages/docs/' + if (!normalized.startsWith(prefix) || !/\.mdx?$/.test(normalized)) return + + let route = normalized.slice('src/pages/'.length).replace(/\.mdx?$/, '') + if (route === 'docs/index') route = 'docs' + else if (route.endsWith('/index')) route = route.slice(0, -'/index'.length) + if (route.split('/').some((segment) => segment.startsWith('_'))) return + + return `/${route}` +} + +async function buildRelatedDocsManifest(rootDirectory: string): Promise { + const localRoutes = await discoverLocalDocsRoutes(rootDirectory) + const localSourceUrls = localRoutes.flatMap((route) => { + const source = canonicalDocsUrl(route) + return source ? [source] : [] + }) + const cached = await readCache(rootDirectory) + + if ( + cached && + Date.now() - cached.generatedAt < cacheMaxAgeMs && + localSourceUrls.every((source) => cached.sourceUrls.includes(source)) + ) { + return cached.manifest + } + + const sitemapSourceUrls = await discoverSitemapDocsUrls() + const sourceUrls = [...new Set([...localSourceUrls, ...sitemapSourceUrls])].sort() + const waitForStart = createStartRateLimiter(requestStartIntervalMs) + const manifest: RelatedDocsManifest = {} + let successfulRequests = 0 + let consecutiveFailures = 0 + let circuitOpen = false + + await mapWithConcurrency(sourceUrls, maxConcurrency, async (sourceUrl) => { + if (circuitOpen) { + const route = new URL(sourceUrl).pathname.slice('/developers'.length) || '/' + manifest[route] = cached?.manifest[route] ?? [] + return + } + + await waitForStart() + if (circuitOpen) { + const route = new URL(sourceUrl).pathname.slice('/developers'.length) || '/' + manifest[route] = cached?.manifest[route] ?? [] + return + } + + const result = await fetchRelatedDocs(sourceUrl) + if (result.ok) successfulRequests += 1 + if (result.retryableFailure) { + consecutiveFailures += 1 + if (consecutiveFailures >= circuitBreakerFailureThreshold && !circuitOpen) { + circuitOpen = true + console.warn( + `[Graphite ILAPI] Skipping remaining docs requests after ${consecutiveFailures} consecutive upstream failures.`, + ) + } + } else { + consecutiveFailures = 0 + } + + const route = new URL(sourceUrl).pathname.slice('/developers'.length) || '/' + manifest[route] = result.ok ? result.links : (cached?.manifest[route] ?? []) + }) + + if (isCompleteManifestRefresh(successfulRequests, sourceUrls.length)) { + await writeCache(rootDirectory, { + generatedAt: Date.now(), + manifest, + sourceUrls, + version: cacheVersion, + }) + } else if (successfulRequests === 0 && cached) { + return cached.manifest + } + + return manifest +} + +export function isCompleteManifestRefresh( + successfulRequests: number, + totalRequests: number, +): boolean { + return totalRequests > 0 && successfulRequests === totalRequests +} + +async function discoverLocalDocsRoutes(rootDirectory: string): Promise { + const pagesDirectory = path.join(rootDirectory, 'src/pages/docs') + const files = await filesWithin(pagesDirectory) + return files.flatMap((file) => { + const route = docsPageRouteFromFile(path.relative(rootDirectory, file)) + return route ? [route] : [] + }) +} + +async function discoverSitemapDocsUrls(): Promise { + try { + const response = await fetch(productionSitemapUrl, { + headers: { accept: 'application/xml,text/xml' }, + signal: AbortSignal.timeout(requestTimeoutMs), + }) + if (!response.ok) return [] + + const xml = await response.text() + const urls: string[] = [] + for (const match of xml.matchAll(/\s*([^<]+?)\s*<\/loc>/g)) { + const rawUrl = match[1]?.replaceAll('&', '&') + if (!rawUrl) continue + + try { + const url = new URL(rawUrl) + if ( + url.origin === TEMPO_ORIGIN && + (url.pathname === PUBLIC_DOCS_PREFIX || url.pathname.startsWith(`${PUBLIC_DOCS_PREFIX}/`)) + ) { + urls.push(`${url.origin}${url.pathname}${url.search}`) + } + } catch { + // Ignore invalid sitemap entries. + } + } + return urls + } catch { + return [] + } +} + +function fetchRelatedDocs(sourceUrl: string): Promise { + const cached = requestPromises.get(sourceUrl) + if (cached) return cached + + const request = (async (): Promise => { + try { + const endpoint = new URL(GRAPHITE_RELATED_DOCS_ENDPOINT) + endpoint.searchParams.set('url', sourceUrl) + const response = await fetch(endpoint, { signal: AbortSignal.timeout(requestTimeoutMs) }) + if (response.status === 204) { + return { links: [], ok: true, retryableFailure: false } + } + if (!response.ok) { + const retryableFailure = response.status === 429 || response.status >= 500 + return { links: [], ok: !retryableFailure, retryableFailure } + } + + const rawBody = await readResponseText(response, maxResponseBytes) + if (rawBody === undefined) { + return { links: [], ok: false, retryableFailure: true } + } + const payload: unknown = JSON.parse(rawBody) + if ( + typeof payload !== 'object' || + payload === null || + !Array.isArray((payload as { related_links?: unknown }).related_links) + ) { + return { links: [], ok: false, retryableFailure: true } + } + return { + links: parseGraphiteRelatedDocs(payload, sourceUrl), + ok: true, + retryableFailure: false, + } + } catch { + return { links: [], ok: false, retryableFailure: true } + } + })() + + requestPromises.set(sourceUrl, request) + return request +} + +async function readResponseText(response: Response, maxBytes: number): Promise { + const contentLength = Number(response.headers.get('content-length')) + if (Number.isFinite(contentLength) && contentLength > maxBytes) return + + if (!response.body) { + const text = await response.text() + return new TextEncoder().encode(text).byteLength <= maxBytes ? text : undefined + } + + const reader = response.body.getReader() + const decoder = new TextDecoder() + let totalBytes = 0 + let text = '' + + while (true) { + const { done, value } = await reader.read() + if (done) break + totalBytes += value.byteLength + if (totalBytes > maxBytes) { + await reader.cancel() + return + } + text += decoder.decode(value, { stream: true }) + } + + return text + decoder.decode() +} + +function createStartRateLimiter(intervalMs: number): () => Promise { + let nextStartAt = 0 + let queue = Promise.resolve() + + return () => { + const ready = queue.then(async () => { + const delay = Math.max(0, nextStartAt - Date.now()) + if (delay > 0) await new Promise((resolve) => setTimeout(resolve, delay)) + nextStartAt = Date.now() + intervalMs + }) + queue = ready.catch(() => undefined) + return ready + } +} + +async function mapWithConcurrency( + values: T[], + concurrency: number, + mapper: (value: T) => Promise, +): Promise { + let cursor = 0 + + async function worker() { + while (cursor < values.length) { + const index = cursor + cursor += 1 + const value = values[index] + if (value !== undefined) await mapper(value) + } + } + + await Promise.all(Array.from({ length: Math.min(concurrency, values.length) }, () => worker())) +} + +async function filesWithin(directory: string): Promise { + try { + const entries = await fs.readdir(directory, { withFileTypes: true }) + const files = await Promise.all( + entries.map(async (entry) => { + const entryPath = path.join(directory, entry.name) + if (entry.isDirectory()) return filesWithin(entryPath) + return entry.isFile() ? [entryPath] : [] + }), + ) + return files.flat() + } catch { + return [] + } +} + +async function readCache(rootDirectory: string): Promise { + try { + const contents = await fs.readFile(cachePath(rootDirectory), 'utf8') + const cache: unknown = JSON.parse(contents) + if (!isCacheFile(cache)) return + return cache + } catch { + return + } +} + +async function writeCache(rootDirectory: string, cache: CacheFile): Promise { + try { + const filePath = cachePath(rootDirectory) + await fs.mkdir(path.dirname(filePath), { recursive: true }) + await fs.writeFile(filePath, JSON.stringify(cache), 'utf8') + } catch { + // Caching is an optimization. A read-only build filesystem must not fail the build. + } +} + +function cachePath(rootDirectory: string): string { + return path.join(rootDirectory, 'node_modules/.cache/graphite-related-docs.json') +} + +function isCacheFile(value: unknown): value is CacheFile { + if (typeof value !== 'object' || value === null) return false + const cache = value as Partial + return ( + cache.version === cacheVersion && + typeof cache.generatedAt === 'number' && + Array.isArray(cache.sourceUrls) && + cache.sourceUrls.every((source) => typeof source === 'string') && + typeof cache.manifest === 'object' && + cache.manifest !== null + ) +} diff --git a/src/components/RelatedDocsLinks.tsx b/src/components/RelatedDocsLinks.tsx new file mode 100644 index 00000000..ab865335 --- /dev/null +++ b/src/components/RelatedDocsLinks.tsx @@ -0,0 +1,41 @@ +'use client' + +import relatedDocsManifest from 'virtual:graphite-related-docs' +import { Link, useRouter } from 'waku' +import { relatedDocsForRoute } from '../lib/graphite-related-docs' + +export default function RelatedDocsLinks() { + const { path } = useRouter() + const links = relatedDocsForRoute(relatedDocsManifest, path ?? '/') + if (links.length === 0) return null + + return ( +
+ +
    + {links.map((link) => ( +
  • + + {link.title} + {link.description && ( + + {link.description} + + )} + +
    • + ))} +
+
+ ) +} diff --git a/src/lib/graphite-related-docs-plugin.test.ts b/src/lib/graphite-related-docs-plugin.test.ts new file mode 100644 index 00000000..eed6c553 --- /dev/null +++ b/src/lib/graphite-related-docs-plugin.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' +import { + docsPageRouteFromFile, + isCompleteManifestRefresh, +} from '../../scripts/graphite-related-docs-plugin' + +describe('docsPageRouteFromFile', () => { + it.each([ + ['src/pages/docs/index.mdx', '/docs'], + ['src/pages/docs/tools.mdx', '/docs/tools'], + ['src/pages/docs/guide/payments/index.mdx', '/docs/guide/payments'], + ['src/pages/docs/guide/payments/send-a-payment.mdx', '/docs/guide/payments/send-a-payment'], + ['src/pages/docs/changelog.md', '/docs/changelog'], + ])('maps %s to %s', (filePath, expected) => { + expect(docsPageRouteFromFile(filePath)).toBe(expected) + }) + + it.each([ + 'src/pages/docs/_layout.tsx', + 'src/pages/index.tsx', + 'src/pages/docs/not-markdown.txt', + ])('ignores %s', (filePath) => { + expect(docsPageRouteFromFile(filePath)).toBeUndefined() + }) +}) + +describe('isCompleteManifestRefresh', () => { + it('persists only a complete non-empty refresh', () => { + expect(isCompleteManifestRefresh(221, 221)).toBe(true) + expect(isCompleteManifestRefresh(220, 221)).toBe(false) + expect(isCompleteManifestRefresh(0, 0)).toBe(false) + }) +}) diff --git a/src/lib/graphite-related-docs.test.ts b/src/lib/graphite-related-docs.test.ts new file mode 100644 index 00000000..051d430a --- /dev/null +++ b/src/lib/graphite-related-docs.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from 'vitest' +import { + canonicalDocsUrl, + normalizeDocsRoutePath, + parseGraphiteRelatedDocs, + relatedDocsForRoute, +} from './graphite-related-docs' + +const source = 'https://tempo.xyz/developers/docs/guide/payments/send-a-payment' + +describe('parseGraphiteRelatedDocs', () => { + it('keeps all valid related and random links in response order', () => { + expect( + parseGraphiteRelatedDocs( + { + related_links: [ + { + type: 'related', + title: 'Accept a payment', + description: 'Accept stablecoin payments.', + url: 'https://tempo.xyz/developers/docs/guide/payments/accept-a-payment', + }, + { + type: 'random', + title: 'Tempo transactions', + url: 'https://tempo.xyz/developers/docs/protocol/transactions', + }, + ], + }, + source, + ), + ).toEqual([ + { + type: 'related', + title: 'Accept a payment', + description: 'Accept stablecoin payments.', + href: '/docs/guide/payments/accept-a-payment', + }, + { + type: 'random', + title: 'Tempo transactions', + href: '/docs/protocol/transactions', + }, + ]) + }) + + it('rejects untrusted, non-docs, self, duplicate, and malformed links', () => { + expect( + parseGraphiteRelatedDocs( + { + related_links: [ + { type: 'related', title: 'Self', url: `${source}/` }, + { + type: 'related', + title: 'External', + url: 'https://example.com/developers/docs/external', + }, + { + type: 'related', + title: 'Wrong protocol', + url: 'http://tempo.xyz/developers/docs/wrong-protocol', + }, + { + type: 'related', + title: 'Marketing page', + url: 'https://tempo.xyz/developers/blog/post', + }, + { + type: 'other', + title: 'Unknown type', + url: 'https://tempo.xyz/developers/docs/unknown-type', + }, + { + type: 'related', + title: 'Valid', + url: 'https://tempo.xyz/developers/docs/tools', + }, + { + type: 'random', + title: 'Duplicate', + url: 'https://tempo.xyz/developers/docs/tools/', + }, + { + type: 'related', + title: ' ', + url: 'https://tempo.xyz/developers/docs/no-title', + }, + { + type: 'related', + title: 'x'.repeat(241), + url: 'https://tempo.xyz/developers/docs/oversized-title', + }, + { + type: 'related', + title: 'Control\u0000character', + url: 'https://tempo.xyz/developers/docs/control-character', + }, + ], + }, + source, + ), + ).toEqual([{ type: 'related', title: 'Valid', href: '/docs/tools' }]) + }) + + it('fails open for invalid response shapes and source URLs', () => { + expect(parseGraphiteRelatedDocs(null, source)).toEqual([]) + expect(parseGraphiteRelatedDocs({ related_links: 'invalid' }, source)).toEqual([]) + expect(parseGraphiteRelatedDocs({ related_links: [] }, 'not a URL')).toEqual([]) + }) + + it('keeps the docs root and drops an oversized optional description', () => { + expect( + parseGraphiteRelatedDocs( + { + related_links: [ + { + type: 'related', + title: 'Documentation home', + description: 'x'.repeat(501), + url: 'https://tempo.xyz/developers/docs', + }, + ], + }, + source, + ), + ).toEqual([ + { + type: 'related', + title: 'Documentation home', + href: '/docs', + }, + ]) + }) +}) + +describe('docs URL normalization', () => { + it.each([ + ['/developers/docs/tools', '/docs/tools'], + ['/docs/tools/', '/docs/tools'], + ['/developers', '/'], + ['/docs', '/docs'], + ])('normalizes %s to %s', (input, expected) => { + expect(normalizeDocsRoutePath(input)).toBe(expected) + }) + + it('builds only canonical public docs URLs', () => { + expect(canonicalDocsUrl('/docs/tools')).toBe('https://tempo.xyz/developers/docs/tools') + expect(canonicalDocsUrl('/developers/docs/tools')).toBe( + 'https://tempo.xyz/developers/docs/tools', + ) + expect(canonicalDocsUrl('/blog')).toBeUndefined() + }) + + it('selects new links when the Vocs route changes', () => { + const manifest = { + '/docs/tools': [{ type: 'related' as const, title: 'SDKs', href: '/docs/sdk' }], + '/docs/sdk': [{ type: 'random' as const, title: 'Tools', href: '/docs/tools' }], + } + + expect(relatedDocsForRoute(manifest, '/docs/tools')).toEqual(manifest['/docs/tools']) + expect(relatedDocsForRoute(manifest, '/developers/docs/sdk')).toEqual(manifest['/docs/sdk']) + }) +}) diff --git a/src/lib/graphite-related-docs.ts b/src/lib/graphite-related-docs.ts new file mode 100644 index 00000000..ebc16462 --- /dev/null +++ b/src/lib/graphite-related-docs.ts @@ -0,0 +1,120 @@ +export const GRAPHITE_RELATED_DOCS_ENDPOINT = + 'https://ilapi.graphite.io/tempoxyz/docs/related-links' +export const PUBLIC_DEVELOPERS_PREFIX = '/developers' +export const PUBLIC_DOCS_PREFIX = `${PUBLIC_DEVELOPERS_PREFIX}/docs` +export const TEMPO_ORIGIN = 'https://tempo.xyz' +const MAX_LINKS = 32 +const MAX_TITLE_CHARS = 240 +const MAX_DESCRIPTION_CHARS = 500 + +export type RelatedDocsLink = { + description?: string + href: string + title: string + type: 'random' | 'related' +} + +export type RelatedDocsManifest = Record + +export function normalizeDocsRoutePath(pathname: string): string { + let normalized = pathname || '/' + + if (normalized === PUBLIC_DEVELOPERS_PREFIX) normalized = '/' + else if (normalized.startsWith(`${PUBLIC_DEVELOPERS_PREFIX}/`)) { + normalized = normalized.slice(PUBLIC_DEVELOPERS_PREFIX.length) || '/' + } + + if (normalized.length > 1) normalized = normalized.replace(/\/+$/, '') + return normalized || '/' +} + +export function canonicalDocsUrl(routePath: string): string | undefined { + const normalized = normalizeDocsRoutePath(routePath) + if (normalized !== '/docs' && !normalized.startsWith('/docs/')) return + return `${TEMPO_ORIGIN}${PUBLIC_DEVELOPERS_PREFIX}${normalized}` +} + +export function relatedDocsForRoute( + manifest: RelatedDocsManifest, + routePath: string, +): RelatedDocsLink[] { + return manifest[normalizeDocsRoutePath(routePath)] ?? [] +} + +export function parseGraphiteRelatedDocs( + payload: unknown, + sourceCanonicalUrl: string, +): RelatedDocsLink[] { + if (!isRecord(payload) || !Array.isArray(payload.related_links)) return [] + + const source = parseTempoUrl(sourceCanonicalUrl) + if (!source) return [] + + const sourcePath = normalizeComparablePath(source.pathname) + const seen = new Set() + const links: RelatedDocsLink[] = [] + + for (const candidate of payload.related_links) { + if (!isRecord(candidate)) continue + if (candidate.type !== 'related' && candidate.type !== 'random') continue + + const title = boundedText(candidate.title, MAX_TITLE_CHARS) + const target = typeof candidate.url === 'string' ? parseTempoUrl(candidate.url) : undefined + if ( + !title || + !target || + (target.pathname !== PUBLIC_DOCS_PREFIX && + !target.pathname.startsWith(`${PUBLIC_DOCS_PREFIX}/`)) + ) { + continue + } + if (normalizeComparablePath(target.pathname) === sourcePath) continue + + const href = `${target.pathname.slice(PUBLIC_DEVELOPERS_PREFIX.length)}${target.search}${target.hash}` + const targetKey = `${normalizeComparablePath(target.pathname)}${target.search}` + if (seen.has(targetKey)) continue + seen.add(targetKey) + + const description = boundedText(candidate.description, MAX_DESCRIPTION_CHARS) + links.push({ + ...(description ? { description } : {}), + href, + title, + type: candidate.type, + }) + if (links.length >= MAX_LINKS) break + } + + return links +} + +function parseTempoUrl(value: string): URL | undefined { + try { + const url = new URL(value) + if (url.origin !== TEMPO_ORIGIN || url.username || url.password) return + return url + } catch { + return + } +} + +function normalizeComparablePath(pathname: string): string { + const normalized = pathname.length > 1 ? pathname.replace(/\/+$/, '') : pathname + return normalized || '/' +} + +function boundedText(value: unknown, maxChars: number): string | undefined { + if (typeof value !== 'string') return + const normalized = value.trim() + if (!normalized || normalized.length > maxChars) return + + for (const character of normalized) { + const code = character.charCodeAt(0) + if (code <= 31 || code === 127) return + } + return normalized +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/src/pages/_slots.tsx b/src/pages/_slots.tsx new file mode 100644 index 00000000..317dca0e --- /dev/null +++ b/src/pages/_slots.tsx @@ -0,0 +1,4 @@ +export { default as Footer } from '../components/RelatedDocsLinks' + +export const OutlineFooter = undefined +export const SidebarHeader = undefined diff --git a/vite.config.ts b/vite.config.ts index 18724075..b517266d 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -6,6 +6,7 @@ import Icons from 'unplugin-icons/vite' import { defineConfig, loadEnv, type Plugin, type ResolvedConfig } from 'vite' import mkcert from 'vite-plugin-mkcert' import { vocs } from 'vocs/vite' +import { graphiteRelatedDocsPlugin } from './scripts/graphite-related-docs-plugin' import { resolveBaseUrl } from './src/lib/base-url' import { canonicalizeGeneratedDeveloperLinks } from './src/lib/canonical-developer-links' import { blogPostsPlugin } from './src/marketing/blogPlugin' @@ -26,6 +27,7 @@ export default defineConfig(({ mode }) => { blogPostsPlugin(), marketingPages(), developersProxyBasePath(), + graphiteRelatedDocsPlugin(), vocs(), Icons({ compiler: 'jsx', jsx: 'react' }), react(),