diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 88633b05..b8aebedd 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -44,7 +44,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Type check - run: pnpm types:check + run: pnpm typecheck - name: Lint run: pnpm lint @@ -54,11 +54,14 @@ jobs: DOCS_BASE_PATH: /taskito run: pnpm build + - name: SPA fallback for client-side routes + run: cp build/client/__spa-fallback.html build/client/404.html + - name: Upload Pages artifact if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' uses: actions/upload-pages-artifact@v5 with: - path: docs/out + path: docs/build/client deploy: needs: build diff --git a/docs/.gitignore b/docs/.gitignore index 9e429e49..8c074087 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -23,4 +23,7 @@ yarn-error.log* # others .env*.local .vercel -next-env.d.ts \ No newline at end of file +next-env.d.ts +# React Router +.react-router/ +build/ diff --git a/docs/README.md b/docs/README.md index d0aff640..35e28957 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,47 +1,49 @@ -# docs +# Taskito docs -This is a Next.js application generated with -[Create Fumadocs](https://github.com/fuma-nama/fumadocs). +The Taskito documentation site — a polyglot (Python + Node.js) docs site built with +[React Router v7](https://reactrouter.com) in framework mode, statically prerendered +to HTML for GitHub Pages. -It is a Next.js app with [Static Export](https://nextjs.org/docs/app/guides/static-exports) configured. +## Stack -Run development server: +- **React Router v7** (framework mode, `ssr: false` + `prerender`) — static export to + `build/client/`. +- **MDX** via `@mdx-js/rollup` — the content under `content/docs/**/*.mdx` is the source, + compiled at build and rendered through the component map in `app/components/mdx/`. +- **Shiki** for build-time code highlighting; **Mermaid** (client-side) for diagrams. +- **Tailwind v4** + a ported design system in `app/app.css` (dark + warm-light themes). +- **MiniSearch** for the client-side ⌘K search index. + +## Develop ```bash -npm run dev -# or -pnpm dev -# or -yarn dev +pnpm install +pnpm dev # local dev server +pnpm typecheck # react-router typegen + tsc +pnpm lint # biome +pnpm build # static prerender → build/client/ +pnpm start # serve the built output ``` -Open http://localhost:3000 with your browser to see the result. - -## Explore - -In the project, you can see: - -- `lib/source.ts`: Code for content source adapter, [`loader()`](https://fumadocs.dev/docs/headless/source-api) provides the interface to access your content. -- `lib/layout.shared.tsx`: Shared options for layouts, optional but preferred to keep. - -| Route | Description | -| ------------------------- | ------------------------------------------------------ | -| `app/(home)` | The route group for your landing page and other pages. | -| `app/docs` | The documentation layout and pages. | -| `app/api/search/route.ts` | The Route Handler for search. | +Set `DOCS_BASE_PATH=/taskito` to build under the GitHub Pages base path (CI does this). -### Fumadocs MDX +## Layout -A `source.config.ts` config file has been included, you can customise different options like frontmatter schema. - -Read the [Introduction](https://fumadocs.dev/docs/mdx) for further details. - -## Learn More - -To learn more about Next.js and Fumadocs, take a look at the following -resources: +``` +content/docs/**/*.mdx the documentation source (+ meta.json nav) +app/ + root.tsx document shell, fonts, no-flash theme + routes.ts route config (landing, llms.txt, docs catch-all) + routes/ home, docs-layout, docs.$ (MDX page), llms[.txt] + components/ + landing/ polyglot landing sections + docs/ sidebar, SDK switch, TOC, prev/next, search modal + mdx/ Callout/Tabs/Card shims + MDX component map + ui/ nav, theme toggle, RawHtml + animated/ diagram components (reused in MDX) + lib/ content glob, nav tree, search index, theme, highlighters +``` -- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js - features and API. -- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. -- [Fumadocs](https://fumadocs.dev) - learn about Fumadocs +Adding a page: drop an `.mdx` file under `content/docs/`, add it to the directory's +`meta.json`. It is picked up, prerendered, indexed for search, and slotted into the +sidebar automatically. diff --git a/docs/app/app.css b/docs/app/app.css new file mode 100644 index 00000000..c066aaee --- /dev/null +++ b/docs/app/app.css @@ -0,0 +1,12 @@ +@import "tailwindcss"; + +/* Design system, split by concern. tokens first — every module uses var(--*). */ +@import "./styles/tokens.css"; +@import "./styles/atoms.css"; +@import "./styles/cmdk.css"; +@import "./styles/docs.css"; +@import "./styles/changelog.css"; +@import "./styles/diagrams.css"; +@import "./styles/landing.css"; +@import "./styles/shiki.css"; +@import "./styles/sdk.css"; diff --git a/docs/app/components/diagrams/arch-stack.tsx b/docs/app/components/diagrams/arch-stack.tsx new file mode 100644 index 00000000..8030b5f9 --- /dev/null +++ b/docs/app/components/diagrams/arch-stack.tsx @@ -0,0 +1,166 @@ +import { type CSSProperties, Fragment, type ReactNode } from "react"; + +// Layered architecture stack — exact port of the prototype's `.archstack` HTML +// (semantic layers + PyO3/store boundaries as direct children). + +type Kind = "py" | "rust" | "store" | ""; + +interface Layer { + tag: string; + /** Pill colour: `.ltag t-*`. */ + tagKind?: Kind; + /** `.layer` modifier; defaults to `tagKind`. The prototype sometimes pairs a + * `rust` layer surface with a `py` pill (e.g. the resource-proxy layer). */ + layerKind?: Kind; + style?: CSSProperties; + title: string; + body: ReactNode; + role?: string; +} + +function ArchStack({ + layers, + bounds, + legend, +}: { + layers: Layer[]; + bounds: ReactNode[]; // one fewer than layers + legend?: ReactNode; +}) { + return ( +
+ {legend ?
{legend}
: null} + {layers.map((l, i) => ( + +
+ {l.tag} +
+
{l.title}
+
{l.body}
+
+ {l.role ? {l.role} : null} +
+ {i < bounds.length ?
{bounds[i]}
: null} +
+ ))} +
+ ); +} + +/** `architecture/overview.mdx` — the Python / Rust / Store stack. */ +export function ArchitectureStack() { + return ( + + + you call down — enqueue + jobs + + + results & state flow back + up + + + } + layers={[ + { + tag: "Python", + tagKind: "py", + title: "User-facing API", + body: ( + <> + Queue, @task, .delay(), + results, workflows, resources — the surface you write against. + + ), + role: "what you write", + }, + { + tag: "Rust", + tagKind: "rust", + title: "Engine", + body: "Scheduler, dispatcher, worker pool, rate limiter — Tokio runtime, OS-thread pool, near-zero Python overhead.", + role: "where the work runs", + }, + { + tag: "Store", + tagKind: "store", + title: "Storage layer", + body: "SQLite (bundled) or Postgres — jobs, results, schedules, rate-limit state, all in one place.", + role: "where state lives", + }, + ]} + bounds={[ + <> + + PyO3 boundary + + , + <> + reads & writes + + , + ]} + /> + ); +} + +/** `architecture/resources.mdx` — the 3-layer resource pipeline. */ +export function ResourcePipeline() { + return ( + + The ArgumentInterceptor walks every argument before + serialization. CONVERT types become JSON-safe markers, REDIRECT + types become DI placeholders, PROXY types are deconstructed, + REJECT types raise in strict mode. + + ), + }, + { + tag: "Layer 2", + tagKind: "py", + title: "Worker resource runtime", + body: ( + <> + ResourceRuntime initializes resources at worker + startup in topological order, then injects requested ones (via{" "} + inject= or Inject["name"]) as kwargs. + Task-scoped resources come from a semaphore pool. + + ), + }, + { + tag: "Layer 3", + tagKind: "py", + layerKind: "rust", + style: { + borderColor: "var(--indigo-line)", + background: + "linear-gradient(100deg,var(--indigo-soft),var(--panel))", + }, + title: "Resource proxies", + body: ( + <> + ProxyHandlers deconstruct live objects (file handles, + HTTP sessions, cloud clients) into a JSON recipe and reconstruct + them on the worker. Recipes are optionally HMAC-signed for tamper + detection. + + ), + }, + ]} + bounds={["enqueue → worker", "before task()"]} + /> + ); +} diff --git a/docs/app/components/diagrams/failure-timeline.tsx b/docs/app/components/diagrams/failure-timeline.tsx new file mode 100644 index 00000000..322dd34a --- /dev/null +++ b/docs/app/components/diagrams/failure-timeline.tsx @@ -0,0 +1,54 @@ +import type { ReactNode } from "react"; + +// Crash-recovery timeline — ports the prototype's `.timeline` (`.tl`/`.tt`/`.tb`). +// `architecture/failure-model.mdx`. +const STEPS: { at: string; body: ReactNode }[] = [ + { + at: "t = 0s", + body: ( + <> + Worker claims a job — status pending → running,{" "} + started_at recorded. + + ), + }, + { + at: "t = crash", + body: ( + <> + Worker dies mid-task. The row is left in running with no result. + + ), + }, + { + at: "t = timeout_ms", + body: ( + <> + The stale reaper (runs ~every 5s) finds the job has exceeded{" "} + timeout_ms and marks it failed. + + ), + }, + { + at: "t + retry", + body: ( + <> + If retries remain → re-enqueued as pending with backoff. + Otherwise → dead-letter. + + ), + }, +]; + +export function CrashRecoveryTimeline() { + return ( +
+ {STEPS.map((s) => ( +
+
{s.at}
+
{s.body}
+
+ ))} +
+ ); +} diff --git a/docs/app/components/diagrams/index.ts b/docs/app/components/diagrams/index.ts new file mode 100644 index 00000000..fe3e30e9 --- /dev/null +++ b/docs/app/components/diagrams/index.ts @@ -0,0 +1,12 @@ +// Single barrel for every diagram component used in MDX. All architecture +// diagrams below are prototype-faithful ports (arch.js + arch-*.html). + +export { Mermaid } from "@/components/mermaid"; +export { ArchitectureStack, ResourcePipeline } from "./arch-stack"; +export { CrashRecoveryTimeline } from "./failure-timeline"; +export { JobStateMachine } from "./lifecycle-graph"; +export { MeshDemo } from "./mesh-demo"; +export { SchedulerPollLoop } from "./poll-cycle"; +export { SchemaGrid } from "./schema-grid"; +export { Seq } from "./seq"; +export { WorkerDispatch } from "./worker-fork"; diff --git a/docs/app/components/diagrams/lifecycle-graph.tsx b/docs/app/components/diagrams/lifecycle-graph.tsx new file mode 100644 index 00000000..fc639ffd --- /dev/null +++ b/docs/app/components/diagrams/lifecycle-graph.tsx @@ -0,0 +1,170 @@ +// Job state machine — faithful port of arch.js `lifecycle()` (static semantic-color +// SVG). `architecture/job-lifecycle.mdx`. + +const ARROWS = [ + { id: "lcar", stroke: "var(--line3)", w: 1.6 }, + { id: "lcok", stroke: "var(--grn)", w: 1.8 }, + { id: "lcwarn", stroke: "var(--amber)", w: 1.8 }, + { id: "lcind", stroke: "var(--indigo)", w: 1.8 }, +]; + +function LcNode({ + x, + y, + label, + code, + cls, +}: { + x: number; + y: number; + label: string; + code: string; + cls: string; +}) { + return ( + + + + + {label} + + + status {code} + + + ); +} + +export function JobStateMachine() { + return ( +
+ + + {ARROWS.map((a) => ( + + + + ))} + + + + + dispatch + + + + success + + + + error + + + + exhausted + + + + retry · backoff + + + + cancel + + + + + + + + + +
+ ); +} diff --git a/docs/app/components/diagrams/mesh-demo.tsx b/docs/app/components/diagrams/mesh-demo.tsx new file mode 100644 index 00000000..f47f1d68 --- /dev/null +++ b/docs/app/components/diagrams/mesh-demo.tsx @@ -0,0 +1,518 @@ +import { useEffect, useRef } from "react"; + +// Interactive mesh work-stealing demo — faithful port of arch.js `meshdemo()`. +// The proven imperative animation runs inside one effect against a ref'd stage; +// React owns mount/cleanup (timers + resize listener + DOM). Client-only (no SSR), +// so the prerendered page ships an empty stage that hydrates here. + +interface Region { + code: string; + city: string; + lat: number; + lng: number; + off: [number, number]; +} +interface Worker { + el: HTMLDivElement; + dot: HTMLSpanElement; + region: Region; + qEl: HTMLElement; + nEl: HTMLElement; + pulse: HTMLElement; + jobs: number; + x: number; + y: number; + cx: number; + cy: number; +} +interface MeshLink { + a: number; + b: number; + el: SVGPathElement; + lab: HTMLDivElement; + ms: number; + geo?: { cx: number; cy: number; mx: number; my: number }; +} + +const REGIONS: Region[] = [ + { + code: "us-east-1", + city: "N. Virginia", + lat: 38.9, + lng: -77.0, + off: [-2, 64], + }, + { code: "us-west-2", city: "Oregon", lat: 45.5, lng: -122.7, off: [-6, -66] }, + { code: "eu-west-1", city: "Ireland", lat: 53.3, lng: -6.2, off: [6, 64] }, + { code: "ap-south-1", city: "Mumbai", lat: 19.0, lng: 72.8, off: [0, 66] }, + { + code: "ap-northeast-1", + city: "Tokyo", + lat: 35.6, + lng: 139.7, + off: [-14, -60], + }, +]; + +const LANDS: [number, number][][] = [ + [ + [-140, 60], + [-125, 49], + [-123, 38], + [-117, 32], + [-97, 25], + [-81, 25], + [-80, 31], + [-70, 44], + [-60, 48], + [-58, 52], + [-78, 62], + [-110, 60], + [-140, 60], + ], + [ + [-10, 43], + [-9, 37], + [4, 40], + [18, 41], + [28, 41], + [31, 46], + [41, 48], + [30, 60], + [8, 58], + [-6, 57], + [-10, 50], + ], + [ + [-16, 14], + [-9, 28], + [11, 33], + [25, 32], + [34, 30], + [43, 12], + [48, 9], + [40, 5], + [14, 5], + [-9, 6], + [-16, 14], + ], + [ + [33, 46], + [46, 40], + [61, 38], + [76, 33], + [90, 24], + [101, 22], + [120, 31], + [135, 35], + [143, 45], + [139, 52], + [118, 53], + [92, 55], + [66, 55], + [46, 52], + [33, 46], + ], +]; + +export function MeshDemo() { + const stageRef = useRef(null); + + useEffect(() => { + const stage = stageRef.current; + if (!stage) { + return; + } + const REDUCE = + window.matchMedia?.("(prefers-reduced-motion: reduce)").matches ?? false; + const SVGNS = "http://www.w3.org/2000/svg"; + const H = 480; + const N = REGIONS.length; + let running = !REDUCE; + let timers: ReturnType[] = []; + let completed = 0; + let steals = 0; + let stolenJobs = 0; + let focus = -1; + const workers: Worker[] = []; + let links: MeshLink[] = []; + + const LNG0 = -150, + LNG1 = 165, + LAT0 = 5, + LAT1 = 62, + padX = 58, + padTop = 66, + padBot = 46; + const proj = (lng: number, lat: number, W: number): [number, number] => { + const mapW = W - padX * 2, + mapH = H - padTop - padBot; + return [ + padX + ((lng - LNG0) / (LNG1 - LNG0)) * mapW, + padTop + ((LAT1 - lat) / (LAT1 - LAT0)) * mapH, + ]; + }; + const haversine = (a: Region, b: Region) => { + const R = 6371, + toR = Math.PI / 180; + const dLat = (b.lat - a.lat) * toR, + dLng = (b.lng - a.lng) * toR, + la1 = a.lat * toR, + la2 = b.lat * toR; + const h = + Math.sin(dLat / 2) ** 2 + + Math.cos(la1) * Math.cos(la2) * Math.sin(dLng / 2) ** 2; + return 2 * R * Math.asin(Math.sqrt(h)); + }; + const latencyMs = (a: Region, b: Region) => + Math.round(haversine(a, b) / 95) + 6; + const arc = (x0: number, y0: number, x1: number, y1: number) => { + const mx = (x0 + x1) / 2, + my = (y0 + y1) / 2, + dx = x1 - x0, + dy = y1 - y0, + len = Math.hypot(dx, dy) || 1; + const bow = -len * 0.16, + cx = mx + (-dy / len) * bow, + cy = my + (dx / len) * bow; + return { + cx, + cy, + mx: 0.25 * x0 + 0.5 * cx + 0.25 * x1, + my: 0.25 * y0 + 0.5 * cy + 0.25 * y1, + }; + }; + + stage.innerHTML = + '
' + + '0 completed' + + '0 steals' + + '0 moved · HTTP
' + + `
` + + '' + + `
` + + '' + + '
region nodemesh link (gossip)work-steal over HTTP
'; + + const q = (sel: string) => stage.querySelector(sel) as T; + const mapSvg = q("#m-map"); + const gGrat = q("#m-grat"); + const gLand = q("#m-land"); + const gLinks = q("#m-links"); + const gConn = q("#m-conn"); + const setText = (sel: string, v: string) => { + const el = q(sel); + if (el) el.textContent = v; + }; + + const makeWorker = (i: number): Worker => { + const r = REGIONS[i]; + const w = document.createElement("div"); + w.className = "mw"; + w.innerHTML = + `
${r.code}${r.city}
` + + '
0 queued
'; + stage.appendChild(w); + const dot = document.createElement("span"); + dot.className = "mnode"; + stage.appendChild(dot); + w.addEventListener("mouseenter", () => setFocus(i)); + w.addEventListener("mouseleave", () => setFocus(-1)); + return { + el: w, + dot, + region: r, + qEl: w.querySelector(".mw-q") as HTMLElement, + nEl: w.querySelector(".mw-n") as HTMLElement, + pulse: w.querySelector(".mw-pulse") as HTMLElement, + jobs: 0, + x: 0, + y: 0, + cx: 0, + cy: 0, + }; + }; + + const buildLinks = () => { + gLinks.innerHTML = ""; + links = []; + for (let i = 0; i < N; i++) { + for (let j = i + 1; j < N; j++) { + const p = document.createElementNS(SVGNS, "path"); + p.setAttribute("class", "link"); + gLinks.appendChild(p); + const lab = document.createElement("div"); + lab.className = "mlat"; + stage.appendChild(lab); + links.push({ + a: i, + b: j, + el: p, + lab, + ms: latencyMs(REGIONS[i], REGIONS[j]), + }); + } + } + }; + const linkBetween = (i: number, j: number) => + links.find((L) => (L.a === i && L.b === j) || (L.a === j && L.b === i)) ?? + null; + + const layout = () => { + stage.style.height = `${H}px`; + const W = stage.clientWidth || 760; + mapSvg.setAttribute("viewBox", `0 0 ${W} ${H}`); + let g = ""; + for (let lng = -120; lng <= 150; lng += 30) { + const a = proj(lng, LAT0, W), + b = proj(lng, LAT1, W); + g += ``; + } + for (let lat = 10; lat <= 60; lat += 10) { + const c = proj(LNG0, lat, W), + d = proj(LNG1, lat, W); + g += ``; + } + gGrat.innerHTML = g; + gLand.innerHTML = LANDS.map( + (poly) => + ``, + ).join(""); + let conn = ""; + for (const w of workers) { + const p = proj(w.region.lng, w.region.lat, W); + w.x = p[0]; + w.y = p[1]; + w.dot.style.left = `${p[0]}px`; + w.dot.style.top = `${p[1]}px`; + const cxp = p[0] + w.region.off[0], + cyp = p[1] + w.region.off[1]; + w.cx = cxp; + w.cy = cyp; + w.el.style.left = `${cxp}px`; + w.el.style.top = `${cyp}px`; + conn += ``; + } + gConn.innerHTML = conn; + for (const L of links) { + const wa = workers[L.a], + wb = workers[L.b], + k = arc(wa.x, wa.y, wb.x, wb.y); + L.el.setAttribute( + "d", + `M ${wa.x.toFixed(1)} ${wa.y.toFixed(1)} Q ${k.cx.toFixed(1)} ${k.cy.toFixed(1)} ${wb.x.toFixed(1)} ${wb.y.toFixed(1)}`, + ); + L.geo = k; + L.lab.textContent = `${L.ms} ms`; + L.lab.style.left = `${k.mx}px`; + L.lab.style.top = `${k.my}px`; + } + }; + + const setFocus = (i: number) => { + focus = i; + if (i < 0) { + mapSvg.classList.remove("focus"); + for (const L of links) { + L.el.classList.remove("lit"); + L.lab.classList.remove("show"); + } + return; + } + mapSvg.classList.add("focus"); + for (const L of links) { + const on = L.a === i || L.b === i; + L.el.classList.toggle("lit", on); + L.lab.classList.toggle("show", on); + } + }; + + const renderQueue = (w: Worker) => { + const cap = 8, + n = w.jobs, + show = Math.min(n, cap); + let html = ""; + for (let k = 0; k < show; k++) html += ""; + if (n > cap) html += `+${n - cap}`; + w.qEl.innerHTML = html; + w.nEl.textContent = String(n); + const hot = n >= 6; + w.el.classList.toggle("hot", hot); + w.el.classList.toggle("idle", n === 0); + w.dot.classList.toggle("hot", hot); + }; + + const flyArc = (ai: number, bi: number, cls: string, cb?: () => void) => { + const wa = workers[ai], + wb = workers[bi], + k = arc(wa.x, wa.y, wb.x, wb.y); + const d = document.createElement("span"); + d.className = `mjob ${cls}`.trim(); + stage.appendChild(d); + const frames: Keyframe[] = []; + const steps = 18; + for (let s = 0; s <= steps; s++) { + const t = s / steps, + u = 1 - t; + const x = u * u * wa.x + 2 * u * t * k.cx + t * t * wb.x; + const y = u * u * wa.y + 2 * u * t * k.cy + t * t * wb.y; + frames.push({ + transform: `translate(${x.toFixed(1)}px,${y.toFixed(1)}px) translate(-50%,-50%)`, + opacity: s === 0 || s === steps ? 0 : 1, + }); + } + d.animate(frames, { duration: 740, easing: "ease-in-out" }).onfinish = + () => { + d.remove(); + cb?.(); + }; + }; + + const weightedTarget = () => { + const r = Math.random(); + if (r < 0.5) return 0; + if (r < 0.8) return 1; + return 2 + Math.floor(Math.random() * (N - 2)); + }; + const spawn = (target?: number) => { + const w = workers[target ?? weightedTarget()]; + if (!w) return; + w.jobs++; + renderQueue(w); + }; + const process = () => { + for (const w of workers) { + if (w.jobs > 0 && Math.random() < 0.6) { + w.jobs--; + renderQueue(w); + completed++; + w.pulse.classList.remove("go"); + void w.pulse.offsetWidth; + w.pulse.classList.add("go"); + } + } + setText("#m-done", String(completed)); + }; + const steal = () => { + let total = 0, + maxI = 0; + workers.forEach((w, i) => { + total += w.jobs; + if (w.jobs > workers[maxI].jobs) maxI = i; + }); + const max = workers[maxI]; + if (total === 0 || max.jobs < 2) return; + const avg = total / N; + const needy = workers + .map((w, i) => ({ w, i })) + .filter((o) => o.w !== max && o.w.jobs < avg) + .sort((a, b) => a.w.jobs - b.w.jobs); + if (!needy.length) return; + const surplus = Math.max(1, Math.round(max.jobs - avg)); + const toMove = Math.min(surplus, max.jobs - 1, needy.length * 2, 5); + let moved = 0; + for (let k = 0; k < toMove; k++) { + const tgt = needy[k % needy.length]; + max.jobs--; + tgt.w.jobs++; + renderQueue(max); + renderQueue(tgt.w); + const ti = tgt.i; + const delay = k * 110; + const L = linkBetween(maxI, ti); + workers[ti].el.classList.add("thief"); + setTimeout(() => workers[ti].el.classList.remove("thief"), 700); + if (L) { + L.el.classList.add("on"); + setTimeout(() => L.el.classList.remove("on"), 760); + L.lab.classList.add("show"); + setTimeout(() => { + if (focus < 0) L.lab.classList.remove("show"); + }, 900); + } + setTimeout(() => flyArc(maxI, ti, "steal"), delay); + moved++; + } + if (moved > 0) { + steals++; + stolenJobs += moved; + setText("#m-steal", String(steals)); + setText("#m-moved", String(stolenJobs)); + } + }; + const gossip = () => { + if (!links.length) return; + const L = links[Math.floor(Math.random() * links.length)]; + L.el.classList.add("on"); + setTimeout(() => L.el.classList.remove("on"), 600); + flyArc( + Math.random() < 0.5 ? L.a : L.b, + Math.random() < 0.5 ? L.b : L.a, + "gos", + ); + }; + const reset = () => { + completed = 0; + steals = 0; + stolenJobs = 0; + setText("#m-done", "0"); + setText("#m-steal", "0"); + setText("#m-moved", "0"); + for (const w of workers) w.jobs = 0; + workers[0].jobs = 7; + workers[1].jobs = 4; + workers.forEach(renderQueue); + }; + const stop = () => { + for (const t of timers) clearInterval(t); + timers = []; + }; + const start = () => { + stop(); + timers.push(setInterval(() => running && spawn(), 380)); + timers.push(setInterval(() => running && process(), 720)); + timers.push(setInterval(() => running && steal(), 1100)); + timers.push(setInterval(() => running && gossip(), 760)); + }; + + for (let i = 0; i < N; i++) workers.push(makeWorker(i)); + buildLinks(); + layout(); + window.addEventListener("resize", layout); + + q("#m-burst").addEventListener("click", () => { + for (let k = 0; k < 10; k++) setTimeout(() => spawn(), k * 70); + }); + const toggle = q("#m-toggle"); + toggle.addEventListener("click", () => { + running = !running; + toggle.textContent = running ? "Pause" : "Play"; + }); + q("#m-reset").addEventListener("click", reset); + + if (!REDUCE) { + workers[0].jobs = 7; + workers[1].jobs = 4; + workers.forEach(renderQueue); + start(); + } else { + workers[0].jobs = 5; + workers[1].jobs = 3; + workers[2].jobs = 1; + workers.forEach(renderQueue); + } + + return () => { + stop(); + window.removeEventListener("resize", layout); + stage.innerHTML = ""; + }; + }, []); + + return
; +} diff --git a/docs/app/components/diagrams/poll-cycle.tsx b/docs/app/components/diagrams/poll-cycle.tsx new file mode 100644 index 00000000..6a7d0d01 --- /dev/null +++ b/docs/app/components/diagrams/poll-cycle.tsx @@ -0,0 +1,92 @@ +// Scheduler poll loop — faithful port of arch.js `pollcycle()` (50ms-tick clock +// with a CSS-animated sweep + cadence legend). `architecture/scheduler.mdx`. + +const CX = 180; +const CY = 180; +const R = 120; + +const TASKS = [ + { + a: -90, + name: "try_dispatch", + cad: "every tick", + accent: "var(--indigo-br)", + }, + { + a: 0, + name: "check_periodic", + cad: "~3s · 60 ticks", + accent: "var(--cyan)", + }, + { a: 90, name: "reap_stale", cad: "~5s · 100 ticks", accent: "var(--amber)" }, + { + a: 180, + name: "auto_cleanup", + cad: "~60s · 1200 ticks", + accent: "var(--mut)", + }, +]; + +export function SchedulerPollLoop() { + return ( +
+ + + + + + + {TASKS.map((t) => { + const rad = (t.a * Math.PI) / 180; + return ( + + ); + })} + + 50ms + + + tick + + +
+ {TASKS.map((t) => ( +
+ + {t.name}() + {t.cad} +
+ ))} +
+
+ ); +} diff --git a/docs/app/components/diagrams/schema-grid.tsx b/docs/app/components/diagrams/schema-grid.tsx new file mode 100644 index 00000000..ba818d95 --- /dev/null +++ b/docs/app/components/diagrams/schema-grid.tsx @@ -0,0 +1,97 @@ +// Database schema — exact port of the prototype's `.schema-grid` (six SQLite +// tables as `.entity` cards with PK/FK badges). `architecture/storage.mdx`. + +type Field = { name: string; key?: "PK" | "FK"; type: string }; +type Entity = { name: string; fields: Field[] }; + +const ENTITIES: Entity[] = [ + { + name: "jobs", + fields: [ + { name: "id", key: "PK", type: "TEXT" }, + { name: "queue", type: "TEXT" }, + { name: "task_name", type: "TEXT" }, + { name: "payload", type: "BLOB" }, + { name: "status", type: "INTEGER" }, + { name: "priority", type: "INTEGER" }, + { name: "scheduled_at", type: "INTEGER" }, + { name: "retry_count", type: "INTEGER" }, + { name: "result", type: "BLOB" }, + { name: "timeout_ms", type: "INTEGER" }, + { name: "unique_key", type: "TEXT" }, + ], + }, + { + name: "dead_letter", + fields: [ + { name: "id", key: "PK", type: "TEXT" }, + { name: "original_job_id", type: "TEXT" }, + { name: "task_name", type: "TEXT" }, + { name: "payload", type: "BLOB" }, + { name: "error", type: "TEXT" }, + { name: "retry_count", type: "INTEGER" }, + { name: "failed_at", type: "INTEGER" }, + ], + }, + { + name: "job_errors", + fields: [ + { name: "id", key: "PK", type: "TEXT" }, + { name: "job_id", key: "FK", type: "TEXT" }, + { name: "attempt", type: "INTEGER" }, + { name: "error", type: "TEXT" }, + { name: "failed_at", type: "INTEGER" }, + ], + }, + { + name: "rate_limits", + fields: [ + { name: "key", key: "PK", type: "TEXT" }, + { name: "tokens", type: "REAL" }, + { name: "max_tokens", type: "REAL" }, + { name: "refill_rate", type: "REAL" }, + { name: "last_refill", type: "INTEGER" }, + ], + }, + { + name: "periodic_tasks", + fields: [ + { name: "name", key: "PK", type: "TEXT" }, + { name: "task_name", type: "TEXT" }, + { name: "cron_expr", type: "TEXT" }, + { name: "args", type: "BLOB" }, + { name: "queue", type: "TEXT" }, + { name: "enabled", type: "BOOLEAN" }, + { name: "next_run", type: "INTEGER" }, + ], + }, + { + name: "workers", + fields: [ + { name: "worker_id", key: "PK", type: "TEXT" }, + { name: "last_heartbeat", type: "INTEGER" }, + { name: "queues", type: "TEXT" }, + { name: "status", type: "TEXT" }, + ], + }, +]; + +export function SchemaGrid() { + return ( +
+ {ENTITIES.map((e) => ( +
+
{e.name}
+ {e.fields.map((f) => ( +
+ {f.name} + {f.key === "PK" ? PK : null} + {f.key === "FK" ? FK : null} + {f.type} +
+ ))} +
+ ))} +
+ ); +} diff --git a/docs/app/components/diagrams/seq.tsx b/docs/app/components/diagrams/seq.tsx new file mode 100644 index 00000000..601c7b3e --- /dev/null +++ b/docs/app/components/diagrams/seq.tsx @@ -0,0 +1,45 @@ +// Sequence strip — exact port of the prototype's `.seq` (paired actor heads + +// directional message rows). Used on `architecture/mesh.mdx` for the local +// deque and the work-stealing protocol. + +type SeqRow = + | { self: string } + | { left?: string; right?: string; rev?: boolean }; + +export function Seq({ heads, rows }: { heads: string[]; rows: SeqRow[] }) { + return ( +
+
+ {heads.map((h) => ( +
+ {h} +
+ ))} +
+
+ {rows.map((row, i) => + "self" in row ? ( + // biome-ignore lint/suspicious/noArrayIndexKey: fixed sequence rows +
+ {row.self} +
+ ) : ( +
+ {row.left != null ? ( + {row.left} + ) : null} + + {row.right != null ? ( + {row.right} + ) : null} +
+ ), + )} +
+
+ ); +} diff --git a/docs/app/components/diagrams/worker-fork.tsx b/docs/app/components/diagrams/worker-fork.tsx new file mode 100644 index 00000000..1820d8f0 --- /dev/null +++ b/docs/app/components/diagrams/worker-fork.tsx @@ -0,0 +1,52 @@ +// Worker pool — exact port of the prototype's `.archstack` + `.fork-route` + +// `.archfork` (scheduler routes by task type; sync OS-thread pool vs async pool). +// `architecture/worker-pool.mdx`. +export function WorkerDispatch() { + return ( +
+
+ Rust +
+
Scheduler
+
+ Dequeues a job, applies rate limits, then routes it{" "} + by task type — sync functions to the thread pool,{" "} + async def functions to the async runtime. +
+
+ routes by task type +
+
sync def  ·  async def
+
+
+
bounded mpsc · workers × 2
+
+ Sync pool +
+
OS-thread workers
+
+ Each sync worker is a Rust std::thread. The GIL is + acquired per task via Python::with_gil() — + independent across workers. +
+
+
+
+
+
bypasses the thread pool
+
+ Async pool +
+
NativeAsyncPool
+
+ async def tasks are dispatched to an{" "} + AsyncTaskExecutor on a Python daemon thread;{" "} + PyResultSender bridges results back. +
+
+
+
+
+
+ ); +} diff --git a/docs/app/components/docs/index.ts b/docs/app/components/docs/index.ts new file mode 100644 index 00000000..de68056c --- /dev/null +++ b/docs/app/components/docs/index.ts @@ -0,0 +1,4 @@ +export { PrevNext } from "./prev-next"; +export { SearchModal } from "./search-modal"; +export { Sidebar } from "./sidebar"; +export { Toc } from "./toc"; diff --git a/docs/app/components/docs/prev-next.tsx b/docs/app/components/docs/prev-next.tsx new file mode 100644 index 00000000..f78fa6aa --- /dev/null +++ b/docs/app/components/docs/prev-next.tsx @@ -0,0 +1,36 @@ +import { Link, useLocation } from "react-router"; +import { useActiveSdk } from "@/hooks"; +import { flatNav } from "@/lib"; + +/** Previous/next page links derived from the flattened SDK nav order. */ +export function PrevNext() { + const { pathname } = useLocation(); + const current = pathname.replace(/\/$/, "") || "/"; + const pages = flatNav(useActiveSdk()); + const i = pages.findIndex((p) => p.href === current); + if (i === -1) { + return null; + } + const prev = pages[i - 1]; + const next = pages[i + 1]; + return ( + + ); +} diff --git a/docs/app/components/docs/search-modal.tsx b/docs/app/components/docs/search-modal.tsx new file mode 100644 index 00000000..63b7e5db --- /dev/null +++ b/docs/app/components/docs/search-modal.tsx @@ -0,0 +1,223 @@ +import { Fragment, useEffect, useMemo, useRef, useState } from "react"; +import { useNavigate } from "react-router"; +import type { Sdk } from "@/hooks"; +import { type SearchHit, searchDocs } from "@/lib/search"; + +// Section glyphs mirror the prototype's command-palette icons. +const SECTION_ICON: { match: string; path: string }[] = [ + { match: "getting", path: "M13 2L3 14h7l-1 8 10-12h-7l1-8z" }, + { + match: "guide", + path: "M4 19.5A2.5 2.5 0 0 1 6.5 17H20M4 19.5A2.5 2.5 0 0 0 6.5 22H20V2H6.5A2.5 2.5 0 0 0 4 4.5v15z", + }, + { + match: "architecture", + path: "M12 2L2 7l10 5 10-5-10-5zM2 17l10 5 10-5M2 12l10 5 10-5", + }, + { match: "api", path: "M16 18l6-6-6-6M8 6l-6 6 6 6" }, + { match: "node", path: "M12 2l8.5 5v10L12 22 3.5 17V7z" }, + { match: "reference", path: "M16 18l6-6-6-6M8 6l-6 6 6 6" }, +]; + +function sectionIconPath(section: string): string { + const key = section.toLowerCase(); + return ( + SECTION_ICON.find((s) => key.includes(s.match))?.path ?? + "M12 2a10 10 0 1 0 0 20 10 10 0 0 0 0-20z" + ); +} + +function SectionIcon({ section }: { section: string }) { + return ( + + ); +} + +interface Group { + section: string; + items: SearchHit[]; +} + +/** Group ranked hits by section, preserving first-appearance order. */ +function groupBySection(hits: SearchHit[]): Group[] { + const groups: Group[] = []; + const byName = new Map(); + for (const hit of hits) { + let group = byName.get(hit.section); + if (!group) { + group = { section: hit.section, items: [] }; + byName.set(hit.section, group); + groups.push(group); + } + group.items.push(hit); + } + return groups; +} + +/** ⌘K command palette: browse-by-section on open, ranked search as you type. */ +export function SearchModal({ + open, + onClose, + sdk, +}: { + open: boolean; + onClose: () => void; + sdk?: Sdk; +}) { + const navigate = useNavigate(); + const [query, setQuery] = useState(""); + const [active, setActive] = useState(0); + const listRef = useRef(null); + + const groups = useMemo( + () => groupBySection(searchDocs(query, sdk)), + [query, sdk], + ); + const flat = useMemo(() => groups.flatMap((g) => g.items), [groups]); + + // biome-ignore lint/correctness/useExhaustiveDependencies: reset selection as results change + useEffect(() => { + setActive(0); + }, [query]); + + useEffect(() => { + if (!open) { + setQuery(""); + } + }, [open]); + + // Keep the active row in view during keyboard navigation. + useEffect(() => { + listRef.current + ?.querySelector(`[data-idx="${active}"]`) + ?.scrollIntoView({ block: "nearest" }); + }, [active]); + + if (!open) { + return null; + } + + const go = (slug: string) => { + onClose(); + navigate(slug); + }; + + const onKeyDown = (e: React.KeyboardEvent) => { + if (e.key === "Escape") { + onClose(); + } else if (e.key === "ArrowDown") { + e.preventDefault(); + setActive((i) => Math.min(i + 1, flat.length - 1)); + } else if (e.key === "ArrowUp") { + e.preventDefault(); + setActive((i) => Math.max(i - 1, 0)); + } else if (e.key === "Enter" && flat[active]) { + go(flat[active].id); + } + }; + + let idx = -1; + return ( +
+ +
+
+ {flat.length === 0 ? ( +
No results for “{query}”.
+ ) : ( + groups.map((group) => ( + +
{group.section}
+ {group.items.map((hit) => { + idx += 1; + const i = idx; + return ( + + ); + })} +
+ )) + )} +
+
+ + + navigate + + + open + + + esc close + +
+
+ + ); +} diff --git a/docs/app/components/docs/sidebar.tsx b/docs/app/components/docs/sidebar.tsx new file mode 100644 index 00000000..391a9cdb --- /dev/null +++ b/docs/app/components/docs/sidebar.tsx @@ -0,0 +1,213 @@ +import { useEffect, useState } from "react"; +import { Link, useLocation, useNavigate } from "react-router"; +import { useActiveSdk, useSdk } from "@/hooks"; +import { + forcedSdkForPath, + type NavNode, + navForSdk, + type Sdk, + sdkSwitchTarget, +} from "@/lib"; + +function containsHref(node: NavNode, current: string): boolean { + return ( + node.href === current || + (node.children?.some((c) => containsHref(c, current)) ?? false) + ); +} + +const SDK_LABELS: { id: Sdk; label: string }[] = [ + { id: "python", label: "Python" }, + { id: "node", label: "Node.js" }, +]; + +/** Global SDK toggle. Sets the shared store (flips inline variants + this nav); + * on an SDK-specific page it also navigates to the counterpart page, on a shared + * page it stays put. A labelled control — it switches the whole page, not a panel. */ +function SdkSwitch({ sdk, current }: { sdk: Sdk; current: string }) { + const { setSdk } = useSdk(); + const navigate = useNavigate(); + + function select(target: Sdk) { + if (target === sdk) { + return; + } + setSdk(target); + if (forcedSdkForPath(current)) { + navigate(sdkSwitchTarget(current, target)); + } + } + + return ( +
+ {SDK_LABELS.map(({ id, label }) => ( + + ))} +
+ ); +} + +function Caret({ + open, + onToggle, + title, +}: { + open: boolean; + onToggle: () => void; + title: string; +}) { + return ( + + ); +} + +function NavLink({ node, current }: { node: NavNode; current: string }) { + if (!node.href) { + return {node.title}; + } + return ( + + {node.title} + + ); +} + +/** A subsection with children — collapsible, auto-opens around the active page. */ +function NavSection({ node, current }: { node: NavNode; current: string }) { + const active = node.children?.some((c) => containsHref(c, current)) ?? false; + const [open, setOpen] = useState(active); + useEffect(() => { + if (active) { + setOpen(true); + } + }, [active]); + return ( +
+
+ + setOpen((o) => !o)} + title={node.title} + /> +
+ {open ? : null} +
+ ); +} + +function NavTree({ nodes, current }: { nodes: NavNode[]; current: string }) { + return ( +
+ {nodes.map((node) => + node.children?.length ? ( + + ) : ( + + ), + )} +
+ ); +} + +/** Top-level group — collapsible, default-open for the section holding the page. */ +function NavGroup({ group, current }: { group: NavNode; current: string }) { + const active = containsHref(group, current); + const [open, setOpen] = useState(active); + useEffect(() => { + if (active) { + setOpen(true); + } + }, [active]); + const hasChildren = Boolean(group.children?.length); + return ( +
+
+ {hasChildren ? ( + + ) : group.href ? ( + + {group.title} + + ) : ( + {group.title} + )} + {hasChildren ? ( + setOpen((o) => !o)} + title={group.title} + /> + ) : null} +
+ {hasChildren && open ? ( + + ) : null} +
+ ); +} + +export function Sidebar({ onSearch }: { onSearch?: () => void }) { + const { pathname } = useLocation(); + const current = pathname.replace(/\/$/, "") || "/"; + const sdk = useActiveSdk(); + return ( + + ); +} diff --git a/docs/app/components/docs/toc.tsx b/docs/app/components/docs/toc.tsx new file mode 100644 index 00000000..5c9f0f77 --- /dev/null +++ b/docs/app/components/docs/toc.tsx @@ -0,0 +1,96 @@ +import { useEffect, useState } from "react"; +import { useLocation } from "react-router"; + +interface Heading { + id: string; + text: string; + level: number; +} + +function readHeadings(article: Element): Heading[] { + return [...article.querySelectorAll("h2[id], h3[id]")].map( + (el) => ({ + id: el.id, + text: el.textContent ?? "", + level: el.tagName === "H3" ? 3 : 2, + }), + ); +} + +/** On-this-page TOC, built from the rendered article headings with scroll-spy. */ +export function Toc() { + const { pathname } = useLocation(); + const [headings, setHeadings] = useState([]); + const [active, setActive] = useState(""); + + // The article is a lazily-loaded MDX chunk, so its headings are NOT in the DOM + // when this effect first runs after a client-side navigation. Re-scan on every + // article mutation (via MutationObserver) so the TOC fills in once the page + // content actually mounts, and re-arm scroll-spy whenever the heading set changes. + // biome-ignore lint/correctness/useExhaustiveDependencies: re-scan when the path changes + useEffect(() => { + const article = document.querySelector(".article"); + if (!article) { + setHeadings([]); + return; + } + let spy: IntersectionObserver | null = null; + let lastKey = ""; + + const scan = () => { + const next = readHeadings(article); + const key = next.map((h) => h.id).join("|"); + if (key === lastKey) { + return; + } + lastKey = key; + setHeadings(next); + spy?.disconnect(); + if (!next.length) { + return; + } + spy = new IntersectionObserver( + (entries) => { + for (const entry of entries) { + if (entry.isIntersecting) { + setActive(entry.target.id); + } + } + }, + { rootMargin: "-80px 0px -70% 0px" }, + ); + for (const el of article.querySelectorAll("h2[id], h3[id]")) { + spy.observe(el); + } + }; + + scan(); + const content = new MutationObserver(scan); + content.observe(article, { childList: true, subtree: true }); + return () => { + content.disconnect(); + spy?.disconnect(); + }; + }, [pathname]); + + if (headings.length === 0) { + return