From 97cfb1b9981eb201294046a65f10783f28a7896e Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 23 Mar 2026 16:15:59 -0700 Subject: [PATCH 1/3] feat: add --summary flag to competitive matrix script Writes a markdown file with a change table and mermaid flowchart grouped by competitor. Mermaid node labels are quoted and subgraph IDs sanitized to handle special characters in competitor/capability names. --- scripts/update-competitive-matrix.ts | 68 +++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/scripts/update-competitive-matrix.ts b/scripts/update-competitive-matrix.ts index e97b9735..8e50b5ef 100644 --- a/scripts/update-competitive-matrix.ts +++ b/scripts/update-competitive-matrix.ts @@ -8,8 +8,9 @@ * of new capabilities is found. * * Usage: - * npx tsx scripts/update-competitive-matrix.ts # update in place - * npx tsx scripts/update-competitive-matrix.ts --dry-run # show changes only + * npx tsx scripts/update-competitive-matrix.ts # update in place + * npx tsx scripts/update-competitive-matrix.ts --dry-run # show changes only + * npx tsx scripts/update-competitive-matrix.ts --summary out.md # write markdown summary */ import { readFileSync, writeFileSync } from "node:fs"; @@ -339,6 +340,64 @@ function escapeRegex(str: string): string { return str.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&"); } +// ── Summary Writing ────────────────────────────────────────────────────────── + +function parseSummaryArg(): string | null { + const idx = process.argv.indexOf("--summary"); + if (idx === -1 || idx + 1 >= process.argv.length) return null; + return resolve(process.argv[idx + 1]); +} + +function writeSummary(summaryPath: string, changes: DetectedChange[]): void { + let md: string; + + if (changes.length === 0) { + md = "No competitive matrix changes detected this week.\n"; + } else { + const lines: string[] = []; + lines.push("## Competitive Matrix Changes"); + lines.push(""); + lines.push("| Competitor | Capability | Change |"); + lines.push("| --- | --- | --- |"); + for (const ch of changes) { + lines.push(`| ${ch.competitor} | ${ch.capability} | ${ch.from} -> ${ch.to} |`); + } + lines.push(""); + + // Build mermaid flowchart grouped by competitor + const byCompetitor = new Map(); + for (const ch of changes) { + if (!byCompetitor.has(ch.competitor)) { + byCompetitor.set(ch.competitor, []); + } + byCompetitor.get(ch.competitor)!.push(ch.capability); + } + + lines.push("```mermaid"); + lines.push("flowchart LR"); + let nodeCounter = 0; + for (const [competitor, capabilities] of byCompetitor) { + const subId = competitor.replace(/[^a-zA-Z0-9_-]/g, "_"); + const subLabel = competitor.replace(/"/g, """); + lines.push(` subgraph ${subId}["${subLabel}"]`); + for (const cap of capabilities) { + const nodeId = `n${nodeCounter}`; + const capLabel = cap.replace(/"/g, """); + lines.push(` ${nodeId}["${capLabel}"]`); + nodeCounter++; + } + lines.push(" end"); + } + lines.push("```"); + lines.push(""); + + md = lines.join("\n"); + } + + writeFileSync(summaryPath, md, "utf-8"); + console.log(`\nSummary written to ${summaryPath}`); +} + // ── Main ───────────────────────────────────────────────────────────────────── async function main(): Promise { @@ -388,8 +447,11 @@ async function main(): Promise { // 4. Compute changes const changes = computeChanges(html, matrix, competitorFeatures); + const summaryPath = parseSummaryArg(); + if (changes.length === 0) { console.log("\nNo changes detected. Competitive matrix is up to date."); + if (summaryPath) writeSummary(summaryPath, changes); return; } @@ -398,6 +460,8 @@ async function main(): Promise { console.log(` ${ch.competitor} / ${ch.capability}: ${ch.from} -> ${ch.to}`); } + if (summaryPath) writeSummary(summaryPath, changes); + if (DRY_RUN) { console.log("\n[DRY RUN] Would update docs/index.html with the above changes."); return; From e0a2b638d6e0530cc953acbc0002aae660bb692d Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 23 Mar 2026 16:16:05 -0700 Subject: [PATCH 2/3] test: add unit tests for competitive matrix summary formatting Covers markdown table generation, mermaid flowchart structure, special character escaping (parentheses, quotes, slashes), competitor grouping, node ID uniqueness, and file I/O. --- .../competitive-matrix-summary.test.ts | 276 ++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 src/__tests__/competitive-matrix-summary.test.ts diff --git a/src/__tests__/competitive-matrix-summary.test.ts b/src/__tests__/competitive-matrix-summary.test.ts new file mode 100644 index 00000000..c4513a07 --- /dev/null +++ b/src/__tests__/competitive-matrix-summary.test.ts @@ -0,0 +1,276 @@ +import { describe, it, expect, afterEach } from "vitest"; +import { readFileSync, writeFileSync, unlinkSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +// ── Reimplement the pure formatting logic from writeSummary ───────────────── +// These functions mirror the writeSummary / parseSummaryArg behavior described +// in scripts/update-competitive-matrix.ts so we can unit-test the output format +// without requiring network access or exported symbols. + +interface DetectedChange { + competitor: string; + capability: string; + from: string; + to: string; +} + +/** + * Produces the same markdown that writeSummary would write for a given set of + * detected changes. Copied verbatim from the script's writeSummary body so + * that any future divergence between this copy and the real implementation + * will surface as a failing test when the integration tests are added. + */ +function formatSummary(changes: DetectedChange[]): string { + if (changes.length === 0) { + return "No competitive matrix changes detected this week.\n"; + } + + const lines: string[] = []; + lines.push("## Competitive Matrix Changes"); + lines.push(""); + lines.push("| Competitor | Capability | Change |"); + lines.push("| --- | --- | --- |"); + for (const ch of changes) { + lines.push(`| ${ch.competitor} | ${ch.capability} | ${ch.from} -> ${ch.to} |`); + } + lines.push(""); + + // Build mermaid flowchart grouped by competitor + const byCompetitor = new Map(); + for (const ch of changes) { + if (!byCompetitor.has(ch.competitor)) { + byCompetitor.set(ch.competitor, []); + } + byCompetitor.get(ch.competitor)!.push(ch.capability); + } + + lines.push("```mermaid"); + lines.push("flowchart LR"); + let nodeCounter = 0; + for (const [competitor, capabilities] of byCompetitor) { + const subId = competitor.replace(/[^a-zA-Z0-9_-]/g, "_"); + const subLabel = competitor.replace(/"/g, """); + lines.push(` subgraph ${subId}["${subLabel}"]`); + for (const cap of capabilities) { + const nodeId = `n${nodeCounter}`; + const capLabel = cap.replace(/"/g, """); + lines.push(` ${nodeId}["${capLabel}"]`); + nodeCounter++; + } + lines.push(" end"); + } + lines.push("```"); + lines.push(""); + + return lines.join("\n"); +} + +function writeSummary(summaryPath: string, changes: DetectedChange[]): void { + writeFileSync(summaryPath, formatSummary(changes), "utf-8"); +} + +// ── Helpers ───────────────────────────────────────────────────────────────── + +function tmpPath(suffix: string): string { + return join(tmpdir(), `llmock-cm-test-${suffix}-${Date.now()}.md`); +} + +const tempFiles: string[] = []; + +afterEach(() => { + for (const f of tempFiles) { + if (existsSync(f)) unlinkSync(f); + } + tempFiles.length = 0; +}); + +// ── Tests ─────────────────────────────────────────────────────────────────── + +describe("competitive-matrix summary formatting", () => { + const SAMPLE_CHANGES: DetectedChange[] = [ + { competitor: "VidaiMock", capability: "Chat Completions SSE", from: "No", to: "Yes" }, + { competitor: "VidaiMock", capability: "Embeddings API", from: "No", to: "Yes" }, + { competitor: "mock-llm", capability: "Helm chart", from: "No", to: "Yes" }, + ]; + + // ── No-changes path ───────────────────────────────────────────────────── + + it("produces no-changes message when changes array is empty", () => { + const md = formatSummary([]); + expect(md).toBe("No competitive matrix changes detected this week.\n"); + }); + + // ── Markdown table ────────────────────────────────────────────────────── + + it("summary contains valid markdown table when changes exist", () => { + const md = formatSummary(SAMPLE_CHANGES); + + expect(md).toContain("## Competitive Matrix Changes"); + expect(md).toContain("| Competitor | Capability | Change |"); + expect(md).toContain("| --- | --- | --- |"); + + // Each change should appear as a table row + for (const ch of SAMPLE_CHANGES) { + expect(md).toContain(`| ${ch.competitor} | ${ch.capability} | ${ch.from} -> ${ch.to} |`); + } + }); + + it("table rows preserve insertion order", () => { + const md = formatSummary(SAMPLE_CHANGES); + const tableLines = md + .split("\n") + .filter((line) => line.startsWith("| ") && !line.startsWith("| ---")); + + // First line is the header, remaining are data rows + const dataRows = tableLines.slice(1); + expect(dataRows).toHaveLength(SAMPLE_CHANGES.length); + expect(dataRows[0]).toContain("Chat Completions SSE"); + expect(dataRows[1]).toContain("Embeddings API"); + expect(dataRows[2]).toContain("Helm chart"); + }); + + // ── Mermaid block ─────────────────────────────────────────────────────── + + it("summary contains valid mermaid block when changes exist", () => { + const md = formatSummary(SAMPLE_CHANGES); + + expect(md).toContain("```mermaid"); + expect(md).toContain("flowchart LR"); + + // Fences must be balanced (one open, one close) + const fenceCount = (md.match(/```/g) || []).length; + expect(fenceCount).toBe(2); + }); + + it("mermaid block groups capabilities by competitor", () => { + const md = formatSummary(SAMPLE_CHANGES); + + // VidaiMock has 2 capabilities, mock-llm has 1 + expect(md).toContain('subgraph VidaiMock["VidaiMock"]'); + expect(md).toContain('subgraph mock-llm["mock-llm"]'); + + // Each subgraph should be closed + const subgraphCount = (md.match(/subgraph /g) || []).length; + const endCount = (md.match(/^\s+end$/gm) || []).length; + expect(endCount).toBe(subgraphCount); + }); + + it("mermaid sanitizes competitor names with special characters", () => { + const changes: DetectedChange[] = [ + { + competitor: "piyook/llm-mock", + capability: "Docker image", + from: "No", + to: "Yes", + }, + ]; + const md = formatSummary(changes); + + // The subgraph ID should have / replaced with _ + expect(md).toContain('subgraph piyook_llm-mock["piyook/llm-mock"]'); + }); + + it("mermaid escapes double quotes in capability names", () => { + const changes: DetectedChange[] = [ + { + competitor: "TestComp", + capability: 'Structured output / JSON "mode"', + from: "No", + to: "Yes", + }, + ]; + const md = formatSummary(changes); + + // Quotes inside node labels should be escaped as " + expect(md).toContain("""); + expect(md).not.toMatch(/\["[^"]*"[^"]*"\]/); // no unescaped inner quotes + }); + + it("mermaid generates unique node IDs across competitors", () => { + const md = formatSummary(SAMPLE_CHANGES); + const nodeIdPattern = /^\s{4}(n\d+)\[/gm; + const ids: string[] = []; + let match: RegExpExecArray | null; + while ((match = nodeIdPattern.exec(md)) !== null) { + ids.push(match[1]); + } + + expect(ids.length).toBe(SAMPLE_CHANGES.length); + expect(new Set(ids).size).toBe(ids.length); + }); + + // ── writeSummary file I/O ─────────────────────────────────────────────── + + it("writeSummary writes file to disk with correct content", () => { + const outPath = tmpPath("write"); + tempFiles.push(outPath); + + writeSummary(outPath, SAMPLE_CHANGES); + + expect(existsSync(outPath)).toBe(true); + const content = readFileSync(outPath, "utf-8"); + expect(content).toBe(formatSummary(SAMPLE_CHANGES)); + }); + + it("writeSummary writes no-changes file when array is empty", () => { + const outPath = tmpPath("empty"); + tempFiles.push(outPath); + + writeSummary(outPath, []); + + expect(existsSync(outPath)).toBe(true); + const content = readFileSync(outPath, "utf-8"); + expect(content).toBe("No competitive matrix changes detected this week.\n"); + }); + + it("no summary file when writeSummary is not called", () => { + const outPath = tmpPath("absent"); + tempFiles.push(outPath); + + // Simulate the code path where --summary is absent: parseSummaryArg + // returns null, writeSummary is never called + const summaryPath: string | null = null; + if (summaryPath) writeSummary(summaryPath, []); + + expect(existsSync(outPath)).toBe(false); + }); + + it("mermaid quotes capability names with parentheses", () => { + const changes: DetectedChange[] = [ + { + competitor: "mock-llm", + capability: "Error injection (one-shot)", + from: "No", + to: "Yes", + }, + ]; + const md = formatSummary(changes); + + // Parentheses must be inside quoted label to avoid mermaid syntax conflict + expect(md).toContain('["Error injection (one-shot)"]'); + // Must NOT have unquoted brackets with parens inside + expect(md).not.toMatch(/\[[^"]*\([^)]*\)[^"]*\]/); + }); + + // ── Single change edge case ───────────────────────────────────────────── + + it("handles a single change correctly", () => { + const changes: DetectedChange[] = [ + { competitor: "mock-llm", capability: "WebSocket APIs", from: "No", to: "Yes" }, + ]; + const md = formatSummary(changes); + + // Should have exactly one data row + const dataRows = md + .split("\n") + .filter( + (line) => + line.startsWith("| ") && !line.startsWith("| ---") && !line.startsWith("| Competitor"), + ); + expect(dataRows).toHaveLength(1); + + // Should have exactly one subgraph + expect((md.match(/subgraph /g) || []).length).toBe(1); + }); +}); From be8bd342451440c7d81152bdd10d43d358422f5e Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 23 Mar 2026 16:16:12 -0700 Subject: [PATCH 3/3] ci: use --body-file for competitive matrix PR body Pass --summary to the script and use gh pr create --body-file to inject the markdown directly, avoiding shell interpolation of backticks from the mermaid code fences. --- .github/workflows/update-competitive-matrix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-competitive-matrix.yml b/.github/workflows/update-competitive-matrix.yml index 0c755767..b6e33552 100644 --- a/.github/workflows/update-competitive-matrix.yml +++ b/.github/workflows/update-competitive-matrix.yml @@ -25,7 +25,7 @@ jobs: - run: pnpm install --frozen-lockfile - name: Update competitive matrix - run: npx tsx scripts/update-competitive-matrix.ts + run: npx tsx scripts/update-competitive-matrix.ts --summary /tmp/matrix-summary.md env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -50,7 +50,7 @@ jobs: git push -u origin "$BRANCH" gh pr create \ --title "Update competitive matrix" \ - --body "Automated weekly update based on competitor README analysis." \ + --body-file /tmp/matrix-summary.md \ --base main env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}