From 9f1bcc234c552a1e6285a2464b2c4e4dce812831 Mon Sep 17 00:00:00 2001 From: Mark Probst Date: Wed, 22 Jul 2026 16:30:32 -0400 Subject: [PATCH] feat(ci): group output differences by test case --- OUTPUT-DIFFS.md | 6 +-- script/output-diff.ts | 99 +++++++++++++++++++++++++++-------- test/unit/output-diff.test.ts | 25 ++++++++- 3 files changed, 103 insertions(+), 27 deletions(-) diff --git a/OUTPUT-DIFFS.md b/OUTPUT-DIFFS.md index 6e032b57b..285767748 100644 --- a/OUTPUT-DIFFS.md +++ b/OUTPUT-DIFFS.md @@ -63,10 +63,10 @@ The report is static, self-contained HTML with: - summary cards and base/merge/head commit metadata; - a prominent link back to the pull request; -- filtering by file status and text search; -- navigation grouped by fixture/target; +- filtering by target, file status, and text search; +- generated files grouped by input test case, with each target clearly labeled; - collapsible, GitHub-style unified diffs with old/new line numbers; and -- per-file insertion/deletion totals. +- per-test and per-file insertion/deletion totals. Generated source and paths are always HTML-escaped. The page has a restrictive content-security policy and makes no third-party network requests. diff --git a/script/output-diff.ts b/script/output-diff.ts index c0aee906f..f98efc4bd 100644 --- a/script/output-diff.ts +++ b/script/output-diff.ts @@ -303,28 +303,73 @@ export function renderOutputDiffReport(args: { if (sections.length !== result.files.length) { throw new Error("Patch and result file counts differ"); } - const fixtures = Array.from( - new Set(result.files.map((file) => file.path.split(path.sep)[0])), + const renderedFiles = result.files.map((file, index) => { + const parts = file.path.split("/"); + const target = parts[0] ?? file.path; + const optionsIndex = parts.findIndex( + (part, partIndex) => + partIndex > 1 && + (part === "default" || /--[0-9a-f]{12}$/.test(part)), + ); + const test = + optionsIndex < 0 + ? "Other generated output" + : parts.slice(1, optionsIndex).join("/"); + const output = + optionsIndex < 0 + ? file.path + : parts.slice(optionsIndex).join(" / "); + return { file, index, output, section: sections[index], target, test }; + }); + const targets = Array.from( + new Set(renderedFiles.map(({ target }) => target)), ).sort(); - const fileHTML = result.files - .map((file, index) => { - const fixture = file.path.split(path.sep)[0]; - const statusLetter = - file.status === "added" - ? "A" - : file.status === "deleted" - ? "D" - : "M"; - return `
-${statusLetter}${escapeHTML(file.path)}+${formatNumber(file.additions)} −${formatNumber(file.deletions)} -
${renderPatchLines(sections[index])}
+ const groupedFiles = new Map(); + for (const renderedFile of renderedFiles) { + const files = groupedFiles.get(renderedFile.test) ?? []; + files.push(renderedFile); + groupedFiles.set(renderedFile.test, files); + } + const groups = Array.from(groupedFiles).sort(([left], [right]) => + left.localeCompare(right), + ); + const groupsHTML = groups + .map(([test, files]) => { + const additions = files.reduce( + (sum, { file }) => sum + file.additions, + 0, + ); + const deletions = files.reduce( + (sum, { file }) => sum + file.deletions, + 0, + ); + const fileHTML = files + .sort(({ file: left }, { file: right }) => + left.path.localeCompare(right.path), + ) + .map(({ file, index, output, section, target }) => { + const statusLetter = + file.status === "added" + ? "A" + : file.status === "deleted" + ? "D" + : "M"; + return `
+${statusLetter}${escapeHTML(target)}${escapeHTML(output)}+${formatNumber(file.additions)} −${formatNumber(file.deletions)} +
${renderPatchLines(section)}
`; + }) + .join("\n"); + return `
+
Test case

${escapeHTML(test)}

${formatNumber(files.length)} generated ${files.length === 1 ? "file" : "files"} · +${formatNumber(additions)} −${formatNumber(deletions)}
+${fileHTML} +
`; }) .join("\n"); - const fixtureOptions = fixtures + const targetOptions = targets .map( - (fixture) => - ``, + (target) => + ``, ) .join(""); const summary = result.summary; @@ -373,11 +418,20 @@ input:focus, select:focus, button:focus { border-color: var(--blue); outline: 2p button { cursor: pointer } button:hover { background: var(--panel-hover) } input { flex: 1; min-width: 240px } -.file { border: 1px solid var(--border); border-radius: 6px; background: var(--diff-bg); margin: 16px 0; overflow: hidden } +.test-group { border: 1px solid var(--border); border-radius: 8px; margin: 20px 0; padding: 12px; background: var(--panel) } +.test-group[hidden] { display: none } +.test-header { display: flex; align-items: end; gap: 16px; justify-content: space-between; padding: 2px 4px 10px } +.test-label { color: var(--muted); font-size: 11px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase } +h2 { margin: 1px 0 0; font: 600 16px/1.4 ui-monospace, SFMono-Regular, Consolas, "Liberation Mono", monospace; overflow-wrap: anywhere } +.test-counts { color: var(--muted); white-space: nowrap; font: 12px/1.5 ui-monospace, SFMono-Regular, Consolas, monospace } +.test-counts b { color: var(--green) } +.test-counts i { color: var(--red); font-style: normal } +.file { border: 1px solid var(--border); border-radius: 6px; background: var(--diff-bg); margin: 8px 0 0; overflow: hidden } .file[hidden] { display: none } -.file summary { cursor: pointer; display: flex; align-items: center; gap: 10px; min-height: 46px; padding: 8px 12px; background: var(--panel); font-weight: 600 } +.file summary { cursor: pointer; display: flex; align-items: center; gap: 10px; min-height: 46px; padding: 8px 12px; background: var(--bg); font-weight: 600 } .file summary:hover { background: var(--panel-hover) } .file[open] summary { border-bottom: 1px solid var(--border) } +.target { flex: 0 0 auto; border: 1px solid var(--border); border-radius: 2em; padding: 1px 8px; color: var(--muted); font: 600 11px/1.5 ui-monospace, SFMono-Regular, Consolas, "Liberation Mono", monospace } .file-path { font: 600 13px/1.5 ui-monospace, SFMono-Regular, Consolas, "Liberation Mono", monospace; overflow-wrap: anywhere } .counts { margin-left: auto; white-space: nowrap; font: 12px/1.5 ui-monospace, SFMono-Regular, Consolas, monospace } .counts b { color: var(--green) } @@ -426,6 +480,7 @@ footer { padding: 30px 0 50px; color: var(--muted) }
quicktype output changed between the PR base and tested PR merge revisions.
← Back to the pull request
+
${formatNumber(groups.length)}test cases
${formatNumber(summary.files)}files differ
${formatNumber(summary.modified)}modified
${formatNumber(summary.added)}new
@@ -438,17 +493,17 @@ footer { padding: 30px 0 50px; color: var(--muted) }
- +
-
${fileHTML}
+
${groupsHTML}
No generated files match these filters.
Generated by quicktype CI. This report is immutable for the tested PR and head SHAs.
`; diff --git a/test/unit/output-diff.test.ts b/test/unit/output-diff.test.ts index 0c5ba823d..1db5bfb5c 100644 --- a/test/unit/output-diff.test.ts +++ b/test/unit/output-diff.test.ts @@ -157,9 +157,24 @@ describe("generated-output comparison", () => { const head = path.join(root, "head"); fs.mkdirSync(base); fs.mkdirSync(head); - fs.writeFileSync(path.join(base, "unsafe.ts"), "safe\n"); + const typescriptOutput = + "schema-typescript/test/inputs/schema/unsafe.schema/default/QuickType.ts"; + const rustOutput = + "schema-rust/test/inputs/schema/unsafe.schema/default/quicktype.rs"; + const otherOutput = + "schema-rust/test/inputs/schema/other.schema/default/quicktype.rs"; + for (const output of [typescriptOutput, rustOutput, otherOutput]) { + fs.mkdirSync(path.dirname(path.join(base, output)), { + recursive: true, + }); + fs.mkdirSync(path.dirname(path.join(head, output)), { + recursive: true, + }); + fs.writeFileSync(path.join(base, output), "safe\n"); + fs.writeFileSync(path.join(head, output), "changed\n"); + } fs.writeFileSync( - path.join(head, "unsafe.ts"), + path.join(head, typescriptOutput), "\n", ); const { patch, result } = compareOutputSnapshots(base, head); @@ -180,6 +195,12 @@ describe("generated-output comparison", () => { ); expect(html).toContain("<script>alert(1)</script>"); expect(html).not.toContain(""); + expect(html.match(/class="test-group"/g)).toHaveLength(2); + expect( + html.match(/test\/inputs\/schema\/unsafe\.schema<\/h2>/g), + ).toHaveLength(1); + expect(html).toContain(">schema-rust"); + expect(html).toContain(">schema-typescript"); expect(html).toContain('class="diff-table"'); expect(html).toContain(">Expand all"); expect(html).toContain(