diff --git a/package-lock.json b/package-lock.json index edccbe8993..bc9e66139d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "libtess": "^1.2.2", "omggif": "^1.0.10", "pako": "^2.1.0", + "pixelmatch": "^7.1.0", "zod": "^3.23.8" }, "devDependencies": { @@ -8619,6 +8620,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pixelmatch": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-7.1.0.tgz", + "integrity": "sha512-1wrVzJ2STrpmONHKBy228LM1b84msXDUoAzVEl0R8Mz4Ce6EPr+IVtxm8+yvrqLYMHswREkjYFaMxnyGnaY3Ng==", + "dependencies": { + "pngjs": "^7.0.0" + }, + "bin": { + "pixelmatch": "bin/pixelmatch" + } + }, "node_modules/pkg-dir": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-5.0.0.tgz", @@ -8642,6 +8654,14 @@ "semver-compare": "^1.0.0" } }, + "node_modules/pngjs": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", + "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "engines": { + "node": ">=14.19.0" + } + }, "node_modules/postcss": { "version": "8.5.1", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.1.tgz", diff --git a/package.json b/package.json index e34a8ef5c7..effe967822 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "libtess": "^1.2.2", "omggif": "^1.0.10", "pako": "^2.1.0", + "pixelmatch": "^7.1.0", "zod": "^3.23.8" }, "devDependencies": { diff --git a/test/unit/visual/visualTest.js b/test/unit/visual/visualTest.js index 068e8c7247..120ce79565 100644 --- a/test/unit/visual/visualTest.js +++ b/test/unit/visual/visualTest.js @@ -2,6 +2,7 @@ import p5 from '../../../src/app.js'; import { server } from '@vitest/browser/context' import { THRESHOLD, DIFFERENCE, ERODE } from '../../../src/core/constants.js'; const { readFile, writeFile } = server.commands +import pixelmatch from 'pixelmatch'; // By how much can each color channel value (0-255) differ before // we call it a mismatch? This should be large enough to not trigger @@ -86,17 +87,60 @@ export function visualSuite( }); } +/** + * Image Diff Algorithm for p5.js Visual Tests + * + * This algorithm addresses the challenge of cross-platform rendering differences in p5.js visual tests. + * Different operating systems and browsers render graphics with subtle variations, particularly with + * anti-aliasing, text rendering, and sub-pixel positioning. This can cause false negatives in tests + * when the visual differences are acceptable rendering variations rather than actual bugs. + * + * Key components of the approach: + * + * 1. Initial pixel-by-pixel comparison: + * - Uses pixelmatch to identify differences between expected and actual images + * - Sets a moderate threshold (0.5) to filter out minor color/intensity variations + * - Produces a diff image with red pixels marking differences + * + * 2. Cluster identification using BFS (Breadth-First Search): + * - Groups connected difference pixels into clusters + * - Uses a queue-based BFS algorithm to find all connected pixels + * - Defines connectivity based on 8-way adjacency (all surrounding pixels) + * + * 3. Cluster categorization by type: + * - Analyzes each pixel's neighborhood characteristics + * - Specifically identifies "line shift" clusters - differences that likely represent + * the same visual elements shifted by 1px due to platform rendering differences + * - Line shifts are identified when >80% of pixels in a cluster have ≤2 neighboring diff pixels + * + * 4. Intelligent failure criteria: + * - Filters out clusters smaller than MIN_CLUSTER_SIZE pixels (noise reduction) + * - Applies different thresholds for regular differences vs. line shifts + * - Considers both the total number of significant pixels and number of distinct clusters + * + * This approach balances the need to catch genuine visual bugs (like changes to shape geometry, + * colors, or positioning) while tolerating acceptable cross-platform rendering variations. + * + * Parameters: + * - MIN_CLUSTER_SIZE: Minimum size for a cluster to be considered significant (default: 4) + * - MAX_TOTAL_DIFF_PIXELS: Maximum allowed non-line-shift difference pixels (default: 40) + * Note: These can be adjusted for further updation + * + * Note for contributors: When running tests locally, you may not see these differences as they + * mainly appear when tests run on different operating systems or browser rendering engines. + * However, the same code may produce slightly different renderings on CI environments, particularly + * with text positioning, thin lines, or curved shapes. This algorithm helps distinguish between + * these acceptable variations and actual visual bugs. + */ + export async function checkMatch(actual, expected, p5) { let scale = Math.min(MAX_SIDE/expected.width, MAX_SIDE/expected.height); - - // Long screenshots end up super tiny when fit to a small square, so we - // can double the max side length for these const ratio = expected.width / expected.height; const narrow = ratio !== 1; if (narrow) { scale *= 2; } - + for (const img of [actual, expected]) { img.resize( Math.ceil(img.width * scale), @@ -104,39 +148,197 @@ export async function checkMatch(actual, expected, p5) { ); } - const expectedWithBg = p5.createGraphics(expected.width, expected.height); - expectedWithBg.pixelDensity(1); - expectedWithBg.background(BG); - expectedWithBg.image(expected, 0, 0); - - const cnv = p5.createGraphics(actual.width, actual.height); - cnv.pixelDensity(1); - cnv.background(BG); - cnv.image(actual, 0, 0); - cnv.blendMode(DIFFERENCE); - cnv.image(expectedWithBg, 0, 0); - for (let i = 0; i < shiftThreshold; i++) { - cnv.filter(ERODE, false); + // Ensure both images have the same dimensions + const width = expected.width; + const height = expected.height; + + // Create canvases with background color + const actualCanvas = p5.createGraphics(width, height); + const expectedCanvas = p5.createGraphics(width, height); + actualCanvas.pixelDensity(1); + expectedCanvas.pixelDensity(1); + + actualCanvas.background(BG); + expectedCanvas.background(BG); + + actualCanvas.image(actual, 0, 0); + expectedCanvas.image(expected, 0, 0); + + // Load pixel data + actualCanvas.loadPixels(); + expectedCanvas.loadPixels(); + + // Create diff output canvas + const diffCanvas = p5.createGraphics(width, height); + diffCanvas.pixelDensity(1); + diffCanvas.loadPixels(); + + // Run pixelmatch + const diffCount = pixelmatch( + actualCanvas.pixels, + expectedCanvas.pixels, + diffCanvas.pixels, + width, + height, + { + threshold: 0.5, + includeAA: false, + alpha: 0.1 + } + ); + + // If no differences, return early + if (diffCount === 0) { + actualCanvas.remove(); + expectedCanvas.remove(); + diffCanvas.updatePixels(); + return { ok: true, diff: diffCanvas }; } - const diff = cnv.get(); - cnv.remove(); - diff.loadPixels(); - expectedWithBg.remove(); - - let ok = true; - for (let i = 0; i < diff.pixels.length; i += 4) { - let diffSum = 0; - for (let off = 0; off < 3; off++) { - diffSum += diff.pixels[i+off] + + // Post-process to identify and filter out isolated differences + const visited = new Set(); + const clusterSizes = []; + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const pos = (y * width + x) * 4; + + // If this is a diff pixel (red in pixelmatch output) and not yet visited + if ( + diffCanvas.pixels[pos] === 255 && + diffCanvas.pixels[pos + 1] === 0 && + diffCanvas.pixels[pos + 2] === 0 && + !visited.has(pos) + ) { + // Find the connected cluster size using BFS + const clusterSize = findClusterSize(diffCanvas.pixels, x, y, width, height, 1, visited); + clusterSizes.push(clusterSize); + } } - diffSum /= 3; - if (diffSum > COLOR_THRESHOLD) { - ok = false; - break; + } + + // Define significance thresholds + const MIN_CLUSTER_SIZE = 4; // Minimum pixels in a significant cluster + const MAX_TOTAL_DIFF_PIXELS = 40; // Maximum total different pixels + + // Determine if the differences are significant + const nonLineShiftClusters = clusterSizes.filter(c => !c.isLineShift && c.size >= MIN_CLUSTER_SIZE); + + // Calculate significant differences excluding line shifts + const significantDiffPixels = nonLineShiftClusters.reduce((sum, c) => sum + c.size, 0); + + // Update the diff canvas + diffCanvas.updatePixels(); + + // Clean up canvases + actualCanvas.remove(); + expectedCanvas.remove(); + + // Determine test result + const ok = ( + diffCount === 0 || + ( + significantDiffPixels === 0 || + ( + (significantDiffPixels <= MAX_TOTAL_DIFF_PIXELS) && + (nonLineShiftClusters.length <= 2) // Not too many significant clusters + ) + ) + ); + + return { + ok, + diff: diffCanvas, + details: { + totalDiffPixels: diffCount, + significantDiffPixels, + clusters: clusterSizes + } + }; +} + +/** + * Find the size of a connected cluster of diff pixels using BFS + */ +function findClusterSize(pixels, startX, startY, width, height, radius, visited) { + const queue = [{x: startX, y: startY}]; + let size = 0; + const clusterPixels = []; + + while (queue.length > 0) { + const {x, y} = queue.shift(); + const pos = (y * width + x) * 4; + + // Skip if already visited + if (visited.has(pos)) continue; + + // Skip if not a diff pixel + if (pixels[pos] !== 255 || pixels[pos + 1] !== 0 || pixels[pos + 2] !== 0) continue; + + // Mark as visited + visited.add(pos); + size++; + clusterPixels.push({x, y}); + + // Add neighbors to queue + for (let dy = -radius; dy <= radius; dy++) { + for (let dx = -radius; dx <= radius; dx++) { + const nx = x + dx; + const ny = y + dy; + + // Skip if out of bounds + if (nx < 0 || nx >= width || ny < 0 || ny >= height) continue; + + // Skip if already visited + const npos = (ny * width + nx) * 4; + if (!visited.has(npos)) { + queue.push({x: nx, y: ny}); + } + } + } + } + + let isLineShift = false; + if (clusterPixels.length > 0) { + // Count pixels with limited neighbors (line-like characteristic) + let linelikePixels = 0; + + for (const {x, y} of clusterPixels) { + // Count neighbors + let neighbors = 0; + for (let dy = -1; dy <= 1; dy++) { + for (let dx = -1; dx <= 1; dx++) { + if (dx === 0 && dy === 0) continue; // Skip self + + const nx = x + dx; + const ny = y + dy; + + // Skip if out of bounds + if (nx < 0 || nx >= width || ny < 0 || ny >= height) continue; + + const npos = (ny * width + nx) * 4; + // Check if neighbor is a diff pixel + if (pixels[npos] === 255 && pixels[npos + 1] === 0 && pixels[npos + 2] === 0) { + neighbors++; + } + } + } + + // Line-like pixels typically have 1-2 neighbors + if (neighbors <= 2) { + linelikePixels++; + } } + + // If most pixels (>80%) in the cluster have ≤2 neighbors, it's likely a line shift + isLineShift = linelikePixels / clusterPixels.length > 0.8; } - return { ok, diff }; + return { + size, + pixels: clusterPixels, + isLineShift + }; } /**