diff --git a/packages/cli-exec/src/baseline.js b/packages/cli-exec/src/baseline.js new file mode 100644 index 000000000..6f92d6976 --- /dev/null +++ b/packages/cli-exec/src/baseline.js @@ -0,0 +1,320 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import url from 'url'; +import { createResource, createRootResource } from '@percy/cli-command/utils'; + +// Drop-in baseline seeding — the `percy exec` side. +// +// Provider contract (framework knowledge stays in the SDK package, as with command discovery): +// any installed Percy SDK can declare +// +// "@percy/cli": { "baselineProvider": "./path/to/module.js" } +// +// in its package.json. The module's default export must be: +// +// { +// buildSource, // drop-in source tag for the head build (e.g. 'playwright-dropin') +// async discoverBaselines({ cwd, log }) +// // -> { baselines: [{ filepath, name, browserFamily, width, height }], degraded?, reason? } +// } +// +// On an empty Percy project, `percy exec` uploads committed baseline screenshots as build #1 +// (auto-approved server-side, the user's suite never runs for it) before starting the head +// build. On an established project nothing is seeded; the user is pointed at the explicit +// `percy playwright:setup-baseline` command instead. + +// Parallel seed-upload cap: fast on large baseline sets without stampeding the API. +const SEED_CONCURRENCY = 8; + +const BASELINE_SOURCE = 'playwright-dropin-baseline'; + +// Path hygiene at the fs boundary: directory-entry names must be single path components (a name +// containing a separator or dot-segment never comes from an honest readdir) and path strings +// are NUL-stripped. +export function sanitizePath(p) { + return String(p).replace(/\0/g, ''); +} + +export function sanitizeDirentName(name) { + let clean = String(name).replace(/\0/g, ''); + if (!clean || clean === '.' || clean === '..') return null; + if (clean.includes('/') || clean.includes('\\')) return null; + return clean; +} + +// Collect @percy/* (and percy-cli-*) package roots from the nearest node_modules — the same +// semantics as @percy/cli's command-discovery walk (findModulePackages): stop at the FIRST +// node_modules at or above `dir`, never cross the home directory, and degrade to [] on any +// filesystem error so discovery can never break `percy exec`. +function findPercyPackages(dir, log) { + try { + dir = sanitizePath(dir); + + // not given node_modules or a directory that contains node_modules, look up + if (path.basename(dir) !== 'node_modules') { + let modulesDir = path.join(dir, 'node_modules'); + let next = fs.existsSync(modulesDir) ? modulesDir : path.dirname(dir); + if (next === dir || next === os.homedir()) return []; + return findPercyPackages(next, log); + } + + let found = []; + + for (let entry of fs.readdirSync(dir)) { + let name = sanitizeDirentName(entry); + // istanbul ignore next: readdir yields single path components — defense-in-depth only + if (name === null) continue; + + if (name === '@percy') { + for (let scopedEntry of fs.readdirSync(path.join(dir, name))) { + let scoped = sanitizeDirentName(scopedEntry); + // istanbul ignore next: readdir yields single path components — defense-in-depth only + if (scoped === null) continue; + found.push(path.join(dir, name, scoped)); + } + } else if (name.startsWith('percy-cli-')) { + found.push(path.join(dir, name)); + } + } + + return found; + } catch (err) { + log?.debug(`Baseline provider discovery walk failed: ${err.message}`); + return []; + } +} + +// Find the first installed package declaring a baseline provider and import it. Returns null +// when none is installed or when the drop-in is disabled (PERCY_DROPIN_DISABLE is the same +// switch the SDK override honors, so one env var turns off both the matcher and the seeding). +export async function findBaselineProvider({ cwd = process.cwd(), log } = {}) { + if (process.env.PERCY_DROPIN_DISABLE === 'true') { + log?.debug('Drop-in disabled via PERCY_DROPIN_DISABLE — skipping baseline provider discovery'); + return null; + } + for (let pkgPath of findPercyPackages(cwd, log)) { + let pkgFile = path.join(sanitizePath(pkgPath), 'package.json'); + + try { + if (!fs.existsSync(pkgFile)) continue; + let pkg = JSON.parse(fs.readFileSync(pkgFile, 'utf-8')); + let providerPath = pkg['@percy/cli']?.baselineProvider; + if (!providerPath) continue; + + // Confine the provider module to the declaring package — a package.json pointing outside + // its own root (../../x.js) is malformed at best and gets skipped. + let resolved = path.resolve(sanitizePath(pkgPath), sanitizePath(providerPath)); + if (!resolved.startsWith(path.resolve(sanitizePath(pkgPath)) + path.sep)) { + log?.debug(`Skipping baseline provider from ${pkgPath}: provider path escapes the package`); + continue; + } + + let module = await import(url.pathToFileURL(resolved).href); + let provider = module.default || module; + + if (typeof provider.discoverBaselines === 'function') { + return { ...provider, packageName: pkg.name }; + } + } catch (err) { + log?.debug(`Skipping baseline provider from ${pkgPath}: ${err.message}`); + } + } + + return null; +} + +// The seed build keeps processing (renders + auto-approval) after finalize. The head build must +// not start until it reaches a terminal state — head snapshots select their baseline as they are +// processed, and an unapproved seed means the whole first run shows as new instead of diffing. +// The timeout matches the pipeline latency budget (~99% of builds finish under 5 minutes) — +// a seed of committed screenshots still renders server-side, so first runs can hold for minutes. +export async function waitForSeedBuild(client, buildId, { log, timeout = 600000, interval = 5000 }) { + let deadline = Date.now() + timeout; + let state = 'pending'; + let polls = 0; + + for (;;) { + ({ state } = (await client.getBuild(buildId)).data.attributes); + if (state !== 'pending' && state !== 'processing') return state; + if (Date.now() >= deadline) return state; + // A visible heartbeat every ~30s so a multi-minute first-run hold doesn't look like a hang. + log[polls++ % 6 === 0 ? 'info' : 'debug']( + `Baseline build still ${state} — waiting for it to finish before tests start`); + await new Promise(resolve => setTimeout(resolve, interval)); + } +} + +// Establish the project's baseline from committed screenshots BEFORE the head build starts. +// Never throws — a seeding problem must not break `percy exec`. Returns true when a baseline +// build was created and finalized. +export async function maybeSeedBaseline(percy, provider, { log, waitTimeout, waitInterval }) { + try { + // Parallel shards would race each other seeding; the head build dedup doesn't apply to the + // separate seed build, so leave parallel runs to the explicit setup command. + if (process.env.PERCY_PARALLEL_TOTAL) { + log.debug('Skipping baseline seeding for a parallel build'); + return false; + } + + let { baselines = [], degraded, reason } = + (await provider.discoverBaselines({ cwd: process.cwd(), log })) || {}; + + if (degraded) { + log.debug(`Baseline discovery degraded (${reason}) — nothing seeded`); + return false; + } + // A provider yielding sparse/null entries must not wedge the upload workers. + baselines = baselines.filter(Boolean); + if (!baselines.length) return false; + + // First-ness is decided by the API, never locally: on an established project it answers + // with the baseline-skipped sentinel (no build persisted), on an empty project it creates + // build #1 as the baseline. + let res = await percy.client.createBuild({ + projectType: percy.projectType, + source: BASELINE_SOURCE, + dropinBaselineCandidate: true + }); + + // Compat guard for APIs that predate the candidate attribute (which decide first-ness + // server-side): only ever seed build #1 — an old API ignores the attribute and hands back + // a NORMAL build, so anything past #1 abandons the build unused and points at the explicit + // setup command (which works at any build number) instead of polluting history. + if (!res?.data?.id || res.data.attributes?.['build-number'] !== 1) { + log.info(`Found ${baselines.length} committed baseline snapshot(s), but this project ` + + 'already has builds — to (re)establish the baseline from them, run: ' + + 'npx percy playwright:setup-baseline'); + return false; + } + + let buildId = res.data.id; + log.info(`New Percy project with ${baselines.length} committed baseline snapshot(s) ` + + 'detected — establishing your baseline (build #1) before running tests'); + + let seeded = await uploadBaselines(percy.client, buildId, baselines, { + log, projectType: percy.projectType + }); + + // Nothing uploaded (e.g. a transient network failure across the board): do NOT finalize — + // an empty auto-approved build #1 would establish the project with a blank baseline and + // permanently disable the auto-seed path. The unfinalized build expires server-side. + if (seeded === 0) { + log.warn(`Could not upload any of the ${baselines.length} committed baseline snapshot(s) — ` + + 'baseline not established. Once connectivity recovers, run: ' + + 'npx percy playwright:setup-baseline'); + return false; + } + + await percy.client.finalizeBuild(buildId); + + let state; + try { + state = await waitForSeedBuild(percy.client, buildId, { + log, timeout: waitTimeout, interval: waitInterval + }); + } catch (err) { + log.debug(`Baseline build wait failed: ${err.message}`); + } + + if (state === 'finished') { + log.info(`Baseline established from ${seeded}/${baselines.length} committed snapshot(s) ` + + 'and auto-approved — this run diffs against it.'); + } else { + log.warn(`Baseline build did not finish processing in time (state: ${state || 'unknown'}) — ` + + 'snapshots in this run may show as new instead of diffing against the baseline'); + } + return true; + } catch (err) { + log.warn('Skipping baseline setup'); + log.debug(err.message); + return false; + } +} + +// Clamp a pixel dimension to the API's accepted snapshot range (same as `percy upload`). +function clampDimension(value, fallback) { + return Math.max(10, Math.min(value || fallback, 2000)); +} + +// A committed baseline PNG becomes a WEB snapshot: a generated root DOM displaying the image at +// its native size plus the image resource — the exact shape `percy upload` uses for web projects, +// so Percy renders the baseline in the project's own browsers and it pairs with the head run's +// DOM snapshots by (name, browser, width). +async function imageSnapshotResources({ name, filepath, width, height }) { + let rootUrl = `http://local/${encodeURIComponent(name)}`; + let imageUrl = `http://local/${encodeURIComponent(name)}.png`; + let content = await fs.promises.readFile(filepath); + + return [ + createRootResource(rootUrl, ` + + + + + ${name} + + + + + + + `), + createResource(imageUrl, content, 'image/png') + ]; +} + +// Bounded-concurrency upload of committed baseline files into `buildId`, shaped by project type: +// • web — each PNG becomes a rendered WEB SNAPSHOT (root DOM + image resource); web projects +// reject bare comparison tiles ("root resource" validation), and rendering server-side makes +// the baseline pair with the head run's DOM snapshots. +// • app — each PNG uploads straight through the COMPARISON ingest (tag + tile); no render flow +// is triggered, exactly how App Percy ingests screenshots. The tag width is the identity +// width (project viewport) so it pairs with the head run's uploads; height comes from the +// PNG bytes. +// Per-file failures are skipped (a partial baseline beats none) and reported in the count. +export async function uploadBaselines(client, buildId, baselines, { log, projectType = 'web' }) { + let queue = [...baselines]; + let seeded = 0; + + let uploadOne = async b => { + if (projectType === 'app') { + await client.sendComparison(buildId, { + name: b.name, + // Mirror the drop-in SDK's tag shape exactly (incl. browserName) — comparison pairing + // matches on the canonical tag row, so any attribute difference orphans the baseline. + tag: { name: b.browserFamily, browserName: b.browserFamily, width: b.width, height: b.height }, + tiles: [{ filepath: b.filepath }] + }); + } else { + await client.sendSnapshot(buildId, { + name: b.name, + widths: [clampDimension(b.width, 1280)], + minHeight: clampDimension(b.height, 1024), + resources: await imageSnapshotResources(b) + }); + } + }; + + let worker = async () => { + for (let b = queue.shift(); b; b = queue.shift()) { + try { + await uploadOne(b); + seeded += 1; + log.progress(`Uploading baseline snapshots: ${seeded}/${baselines.length}`, true); + } catch (err) { + log.warn(`Skipped baseline snapshot "${b.name}": ${err.message}`); + } + } + }; + + await Promise.all( + Array.from({ length: Math.min(SEED_CONCURRENCY, baselines.length) }, worker) + ); + + return seeded; +} diff --git a/packages/cli-exec/src/exec.js b/packages/cli-exec/src/exec.js index cfa6d81ac..d6852fecd 100644 --- a/packages/cli-exec/src/exec.js +++ b/packages/cli-exec/src/exec.js @@ -5,6 +5,8 @@ import stop from './stop.js'; import ping from './ping.js'; import replay from './replay.js'; import { waitForTimeout } from '@percy/client/utils'; +import { yieldTo } from '@percy/cli-command/utils'; +import { findBaselineProvider, maybeSeedBaseline } from './baseline.js'; export const exec = command('exec', { description: 'Start and stop Percy around a supplied command', @@ -76,6 +78,24 @@ export const exec = command('exec', { } else { log.debug('Skipping percy project attribute calculation'); } + + // Drop-in baseline seeding: when an installed SDK declares a baseline provider and the + // Percy project is empty, committed baseline screenshots are uploaded as an auto-approved + // build #1 before the head build starts. Never throws. + if (percy.projectType === 'web' || percy.projectType === 'app') { + let provider = await findBaselineProvider({ log }); + + if (provider) { + // Tag the head build so the API can key drop-in behavior on its source. + if (provider.buildSource && !process.env.PERCY_BUILD_SOURCE) { + process.env.PERCY_BUILD_SOURCE = provider.buildSource; + } + // yieldTo keeps the runner tick-responsive — a bare yield would make the up-to-10min + // seed wait the one place Ctrl-C can't unwind until the promise settles. + yield* yieldTo(maybeSeedBaseline(percy, provider, { log })); + } + } + yield* percy.yield.start(); } catch (error) { if (error.name === 'AbortError') throw error; diff --git a/packages/cli-exec/src/index.js b/packages/cli-exec/src/index.js index b119c3dd2..12a8f8109 100644 --- a/packages/cli-exec/src/index.js +++ b/packages/cli-exec/src/index.js @@ -3,3 +3,6 @@ export { start } from './start.js'; export { stop } from './stop.js'; export { ping } from './ping.js'; export { replay } from './replay.js'; +// Baseline seeding building blocks, reused by SDK-contributed setup commands +// (e.g. @percy/playwright's `percy playwright:setup-baseline`). +export { findBaselineProvider, maybeSeedBaseline, uploadBaselines } from './baseline.js'; diff --git a/packages/cli-exec/test/baseline.test.js b/packages/cli-exec/test/baseline.test.js new file mode 100644 index 000000000..beb6c45b1 --- /dev/null +++ b/packages/cli-exec/test/baseline.test.js @@ -0,0 +1,436 @@ +import fs from 'fs'; +import path from 'path'; +import os from 'os'; +import { + findBaselineProvider, + maybeSeedBaseline, + uploadBaselines, + sanitizePath, + sanitizeDirentName +} from '../src/baseline.js'; + +// Collecting fake logger — baseline.js only needs these four methods. +function fakeLog() { + let entries = { info: [], warn: [], debug: [], progress: [] }; + return { + entries, + info: m => entries.info.push(m), + warn: m => entries.warn.push(m), + debug: m => entries.debug.push(m), + progress: m => entries.progress.push(m) + }; +} + +let tmpPngDir; +let BASELINES; + +beforeAll(() => { + // Real files on disk — uploadBaselines reads image content to build snapshot resources. + tmpPngDir = fs.mkdtempSync(path.join(os.tmpdir(), 'percy-baseline-pngs-')); + for (let name of ['a', 'b']) { + fs.writeFileSync(path.join(tmpPngDir, `${name}.png`), Buffer.from(`png-${name}`)); + } + BASELINES = [ + { filepath: path.join(tmpPngDir, 'a.png'), name: 'home', browserFamily: 'chromium', width: 1280, height: 720 }, + { filepath: path.join(tmpPngDir, 'b.png'), name: 'cart', browserFamily: 'chromium', width: 1280, height: 800 } + ]; +}); + +afterAll(() => { + fs.rmSync(tmpPngDir, { recursive: true, force: true }); +}); + +function fakeClient({ established = false, failUploads = false, buildStates = ['finished'], buildNumber = 1 } = {}) { + let calls = { createBuild: [], sendSnapshot: [], sendComparison: [], finalizeBuild: [], getBuild: [] }; + let states = [...buildStates]; + return { + calls, + async getBuild(buildId) { + calls.getBuild.push(buildId); + let state = states.length > 1 ? states.shift() : states[0]; + return { data: { id: buildId, attributes: { state } } }; + }, + async createBuild(options) { + calls.createBuild.push(options); + // Established projects answer with the baseline-skipped sentinel (no data). + if (established) return { 'baseline-skipped': true, 'already-established': true }; + return { data: { id: 'seed-build-1', attributes: { 'build-number': buildNumber } } }; + }, + async sendSnapshot(buildId, options) { + calls.sendSnapshot.push({ buildId, options }); + if (failUploads) throw new Error('upload failed'); + }, + async sendComparison(buildId, options) { + calls.sendComparison.push({ buildId, options }); + if (failUploads) throw new Error('upload failed'); + }, + async finalizeBuild(buildId) { + calls.finalizeBuild.push(buildId); + } + }; +} + +describe('exec baseline seeding', () => { + beforeEach(() => { + delete process.env.PERCY_PARALLEL_TOTAL; + delete process.env.PERCY_DROPIN_DISABLE; + }); + + describe('maybeSeedBaseline', () => { + it('seeds an empty project: creates the baseline build, uploads, finalizes', async () => { + let client = fakeClient(); + let log = fakeLog(); + let provider = { discoverBaselines: async () => ({ baselines: BASELINES }) }; + + let seeded = await maybeSeedBaseline({ client, projectType: 'web' }, provider, { log }); + + expect(seeded).toBe(true); + expect(client.calls.createBuild[0]).toEqual(jasmine.objectContaining({ + source: 'playwright-dropin-baseline', + dropinBaselineCandidate: true + })); + expect(client.calls.sendSnapshot.length).toBe(2); + expect(client.calls.sendSnapshot[0].buildId).toBe('seed-build-1'); + expect(client.calls.sendSnapshot.map(c => c.options.name).sort()) + .toEqual(['cart', 'home']); + // Web-snapshot shape: root DOM + image resource, widths from the identity width. + let first = client.calls.sendSnapshot.find(c => c.options.name === 'home').options; + expect(first.widths).toEqual([1280]); + expect(first.minHeight).toBe(720); + expect(first.resources.length).toBe(2); + expect(first.resources[0].root).toBe(true); + expect(first.resources[0].content).toContain(' { + let client = fakeClient({ buildStates: ['pending', 'processing', 'finished'] }); + let log = fakeLog(); + let provider = { discoverBaselines: async () => ({ baselines: BASELINES }) }; + + let seeded = await maybeSeedBaseline( + { client, projectType: 'web' }, provider, { log, waitInterval: 1 }); + + expect(seeded).toBe(true); + expect(client.calls.getBuild).toEqual(['seed-build-1', 'seed-build-1', 'seed-build-1']); + expect(log.entries.debug.join('\n')).toContain('Baseline build still processing'); + expect(log.entries.info.join('\n')).toContain('auto-approved'); + }); + + it('warns instead of blocking forever when the seed build stays processing', async () => { + let client = fakeClient({ buildStates: ['processing'] }); + let log = fakeLog(); + let provider = { discoverBaselines: async () => ({ baselines: BASELINES }) }; + + let seeded = await maybeSeedBaseline( + { client, projectType: 'web' }, provider, { log, waitTimeout: 0 }); + + expect(seeded).toBe(true); + expect(log.entries.warn.join('\n')).toContain('did not finish processing in time'); + expect(log.entries.info.join('\n')).not.toContain('auto-approved'); + }); + + it('degrades to a warning when the seed build status cannot be read', async () => { + let client = fakeClient(); + client.getBuild = async () => { throw new Error('status boom'); }; + let log = fakeLog(); + let provider = { discoverBaselines: async () => ({ baselines: BASELINES }) }; + + let seeded = await maybeSeedBaseline({ client, projectType: 'web' }, provider, { log }); + + expect(seeded).toBe(true); + expect(log.entries.debug.join('\n')).toContain('status boom'); + expect(log.entries.warn.join('\n')).toContain('did not finish processing in time'); + }); + + it('abandons the seed when the API hands back a non-first build (pre-candidate API)', async () => { + let client = fakeClient({ buildNumber: 3 }); + let log = fakeLog(); + let provider = { discoverBaselines: async () => ({ baselines: BASELINES }) }; + + let seeded = await maybeSeedBaseline({ client, projectType: 'web' }, provider, { log }); + + expect(seeded).toBe(false); + expect(client.calls.sendSnapshot.length).toBe(0); + expect(client.calls.finalizeBuild.length).toBe(0); + expect(log.entries.info.join('\n')).toContain('already has builds'); + }); + + it('filters out falsy baseline entries from the provider', async () => { + let client = fakeClient(); + let log = fakeLog(); + let provider = { discoverBaselines: async () => ({ baselines: [null, ...BASELINES, undefined] }) }; + + let seeded = await maybeSeedBaseline({ client, projectType: 'web' }, provider, { log }); + + expect(seeded).toBe(true); + expect(client.calls.sendSnapshot.length).toBe(BASELINES.length); + }); + + it('never finalizes a zero-upload seed (a blank baseline would burn the auto-seed path)', async () => { + let client = fakeClient({ failUploads: true }); + let log = fakeLog(); + let provider = { discoverBaselines: async () => ({ baselines: BASELINES }) }; + + let seeded = await maybeSeedBaseline({ client, projectType: 'web' }, provider, { log }); + + expect(seeded).toBe(false); + expect(client.calls.finalizeBuild.length).toBe(0); + expect(log.entries.warn.join('\n')).toContain('baseline not established'); + expect(log.entries.warn.join('\n')).toContain('percy playwright:setup-baseline'); + }); + + it('skips an established project and points at the setup command', async () => { + let client = fakeClient({ established: true }); + let log = fakeLog(); + let provider = { discoverBaselines: async () => ({ baselines: BASELINES }) }; + + let seeded = await maybeSeedBaseline({ client, projectType: 'web' }, provider, { log }); + + expect(seeded).toBe(false); + expect(client.calls.sendSnapshot.length).toBe(0); + expect(client.calls.finalizeBuild.length).toBe(0); + expect(log.entries.info.join('\n')).toContain('already has builds'); + expect(log.entries.info.join('\n')).toContain('percy playwright:setup-baseline'); + }); + + it('does nothing when discovery finds no baselines', async () => { + let client = fakeClient(); + let log = fakeLog(); + let provider = { discoverBaselines: async () => ({ baselines: [] }) }; + + let seeded = await maybeSeedBaseline({ client, projectType: 'web' }, provider, { log }); + + expect(seeded).toBe(false); + expect(client.calls.createBuild.length).toBe(0); + }); + + it('tolerates a provider returning nothing at all', async () => { + let client = fakeClient(); + let provider = { discoverBaselines: async () => null }; + + let seeded = await maybeSeedBaseline({ client, projectType: 'web' }, provider, { log: fakeLog() }); + + expect(seeded).toBe(false); + expect(client.calls.createBuild.length).toBe(0); + }); + + it('does nothing when discovery degrades', async () => { + let client = fakeClient(); + let log = fakeLog(); + let provider = { + discoverBaselines: async () => ({ baselines: [], degraded: true, reason: 'custom_template' }) + }; + + let seeded = await maybeSeedBaseline({ client, projectType: 'web' }, provider, { log }); + + expect(seeded).toBe(false); + expect(client.calls.createBuild.length).toBe(0); + expect(log.entries.debug.join('\n')).toContain('custom_template'); + }); + + it('skips parallel builds', async () => { + process.env.PERCY_PARALLEL_TOTAL = '-1'; + let client = fakeClient(); + let log = fakeLog(); + let provider = { discoverBaselines: async () => ({ baselines: BASELINES }) }; + + let seeded = await maybeSeedBaseline({ client, projectType: 'web' }, provider, { log }); + + expect(seeded).toBe(false); + expect(client.calls.createBuild.length).toBe(0); + }); + + it('never throws — a seeding error degrades to a warning', async () => { + let log = fakeLog(); + let provider = { discoverBaselines: async () => { throw new Error('boom'); } }; + + let seeded = await maybeSeedBaseline({ client: fakeClient(), projectType: 'web' }, provider, { log }); + + expect(seeded).toBe(false); + expect(log.entries.warn.join('\n')).toContain('Skipping baseline setup'); + expect(log.entries.debug.join('\n')).toContain('boom'); + }); + }); + + describe('uploadBaselines', () => { + it('skips per-file failures and reports the seeded count', async () => { + let client = fakeClient({ failUploads: true }); + let log = fakeLog(); + + let seeded = await uploadBaselines(client, 'b1', BASELINES, { log }); + + expect(seeded).toBe(0); + expect(log.entries.warn.length).toBe(2); + expect(log.entries.warn[0]).toContain('Skipped baseline snapshot'); + }); + + it('clamps missing dimensions to the API defaults on the web path', async () => { + let client = fakeClient(); + let noDims = [{ filepath: BASELINES[0].filepath, name: 'no-dims', browserFamily: 'chromium' }]; + + let seeded = await uploadBaselines(client, 'b1', noDims, { log: fakeLog() }); + + expect(seeded).toBe(1); + let options = client.calls.sendSnapshot[0].options; + expect(options.widths).toEqual([1280]); + expect(options.minHeight).toBe(1024); + }); + + it('app projects upload through the comparison ingest (tag + tile, no render flow)', async () => { + let client = fakeClient(); + let log = fakeLog(); + + let seeded = await uploadBaselines(client, 'b1', BASELINES, { log, projectType: 'app' }); + + expect(seeded).toBe(2); + expect(client.calls.sendSnapshot.length).toBe(0); + expect(client.calls.sendComparison.length).toBe(2); + let home = client.calls.sendComparison.find(c => c.options.name === 'home').options; + expect(home.tag).toEqual({ name: 'chromium', browserName: 'chromium', width: 1280, height: 720 }); + expect(home.tiles.length).toBe(1); + expect(home.tiles[0].filepath.endsWith('a.png')).toBe(true); + }); + }); + + describe('path hygiene helpers', () => { + it('sanitizePath strips NUL bytes', () => { + expect(sanitizePath('/repo\0/x')).toBe('/repo/x'); + }); + + it('sanitizeDirentName rejects non-single-component names', () => { + expect(sanitizeDirentName('percy-tool')).toBe('percy-tool'); + expect(sanitizeDirentName('')).toBe(null); + expect(sanitizeDirentName('.')).toBe(null); + expect(sanitizeDirentName('..')).toBe(null); + expect(sanitizeDirentName('a/b')).toBe(null); + expect(sanitizeDirentName('a\\b')).toBe(null); + }); + }); + + describe('findBaselineProvider', () => { + let tmpDir; + + beforeEach(() => { + // A fake installed SDK declaring a baseline provider, discovered by the node_modules walk. + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'percy-baseline-')); + let pkgDir = path.join(tmpDir, 'node_modules', '@percy', 'fake-sdk'); + fs.mkdirSync(pkgDir, { recursive: true }); + fs.writeFileSync(path.join(pkgDir, 'package.json'), JSON.stringify({ + name: '@percy/fake-sdk', + '@percy/cli': { baselineProvider: 'provider.cjs' } + })); + fs.writeFileSync(path.join(pkgDir, 'provider.cjs'), [ + 'module.exports = {', + " buildSource: 'playwright-dropin',", + ' discoverBaselines: async () => ({ baselines: [] })', + '};' + ].join('\n')); + + // Neighbors the walk must tolerate, named to sort BEFORE the valid package so every + // skip branch executes on the way to it: a bare dir without package.json, a provider + // module (ESM, named export only) that lacks discoverBaselines, a scoped package without + // the provider key, and a top-level percy-* package (the non-scoped collection branch). + let emptyDir = path.join(tmpDir, 'node_modules', '@percy', 'aa-empty-dir'); + fs.mkdirSync(emptyDir, { recursive: true }); + let noFn = path.join(tmpDir, 'node_modules', '@percy', 'ab-no-fn-provider'); + fs.mkdirSync(noFn, { recursive: true }); + fs.writeFileSync(path.join(noFn, 'package.json'), JSON.stringify({ + name: '@percy/ab-no-fn-provider', + '@percy/cli': { baselineProvider: 'provider.mjs' } + })); + fs.writeFileSync(path.join(noFn, 'provider.mjs'), 'export const notAProvider = true;\n'); + let plainScoped = path.join(tmpDir, 'node_modules', '@percy', 'plain-sdk'); + fs.mkdirSync(plainScoped, { recursive: true }); + fs.writeFileSync(path.join(plainScoped, 'package.json'), JSON.stringify({ name: '@percy/plain-sdk' })); + let topLevel = path.join(tmpDir, 'node_modules', 'percy-cli-plain-tool'); + fs.mkdirSync(topLevel, { recursive: true }); + fs.writeFileSync(path.join(topLevel, 'package.json'), JSON.stringify({ name: 'percy-cli-plain-tool' })); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('finds a provider declared by an installed @percy package', async () => { + let provider = await findBaselineProvider({ cwd: tmpDir, log: fakeLog() }); + + expect(provider).not.toBeNull(); + expect(provider.packageName).toBe('@percy/fake-sdk'); + expect(provider.buildSource).toBe('playwright-dropin'); + expect(typeof provider.discoverBaselines).toBe('function'); + }); + + it('returns null when no package declares a provider', async () => { + let bare = fs.mkdtempSync(path.join(os.tmpdir(), 'percy-noprov-')); + try { + expect(await findBaselineProvider({ cwd: bare, log: fakeLog() })).toBeNull(); + } finally { + fs.rmSync(bare, { recursive: true, force: true }); + } + }); + + it('returns null when the drop-in is disabled via PERCY_DROPIN_DISABLE', async () => { + process.env.PERCY_DROPIN_DISABLE = 'true'; + try { + let log = fakeLog(); + expect(await findBaselineProvider({ cwd: tmpDir, log })).toBeNull(); + expect(log.entries.debug.join('\n')).toContain('PERCY_DROPIN_DISABLE'); + // ...and works without a logger at all + expect(await findBaselineProvider({ cwd: tmpDir })).toBeNull(); + // ...and with no options at all (cwd defaults; disable short-circuits before any walk) + expect(await findBaselineProvider()).toBeNull(); + } finally { + delete process.env.PERCY_DROPIN_DISABLE; + } + }); + + it('skips a package whose provider module fails to load', async () => { + let pkgDir = path.join(tmpDir, 'node_modules', '@percy', 'fake-sdk'); + fs.writeFileSync(path.join(pkgDir, 'provider.cjs'), 'throw new Error("bad module");'); + + let log = fakeLog(); + expect(await findBaselineProvider({ cwd: tmpDir, log })).toBeNull(); + expect(log.entries.debug.join('\n')).toContain('bad module'); + // ...and without a logger the failure is still non-fatal + expect(await findBaselineProvider({ cwd: tmpDir })).toBeNull(); + }); + + it('skips a package whose provider path escapes the package root', async () => { + let pkgDir = path.join(tmpDir, 'node_modules', '@percy', 'fake-sdk'); + fs.writeFileSync(path.join(pkgDir, 'package.json'), JSON.stringify({ + name: '@percy/fake-sdk', + '@percy/cli': { baselineProvider: '../../../outside.cjs' } + })); + fs.writeFileSync(path.join(tmpDir, 'outside.cjs'), 'module.exports = { discoverBaselines: async () => ({}) };'); + + let log = fakeLog(); + expect(await findBaselineProvider({ cwd: tmpDir, log })).toBeNull(); + expect(log.entries.debug.join('\n')).toContain('escapes the package'); + }); + + it('degrades to no provider when the walk hits an unreadable node_modules', async () => { + let broken = fs.mkdtempSync(path.join(os.tmpdir(), 'percy-broken-')); + try { + // node_modules as a FILE: existsSync passes, readdirSync throws ENOTDIR. + fs.writeFileSync(path.join(broken, 'node_modules'), 'not a directory'); + + let log = fakeLog(); + expect(await findBaselineProvider({ cwd: broken, log })).toBeNull(); + expect(log.entries.debug.join('\n')).toContain('discovery walk failed'); + } finally { + fs.rmSync(broken, { recursive: true, force: true }); + } + }); + + it('never walks past the home directory', async () => { + // A nonexistent path under the real home: the walk ascends and must stop AT home + // without reading anything above (or inside) it. + let cwd = path.join(os.homedir(), 'percy-nonexistent-zz', 'a', 'b'); + expect(await findBaselineProvider({ cwd, log: fakeLog() })).toBeNull(); + }); + }); +}); diff --git a/packages/cli-exec/test/exec.test.js b/packages/cli-exec/test/exec.test.js index 792fa72c7..b6cf60c3b 100644 --- a/packages/cli-exec/test/exec.test.js +++ b/packages/cli-exec/test/exec.test.js @@ -1,3 +1,5 @@ +import fs from 'fs'; +import path from 'path'; import { logger, api, setupTest } from '@percy/cli-command/test/helpers'; import exec from '@percy/cli-exec'; describe('percy exec', () => { @@ -57,6 +59,55 @@ describe('percy exec', () => { }); }); + describe('with a baseline provider installed', () => { + // A real package in this package's node_modules, discovered by the provider walk. + const providerRoot = path.join(process.cwd(), 'node_modules', '@percy', 'exec-test-baseline-provider'); + + beforeEach(async () => { + await fs.promises.mkdir(providerRoot, { recursive: true }); + await fs.promises.writeFile(path.join(providerRoot, 'package.json'), JSON.stringify({ + name: '@percy/exec-test-baseline-provider', + '@percy/cli': { baselineProvider: 'provider.cjs' } + })); + await fs.promises.writeFile(path.join(providerRoot, 'provider.cjs'), [ + 'module.exports = {', + " buildSource: 'playwright-dropin',", + ' discoverBaselines: async () => ({ baselines: [] })', + '};' + ].join('\n')); + }); + + afterEach(async () => { + await fs.promises.rm(providerRoot, { recursive: true, force: true }); + delete process.env.PERCY_BUILD_SOURCE; + }); + + it('tags the head build source and skips seeding when no baselines are found', async () => { + await exec(['--', 'node', '--eval', '']); + + expect(process.env.PERCY_BUILD_SOURCE).toBe('playwright-dropin'); + expect(logger.stdout).toEqual(jasmine.arrayContaining([ + '[percy] Percy has started!' + ])); + }); + + it('preserves a user-set PERCY_BUILD_SOURCE', async () => { + process.env.PERCY_BUILD_SOURCE = 'playwright-dropin-baseline'; + + await exec(['--', 'node', '--eval', '']); + + expect(process.env.PERCY_BUILD_SOURCE).toBe('playwright-dropin-baseline'); + }); + + it('skips provider discovery entirely for non web/app tokens', async () => { + process.env.PERCY_TOKEN = 'auto_PERCY_TOKEN'; // tokenType() -> automate + + await exec(['--', 'node', '--eval', '']); + + expect(process.env.PERCY_BUILD_SOURCE).toBeUndefined(); + }); + }); + it('logs an error when no command is provided', async () => { await expectAsync(exec()).toBeRejected(); diff --git a/packages/client/src/client.js b/packages/client/src/client.js index a4aa6f396..51dfaad8d 100644 --- a/packages/client/src/client.js +++ b/packages/client/src/client.js @@ -23,6 +23,10 @@ const { PERCY_CLIENT_API_URL = 'https://percy.io/api/v1' } = process.env; let pkg = getPackageJSON(import.meta.url); // minimum polling interval milliseconds const MIN_POLLING_INTERVAL = 1_000; +// Allow-listed drop-in build sources. The API keys drop-in behavior (baseline auto-approval, +// telemetry) on these; anything else keeps the computed default so a stray env var can't +// inject an arbitrary source. +export const DROPIN_BUILD_SOURCES = ['playwright-dropin', 'playwright-dropin-baseline']; const INVALID_TOKEN_ERROR_MESSAGE = 'Unable to retrieve snapshot details with write access token. Kindly use a full access token for retrieving snapshot details with Synchronous CLI.'; // Validate ID arguments @@ -313,7 +317,21 @@ export class PercyClient { // Creates a build with optional build resources. Only one build can be // created at a time per instance so snapshots and build finalization can be // done more seamlessly without manually tracking build ids - async createBuild({ resources = [], projectType, cliStartTime = null } = {}) { + // + // Drop-in baseline options (Playwright drop-in): + // source — override the build source with an allow-listed drop-in value. + // dropinBaselineCandidate — ask the API to treat the build as the project's baseline IF it is + // the project's first build (the server decides; a no-op on established projects). + // dropinBaselineSetup — explicit `percy playwright:setup-baseline`: the build IS the baseline + // regardless of build number (deliberate user-triggered re-baseline). + async createBuild({ + resources = [], + projectType, + cliStartTime = null, + source: sourceOverride, + dropinBaselineCandidate, + dropinBaselineSetup + } = {}) { this.log.debug('Creating a new build...'); let visualConfig = parseVisualConfigFromEnv(this.log); let source = 'user_created'; @@ -322,8 +340,16 @@ export class PercyClient { source = 'bstack_sdk_created'; } else if (process.env.PERCY_AUTO_ENABLED_GROUP_BUILD === 'true') { source = 'auto_enabled_group'; + } else if (DROPIN_BUILD_SOURCES.includes(process.env.PERCY_BUILD_SOURCE)) { + source = process.env.PERCY_BUILD_SOURCE; } + // Allow-listed explicit override (used by the baseline seeding flows). + if (DROPIN_BUILD_SOURCES.includes(sourceOverride)) source = sourceOverride; + + dropinBaselineCandidate ??= process.env.PERCY_DROPIN_BASELINE_CANDIDATE === 'true'; + dropinBaselineSetup ??= process.env.PERCY_DROPIN_BASELINE_SETUP === 'true'; + let tagsArr = tagsList(this.labels); // PER-9724: internal-only priority request. Set by internal product @@ -357,6 +383,8 @@ export class PercyClient { 'skip-base-build': this.config.percy?.skipBaseBuild, 'testhub-build-uuid': this.env.testhubBuildUuid, 'testhub-build-run-id': this.env.testhubBuildRunId, + ...(dropinBaselineCandidate ? { 'dropin-baseline-candidate': true } : {}), + ...(dropinBaselineSetup ? { 'dropin-baseline-setup': true } : {}), ...(visualConfig ? { 'visual-config': visualConfig } : {}), ...(priority ? { priority: true } : {}) }, diff --git a/packages/client/test/client.test.js b/packages/client/test/client.test.js index 54406c81b..d5149b8da 100644 --- a/packages/client/test/client.test.js +++ b/packages/client/test/client.test.js @@ -199,6 +199,9 @@ describe('PercyClient', () => { delete process.env.PERCY_AUTO_ENABLED_GROUP_BUILD; delete process.env.PERCY_ORIGINATED_SOURCE; delete process.env.PERCY_VISUAL_CONFIG; + delete process.env.PERCY_BUILD_SOURCE; + delete process.env.PERCY_DROPIN_BASELINE_CANDIDATE; + delete process.env.PERCY_DROPIN_BASELINE_SETUP; delete process.env.PERCY_PRIORITY; }); @@ -622,6 +625,64 @@ describe('PercyClient', () => { })); }); + it('tags an allow-listed drop-in source from PERCY_BUILD_SOURCE', async () => { + process.env.PERCY_BUILD_SOURCE = 'playwright-dropin'; + await expectAsync(client.createBuild({ projectType: 'web' })).toBeResolved(); + + let attrs = api.requests['/builds'][0].body.data.attributes; + expect(attrs.source).toEqual('playwright-dropin'); + expect(attrs['dropin-baseline-candidate']).toBeUndefined(); + expect(attrs['dropin-baseline-setup']).toBeUndefined(); + }); + + it('ignores a non-allow-listed PERCY_BUILD_SOURCE', async () => { + process.env.PERCY_BUILD_SOURCE = 'not-a-dropin-source'; + await expectAsync(client.createBuild({ projectType: 'web' })).toBeResolved(); + + expect(api.requests['/builds'][0].body.data.attributes.source) + .toEqual('user_created'); + }); + + it('sends the drop-in baseline candidate attribute from the env', async () => { + process.env.PERCY_DROPIN_BASELINE_CANDIDATE = 'true'; + await expectAsync(client.createBuild({ projectType: 'web' })).toBeResolved(); + + expect(api.requests['/builds'][0].body.data.attributes['dropin-baseline-candidate']) + .toEqual(true); + }); + + it('sends the drop-in baseline setup attribute from the env', async () => { + process.env.PERCY_DROPIN_BASELINE_SETUP = 'true'; + await expectAsync(client.createBuild({ projectType: 'web' })).toBeResolved(); + + expect(api.requests['/builds'][0].body.data.attributes['dropin-baseline-setup']) + .toEqual(true); + }); + + it('accepts explicit drop-in baseline options (used by the seeding flows)', async () => { + await expectAsync(client.createBuild({ + projectType: 'web', + source: 'playwright-dropin-baseline', + dropinBaselineCandidate: true, + dropinBaselineSetup: true + })).toBeResolved(); + + let attrs = api.requests['/builds'][0].body.data.attributes; + expect(attrs.source).toEqual('playwright-dropin-baseline'); + expect(attrs['dropin-baseline-candidate']).toEqual(true); + expect(attrs['dropin-baseline-setup']).toEqual(true); + }); + + it('ignores a non-allow-listed explicit source option', async () => { + await expectAsync(client.createBuild({ + projectType: 'web', + source: 'design' + })).toBeResolved(); + + expect(api.requests['/builds'][0].body.data.attributes.source) + .toEqual('user_created'); + }); + it('creates a new build with visual-config from PERCY_VISUAL_CONFIG', async () => { process.env.PERCY_VISUAL_CONFIG = JSON.stringify({ diffSensitivity: 3, diff --git a/packages/core/src/snapshot.js b/packages/core/src/snapshot.js index 703f86ab5..ec476a660 100644 --- a/packages/core/src/snapshot.js +++ b/packages/core/src/snapshot.js @@ -408,9 +408,13 @@ export function createSnapshotsQueue(percy) { let { data } = await percy.client.createBuild({ projectType: percy.projectType, cliStartTime: percy.cliStartTime }); let url = data.attributes['web-url']; let number = data.attributes['build-number']; + // Server-decided build source, exposed via /percy/healthcheck so SDKs can key on it. + // Only assigned when the API returned one — a `source: undefined` key would drop out + // of JSON responses, breaking shape equality for clients. + let source = data.attributes.source; let usageWarning = data.attributes['usage-warning']; percy.client.buildType = data.attributes?.type; - Object.assign(build, { id: data.id, url, number }); + Object.assign(build, { id: data.id, url, number }, source ? { source } : {}); // Display usage warning if present if (usageWarning) { diff --git a/packages/core/test/api.test.js b/packages/core/test/api.test.js index 771b81927..a6e8123f2 100644 --- a/packages/core/test/api.test.js +++ b/packages/core/test/api.test.js @@ -1,7 +1,7 @@ import os from 'os'; import path from 'path'; import PercyConfig from '@percy/config'; -import { logger, setupTest, fs } from './helpers/index.js'; +import { logger, api, setupTest, fs } from './helpers/index.js'; import Percy from '@percy/core'; import WebdriverUtils from '@percy/webdriver-utils'; import { getPercyDomPath, _applyHttpReadOnlyStripping } from '../src/api.js'; @@ -80,6 +80,29 @@ describe('API Server', () => { }); }); + it('includes the server-decided build source in /healthcheck when present', async () => { + api.reply('/builds', () => [201, { + data: { + id: '123', + attributes: { + 'build-number': 1, + 'web-url': 'https://percy.io/test/test/123', + source: 'playwright-dropin-baseline' + } + } + }]); + + await percy.start(); + + let [data] = await request('/percy/healthcheck', true); + expect(data.build).toEqual({ + id: '123', + number: 1, + url: 'https://percy.io/test/test/123', + source: 'playwright-dropin-baseline' + }); + }); + it('has a /config endpoint that returns loaded config options', async () => { await percy.start(); diff --git a/packages/core/test/utils.test.js b/packages/core/test/utils.test.js index 083da6671..36330fa3a 100644 --- a/packages/core/test/utils.test.js +++ b/packages/core/test/utils.test.js @@ -919,6 +919,12 @@ describe('utils', () => { expect(isMetadataIP('93.184.216.34')).toBeNull(); }); + it('tolerates a colon-containing host that is not parseable IPv6 (falls back to the raw value)', () => { + // canonicalHost wraps the value in http://[...] to normalize IPv6; an + // unparseable value must fall back to the raw string, not throw. + expect(isMetadataIP('fd00:zz::254')).toBeNull(); + }); + it('returns null for an empty/absent connected address', () => { expect(isMetadataIP('')).toBeNull(); expect(isMetadataIP(undefined)).toBeNull();