diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index 169ba7fae0192..26bfa4803d384 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -107,12 +107,23 @@ export default defineConfig({ }); ``` +You can omit test tags that are automatically appended to test titles: + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + reporter: [['list', { omitTags: true }]], +}); +``` + List report supports the following configuration options and environment variables: | Environment Variable Name | Reporter Config Option| Description | Default |---|---|---|---| | `PLAYWRIGHT_LIST_PRINT_STEPS` | `printSteps` | Whether to print each step on its own line. | `false` | `PLAYWRIGHT_LIST_PRINT_FAILURES_INLINE` | `printFailuresInline` | Whether to print failure details immediately after a failed test instead of at the end. | `false` +| `PLAYWRIGHT_LIST_OMIT_TAGS` | `omitTags` | Whether to omit test tags that are automatically appended to test titles. | `false` | `PLAYWRIGHT_FORCE_TTY` | | Whether to produce output suitable for a live terminal. Supports `true`, `1`, `false`, `0`, `[WIDTH]`, and `[WIDTH]x[HEIGHT]`. `[WIDTH]` and `[WIDTH]x[HEIGHT]` specifies the TTY dimensions. | `true` when terminal is in TTY mode, `false` otherwise. | `FORCE_COLOR` | | Whether to produce colored output. | `true` when terminal is in TTY mode, `false` otherwise. | `NO_COLOR` | | Whether to disable colored output ([no-color.org](https://no-color.org/)). Any non-empty value disables colors. | unset @@ -152,6 +163,7 @@ Line report supports the following configuration options and environment variabl | Environment Variable Name | Reporter Config Option| Description | Default |---|---|---|---| +| `PLAYWRIGHT_LINE_OMIT_TAGS` | `omitTags` | Whether to omit test tags that are automatically appended to test titles. | `false` | `PLAYWRIGHT_FORCE_TTY` | | Whether to produce output suitable for a live terminal. Supports `true`, `1`, `false`, `0`, `[WIDTH]`, and `[WIDTH]x[HEIGHT]`. `[WIDTH]` and `[WIDTH]x[HEIGHT]` specifies the TTY dimensions. | `true` when terminal is in TTY mode, `false` otherwise. | `FORCE_COLOR` | | Whether to produce colored output. | `true` when terminal is in TTY mode, `false` otherwise. | `NO_COLOR` | | Whether to disable colored output ([no-color.org](https://no-color.org/)). Any non-empty value disables colors. | unset @@ -195,6 +207,7 @@ Dot report supports the following configuration options and environment variable | Environment Variable Name | Reporter Config Option| Description | Default |---|---|---|---| +| `PLAYWRIGHT_DOT_OMIT_TAGS` | `omitTags` | Whether to omit test tags that are automatically appended to test titles. | `false` | `PLAYWRIGHT_FORCE_TTY` | | Whether to produce output suitable for a live terminal. Supports `true`, `1`, `false`, `0`, `[WIDTH]`, and `[WIDTH]x[HEIGHT]`. `[WIDTH]` and `[WIDTH]x[HEIGHT]` specifies the TTY dimensions. | `true` when terminal is in TTY mode, `false` otherwise. | `FORCE_COLOR` | | Whether to produce colored output. | `true` when terminal is in TTY mode, `false` otherwise. | `NO_COLOR` | | Whether to disable colored output ([no-color.org](https://no-color.org/)). Any non-empty value disables colors. | unset @@ -414,6 +427,7 @@ JUnit report supports following configuration options and environment variables: | `PLAYWRIGHT_JUNIT_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `PLAYWRIGHT_JUNIT_OUTPUT_DIR` and `PLAYWRIGHT_JUNIT_OUTPUT_NAME` will be ignored. | JUnit report is printed to the stdout. | `PLAYWRIGHT_JUNIT_STRIP_ANSI` | `stripANSIControlSequences` | Whether to remove ANSI control sequences from the text before writing it in the report. | By default output text is added as is. | `PLAYWRIGHT_JUNIT_INCLUDE_PROJECT_IN_TEST_NAME` | `includeProjectInTestName` | Whether to include Playwright project name in every test case as a name prefix. | By default not included. +| `PLAYWRIGHT_JUNIT_OMIT_TAGS` | `omitTags` | Whether to omit test tags that are automatically appended to failure details. | `false` | `PLAYWRIGHT_JUNIT_SUITE_ID` | | Value of the `id` attribute on the root `` report entry. | Empty string. | `PLAYWRIGHT_JUNIT_SUITE_NAME` | | Value of the `name` attribute on the root `` report entry. | Empty string. @@ -435,6 +449,8 @@ export default defineConfig({ }); ``` +The `github` reporter accepts `omitTags` (or the `PLAYWRIGHT_GITHUB_OMIT_TAGS` environment variable) to suppress test tags in its annotations, for example `reporter: [['github', { omitTags: true }]]`. + ## Custom reporters You can create a custom reporter by implementing a class with some of the reporter methods. Learn more about the [Reporter] API. diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index 2cfd7ac60d2b3..83536e46341e8 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -163,6 +163,7 @@ export type TerminalReporterOptions = { screen?: TerminalScreen; omitFailures?: boolean; includeTestId?: boolean; + omitTags?: boolean; }; export class TerminalReporter implements ReporterV2 { @@ -366,7 +367,7 @@ export class TerminalReporter implements ReporterV2 { } formatTestHeader(test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error' } = {}): string { - return formatTestHeader(this.screen, this.config, test, { ...options, includeTestId: this._options.includeTestId }); + return formatTestHeader(this.screen, this.config, test, { ...options, includeTestId: this._options.includeTestId, omitTags: this._options.omitTags }); } formatFailure(test: TestCase, index?: number): string { @@ -407,7 +408,7 @@ export function formatFailure(screen: Screen, config: FullConfig, test: TestCase if (!errors.length) continue; if (!printedHeader) { - const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error', includeTestId: options?.includeTestId }); + const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error', includeTestId: options?.includeTestId, omitTags: options?.omitTags }); lines.push(screen.colors.red(header)); printedHeader = true; } @@ -539,7 +540,7 @@ export function stepSuffix(step: TestStep | undefined) { return stepTitles.map(t => t.split('\n')[0]).map(t => ' › ' + t).join(''); } -function formatTestTitle(screen: Screen, config: FullConfig, test: TestCase, step?: TestStep, options: { includeTestId?: boolean } = {}): string { +function formatTestTitle(screen: Screen, config: FullConfig, test: TestCase, step?: TestStep, options: { includeTestId?: boolean, omitTags?: boolean } = {}): string { // root, project, file, ...describes, test const [, projectName, , ...titles] = test.titlePath(); const location = `${relativeTestPath(screen, config, test)}:${test.location.line}:${test.location.column}`; @@ -547,11 +548,11 @@ function formatTestTitle(screen: Screen, config: FullConfig, test: TestCase, ste const projectLabel = options.includeTestId ? `project=` : ''; const projectTitle = projectName ? `[${projectLabel}${projectName}] › ` : ''; const testTitle = `${testId}${projectTitle}${location} › ${titles.join(' › ')}`; - const extraTags = test.tags.filter(t => !testTitle.includes(t) && !config.tags.includes(t)); + const extraTags = options.omitTags ? [] : test.tags.filter(t => !testTitle.includes(t) && !config.tags.includes(t)); return `${testTitle}${stepSuffix(step)}${extraTags.length ? ' ' + extraTags.join(' ') : ''}`; } -function formatTestHeader(screen: Screen, config: FullConfig, test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error', includeTestId?: boolean } = {}): string { +function formatTestHeader(screen: Screen, config: FullConfig, test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error', includeTestId?: boolean, omitTags?: boolean } = {}): string { const title = formatTestTitle(screen, config, test, undefined, options); const header = `${options.indent || ''}${options.index ? options.index + ') ' : ''}${title}`; let fullHeader = header; diff --git a/packages/playwright/src/reporters/dot.ts b/packages/playwright/src/reporters/dot.ts index 723ee6ad93159..dd3812b44a8ae 100644 --- a/packages/playwright/src/reporters/dot.ts +++ b/packages/playwright/src/reporters/dot.ts @@ -14,13 +14,21 @@ * limitations under the License. */ +import { getAsBooleanFromENV } from '@utils/env'; + import { markErrorsAsReported, TerminalReporter } from './base'; +import type { DotReporterOptions } from '../../types/test'; import type { FullResult, Suite, TestCase, TestError, TestResult } from '../../types/testReporter'; +import type { CommonReporterOptions, TerminalReporterOptions } from './base'; class DotReporter extends TerminalReporter { private _counter = 0; + constructor(options?: DotReporterOptions & CommonReporterOptions & TerminalReporterOptions) { + super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_DOT_OMIT_TAGS', options?.omitTags) }); + } + override onBegin(suite: Suite) { super.onBegin(suite); this.writeLine(this.generateStartingMessage()); diff --git a/packages/playwright/src/reporters/github.ts b/packages/playwright/src/reporters/github.ts index 6975adb1dd098..e21fae59e6d06 100644 --- a/packages/playwright/src/reporters/github.ts +++ b/packages/playwright/src/reporters/github.ts @@ -18,10 +18,12 @@ import path from 'path'; import { noColors } from '@isomorphic/colors'; import { msToString } from '@isomorphic/formatUtils'; +import { getAsBooleanFromENV } from '@utils/env'; import { TerminalReporter, formatResultFailure, formatRetry } from './base'; import { stripAnsiEscapes } from '../util'; +import type { TerminalReporterOptions } from './base'; import type { FullResult, TestCase, TestError, TestResult } from '../../types/testReporter'; type GitHubLogType = 'debug' | 'notice' | 'warning' | 'error'; @@ -71,8 +73,8 @@ export class GitHubReporter extends TerminalReporter { githubLogger = new GitHubLogger(); private _failedTestCount = 0; - constructor(options: { omitFailures?: boolean } = {}) { - super(options); + constructor(options: TerminalReporterOptions = {}) { + super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_GITHUB_OMIT_TAGS', options.omitTags) }); this.screen = { ...this.screen, colors: noColors }; } diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.ts index c5a516274eda0..82df28228344b 100644 --- a/packages/playwright/src/reporters/junit.ts +++ b/packages/playwright/src/reporters/junit.ts @@ -39,11 +39,13 @@ class JUnitReporter implements ReporterV2 { private stripANSIControlSequences = false; private includeProjectInTestName = false; private includeRetries = false; + private omitTags = false; constructor(options: JUnitReporterOptions & CommonReporterOptions) { this.stripANSIControlSequences = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_STRIP_ANSI', !!options.stripANSIControlSequences); this.includeProjectInTestName = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_PROJECT_IN_TEST_NAME', !!options.includeProjectInTestName); this.includeRetries = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_RETRIES', !!options.includeRetries); + this.omitTags = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_OMIT_TAGS', !!options.omitTags); this.configDir = options.configDir; this.resolvedOutputFile = resolveOutputFile('JUNIT', options)?.outputFile; } @@ -222,7 +224,7 @@ class JUnitReporter implements ReporterV2 { entry.children!.push({ name: errorInfo.elementName, attributes: { message: errorInfo.message, type: errorInfo.type }, - text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test)) + text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test, undefined, { omitTags: this.omitTags })) }); return errorInfo.elementName; } @@ -232,7 +234,7 @@ class JUnitReporter implements ReporterV2 { message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`, type: 'FAILURE', }, - text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test)) + text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test, undefined, { omitTags: this.omitTags })) }); return 'failure'; } diff --git a/packages/playwright/src/reporters/line.ts b/packages/playwright/src/reporters/line.ts index 81f408f86a44f..c703d3cc01e18 100644 --- a/packages/playwright/src/reporters/line.ts +++ b/packages/playwright/src/reporters/line.ts @@ -14,9 +14,13 @@ * limitations under the License. */ +import { getAsBooleanFromENV } from '@utils/env'; + import { markErrorsAsReported, TerminalReporter } from './base'; +import type { LineReporterOptions } from '../../types/test'; import type { FullResult, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; +import type { CommonReporterOptions, TerminalReporterOptions } from './base'; class LineReporter extends TerminalReporter { private _current = 0; @@ -24,6 +28,10 @@ class LineReporter extends TerminalReporter { private _lastTest: TestCase | undefined; private _didBegin = false; + constructor(options?: LineReporterOptions & CommonReporterOptions & TerminalReporterOptions) { + super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_LINE_OMIT_TAGS', options?.omitTags) }); + } + override onBegin(suite: Suite) { super.onBegin(suite); const startingMessage = this.generateStartingMessage(); diff --git a/packages/playwright/src/reporters/list.ts b/packages/playwright/src/reporters/list.ts index 58dff8deeca26..eb128a33dbd24 100644 --- a/packages/playwright/src/reporters/list.ts +++ b/packages/playwright/src/reporters/list.ts @@ -43,7 +43,7 @@ class ListReporter extends TerminalReporter { private _paused = new Set(); constructor(options?: ListReporterOptions & CommonReporterOptions & TerminalReporterOptions) { - super(options); + super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_LIST_OMIT_TAGS', options?.omitTags) }); this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options?.printSteps); this._printFailuresInline = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_FAILURES_INLINE', options?.printFailuresInline); } diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 2e7180bde3d94..12667c34658a0 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -19,8 +19,11 @@ import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, export * from 'playwright-core'; export type BlobReporterOptions = { outputDir?: string, fileName?: string }; -export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: boolean }; -export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean }; +export type DotReporterOptions = { omitTags?: boolean }; +export type LineReporterOptions = { omitTags?: boolean }; +export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: boolean, omitTags?: boolean }; +export type GitHubReporterOptions = { omitTags?: boolean }; +export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean, omitTags?: boolean }; export type JsonReporterOptions = { outputFile?: string }; export type HtmlReporterOptions = { outputFolder?: string; @@ -37,10 +40,10 @@ export type HtmlReporterOptions = { export type ReporterDescription = Readonly< ['blob'] | ['blob', BlobReporterOptions] | - ['dot'] | - ['line'] | + ['dot'] | ['dot', DotReporterOptions] | + ['line'] | ['line', LineReporterOptions] | ['list'] | ['list', ListReporterOptions] | - ['github'] | + ['github'] | ['github', GitHubReporterOptions] | ['junit'] | ['junit', JUnitReporterOptions] | ['json'] | ['json', JsonReporterOptions] | ['html'] | ['html', HtmlReporterOptions] | diff --git a/tests/playwright-test/reporter-base.spec.ts b/tests/playwright-test/reporter-base.spec.ts index afd5aa0543a9a..fb8b8176b7133 100644 --- a/tests/playwright-test/reporter-base.spec.ts +++ b/tests/playwright-test/reporter-base.spec.ts @@ -486,5 +486,106 @@ for (const useIntermediateMergeReport of [false, true] as const) { expect(text).toContain('› passes @bar1 @bar2 ('); expect(text).toContain('› passes @baz1 @baz2 ('); }); + + test('should omit tags when omitTags is set', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + reporter: [['list', { omitTags: true }]], + }; + `, + 'a.test.ts': ` + const { test, expect } = require('@playwright/test'); + test('passes', { tag: ['@foo1', '@foo2'] }, async ({}) => { + expect(0).toBe(0); + }); + test('passes @bar1 @bar2', async ({}) => { + expect(0).toBe(0); + }); + test('passes @baz1', { tag: ['@baz2'] }, async ({}) => { + expect(0).toBe(0); + }); + `, + }); + const text = stripAnsi(result.output); + // Tags appended from the `tag` annotation are omitted. + expect(text).toContain('› passes ('); + expect(text).not.toContain('@foo1'); + expect(text).not.toContain('@foo2'); + expect(text).not.toContain('@baz2'); + // Tags authored directly in the title are preserved. + expect(text).toContain('› passes @bar1 @bar2 ('); + expect(text).toContain('› passes @baz1 ('); + expect(result.exitCode).toBe(0); + }); + + test('should omit tags from failures when omitTags is set', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + reporter: [['list', { omitTags: true }]], + }; + `, + 'a.test.ts': ` + const { test, expect } = require('@playwright/test'); + test('fails', { tag: ['@qux1'] }, async ({}) => { + expect(1).toBe(2); + }); + `, + }); + const text = stripAnsi(result.output); + const titleLines = text.split('\n').filter(line => line.includes('a.test.ts') && line.includes('› fails')); + expect(titleLines.length).toBeGreaterThan(0); + for (const line of titleLines) + expect(line).not.toContain('@qux1'); + expect(result.exitCode).toBe(1); + }); + + test('should omit tags via the PLAYWRIGHT_LIST_OMIT_TAGS environment variable', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + reporter: [['list']], + }; + `, + 'a.test.ts': ` + const { test, expect } = require('@playwright/test'); + test('passes', { tag: ['@foo1', '@foo2'] }, async ({}) => { + expect(0).toBe(0); + }); + test('passes @bar1 @bar2', async ({}) => { + expect(0).toBe(0); + }); + `, + }, undefined, { PLAYWRIGHT_LIST_OMIT_TAGS: '1' }); + const text = stripAnsi(result.output); + // Tags appended from the `tag` annotation are omitted. + expect(text).toContain('› passes ('); + expect(text).not.toContain('@foo1'); + expect(text).not.toContain('@foo2'); + // Tags authored directly in the title are preserved. + expect(text).toContain('› passes @bar1 @bar2 ('); + expect(result.exitCode).toBe(0); + }); + + test('should let PLAYWRIGHT_LIST_OMIT_TAGS override the omitTags option', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + reporter: [['list', { omitTags: true }]], + }; + `, + 'a.test.ts': ` + const { test, expect } = require('@playwright/test'); + test('passes', { tag: ['@foo1', '@foo2'] }, async ({}) => { + expect(0).toBe(0); + }); + `, + }, undefined, { PLAYWRIGHT_LIST_OMIT_TAGS: '0' }); + const text = stripAnsi(result.output); + // The environment variable disables omitTags, so the appended tags are shown. + expect(text).toContain('› passes @foo1 @foo2 ('); + expect(result.exitCode).toBe(0); + }); }); } diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 4a7273f2231a4..2a736cad5d5c5 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -18,8 +18,11 @@ import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, export * from 'playwright-core'; export type BlobReporterOptions = { outputDir?: string, fileName?: string }; -export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: boolean }; -export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean }; +export type DotReporterOptions = { omitTags?: boolean }; +export type LineReporterOptions = { omitTags?: boolean }; +export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: boolean, omitTags?: boolean }; +export type GitHubReporterOptions = { omitTags?: boolean }; +export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean, omitTags?: boolean }; export type JsonReporterOptions = { outputFile?: string }; export type HtmlReporterOptions = { outputFolder?: string; @@ -36,10 +39,10 @@ export type HtmlReporterOptions = { export type ReporterDescription = Readonly< ['blob'] | ['blob', BlobReporterOptions] | - ['dot'] | - ['line'] | + ['dot'] | ['dot', DotReporterOptions] | + ['line'] | ['line', LineReporterOptions] | ['list'] | ['list', ListReporterOptions] | - ['github'] | + ['github'] | ['github', GitHubReporterOptions] | ['junit'] | ['junit', JUnitReporterOptions] | ['json'] | ['json', JsonReporterOptions] | ['html'] | ['html', HtmlReporterOptions] |