feat: add repeat method to tests (#2652)

Author: samuel-sou

docs/api/index.md

@@ -11,8 +11,20 @@ type Awaitable<T> = T | PromiseLike<T>
 type TestFunction = () => Awaitable<void>
 
 interface TestOptions {
+  /**
+   * Will fail the test if it takes too long to execute
+   */
   timeout?: number
+  /**
+   * Will retry the test specific number of times if it fails
+   */
   retry?: number
+  /**
+   * Will repeat the same test several times even if it fails each time
+   * If you have "retry" option and it fails, it will use every retry in each cycle
+   * Useful for debugging random failings
+   */
+  repeats?: number
 }
 ```
 

packages/runner/src/run.ts

@@ -125,55 +125,63 @@ export async function runTest(test: Test, runner: VitestRunner) {
 
   setCurrentTest(test)
 
-  const retry = test.retry || 1
-  for (let retryCount = 0; retryCount < retry; retryCount++) {
-    let beforeEachCleanups: HookCleanupCallback[] = []
-    try {
-      await runner.onBeforeTryTest?.(test, retryCount)
+  const repeats = typeof test.repeats === 'number' ? test.repeats : 1
 
-      beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite])
+  for (let repeatCount = 0; repeatCount < repeats; repeatCount++) {
+    const retry = test.retry || 1
 
-      test.result.retryCount = retryCount
+    for (let retryCount = 0; retryCount < retry; retryCount++) {
+      let beforeEachCleanups: HookCleanupCallback[] = []
+      try {
+        await runner.onBeforeTryTest?.(test, { retry: retryCount, repeats: repeatCount })
 
-      if (runner.runTest) {
-        await runner.runTest(test)
-      }
-      else {
-        const fn = getFn(test)
-        if (!fn)
-          throw new Error('Test function is not found. Did you add it using `setFn`?')
-        await fn()
-      }
+        test.result.retryCount = retryCount
+        test.result.repeatCount = repeatCount
 
-      // some async expect will be added to this array, in case user forget to await theme
-      if (test.promises) {
-        const result = await Promise.allSettled(test.promises)
-        const errors = result.map(r => r.status === 'rejected' ? r.reason : undefined).filter(Boolean)
-        if (errors.length)
-          throw errors
-      }
+        beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite])
 
-      await runner.onAfterTryTest?.(test, retryCount)
+        if (runner.runTest) {
+          await runner.runTest(test)
+        }
+        else {
+          const fn = getFn(test)
+          if (!fn)
+            throw new Error('Test function is not found. Did you add it using `setFn`?')
+          await fn()
+        }
 
-      test.result.state = 'pass'
-    }
-    catch (e) {
-      failTask(test.result, e)
-    }
+        // some async expect will be added to this array, in case user forget to await theme
+        if (test.promises) {
+          const result = await Promise.allSettled(test.promises)
+          const errors = result.map(r => r.status === 'rejected' ? r.reason : undefined).filter(Boolean)
+          if (errors.length)
+            throw errors
+        }
 
-    try {
-      await callSuiteHook(test.suite, test, 'afterEach', runner, [test.context, test.suite])
-      await callCleanupHooks(beforeEachCleanups)
-    }
-    catch (e) {
-      failTask(test.result, e)
-    }
+        await runner.onAfterTryTest?.(test, { retry: retryCount, repeats: repeatCount })
 
-    if (test.result.state === 'pass')
-      break
+        if (!test.repeats)
+          test.result.state = 'pass'
+        else if (test.repeats && retry === retryCount)
+          test.result.state = 'pass'
+      }
+      catch (e) {
+        failTask(test.result, e)
+      }
 
-    // update retry info
-    updateTask(test, runner)
+      try {
+        await callSuiteHook(test.suite, test, 'afterEach', runner, [test.context, test.suite])
+        await callCleanupHooks(beforeEachCleanups)
+      }
+      catch (e) {
+        failTask(test.result, e)
+      }
+
+      if (test.result.state === 'pass')
+        break
+      // update retry info
+      updateTask(test, runner)
+    }
   }
 
   if (test.result.state === 'fail')
@@ -291,30 +299,30 @@ export async function runSuite(suite: Suite, runner: VitestRunner) {
     catch (e) {
       failTask(suite.result, e)
     }
-  }
 
-  suite.result.duration = now() - start
-
-  if (suite.mode === 'run') {
-    if (!hasTests(suite)) {
-      suite.result.state = 'fail'
-      if (!suite.result.error) {
-        const error = processError(new Error(`No test found in suite ${suite.name}`))
-        suite.result.error = error
-        suite.result.errors = [error]
+    if (suite.mode === 'run') {
+      if (!hasTests(suite)) {
+        suite.result.state = 'fail'
+        if (!suite.result.error) {
+          const error = processError(new Error(`No test found in suite ${suite.name}`))
+          suite.result.error = error
+          suite.result.errors = [error]
+        }
+      }
+      else if (hasFailed(suite)) {
+        suite.result.state = 'fail'
+      }
+      else {
+        suite.result.state = 'pass'
       }
     }
-    else if (hasFailed(suite)) {
-      suite.result.state = 'fail'
-    }
-    else {
-      suite.result.state = 'pass'
-    }
-  }
 
-  await runner.onAfterRunSuite?.(suite)
+    updateTask(suite, runner)
 
-  updateTask(suite, runner)
+    suite.result.duration = now() - start
+
+    await runner.onAfterRunSuite?.(suite)
+  }
 }
 
 async function runSuiteChild(c: Task, runner: VitestRunner) {

packages/runner/src/suite.ts

@@ -65,6 +65,14 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
     if (typeof options === 'number')
       options = { timeout: options }
 
+    // inherit repeats and retry from suite
+    if (typeof suiteOptions === 'object') {
+      options = {
+        ...suiteOptions,
+        ...options,
+      }
+    }
+
     const test: Test = {
       id: '',
       type: 'test',
@@ -73,6 +81,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
       suite: undefined!,
       fails: this.fails,
       retry: options?.retry,
+      repeats: options?.repeats,
     } as Omit<Test, 'context'> as Test
 
     if (this.concurrent || concurrent)
@@ -124,6 +133,9 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
   }
 
   function initSuite() {
+    if (typeof suiteOptions === 'number')
+      suiteOptions = { timeout: suiteOptions }
+
     suite = {
       id: '',
       type: 'suite',
@@ -132,6 +144,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
       shuffle,
       tasks: [],
     }
+
     setHooks(suite, createSuiteHooks())
   }
 

packages/runner/src/types/runner.ts

@@ -44,15 +44,15 @@ export interface VitestRunner {
   /**
    * Called before actually running the test function. Already has "result" with "state" and "startTime".
    */
-  onBeforeTryTest?(test: Test, retryCount: number): unknown
+  onBeforeTryTest?(test: Test, options: { retry: number; repeats: number }): unknown
   /**
    * Called after result and state are set.
    */
   onAfterRunTest?(test: Test): unknown
   /**
    * Called right after running the test function. Doesn't have new state yet. Will not be called, if the test function throws.
    */
-  onAfterTryTest?(test: Test, retryCount: number): unknown
+  onAfterTryTest?(test: Test, options: { retry: number; repeats: number }): unknown
 
   /**
    * Called before running a single suite. Doesn't have "result" yet.

packages/runner/src/types/tasks.ts

@@ -16,6 +16,7 @@ export interface TaskBase {
   result?: TaskResult
   retry?: number
   meta?: any
+  repeats?: number
 }
 
 export interface TaskCustom extends TaskBase {
@@ -35,6 +36,7 @@ export interface TaskResult {
   htmlError?: string
   hooks?: Partial<Record<keyof SuiteHooks, TaskState>>
   retryCount?: number
+  repeatCount?: number
 }
 
 export type TaskResultPack = [id: string, result: TaskResult | undefined]
@@ -165,6 +167,12 @@ export interface TestOptions {
    * @default 1
    */
   retry?: number
+  /**
+   * How many times the test will repeat.
+   *
+   * @default 5
+   */
+  repeats?: number
 }
 
 export type TestAPI<ExtraContext = {}> = ChainableTestAPI<ExtraContext> & {

packages/vitest/src/node/reporters/renderers/listRenderer.ts

@@ -111,6 +111,9 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level =
     if (task.mode === 'skip' || task.mode === 'todo')
       suffix += ` ${c.dim(c.gray('[skipped]'))}`
 
+    if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 1)
+      suffix += c.yellow(` (repeat x${task.result.repeatCount})`)
+
     if (task.result?.duration != null) {
       if (task.result.duration > DURATION_LONG)
         suffix += c.yellow(` ${Math.round(task.result.duration)}${c.dim('ms')}`)

test/core/test/repeats.test.ts

@@ -0,0 +1,50 @@
+import { afterAll, describe, expect, test } from 'vitest'
+
+const testNumbers: number[] = []
+
+describe('testing it/test', () => {
+  const result = [1, 1, 1, 1, 1, 2, 2, 2]
+
+  test('test 1', () => {
+    testNumbers.push(1)
+  }, { repeats: 5 })
+
+  test('test 2', () => {
+    testNumbers.push(2)
+  }, { repeats: 3 })
+
+  test.fails('test 3', () => {
+    testNumbers.push(3)
+    expect(testNumbers).toStrictEqual(result)
+  }, { repeats: 1 })
+
+  afterAll(() => {
+    result.push(3)
+    expect(testNumbers).toStrictEqual(result)
+  })
+})
+
+const describeNumbers: number[] = []
+
+describe('testing describe', () => {
+  test('test 1', () => {
+    describeNumbers.push(1)
+  })
+}, { repeats: 3 })
+
+afterAll(() => {
+  expect(describeNumbers).toStrictEqual([1, 1, 1])
+})
+
+const retryNumbers: number[] = []
+
+describe('testing repeats with retry', () => {
+  const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+  test('test 1', () => {
+    retryNumbers.push(1)
+  }, { repeats: 5, retry: 2 })
+
+  afterAll(() => {
+    expect(retryNumbers).toStrictEqual(result)
+  })
+})

test/reporters/tests/__snapshots__/html.test.ts.snap

@@ -84,6 +84,7 @@ exports[`html reporter > resolves to "failing" status for test file "json-fail"
               "afterEach": "pass",
               "beforeEach": "pass",
             },
+            "repeatCount": 0,
             "retryCount": 0,
             "startTime": 0,
             "state": "fail",
@@ -155,6 +156,7 @@ exports[`html reporter > resolves to "passing" status for test file "all-passing
               "afterEach": "pass",
               "beforeEach": "pass",
             },
+            "repeatCount": 0,
             "retryCount": 0,
             "startTime": 0,
             "state": "pass",
@@ -201,6 +203,7 @@ exports[`html reporter > resolves to "passing" status for test file "all-passing
                     "afterEach": "pass",
                     "beforeEach": "pass",
                   },
+                  "repeatCount": 0,
                   "retryCount": 0,
                   "startTime": 0,
                   "state": "pass",