perf!: allow reducing data retention, by adding retainSamples option on bench and task level, default is false

README.md

@@ -157,6 +157,24 @@ bench.concurrency = 'task' // The concurrency mode to determine how tasks are ru
 await bench.run()
 ```
 
+## Retaining Samples
+
+By default, Tinybench does not retain individual samples (latency measurements) for each task to save memory.
+
+You can enable samples retention at the bench level by setting the `retainSamples` option to `true` when creating a `Bench` instance:
+
+```tsts
+const bench = new Bench({ retainSamples: true })
+```
+
+You can also enable samples retention by setting the `retainSamples` option to `true` when adding a task:
+
+```ts
+bench.add('task with samples', () => {
+  // Task logic here
+}, { retainSamples: true })
+```
+
 ## Aborting Benchmarks
 
 Tinybench supports aborting benchmarks using `AbortSignal` at both the bench and task levels:

src/bench.ts

@@ -63,6 +63,8 @@ export class Bench extends EventTarget implements BenchLike {
     options?: RemoveEventListenerOptionsArgument
   ) => void
 
+  readonly retainSamples: boolean
+
   /**
    * The JavaScript runtime environment.
    */
@@ -137,6 +139,7 @@ export class Bench extends EventTarget implements BenchLike {
     this.teardown = restOptions.teardown ?? emptyFunction
     this.throws = restOptions.throws ?? false
     this.signal = restOptions.signal
+    this.retainSamples = restOptions.retainSamples === true
 
     if (this.signal) {
       this.signal.addEventListener(

src/task.ts

@@ -6,22 +6,22 @@ import type {
   Fn,
   FnOptions,
   RemoveEventListenerOptionsArgument,
+  Samples,
   TaskEvents,
   TaskResult,
   TaskResultRuntimeInfo,
 } from './types'
 
 import { BenchEvent } from './event'
-import { withConcurrency } from './utils'
 import {
   getStatisticsSorted,
   invariant,
   isFnAsyncResource,
   isPromiseLike,
   isValidSamples,
-  type Samples,
   sortSamples,
   toError,
+  withConcurrency
 } from './utils'
 
 const hookNames = ['afterAll', 'beforeAll', 'beforeEach', 'afterEach'] as const
@@ -101,6 +101,11 @@ export class Task extends EventTarget {
    */
   #result: TaskResult = notStartedTaskResult
 
+  /**
+   * Retain samples
+   */
+  readonly #retainSamples: boolean
+
   /**
    * The number of times the task function has been executed
    */
@@ -119,6 +124,7 @@ export class Task extends EventTarget {
     this.#fnOpts = fnOpts
     this.#async = fnOpts.async ?? isFnAsyncResource(fn)
     this.#signal = fnOpts.signal
+    this.#retainSamples = fnOpts.retainSamples ?? bench.retainSamples
 
     for (const hookName of hookNames) {
       if (this.#fnOpts[hookName] != null) {
@@ -511,11 +517,11 @@ export class Task extends EventTarget {
 
       sortSamples(latencySamples)
 
-      const latencyStatistics = getStatisticsSorted(latencySamples)
+      const latencyStatistics = getStatisticsSorted(latencySamples, this.#retainSamples)
       const latencyStatisticsMean = latencyStatistics.mean
 
       let totalTime = 0
-      const throughputSamples = [] as unknown as Samples
+      const throughputSamples: Samples | undefined = [] as unknown as Samples
 
       for (const sample of latencySamples) {
         if (sample !== 0) {
@@ -527,7 +533,7 @@ export class Task extends EventTarget {
       }
 
       sortSamples(throughputSamples)
-      const throughputStatistics = getStatisticsSorted(throughputSamples)
+      const throughputStatistics = getStatisticsSorted(throughputSamples, this.#bench.retainSamples)
 
       /* eslint-disable perfectionist/sort-objects */
       this.#result = {

src/types.ts

@@ -41,6 +41,7 @@ export interface BenchLike extends EventTarget {
     listener: EventListener<K> | EventListenerObject<K> | null,
     options?: RemoveEventListenerOptionsArgument
   ) => void
+  retainSamples: boolean
   runtime: JSRuntime
   runtimeVersion: string
   setup: (task: Task, mode: 'run' | 'warmup') => Promise<void> | void
@@ -83,6 +84,12 @@ export interface BenchOptions {
    */
   now?: NowFn
 
+  /**
+   * keep samples for statistics calculation
+   * @default false
+   */
+  retainSamples?: boolean
+
   /**
    * setup function to run before each benchmark task (cycle)
    */
@@ -209,6 +216,11 @@ export interface FnOptions {
    */
   beforeEach?: FnHook
 
+  /**
+   * Retain samples for this task, overriding the bench-level retainSamples option
+   */
+  retainSamples?: boolean
+
   /**
    * An AbortSignal for aborting this specific task
    *
@@ -260,6 +272,13 @@ export interface ResolvedBenchOptions extends BenchOptions {
   warmupTime: NonNullable<BenchOptions['warmupTime']>
 }
 
+/**
+ * A type representing a samples-array with at least one number.
+ */
+export type Samples = [number, ...number[]]
+
+export type SortedSamples = Samples & { readonly __sorted__: unique symbol }
+
 /**
  * The statistics object
  */
@@ -334,10 +353,12 @@ export interface Statistics {
    */
   rme: number
 
+  samples: SortedSamples | undefined
+
   /**
-   * samples
+   * samples count
    */
-  samples: number[]
+  samplesCount: number
 
   /**
    * standard deviation

src/utils.ts

@@ -5,6 +5,8 @@ import type { Task } from './task'
 import type {
   ConsoleTableConverter,
   Fn,
+  Samples,
+  SortedSamples,
   Statistics,
 } from './types'
 
@@ -257,13 +259,6 @@ export const isFnAsyncResource = (fn: Fn | null | undefined): boolean => {
   }
 }
 
-/**
- * A type representing a samples-array with at least one number.
- */
-export type Samples = [number, ...number[]]
-
-export type SortedSamples = Samples & { readonly __sorted__: unique symbol }
-
 /**
  * Checks if a value is a Samples type.
  * @param value - value to check
@@ -409,9 +404,10 @@ export function absoluteDeviationMedian (samples: SortedSamples, median: number)
  * Computes the statistics of a sample.
  * The sample must be sorted.
  * @param samples - the sorted sample
+ * @param retainSamples - whether to keep the samples in the statistics
  * @returns the statistics of the sample
  */
-export const getStatisticsSorted = (samples: SortedSamples): Statistics => {
+export function getStatisticsSorted (samples: SortedSamples, retainSamples = false): Statistics {
   const { mean, vr } = meanAndVariance(samples)
   const sd = Math.sqrt(vr)
   const sem = sd / Math.sqrt(samples.length)
@@ -438,7 +434,8 @@ export const getStatisticsSorted = (samples: SortedSamples): Statistics => {
     p995: quantileSorted(samples, 0.995),
     p999: quantileSorted(samples, 0.999),
     rme,
-    samples,
+    samples: retainSamples ? samples : undefined,
+    samplesCount: samples.length,
     sd,
     sem,
     variance: vr,
@@ -489,7 +486,7 @@ export const defaultConvertTaskResultForConsoleTable: ConsoleTableConverter = (
           'Latency med (ns)': `${formatNumber(mToNs(task.result.latency.p50), 5, 2)} \xb1 ${formatNumber(mToNs(task.result.latency.mad), 5, 2)}`,
           'Throughput avg (ops/s)': `${Math.round(task.result.throughput.mean).toString()} \xb1 ${task.result.throughput.rme.toFixed(2)}%`,
           'Throughput med (ops/s)': `${Math.round(task.result.throughput.p50).toString()} \xb1 ${Math.round(task.result.throughput.mad).toString()}`,
-          Samples: task.result.latency.samples.length,
+          Samples: task.result.latency.samplesCount,
         }
       : state !== 'errored'
         ? {

test/concurrency-iteration-limit.test.ts

@@ -31,7 +31,7 @@ test('iteration limit not exceeded with task concurrency', async () => {
 
   expect(callCount).toBe(10)
   expect(task.runs).toBe(10)
-  expect(task.result.latency.samples.length).toBe(10)
+  expect(task.result.latency.samplesCount).toBe(10)
 })
 
 test('iteration limit not exceeded with high concurrency', async () => {
@@ -62,7 +62,7 @@ test('iteration limit not exceeded with high concurrency', async () => {
 
   expect(callCount).toBe(100)
   expect(task.runs).toBe(100)
-  expect(task.result.latency.samples.length).toBe(100)
+  expect(task.result.latency.samplesCount).toBe(100)
 })
 
 test('iteration limit edge case - limit equals threshold', async () => {

test/statistics.test.ts

@@ -25,7 +25,7 @@ test('statistics (async)', async () => {
   expect(fooTask.result.period).toBeTypeOf('number')
   // latency statistics
   expect(fooTask.result.latency).toBeTypeOf('object')
-  expect(Array.isArray(fooTask.result.latency.samples)).toBe(true)
+  expect(fooTask.result.latency.samplesCount).toBeTypeOf('number')
   expect(fooTask.result.latency.min).toBeTypeOf('number')
   expect(fooTask.result.latency.max).toBeTypeOf('number')
   expect(fooTask.result.latency.mean).toBeTypeOf('number')
@@ -45,7 +45,7 @@ test('statistics (async)', async () => {
   expect(fooTask.result.latency.p999).toBeTypeOf('number')
   // throughput statistics
   expect(fooTask.result.throughput).toBeTypeOf('object')
-  expect(Array.isArray(fooTask.result.throughput.samples)).toBe(true)
+  expect(fooTask.result.throughput.samplesCount).toBeTypeOf('number')
   expect(fooTask.result.throughput.max).toBeTypeOf('number')
   expect(fooTask.result.throughput.mean).toBeTypeOf('number')
   expect(fooTask.result.throughput.variance).toBeTypeOf('number')
@@ -86,7 +86,7 @@ test('statistics (sync)', () => {
   expect(fooTask.result.period).toBeTypeOf('number')
   // latency statistics
   expect(fooTask.result.latency).toBeTypeOf('object')
-  expect(Array.isArray(fooTask.result.latency.samples)).toBe(true)
+  expect(fooTask.result.latency.samplesCount).toBeTypeOf('number')
   expect(fooTask.result.latency.min).toBeTypeOf('number')
   expect(fooTask.result.latency.max).toBeTypeOf('number')
   expect(fooTask.result.latency.mean).toBeTypeOf('number')
@@ -106,7 +106,7 @@ test('statistics (sync)', () => {
   expect(fooTask.result.latency.p999).toBeTypeOf('number')
   // throughput statistics
   expect(fooTask.result.throughput).toBeTypeOf('object')
-  expect(Array.isArray(fooTask.result.throughput.samples)).toBe(true)
+  expect(fooTask.result.throughput.samplesCount).toBeTypeOf('number')
   expect(fooTask.result.throughput.max).toBeTypeOf('number')
   expect(fooTask.result.throughput.mean).toBeTypeOf('number')
   expect(fooTask.result.throughput.variance).toBeTypeOf('number')
@@ -124,3 +124,108 @@ test('statistics (sync)', () => {
   expect(fooTask.result.throughput.p995).toBeTypeOf('number')
   expect(fooTask.result.throughput.p999).toBeTypeOf('number')
 })
+
+test('statistics retainSamples true', () => {
+  const bench = new Bench({ iterations: 32, retainSamples: true, time: 100 })
+  bench.add('foo', () => {
+    // noop
+  })
+  bench.runSync()
+
+  const fooTask = bench.getTask('foo')
+  expect(fooTask).toBeDefined()
+  if (!fooTask) return
+
+  expect(fooTask.result).toBeDefined()
+
+  expect(fooTask.result.state).toBe('completed')
+  if (fooTask.result.state !== 'completed') return
+
+  // latency statistics
+  expect(fooTask.result.latency).toBeTypeOf('object')
+  expect(fooTask.result.latency.samples).toBeTypeOf('object')
+})
+
+test('statistics retainSamples false', () => {
+  const bench = new Bench({ iterations: 32, retainSamples: false, time: 100 })
+  bench.add('foo', () => {
+    // noop
+  })
+  bench.runSync()
+
+  const fooTask = bench.getTask('foo')
+  expect(fooTask).toBeDefined()
+  if (!fooTask) return
+
+  expect(fooTask.result).toBeDefined()
+
+  expect(fooTask.result.state).toBe('completed')
+  if (fooTask.result.state !== 'completed') return
+
+  // latency statistics
+  expect(fooTask.result.latency).toBeTypeOf('object')
+  expect(fooTask.result.latency.samples).toBeTypeOf('undefined')
+})
+
+test('statistics retainSamples default is false', () => {
+  const bench = new Bench({ iterations: 32, time: 100 })
+  bench.add('foo', () => {
+    // noop
+  })
+  bench.runSync()
+
+  const fooTask = bench.getTask('foo')
+  expect(fooTask).toBeDefined()
+  if (!fooTask) return
+
+  expect(fooTask.result).toBeDefined()
+
+  expect(fooTask.result.state).toBe('completed')
+  if (fooTask.result.state !== 'completed') return
+
+  // latency statistics
+  expect(fooTask.result.latency).toBeTypeOf('object')
+  expect(fooTask.result.latency.samples).toBeTypeOf('undefined')
+})
+
+test('statistics retainSamples false on bench level but retainSamples true on task level', () => {
+  const bench = new Bench({ iterations: 32, retainSamples: false, time: 100 })
+  bench.add('foo', () => {
+    // noop
+  }, { retainSamples: true })
+  bench.runSync()
+
+  const fooTask = bench.getTask('foo')
+  expect(fooTask).toBeDefined()
+  if (!fooTask) return
+
+  expect(fooTask.result).toBeDefined()
+
+  expect(fooTask.result.state).toBe('completed')
+  if (fooTask.result.state !== 'completed') return
+
+  // latency statistics
+  expect(fooTask.result.latency).toBeTypeOf('object')
+  expect(fooTask.result.latency.samples).toBeTypeOf('object')
+})
+
+test('statistics retainSamples true on bench level but retainSamples false on task level', () => {
+  const bench = new Bench({ iterations: 32, retainSamples: true, time: 100 })
+  bench.add('foo', () => {
+    // noop
+  }, { retainSamples: false })
+  bench.runSync()
+
+  const fooTask = bench.getTask('foo')
+  expect(fooTask).toBeDefined()
+  if (!fooTask) return
+
+  expect(fooTask.result).toBeDefined()
+
+  expect(fooTask.result.state).toBe('completed')
+  if (fooTask.result.state !== 'completed') return
+
+  // latency statistics
+  expect(fooTask.result.latency).toBeTypeOf('object')
+  expect(fooTask.result.latency.samples).toBeTypeOf('undefined')
+})

test/utils-absolute-deviation-median.test.ts

@@ -1,6 +1,8 @@
 import { describe, expect, it } from 'vitest'
 
-import { absoluteDeviationMedian, Samples, type SortedSamples } from '../src/utils'
+import type { Samples, SortedSamples } from '../src/types'
+
+import { absoluteDeviationMedian } from '../src/utils'
 import { toSortedSamples } from './utils'
 
 // Helper: calculate median of a sorted array