bench: calibrate new ssr benchmarks iteration to reduce variability

benchmarks/ssr/bench-utils.ts

@@ -12,7 +12,6 @@ export interface RunRequestLoopOptions {
   iterations?: number
   buildRequest: (random: () => number, index: number) => Request
   validateResponse?: (response: Response, request: Request) => void
-  validateBody?: (body: string, response: Response, request: Request) => void
 }
 
 const requestInit = {
@@ -37,6 +36,26 @@ function randomSegment(random: () => number) {
 
 export { createDeterministicRandom, randomSegment }
 
+export async function drainResponse(response: Response) {
+  const reader = response.body?.getReader()
+
+  if (!reader) {
+    return
+  }
+
+  try {
+    while (true) {
+      const result = await reader.read()
+
+      if (result.done) {
+        break
+      }
+    }
+  } finally {
+    reader.releaseLock()
+  }
+}
+
 function randomSearchValue(random: () => number) {
   return `q-${randomSegment(random)}`
 }
@@ -70,7 +89,7 @@ export async function runSsrRequestLoop(
       )
     }
 
-    pendingBodyReads.push(response.text().then(() => undefined))
+    pendingBodyReads.push(drainResponse(response))
   }
 
   await Promise.all(pendingBodyReads)
@@ -83,7 +102,6 @@ export async function runRequestLoop(
     iterations = 10,
     buildRequest,
     validateResponse,
-    validateBody,
   }: RunRequestLoopOptions,
 ) {
   const random = createDeterministicRandom(seed)
@@ -110,11 +128,7 @@ export async function runRequestLoop(
       throw error
     }
 
-    pendingBodyReads.push(
-      response.text().then((body) => {
-        validateBody?.(body, response, request)
-      }),
-    )
+    pendingBodyReads.push(drainResponse(response))
   }
 
   await Promise.all(pendingBodyReads)

benchmarks/ssr/scenarios/assets/shared.ts

@@ -7,6 +7,8 @@ export type { StartRequestHandler }
 const benchmarkSeed = 0xdecafbad
 const origin = 'http://localhost'
 const cdnOrigin = 'https://cdn.example.com'
+const inlineCssLoopIterations = 25
+const linkedCssLoopIterations = 30
 
 const inlineRequestInit = {
   method: 'GET',
@@ -39,13 +41,15 @@ function buildAssetsRequest(random: () => number, requestInit: RequestInit) {
 export function runAssetsInlineLoop(handler: StartRequestHandler) {
   return runRequestLoop(handler, {
     seed: benchmarkSeed,
+    iterations: inlineCssLoopIterations,
     buildRequest: (random) => buildAssetsRequest(random, inlineRequestInit),
   })
 }
 
 export function runAssetsLinkedControlLoop(handler: StartRequestHandler) {
   return runRequestLoop(handler, {
     seed: benchmarkSeed,
+    iterations: linkedCssLoopIterations,
     buildRequest: (random) => buildAssetsRequest(random, linkedRequestInit),
   })
 }

benchmarks/ssr/scenarios/before-load/shared-bench.ts

@@ -5,7 +5,7 @@ import type { StartRequestHandler } from '../../bench-utils'
 export type { StartRequestHandler }
 
 const benchmarkSeed = 0xdecafbad
-const beforeLoadLoopIterations = 20
+const beforeLoadLoopIterations = 30
 
 const requestInit = {
   method: 'GET',

benchmarks/ssr/scenarios/control-flow/shared.ts

@@ -6,10 +6,10 @@ export type { StartRequestHandler }
 
 const benchmarkSeed = 0xdecafbad
 const redirectLoopIterations = 100
-const notFoundLoopIterations = 20
-const errorLoopIterations = 20
-const unmatchedLoopIterations = 40
-const routeHeadersLoopIterations = 20
+const notFoundLoopIterations = 45
+const errorLoopIterations = 45
+const unmatchedLoopIterations = 60
+const routeHeadersLoopIterations = 50
 
 // Pinned to the current built handler responses for these control-flow routes.
 const OK_STATUS = 200

benchmarks/ssr/scenarios/global-middleware/bench.ts

@@ -54,8 +54,9 @@ const postHeaders = {
   accept: acceptHeader,
   'content-type': 'application/json',
 } satisfies HeadersInit
-const documentLoopIterations = 20
-const apiLoopIterations = 100
+const documentLoopIterations = 50
+const serverFnLoopIterations = 125
+const serverRouteLoopIterations = 250
 
 export const globalMiddlewareBenchOptions = {
   warmupIterations: 100,
@@ -273,7 +274,7 @@ export function runGlobalMiddlewareServerFnLoop(
 ) {
   return runRequestLoop(handler, {
     seed: benchmarkSeed,
-    iterations: apiLoopIterations,
+    iterations: serverFnLoopIterations,
     buildRequest: (_random, index) =>
       buildPostRequest(context.urls, context.bodies, index),
   })
@@ -284,7 +285,7 @@ export function runGlobalMiddlewareServerRouteLoop(
 ) {
   return runRequestLoop(handler, {
     seed: benchmarkSeed,
-    iterations: apiLoopIterations,
+    iterations: serverRouteLoopIterations,
     buildRequest: buildServerRouteRequest,
   })
 }

benchmarks/ssr/scenarios/head/shared.ts

@@ -7,6 +7,7 @@ export type { StartRequestHandler }
 const benchmarkSeed = 0xdecafbad
 const origin = 'http://localhost'
 const dedupedMetaName = 'head-benchmark-shared'
+const headLoopIterations = 25
 
 const requestInit = {
   method: 'GET',
@@ -31,6 +32,7 @@ function buildHeadRequest(random: () => number) {
 export function runHeadLoop(handler: StartRequestHandler) {
   return runRequestLoop(handler, {
     seed: benchmarkSeed,
+    iterations: headLoopIterations,
     buildRequest: buildHeadRequest,
   })
 }

benchmarks/ssr/scenarios/loaders/shared-bench.ts

@@ -5,6 +5,7 @@ import type { StartRequestHandler } from '../../bench-utils'
 export type { StartRequestHandler }
 
 const benchmarkSeed = 0xdecafbad
+const loadersLoopIterations = 15
 
 const requestInit = {
   method: 'GET',
@@ -67,6 +68,7 @@ export const benchOptions = {
 export function runLoadersLoop(handler: StartRequestHandler) {
   return runRequestLoop(handler, {
     seed: benchmarkSeed,
+    iterations: loadersLoopIterations,
     buildRequest: buildLoaderRequest,
   })
 }

benchmarks/ssr/scenarios/rewrites/shared.ts

@@ -6,6 +6,8 @@ export type { StartRequestHandler }
 
 const benchmarkSeed = 0xdecafbad
 const origin = 'http://localhost'
+const localizedLoopIterations = 25
+const passthroughLoopIterations = 25
 
 const requestInit = {
   method: 'GET',
@@ -37,13 +39,15 @@ function buildPassthroughRequest(random: () => number) {
 export function runRewriteLocalizedLoop(handler: StartRequestHandler) {
   return runRequestLoop(handler, {
     seed: benchmarkSeed,
+    iterations: localizedLoopIterations,
     buildRequest: buildLocalizedRequest,
   })
 }
 
 export function runRewritePassthroughLoop(handler: StartRequestHandler) {
   return runRequestLoop(handler, {
     seed: benchmarkSeed,
+    iterations: passthroughLoopIterations,
     buildRequest: buildPassthroughRequest,
   })
 }

benchmarks/ssr/scenarios/selective-ssr/shared-bench.ts

@@ -5,6 +5,7 @@ import type { StartRequestHandler } from '../../bench-utils'
 export type { StartRequestHandler }
 
 const benchmarkSeed = 0xdecafbad
+const selectiveLoopIterations = 20
 
 const requestInit = {
   method: 'GET',
@@ -87,6 +88,7 @@ export const benchOptions = {
 export function runSelectiveLoop(handler: StartRequestHandler) {
   return runRequestLoop(handler, {
     seed: benchmarkSeed,
+    iterations: selectiveLoopIterations,
     buildRequest: buildSelectiveRequest,
   })
 }

benchmarks/ssr/scenarios/serialization/shared-bench.ts

@@ -5,7 +5,8 @@ import type { StartRequestHandler } from '../../bench-utils'
 export type { StartRequestHandler }
 
 const benchmarkSeed = 0xdecafbad
-const plainSerializationLoopIterations = 20
+const richSerializationLoopIterations = 30
+const plainSerializationLoopIterations = 30
 
 const requestInit = {
   method: 'GET',
@@ -88,6 +89,7 @@ export const serializationBenchOptions = {
 export function runRichSerializationLoop(handler: StartRequestHandler) {
   return runRequestLoop(handler, {
     seed: benchmarkSeed,
+    iterations: richSerializationLoopIterations,
     buildRequest: (random, index) =>
       buildSerializationRequest('rich', random, index),
   })

benchmarks/ssr/scenarios/server-fn-transport/bench.ts

@@ -67,10 +67,9 @@ const commonHeaders = {
   'sec-fetch-site': 'same-origin',
   accept: acceptHeader,
 } satisfies HeadersInit
-const serverFnTransportRequestLoopOptions = {
-  seed: benchmarkSeed,
-  iterations: 100,
-} as const
+const serverFnMultipartLoopIterations = 125
+const serverFnRawResponseLoopIterations = 175
+const serverFnRawStreamLoopIterations = 100
 
 export const serverFnTransportBenchOptions = {
   warmupIterations: 100,
@@ -519,7 +518,8 @@ export function runServerFnMultipartRequestLoop(
   context: ServerFnTransportBenchContext,
 ) {
   return runRequestLoop(handler, {
-    ...serverFnTransportRequestLoopOptions,
+    seed: benchmarkSeed,
+    iterations: serverFnMultipartLoopIterations,
     buildRequest: (_random, index) =>
       buildMultipartRequest(context.urls, context.multipartPayloads, index),
     validateResponse: validateSerializedResponse,
@@ -531,19 +531,11 @@ export function runServerFnRawResponseRequestLoop(
   context: ServerFnTransportBenchContext,
 ) {
   return runRequestLoop(handler, {
-    ...serverFnTransportRequestLoopOptions,
+    seed: benchmarkSeed,
+    iterations: serverFnRawResponseLoopIterations,
     buildRequest: (_random, index) =>
       buildRawResponseRequest(context.urls, context.rawPayloads, index),
     validateResponse: validateRawResponse,
-    validateBody: (body, _response, request) => {
-      const expectedBody = context.rawExpectedBodiesByUrl.get(request.url)
-
-      if (body !== expectedBody) {
-        throw new Error(
-          `Expected raw response body ${expectedBody}, received ${body}`,
-        )
-      }
-    },
   })
 }
 
@@ -552,7 +544,8 @@ export function runServerFnRawStreamRequestLoop(
   context: ServerFnTransportBenchContext,
 ) {
   return runRequestLoop(handler, {
-    ...serverFnTransportRequestLoopOptions,
+    seed: benchmarkSeed,
+    iterations: serverFnRawStreamLoopIterations,
     buildRequest: (_random, index) =>
       buildRawStreamRequest(context.urls, context.streamPayloads, index),
     validateResponse: validateRawStreamResponse,

benchmarks/ssr/scenarios/server-fns/bench.ts

@@ -53,14 +53,12 @@ const documentRequestInit = {
     accept: 'text/html',
   },
 } satisfies RequestInit
-const serverFnRequestLoopOptions = {
-  seed: benchmarkSeed,
-  iterations: 100,
-} as const
-const serverFnSsrRequestLoopOptions = {
-  seed: benchmarkSeed,
-  iterations: 20,
-} as const
+const serverFnGetLoopIterations = 100
+const serverFnPostLoopIterations = 150
+const serverFnRedirectLoopIterations = 125
+const serverFnNotFoundLoopIterations = 200
+const serverFnSendContextLoopIterations = 175
+const serverFnDocumentSsrLoopIterations = 45
 
 export const serverFnBenchOptions = {
   warmupIterations: 100,
@@ -457,7 +455,8 @@ export function runServerFnGetRequestLoop(
   context: ServerFnBenchContext,
 ) {
   return runRequestLoop(handler, {
-    ...serverFnRequestLoopOptions,
+    seed: benchmarkSeed,
+    iterations: serverFnGetLoopIterations,
     buildRequest: (_random, index) =>
       buildGetRequest(context.urls, context.getQueries, index),
   })
@@ -468,7 +467,8 @@ export function runServerFnPostRequestLoop(
   context: ServerFnBenchContext,
 ) {
   return runRequestLoop(handler, {
-    ...serverFnRequestLoopOptions,
+    seed: benchmarkSeed,
+    iterations: serverFnPostLoopIterations,
     buildRequest: (_random, index) =>
       buildPostRequest(context.urls, context.bodies, index),
   })
@@ -479,7 +479,8 @@ export function runServerFnRedirectRequestLoop(
   context: ServerFnBenchContext,
 ) {
   return runRequestLoop(handler, {
-    ...serverFnRequestLoopOptions,
+    seed: benchmarkSeed,
+    iterations: serverFnRedirectLoopIterations,
     buildRequest: (_random, index) =>
       buildRedirectRequest(context.urls, context.bodies, index),
     validateResponse: validateRedirectResponse,
@@ -491,7 +492,8 @@ export function runServerFnNotFoundRequestLoop(
   context: ServerFnBenchContext,
 ) {
   return runRequestLoop(handler, {
-    ...serverFnRequestLoopOptions,
+    seed: benchmarkSeed,
+    iterations: serverFnNotFoundLoopIterations,
     buildRequest: (_random, index) =>
       buildNotFoundRequest(context.urls, context.bodies, index),
     validateResponse: (response) =>
@@ -504,7 +506,8 @@ export function runServerFnSendContextRequestLoop(
   context: ServerFnBenchContext,
 ) {
   return runRequestLoop(handler, {
-    ...serverFnRequestLoopOptions,
+    seed: benchmarkSeed,
+    iterations: serverFnSendContextLoopIterations,
     buildRequest: (_random, index) =>
       buildContextRequest(context.urls, context.contextBodies, index),
     validateResponse: (response) =>
@@ -516,7 +519,8 @@ export function runServerFnDocumentSsrRequestLoop(
   handler: StartRequestHandler,
 ) {
   return runRequestLoop(handler, {
-    ...serverFnSsrRequestLoopOptions,
+    seed: benchmarkSeed,
+    iterations: serverFnDocumentSsrLoopIterations,
     buildRequest: buildSsrCallRequest,
     validateResponse: validateDocumentResponse,
   })

benchmarks/ssr/scenarios/server-routes-middleware/shared.ts

@@ -4,7 +4,7 @@ import type { StartRequestHandler } from '../../bench-utils'
 export type { StartRequestHandler }
 
 const benchmarkSeed = 0xdecafbad
-const loopIterations = 100
+const loopIterations = 150
 
 const apiRequestInit = {
   method: 'GET',

benchmarks/ssr/scenarios/server-routes/shared.ts

@@ -4,7 +4,7 @@ import type { StartRequestHandler } from '../../bench-utils'
 export type { StartRequestHandler }
 
 const benchmarkSeed = 0xdecafbad
-const loopIterations = 100
+const loopIterations = 200
 
 const apiRequestInit = {
   method: 'GET',