fix(benchmarks): run SSR setup imports natively

.github/workflows/client-nav-benchmarks.yml

@@ -47,7 +47,6 @@ jobs:
         uses: TanStack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # main
 
       - name: Run ${{ matrix.benchmark }}:${{ matrix.framework }} CodSpeed benchmark
-        continue-on-error: true
         uses: CodSpeedHQ/action@9d332c4d90b43981c3e55ae8e38e68709996240f # v4.17.0
         with:
           mode: simulation

benchmarks/client-nav/package.json

@@ -3,9 +3,9 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "build:react": "vite build --config ./react/vite.config.ts",
-    "build:solid": "vite build --config ./solid/vite.config.ts",
-    "build:vue": "vite build --config ./vue/vite.config.ts",
+    "build:react": "NODE_ENV=production vite build --config ./react/vite.config.ts",
+    "build:solid": "NODE_ENV=production vite build --config ./solid/vite.config.ts",
+    "build:vue": "NODE_ENV=production vite build --config ./vue/vite.config.ts",
     "test:flame:react": "NODE_ENV=production flame run --md-format=detailed --delay=none --node-options=\"--stack-size=65500\" ./react/speed.flame.ts",
     "test:flame:solid": "NODE_ENV=production flame run --md-format=detailed --delay=none --node-options=\"--stack-size=65500\" ./solid/speed.flame.ts",
     "test:flame:vue": "NODE_ENV=production flame run --md-format=detailed --delay=none --node-options=\"--stack-size=65500\" ./vue/speed.flame.ts",
@@ -30,7 +30,7 @@
   },
   "devDependencies": {
     "@platformatic/flame": "^1.6.0",
-    "@codspeed/vitest-plugin": "^5.0.1",
+    "@codspeed/vitest-plugin": "^5.5.0",
     "@testing-library/react": "^16.2.0",
     "@vitejs/plugin-react": "^6.0.1",
     "@vitejs/plugin-vue": "^6.0.5",

benchmarks/client-nav/react/setup.ts

@@ -3,7 +3,9 @@ import type * as App from './app'
 import { getRequiredLink, waitForRequiredLink } from '../setup-helpers'
 
 const appModulePath = './dist/app.js'
-const { mountTestApp } = (await import(appModulePath)) as typeof App
+const { mountTestApp } = (await import(
+  /* @vite-ignore */ appModulePath
+)) as typeof App
 
 export function setup() {
   if (process.env.NODE_ENV !== 'production') {

benchmarks/client-nav/react/vite.config.ts

@@ -3,6 +3,9 @@ import react from '@vitejs/plugin-react'
 import codspeedPlugin from '@codspeed/vitest-plugin'
 
 export default defineConfig({
+  define: {
+    'process.env.NODE_ENV': JSON.stringify('production'),
+  },
   plugins: [
     !!(process.env.VITEST && process.env.WITH_INSTRUMENTATION) &&
       codspeedPlugin(),

benchmarks/client-nav/solid/setup.ts

@@ -3,7 +3,9 @@ import type * as App from './app'
 import { getRequiredLink, waitForRequiredLink } from '../setup-helpers'
 
 const appModulePath = './dist/app.js'
-const { mountTestApp } = (await import(appModulePath)) as typeof App
+const { mountTestApp } = (await import(
+  /* @vite-ignore */ appModulePath
+)) as typeof App
 
 export function setup() {
   if (process.env.NODE_ENV !== 'production') {

benchmarks/client-nav/solid/vite.config.ts

@@ -3,6 +3,9 @@ import solid from 'vite-plugin-solid'
 import codspeedPlugin from '@codspeed/vitest-plugin'
 
 export default defineConfig({
+  define: {
+    'process.env.NODE_ENV': JSON.stringify('production'),
+  },
   plugins: [
     !!(process.env.VITEST && process.env.WITH_INSTRUMENTATION) &&
       codspeedPlugin(),

benchmarks/client-nav/vue/setup.ts

@@ -3,7 +3,9 @@ import type * as App from './app'
 import { getRequiredLink, waitForRequiredLink } from '../setup-helpers'
 
 const appModulePath = './dist/app.js'
-const { mountTestApp } = (await import(appModulePath)) as typeof App
+const { mountTestApp } = (await import(
+  /* @vite-ignore */ appModulePath
+)) as typeof App
 
 export function setup() {
   if (process.env.NODE_ENV !== 'production') {

benchmarks/ssr/package.json

@@ -3,13 +3,13 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "build:react": "vite build --config ./react/vite.config.ts",
-    "build:solid": "vite build --config ./solid/vite.config.ts",
-    "build:vue": "vite build --config ./vue/vite.config.ts",
-    "test:perf": "vitest bench",
-    "test:perf:react": "vitest bench --config ./react/vite.config.ts ./react/speed.bench.ts",
-    "test:perf:solid": "vitest bench --config ./solid/vite.config.ts ./solid/speed.bench.ts",
-    "test:perf:vue": "vitest bench --config ./vue/vite.config.ts ./vue/speed.bench.ts",
+    "build:react": "NODE_ENV=production vite build --config ./react/vite.config.ts",
+    "build:solid": "NODE_ENV=production vite build --config ./solid/vite.config.ts",
+    "build:vue": "NODE_ENV=production vite build --config ./vue/vite.config.ts",
+    "test:perf": "NODE_ENV=production vitest bench",
+    "test:perf:react": "NODE_ENV=production vitest bench --config ./react/vite.config.ts ./react/speed.bench.ts",
+    "test:perf:solid": "NODE_ENV=production vitest bench --config ./solid/vite.config.ts ./solid/speed.bench.ts",
+    "test:perf:vue": "NODE_ENV=production vitest bench --config ./vue/vite.config.ts ./vue/speed.bench.ts",
     "test:types": "pnpm run test:types:react && pnpm run test:types:solid && pnpm run test:types:vue",
     "test:types:react": "tsc -p ./react/tsconfig.json --noEmit",
     "test:types:solid": "tsc -p ./solid/tsconfig.json --noEmit",
@@ -28,7 +28,7 @@
     "vue": "^3.5.16"
   },
   "devDependencies": {
-    "@codspeed/vitest-plugin": "^5.0.1",
+    "@codspeed/vitest-plugin": "^5.5.0",
     "@vitejs/plugin-react": "^6.0.1",
     "@vitejs/plugin-vue-jsx": "^5.1.5",
     "typescript": "^6.0.2",

benchmarks/ssr/react/speed.bench.ts

@@ -1,49 +1,23 @@
-import { afterAll, beforeAll, bench, describe } from 'vitest'
+import { bench, describe } from 'vitest'
 import { runSsrRequestLoop } from '../bench-utils'
 import type { StartRequestHandler } from '../bench-utils'
 
-const appModulePath = './dist/server/server.js'
+const appModuleUrl = new URL('./dist/server/server.js', import.meta.url).href
 const benchmarkSeed = 0xdecafbad
 
-const uninitializedHandler: StartRequestHandler = {
-  fetch: () => Promise.reject(new Error('Benchmark not initialized')),
-}
-
-let handler = uninitializedHandler
-
-async function setup() {
-  const module = (await import(appModulePath)) as {
-    default: StartRequestHandler
-  }
-
-  handler = module.default
-}
-
-function teardown() {
-  handler = uninitializedHandler
+const { default: handler } = (await import(
+  /* @vite-ignore */ appModuleUrl
+)) as {
+  default: StartRequestHandler
 }
 
 describe('ssr', () => {
-  /**
-   * Running `vitest bench` ignores "suite hooks" like `beforeAll` and `afterAll`,
-   * so we use tinybench's `setup` and `teardown` options to run our setup and teardown logic.
-   *
-   * But CodSpeed calls the benchmarked function directly, bypassing `setup` and `teardown`,
-   * but it does support `beforeAll` and `afterAll`.
-   *
-   * So it looks like we're setting up in duplicate, but in reality, it's only running once per environment, as intended.
-   */
-  beforeAll(setup)
-  afterAll(teardown)
-
   bench(
     'ssr request loop (react)',
     () => runSsrRequestLoop(handler, { seed: benchmarkSeed }),
     {
       warmupIterations: 100,
       time: 10_000,
-      setup,
-      teardown,
       throws: true,
     },
   )

benchmarks/ssr/solid/speed.bench.ts

@@ -1,49 +1,23 @@
-import { afterAll, beforeAll, bench, describe } from 'vitest'
+import { bench, describe } from 'vitest'
 import { runSsrRequestLoop } from '../bench-utils'
 import type { StartRequestHandler } from '../bench-utils'
 
-const appModulePath = './dist/server/server.js'
+const appModuleUrl = new URL('./dist/server/server.js', import.meta.url).href
 const benchmarkSeed = 0xcafebabe
 
-const uninitializedHandler: StartRequestHandler = {
-  fetch: () => Promise.reject(new Error('Benchmark not initialized')),
-}
-
-let handler = uninitializedHandler
-
-async function setup() {
-  const module = (await import(appModulePath)) as {
-    default: StartRequestHandler
-  }
-
-  handler = module.default
-}
-
-function teardown() {
-  handler = uninitializedHandler
+const { default: handler } = (await import(
+  /* @vite-ignore */ appModuleUrl
+)) as {
+  default: StartRequestHandler
 }
 
 describe('ssr', () => {
-  /**
-   * Running `vitest bench` ignores "suite hooks" like `beforeAll` and `afterAll`,
-   * so we use tinybench's `setup` and `teardown` options to run our setup and teardown logic.
-   *
-   * But CodSpeed calls the benchmarked function directly, bypassing `setup` and `teardown`,
-   * but it does support `beforeAll` and `afterAll`.
-   *
-   * So it looks like we're setting up in duplicate, but in reality, it's only running once per environment, as intended.
-   */
-  beforeAll(setup)
-  afterAll(teardown)
-
   bench(
     'ssr request loop (solid)',
     () => runSsrRequestLoop(handler, { seed: benchmarkSeed }),
     {
       warmupIterations: 100,
       time: 10_000,
-      setup,
-      teardown,
       throws: true,
     },
   )

benchmarks/ssr/vue/speed.bench.ts

@@ -1,49 +1,23 @@
-import { afterAll, beforeAll, bench, describe } from 'vitest'
+import { bench, describe } from 'vitest'
 import { runSsrRequestLoop } from '../bench-utils'
 import type { StartRequestHandler } from '../bench-utils'
 
-const appModulePath = './dist/server/server.js'
+const appModuleUrl = new URL('./dist/server/server.js', import.meta.url).href
 const benchmarkSeed = 0xdeadbeef
 
-const uninitializedHandler: StartRequestHandler = {
-  fetch: () => Promise.reject(new Error('Benchmark not initialized')),
-}
-
-let handler = uninitializedHandler
-
-async function setup() {
-  const module = (await import(appModulePath)) as {
-    default: StartRequestHandler
-  }
-
-  handler = module.default
-}
-
-function teardown() {
-  handler = uninitializedHandler
+const { default: handler } = (await import(
+  /* @vite-ignore */ appModuleUrl
+)) as {
+  default: StartRequestHandler
 }
 
 describe('ssr', () => {
-  /**
-   * Running `vitest bench` ignores "suite hooks" like `beforeAll` and `afterAll`,
-   * so we use tinybench's `setup` and `teardown` options to run our setup and teardown logic.
-   *
-   * But CodSpeed calls the benchmarked function directly, bypassing `setup` and `teardown`,
-   * but it does support `beforeAll` and `afterAll`.
-   *
-   * So it looks like we're setting up in duplicate, but in reality, it's only running once per environment, as intended.
-   */
-  beforeAll(setup)
-  afterAll(teardown)
-
   bench(
     'ssr request loop (vue)',
     () => runSsrRequestLoop(handler, { seed: benchmarkSeed }),
     {
       warmupIterations: 100,
       time: 10_000,
-      setup,
-      teardown,
       throws: true,
     },
   )