From b9625b498ae5e6a0acb401f8ae8ddb327be8f101 Mon Sep 17 00:00:00 2001 From: Rob von Behren Date: Thu, 11 Jun 2026 16:27:25 -0700 Subject: [PATCH 1/2] feat: add HTTP/2 load testing infrastructure Adds a loadtest/ directory with scripts for benchmarking HTTP/2 transport throughput against the Runloop API. Includes baselines for raw node:http2, undici, and the SDK client at various concurrency levels. Co-Authored-By: Claude Opus 4.6 --- loadtest/.gitignore | 2 + loadtest/alpn-check.ts | 24 +++++ loadtest/h2-single-conn.ts | 61 +++++++++++++ loadtest/h2-test.ts | 112 +++++++++++++++++++++++ loadtest/loadtest.ts | 159 +++++++++++++++++++++++++++++++++ loadtest/package.json | 16 ++++ loadtest/raw-fetch-test.ts | 84 +++++++++++++++++ loadtest/undici-debug.ts | 72 +++++++++++++++ loadtest/undici-single-conn.ts | 107 ++++++++++++++++++++++ loadtest/undici-test.ts | 93 +++++++++++++++++++ 10 files changed, 730 insertions(+) create mode 100644 loadtest/.gitignore create mode 100644 loadtest/alpn-check.ts create mode 100644 loadtest/h2-single-conn.ts create mode 100644 loadtest/h2-test.ts create mode 100644 loadtest/loadtest.ts create mode 100644 loadtest/package.json create mode 100644 loadtest/raw-fetch-test.ts create mode 100644 loadtest/undici-debug.ts create mode 100644 loadtest/undici-single-conn.ts create mode 100644 loadtest/undici-test.ts diff --git a/loadtest/.gitignore b/loadtest/.gitignore new file mode 100644 index 000000000..504afef81 --- /dev/null +++ b/loadtest/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/loadtest/alpn-check.ts b/loadtest/alpn-check.ts new file mode 100644 index 000000000..9a59813c1 --- /dev/null +++ b/loadtest/alpn-check.ts @@ -0,0 +1,24 @@ +import tls from "node:tls"; + +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const url = new URL(BASE_URL); + +console.log(`Checking ALPN for ${url.hostname}:${url.port || 443}`); + +const socket = tls.connect( + { + host: url.hostname, + port: parseInt(url.port || "443", 10), + ALPNProtocols: ["h2", "http/1.1"], + servername: url.hostname, + }, + () => { + console.log(`Negotiated protocol: ${socket.alpnProtocol}`); + console.log(`TLS version: ${socket.getProtocol()}`); + socket.end(); + }, +); + +socket.on("error", (err) => { + console.error("TLS error:", err.message); +}); diff --git a/loadtest/h2-single-conn.ts b/loadtest/h2-single-conn.ts new file mode 100644 index 000000000..7ceb78ed1 --- /dev/null +++ b/loadtest/h2-single-conn.ts @@ -0,0 +1,61 @@ +import http2 from "node:http2"; + +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const body = JSON.stringify({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: "loadtest-h2s-0", + environment_variables: { TEST_VAR_1: "value_one" }, + launch_parameters: { resource_size_request: "SMALL" }, +}); + +function sendRequest(client: http2.ClientHttp2Session): Promise<{ latencyMs: number; status: number }> { + return new Promise((resolve, reject) => { + const start = performance.now(); + const req = client.request({ + ":method": "POST", + ":path": "/v1/devboxes", + "content-type": "application/json", + authorization: `Bearer ${API_KEY}`, + }); + req.on("response", (headers) => { + const status = headers[":status"] as number; + req.on("data", () => {}); + req.on("end", () => resolve({ latencyMs: performance.now() - start, status })); + }); + req.on("error", reject); + req.end(body); + }); +} + +async function main() { + const url = new URL(BASE_URL); + const client = http2.connect(url.origin); + await new Promise((resolve, reject) => { + client.on("connect", resolve); + client.on("error", reject); + }); + + const maxStreams = client.remoteSettings?.maxConcurrentStreams; + console.log(`Server MAX_CONCURRENT_STREAMS: ${maxStreams}`); + + // Warmup + const w = await sendRequest(client); + console.log(`Warmup: status=${w.status}, latency=${w.latencyMs.toFixed(0)}ms`); + + // Burst 50 requests on single warmed connection + const count = 50; + console.log(`\nBursting ${count} requests on 1 warmed connection...`); + const wallStart = performance.now(); + const results = await Promise.all(Array.from({ length: count }, () => sendRequest(client))); + const wallMs = performance.now() - wallStart; + + const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); + console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`); + console.log(`Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`); + + client.close(); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/loadtest/h2-test.ts b/loadtest/h2-test.ts new file mode 100644 index 000000000..7b31b5db4 --- /dev/null +++ b/loadtest/h2-test.ts @@ -0,0 +1,112 @@ +import http2 from "node:http2"; + +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "10000", 10); +const NUM_CONNECTIONS = parseInt(process.env.NUM_CONNECTIONS ?? "10", 10); +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const body = JSON.stringify({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: "loadtest-h2-0", + environment_variables: { TEST_VAR_1: "value_one", TEST_VAR_2: "value_two" }, + metadata: { test_run: "h2", index: "0" }, + launch_parameters: { resource_size_request: "SMALL", keep_alive_time_seconds: 300 }, +}); + +function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +function connectH2(origin: string): Promise { + return new Promise((resolve, reject) => { + const client = http2.connect(origin); + client.on("connect", () => resolve(client)); + client.on("error", reject); + }); +} + +function sendRequest( + client: http2.ClientHttp2Session, +): Promise<{ latencyMs: number; status: number }> { + return new Promise((resolve, reject) => { + const start = performance.now(); + const req = client.request({ + ":method": "POST", + ":path": "/v1/devboxes", + "content-type": "application/json", + authorization: `Bearer ${API_KEY}`, + }); + req.on("response", (headers) => { + const status = headers[":status"] as number; + req.on("data", () => {}); + req.on("end", () => resolve({ latencyMs: performance.now() - start, status })); + }); + req.on("error", reject); + req.end(body); + }); +} + +async function main() { + console.log( + `HTTP/2 test: ${REQUEST_COUNT} requests, ${NUM_CONNECTIONS} connections to ${BASE_URL}`, + ); + + const url = new URL(BASE_URL); + const clients = await Promise.all( + Array.from({ length: NUM_CONNECTIONS }, () => connectH2(url.origin)), + ); + + const maxStreams = clients[0].remoteSettings?.maxConcurrentStreams; + console.log(`Server MAX_CONCURRENT_STREAMS: ${maxStreams ?? "unknown"}`); + console.log(`${NUM_CONNECTIONS} connections established\n`); + + let completed = 0; + const progressTimer = setInterval(() => { + console.log( + ` progress: ${completed}/${REQUEST_COUNT} (${((completed / REQUEST_COUNT) * 100).toFixed(1)}%)`, + ); + }, 2000); + + const wallStart = performance.now(); + + const promises = Array.from({ length: REQUEST_COUNT }, (_, i) => { + const client = clients[i % NUM_CONNECTIONS]; + return sendRequest(client).then((r) => { + completed++; + return r; + }); + }); + + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + clearInterval(progressTimer); + + for (const c of clients) c.close(); + + const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); + const statusCounts = new Map(); + for (const r of results) { + statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1); + } + + console.log(`\n=== HTTP/2 Results ===`); + console.log(`Requests: ${REQUEST_COUNT}`); + console.log(`Connections: ${NUM_CONNECTIONS}`); + console.log(`Wall clock: ${(wallMs / 1000).toFixed(2)}s`); + console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`); + console.log(`\nLatency (ms):`); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + console.log(`\nStatus codes:`); + for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/loadtest.ts b/loadtest/loadtest.ts new file mode 100644 index 000000000..1c42e970d --- /dev/null +++ b/loadtest/loadtest.ts @@ -0,0 +1,159 @@ +import { Runloop } from "@runloop/api-client"; + +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "100000", 10); +const RUNLOOP_BASE_URL = process.env.RUNLOOP_BASE_URL; +const USE_HTTP2 = process.env.USE_HTTP2 === "1"; +const PROGRESS_INTERVAL_MS = 2000; + +function buildClient(): Runloop { + const opts: ConstructorParameters[0] = { + maxRetries: 0, + timeout: 120_000, + }; + if (RUNLOOP_BASE_URL) { + opts.baseURL = RUNLOOP_BASE_URL; + } + if (USE_HTTP2) { + (opts as any).http2 = true; + } + return new Runloop(opts); +} + +interface RequestResult { + index: number; + latencyMs: number; + status: number | null; + error: string | null; +} + +async function sendRequest( + client: Runloop, + index: number, + runId: string, +): Promise { + const start = performance.now(); + try { + await client.devboxes.create({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: `loadtest-${runId}-${index}`, + environment_variables: { + TEST_VAR_1: "value_one", + TEST_VAR_2: "value_two", + }, + metadata: { + test_run: runId, + index: String(index), + }, + launch_parameters: { + resource_size_request: "SMALL", + keep_alive_time_seconds: 300, + }, + }); + return { + index, + latencyMs: performance.now() - start, + status: 200, + error: null, + }; + } catch (err: any) { + return { + index, + latencyMs: performance.now() - start, + status: err?.status ?? null, + error: err?.message ?? String(err), + }; + } +} + +function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +function printMetrics(results: RequestResult[], wallClockMs: number): void { + const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); + + const statusCounts = new Map(); + for (const r of results) { + const key = r.status != null ? String(r.status) : "network_error"; + statusCounts.set(key, (statusCounts.get(key) ?? 0) + 1); + } + + console.log("\n=== Load Test Results ==="); + console.log(`Requests: ${results.length}`); + console.log(`Wall clock: ${(wallClockMs / 1000).toFixed(2)}s`); + console.log( + `Throughput: ${(results.length / (wallClockMs / 1000)).toFixed(1)} req/s`, + ); + console.log(""); + console.log("Latency (ms):"); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + console.log(""); + console.log("Status codes:"); + for (const [status, count] of [...statusCounts.entries()].sort()) { + console.log(` ${status}: ${count}`); + } +} + +async function main(): Promise { + const fdLimit = await checkFileDescriptorLimit(); + if (!USE_HTTP2 && fdLimit < 10000) { + console.warn( + `\nWARNING: File descriptor limit is ${fdLimit}. For 100k HTTP/1.1 requests, run:\n` + + ` ulimit -n 65536\n` + + `Or use HTTP/2 multiplexing: USE_HTTP2=1\n`, + ); + } + + const client = buildClient(); + const runId = `run-${Date.now()}`; + + console.log(`Starting load test: ${REQUEST_COUNT} concurrent requests`); + console.log(`Run ID: ${runId}`); + console.log(`HTTP mode: ${USE_HTTP2 ? "HTTP/2 (undici)" : "HTTP/1.1 (node-fetch)"}`); + console.log(`Base URL: ${RUNLOOP_BASE_URL ?? "(SDK default)"}`); + console.log(`File descriptor limit: ${fdLimit}`); + console.log(""); + + let completed = 0; + const progressTimer = setInterval(() => { + console.log( + ` progress: ${completed}/${REQUEST_COUNT} (${((completed / REQUEST_COUNT) * 100).toFixed(1)}%)`, + ); + }, PROGRESS_INTERVAL_MS); + + const wallStart = performance.now(); + + const promises = Array.from({ length: REQUEST_COUNT }, (_, i) => + sendRequest(client, i, runId).then((result) => { + completed++; + return result; + }), + ); + + const results = await Promise.all(promises); + + const wallClockMs = performance.now() - wallStart; + clearInterval(progressTimer); + + printMetrics(results, wallClockMs); +} + +async function checkFileDescriptorLimit(): Promise { + try { + const { execSync } = await import("child_process"); + return parseInt(execSync("ulimit -n", { encoding: "utf-8" }).trim(), 10); + } catch { + return -1; + } +} + +main().catch((err) => { + console.error("Fatal error:", err); + process.exit(1); +}); diff --git a/loadtest/package.json b/loadtest/package.json new file mode 100644 index 000000000..91f0a86bd --- /dev/null +++ b/loadtest/package.json @@ -0,0 +1,16 @@ +{ + "name": "runloop-loadtest", + "private": true, + "type": "module", + "scripts": { + "test": "tsx loadtest.ts" + }, + "dependencies": { + "@runloop/api-client": "file:..", + "tsx": "^4.0.0", + "undici": "^7.26.0" + }, + "devDependencies": { + "@types/node": "^25.9.3" + } +} diff --git a/loadtest/raw-fetch-test.ts b/loadtest/raw-fetch-test.ts new file mode 100644 index 000000000..ebcaf2a93 --- /dev/null +++ b/loadtest/raw-fetch-test.ts @@ -0,0 +1,84 @@ +import https from "node:https"; + +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "10000", 10); +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const agent = new https.Agent({ + keepAlive: true, + maxSockets: Infinity, + maxFreeSockets: 4096, +}); + +const body = JSON.stringify({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: "loadtest-raw-0", + environment_variables: { TEST_VAR_1: "value_one", TEST_VAR_2: "value_two" }, + metadata: { test_run: "raw", index: "0" }, + launch_parameters: { resource_size_request: "SMALL", keep_alive_time_seconds: 300 }, +}); + +function makeRequest(index: number): Promise<{ latencyMs: number; status: number }> { + const start = performance.now(); + return new Promise((resolve, reject) => { + const url = new URL("/v1/devboxes", BASE_URL); + const req = https.request( + url, + { + method: "POST", + agent, + headers: { + "content-type": "application/json", + authorization: `Bearer ${API_KEY}`, + "content-length": Buffer.byteLength(body).toString(), + }, + }, + (res) => { + res.resume(); + res.on("end", () => + resolve({ latencyMs: performance.now() - start, status: res.statusCode! }), + ); + }, + ); + req.on("error", reject); + req.end(body); + }); +} + +function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +async function main() { + console.log(`Raw node:https test: ${REQUEST_COUNT} concurrent requests to ${BASE_URL}`); + + const wallStart = performance.now(); + const results = await Promise.all( + Array.from({ length: REQUEST_COUNT }, (_, i) => makeRequest(i)), + ); + const wallMs = performance.now() - wallStart; + + const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); + const statusCounts = new Map(); + for (const r of results) { + statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1); + } + + console.log(`\nWall clock: ${(wallMs / 1000).toFixed(2)}s`); + console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`); + console.log(`\nLatency (ms):`); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + console.log(`\nStatus codes:`); + for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/undici-debug.ts b/loadtest/undici-debug.ts new file mode 100644 index 000000000..ab5286855 --- /dev/null +++ b/loadtest/undici-debug.ts @@ -0,0 +1,72 @@ +import { Agent, fetch } from "undici"; + +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const body = JSON.stringify({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: "loadtest-debug-0", + environment_variables: { TEST_VAR_1: "value_one" }, + launch_parameters: { resource_size_request: "SMALL" }, +}); + +async function main() { + console.log("=== undici HTTP/2 debug ===\n"); + + const dispatcher = new Agent({ + allowH2: true, + connections: 2, + pipelining: 10, + keepAliveTimeout: 600_000, + bodyTimeout: 0, + headersTimeout: 0, + }); + + // Single request to inspect protocol + const res = await fetch(`${BASE_URL}/v1/devboxes`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${API_KEY}`, + }, + body, + dispatcher, + } as any); + + console.log("Status:", res.status); + console.log("HTTP version:", res.headers.get("x-http-version") ?? "(not reported)"); + console.log("\nResponse headers:"); + for (const [k, v] of res.headers) { + console.log(` ${k}: ${v}`); + } + await res.text(); + + // Quick 20-request burst to verify concurrency + console.log("\n--- 20-request burst (2 connections, pipelining=10) ---"); + const wallStart = performance.now(); + const promises = Array.from({ length: 20 }, async () => { + const start = performance.now(); + const r = await fetch(`${BASE_URL}/v1/devboxes`, { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + body, + dispatcher, + } as any); + await r.text(); + return { latencyMs: performance.now() - start, status: r.status }; + }); + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + + console.log(`Wall clock: ${wallMs.toFixed(0)}ms`); + console.log(`Throughput: ${(20 / (wallMs / 1000)).toFixed(1)} req/s`); + const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); + console.log(`Latency: min=${lats[0].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`); + + await dispatcher.close(); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/undici-single-conn.ts b/loadtest/undici-single-conn.ts new file mode 100644 index 000000000..434e7a1d7 --- /dev/null +++ b/loadtest/undici-single-conn.ts @@ -0,0 +1,107 @@ +import { Client, Agent, fetch } from "undici"; + +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const API_KEY = process.env.RUNLOOP_API_KEY!; +const PIPELINING = parseInt(process.env.PIPELINING ?? "100", 10); + +const body = JSON.stringify({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: "loadtest-single-0", + environment_variables: { TEST_VAR_1: "value_one" }, + launch_parameters: { resource_size_request: "SMALL" }, +}); + +async function testClient() { + console.log(`\n--- undici.Client (single connection, pipelining=${PIPELINING}) ---`); + const url = new URL(BASE_URL); + const client = new Client(url.origin, { + allowH2: true, + pipelining: PIPELINING, + bodyTimeout: 0, + headersTimeout: 0, + }); + + // Warm up - single request to establish connection + const warmup = await client.request({ + method: "POST", + path: "/v1/devboxes", + headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + body, + }); + await warmup.body.text(); + console.log(`Warmup: status=${warmup.statusCode}`); + + // Burst 50 requests on single connection + const count = 50; + const wallStart = performance.now(); + const promises = Array.from({ length: count }, async () => { + const start = performance.now(); + const res = await client.request({ + method: "POST", + path: "/v1/devboxes", + headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + body, + }); + await res.body.text(); + return { latencyMs: performance.now() - start, status: res.statusCode }; + }); + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + + const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); + console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`); + console.log(`Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`); + + await client.close(); +} + +async function testAgent() { + console.log(`\n--- undici.Agent (connections=1, pipelining=${PIPELINING}) ---`); + const dispatcher = new Agent({ + allowH2: true, + connections: 1, + pipelining: PIPELINING, + bodyTimeout: 0, + headersTimeout: 0, + }); + + // Warm up + const warmup = await fetch(`${BASE_URL}/v1/devboxes`, { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + body, + dispatcher, + } as any); + await warmup.text(); + console.log(`Warmup: status=${warmup.status}`); + + // Burst 50 requests + const count = 50; + const wallStart = performance.now(); + const promises = Array.from({ length: count }, async () => { + const start = performance.now(); + const r = await fetch(`${BASE_URL}/v1/devboxes`, { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + body, + dispatcher, + } as any); + await r.text(); + return { latencyMs: performance.now() - start, status: r.status }; + }); + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + + const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); + console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`); + console.log(`Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`); + + await dispatcher.close(); +} + +async function main() { + await testClient(); + await testAgent(); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/loadtest/undici-test.ts b/loadtest/undici-test.ts new file mode 100644 index 000000000..0a616e198 --- /dev/null +++ b/loadtest/undici-test.ts @@ -0,0 +1,93 @@ +import { Agent, fetch } from "undici"; + +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "10000", 10); +const NUM_CONNECTIONS = parseInt(process.env.NUM_CONNECTIONS ?? "20", 10); +const PIPELINING = parseInt(process.env.PIPELINING ?? "128", 10); +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const body = JSON.stringify({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: "loadtest-undici-0", + environment_variables: { TEST_VAR_1: "value_one", TEST_VAR_2: "value_two" }, + metadata: { test_run: "undici", index: "0" }, + launch_parameters: { resource_size_request: "SMALL", keep_alive_time_seconds: 300 }, +}); + +function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +async function main() { + const dispatcher = new Agent({ + allowH2: true, + connections: NUM_CONNECTIONS, + pipelining: PIPELINING, + keepAliveTimeout: 600_000, + keepAliveMaxTimeout: 600_000, + bodyTimeout: 0, + headersTimeout: 0, + }); + + console.log( + `undici direct test: ${REQUEST_COUNT} requests, ${NUM_CONNECTIONS} connections, pipelining=${PIPELINING}`, + ); + console.log(`Target: ${BASE_URL}`); + + let completed = 0; + const progressTimer = setInterval(() => { + console.log( + ` progress: ${completed}/${REQUEST_COUNT} (${((completed / REQUEST_COUNT) * 100).toFixed(1)}%)`, + ); + }, 2000); + + const wallStart = performance.now(); + + const promises = Array.from({ length: REQUEST_COUNT }, async () => { + const start = performance.now(); + const res = await fetch(`${BASE_URL}/v1/devboxes`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${API_KEY}`, + }, + body, + dispatcher, + } as any); + await res.text(); + completed++; + return { latencyMs: performance.now() - start, status: res.status }; + }); + + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + clearInterval(progressTimer); + + await dispatcher.close(); + + const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); + const statusCounts = new Map(); + for (const r of results) { + statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1); + } + + console.log(`\n=== undici Direct Results ===`); + console.log(`Requests: ${REQUEST_COUNT}`); + console.log(`Wall clock: ${(wallMs / 1000).toFixed(2)}s`); + console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`); + console.log(`\nLatency (ms):`); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + console.log(`\nStatus codes:`); + for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); From 7c4f6288f40a677720ca29f1c0f79369d1182572 Mon Sep 17 00:00:00 2001 From: Rob von Behren Date: Thu, 11 Jun 2026 17:00:42 -0700 Subject: [PATCH 2/2] fix: handle response-level errors in raw-fetch loadtest Add res.on('error', reject) so the promise settles if the server aborts mid-response, instead of hanging indefinitely. Co-Authored-By: Claude Opus 4.6 --- loadtest/raw-fetch-test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/loadtest/raw-fetch-test.ts b/loadtest/raw-fetch-test.ts index ebcaf2a93..4a2e78adf 100644 --- a/loadtest/raw-fetch-test.ts +++ b/loadtest/raw-fetch-test.ts @@ -38,6 +38,7 @@ function makeRequest(index: number): Promise<{ latencyMs: number; status: number res.on("end", () => resolve({ latencyMs: performance.now() - start, status: res.statusCode! }), ); + res.on("error", reject); }, ); req.on("error", reject);