From aef9e39f32c92b97ef307bd4493c0d09d31e94f9 Mon Sep 17 00:00:00 2001 From: Benchance Date: Wed, 5 Aug 2026 23:50:00 +1000 Subject: [PATCH 1/3] fix(server): stop idle private-memory growth from immortal parent spans Long-lived forkParked roots and PortDiscovery poll loops were inheriting Effect ParentSpan from short-lived startup/make spans. Nested hot-path spans (secret reads, idle poll ticks) kept parenting under those immortal spans and drove ~25-30MB/h private growth while idle. Detach ambient ParentSpan in forkParked, untrace idle port polls and secret gets, and drop PortDiscovery.make's wrapping span. Co-authored-by: Cursor --- apps/server/src/auth/ServerSecretStore.ts | 5 +++- apps/server/src/preview/PortScanner.ts | 17 +++++++++--- apps/server/src/serverActivation.test.ts | 32 ++++++++++++++++++++++- apps/server/src/serverActivation.ts | 27 +++++++++++++++++-- apps/server/src/serverRuntimeStartup.ts | 3 +++ 5 files changed, 76 insertions(+), 8 deletions(-) diff --git a/apps/server/src/auth/ServerSecretStore.ts b/apps/server/src/auth/ServerSecretStore.ts index 5e9890c1ea2..a124e549376 100644 --- a/apps/server/src/auth/ServerSecretStore.ts +++ b/apps/server/src/auth/ServerSecretStore.ts @@ -168,6 +168,9 @@ export const make = Effect.gen(function* () { const resolveSecretPath = (name: string) => path.join(serverConfig.secretsDir, `${name}.bin`); + // Hot path: called every few seconds from parked runtime fibers. Spans here + // previously nested under immortal startup ParentSpans and dominated idle + // trace volume; keep reads cheap and untraced. const get: ServerSecretStore["Service"]["get"] = (name) => fileSystem.readFile(resolveSecretPath(name)).pipe( Effect.map((bytes) => Option.some(Uint8Array.from(bytes))), @@ -181,7 +184,7 @@ export const make = Effect.gen(function* () { }), ), ), - Effect.withSpan("ServerSecretStore.get"), + Effect.withTracerEnabled(false), ); const set: ServerSecretStore["Service"]["set"] = (name, value) => { diff --git a/apps/server/src/preview/PortScanner.ts b/apps/server/src/preview/PortScanner.ts index c306fca2b33..98a60a5a10f 100644 --- a/apps/server/src/preview/PortScanner.ts +++ b/apps/server/src/preview/PortScanner.ts @@ -25,6 +25,7 @@ import * as Schedule from "effect/Schedule"; import * as Scope from "effect/Scope"; import * as ProcessRunner from "../processRunner.ts"; +import { withoutAmbientParentSpan } from "../serverActivation.ts"; export class PortDiscovery extends Context.Service< PortDiscovery, @@ -309,9 +310,17 @@ export const make = Effect.gen(function* PortDiscoveryMake() { ); // Single layer-scoped polling fiber. Ticks are no-ops when no client is - // currently retained, so the cost is one Ref.get every POLL_INTERVAL. - yield* Effect.forkScoped(pollTick().pipe(Effect.repeat(Schedule.spaced(POLL_INTERVAL)))); - + // currently retained. Run detached from any ambient ParentSpan (e.g. + // PortDiscovery.make) and without tracing idle no-ops — otherwise every + // 3s tick parents under an immortal span and grows server private memory. + yield* Effect.forkScoped( + withoutAmbientParentSpan( + pollTick().pipe( + Effect.repeat(Schedule.spaced(POLL_INTERVAL)), + Effect.withTracerEnabled(false), + ), + ), + ); const acquireRetention = Effect.fn("PortDiscovery.retain")(function* () { const wasIdle = yield* Ref.modify(stateRef, (state) => [ state.retainCount === 0, @@ -385,6 +394,6 @@ export const make = Effect.gen(function* PortDiscoveryMake() { registerTerminalProcesses, unregisterTerminal, }); -}).pipe(Effect.withSpan("PortDiscovery.make")); +}); export const layer = Layer.effect(PortDiscovery, make); diff --git a/apps/server/src/serverActivation.test.ts b/apps/server/src/serverActivation.test.ts index a4f942a95b5..83c8017838f 100644 --- a/apps/server/src/serverActivation.test.ts +++ b/apps/server/src/serverActivation.test.ts @@ -1,8 +1,11 @@ import { expect, it } from "@effect/vitest"; +import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Tracer from "effect/Tracer"; -import { forkParked, ServerActivation } from "./serverActivation.ts"; +import { forkParked, ServerActivation, withoutAmbientParentSpan } from "./serverActivation.ts"; it.effect("proves a root is parked before returning and releases it with one gate", () => Effect.scoped( @@ -21,3 +24,30 @@ it.effect("proves a root is parked before returning and releases it with one gat }), ), ); + +it.effect("withoutAmbientParentSpan drops inherited ParentSpan for long-lived roots", () => + Effect.gen(function* () { + const ambient = Tracer.externalSpan({ + traceId: "00000000000000000000000000000001", + spanId: "0000000000000001", + sampled: true, + }); + + const sawParent = yield* Effect.serviceOption(Tracer.ParentSpan).pipe( + Effect.map(Option.isSome), + withoutAmbientParentSpan, + Effect.provideService(Tracer.ParentSpan, ambient), + ); + + expect(sawParent).toBe(false); + + const stillHasParent = yield* Effect.serviceOption(Tracer.ParentSpan).pipe( + Effect.map(Option.isSome), + Effect.provideService(Tracer.ParentSpan, ambient), + ); + expect(stillHasParent).toBe(true); + + const stripped = Context.omit(Tracer.ParentSpan)(Context.make(Tracer.ParentSpan, ambient)); + expect(Context.getOption(stripped, Tracer.ParentSpan)._tag).toBe("None"); + }), +); diff --git a/apps/server/src/serverActivation.ts b/apps/server/src/serverActivation.ts index c068d55e7e7..c178b730394 100644 --- a/apps/server/src/serverActivation.ts +++ b/apps/server/src/serverActivation.ts @@ -2,25 +2,48 @@ import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import type * as Scope from "effect/Scope"; +import * as Tracer from "effect/Tracer"; export class ServerActivation extends Context.Reference | undefined>( "t3/serverActivation", { defaultValue: () => undefined }, ) {} +/** + * Drop any ambient `Tracer.ParentSpan` before running long-lived roots. + * + * Startup phases (and other short-lived `Effect.withSpan` scopes) often call + * `forkParked` while a parent span is still current. Forked fibers inherit + * that `ParentSpan` for their entire lifetime, so every nested span + * (`ServerSecretStore.get`, RPC handlers, …) keeps parenting under an + * immortal startup span and pins the `LocalFileSpan` object graph — steady + * private-memory growth while "idle". + */ +export const withoutAmbientParentSpan = ( + effect: Effect.Effect, +): Effect.Effect => + Effect.updateContext( + effect, + (context) => Context.omit(Tracer.ParentSpan)(context) as Context.Context>, + ); + /** Forks a long-running root before commit and proves it is parked at the activation boundary. */ export const forkParked = ( effect: Effect.Effect, ): Effect.Effect => Effect.gen(function* () { const activation = yield* ServerActivation; + const detached = withoutAmbientParentSpan(effect); if (activation === undefined) { - yield* Effect.forkScoped(effect); + yield* Effect.forkScoped(detached); return; } const parked = yield* Deferred.make(); yield* Effect.forkScoped( - Deferred.succeed(parked, undefined).pipe(Effect.andThen(activation), Effect.andThen(effect)), + Deferred.succeed(parked, undefined).pipe( + Effect.andThen(activation), + Effect.andThen(detached), + ), ); yield* Deferred.await(parked); }); diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index 5db2b75556e..598decd7dd7 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -346,6 +346,9 @@ export const make = (options?: StartupOptions) => ); yield* Effect.logDebug("startup phase: parking orchestration roots at activation"); + // `forkParked` inside these starts detaches ambient ParentSpan so the + // short-lived `server.startup.reactors.start` span can end instead of + // staying pinned on long-lived reactor fibers for the process lifetime. yield* runStartupPhase( "reactors.start", Effect.gen(function* () { From abcf580a6d3c0c4451fb6b3b47efc56755c8d80e Mon Sep 17 00:00:00 2001 From: Benchance Date: Wed, 5 Aug 2026 23:51:45 +1000 Subject: [PATCH 2/3] chore(server): trim comments on idle memory-span fix Co-authored-by: Cursor --- apps/server/src/auth/ServerSecretStore.ts | 3 --- apps/server/src/preview/PortScanner.ts | 5 +---- apps/server/src/serverActivation.ts | 11 +---------- apps/server/src/serverRuntimeStartup.ts | 3 --- 4 files changed, 2 insertions(+), 20 deletions(-) diff --git a/apps/server/src/auth/ServerSecretStore.ts b/apps/server/src/auth/ServerSecretStore.ts index a124e549376..1af1ff9336d 100644 --- a/apps/server/src/auth/ServerSecretStore.ts +++ b/apps/server/src/auth/ServerSecretStore.ts @@ -168,9 +168,6 @@ export const make = Effect.gen(function* () { const resolveSecretPath = (name: string) => path.join(serverConfig.secretsDir, `${name}.bin`); - // Hot path: called every few seconds from parked runtime fibers. Spans here - // previously nested under immortal startup ParentSpans and dominated idle - // trace volume; keep reads cheap and untraced. const get: ServerSecretStore["Service"]["get"] = (name) => fileSystem.readFile(resolveSecretPath(name)).pipe( Effect.map((bytes) => Option.some(Uint8Array.from(bytes))), diff --git a/apps/server/src/preview/PortScanner.ts b/apps/server/src/preview/PortScanner.ts index 98a60a5a10f..863351d745e 100644 --- a/apps/server/src/preview/PortScanner.ts +++ b/apps/server/src/preview/PortScanner.ts @@ -309,10 +309,7 @@ export const make = Effect.gen(function* PortDiscoveryMake() { ), ); - // Single layer-scoped polling fiber. Ticks are no-ops when no client is - // currently retained. Run detached from any ambient ParentSpan (e.g. - // PortDiscovery.make) and without tracing idle no-ops — otherwise every - // 3s tick parents under an immortal span and grows server private memory. + // Idle ticks are no-ops; keep them untraced and detached from ambient ParentSpan (#5410). yield* Effect.forkScoped( withoutAmbientParentSpan( pollTick().pipe( diff --git a/apps/server/src/serverActivation.ts b/apps/server/src/serverActivation.ts index c178b730394..0b60f9e82f3 100644 --- a/apps/server/src/serverActivation.ts +++ b/apps/server/src/serverActivation.ts @@ -9,16 +9,7 @@ export class ServerActivation extends Context.Reference | un { defaultValue: () => undefined }, ) {} -/** - * Drop any ambient `Tracer.ParentSpan` before running long-lived roots. - * - * Startup phases (and other short-lived `Effect.withSpan` scopes) often call - * `forkParked` while a parent span is still current. Forked fibers inherit - * that `ParentSpan` for their entire lifetime, so every nested span - * (`ServerSecretStore.get`, RPC handlers, …) keeps parenting under an - * immortal startup span and pins the `LocalFileSpan` object graph — steady - * private-memory growth while "idle". - */ +// Clear inherited ParentSpan so long-lived forks don't pin short-lived startup spans (#5410). export const withoutAmbientParentSpan = ( effect: Effect.Effect, ): Effect.Effect => diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index 598decd7dd7..5db2b75556e 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -346,9 +346,6 @@ export const make = (options?: StartupOptions) => ); yield* Effect.logDebug("startup phase: parking orchestration roots at activation"); - // `forkParked` inside these starts detaches ambient ParentSpan so the - // short-lived `server.startup.reactors.start` span can end instead of - // staying pinned on long-lived reactor fibers for the process lifetime. yield* runStartupPhase( "reactors.start", Effect.gen(function* () { From 17580be3c67ea841cc71506eab702bed9636a057 Mon Sep 17 00:00:00 2001 From: Benchance Date: Thu, 6 Aug 2026 00:05:54 +1000 Subject: [PATCH 3/3] fix(server): address PR review on idle span-leak fix Keep active PortDiscovery polls traced, tighten withoutAmbientParentSpan's R, and use Option.isNone in the test. Co-authored-by: Cursor --- apps/server/src/preview/PortScanner.ts | 19 +++++++++---------- apps/server/src/serverActivation.test.ts | 2 +- apps/server/src/serverActivation.ts | 7 +++++-- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/apps/server/src/preview/PortScanner.ts b/apps/server/src/preview/PortScanner.ts index 863351d745e..0ff053e9171 100644 --- a/apps/server/src/preview/PortScanner.ts +++ b/apps/server/src/preview/PortScanner.ts @@ -293,9 +293,8 @@ export const make = Effect.gen(function* PortDiscoveryMake() { yield* Effect.forEach(listeners, (listener) => listener(servers), { discard: true }); }); - const pollTick = Effect.fn("PortDiscovery.pollTick")( + const pollTickActive = Effect.fn("PortDiscovery.pollTick")( function* () { - if ((yield* Ref.get(stateRef)).retainCount <= 0) return; const next = yield* scanOnce(); const changed = yield* Ref.modify(stateRef, (state) => serversEqual(state.lastSnapshot, next) @@ -309,14 +308,14 @@ export const make = Effect.gen(function* PortDiscoveryMake() { ), ); - // Idle ticks are no-ops; keep them untraced and detached from ambient ParentSpan (#5410). + // Idle early-return stays outside Effect.fn so no-op ticks create no span; detach ParentSpan (#5410). + const pollTick = Effect.gen(function* () { + if ((yield* Ref.get(stateRef)).retainCount <= 0) return; + yield* pollTickActive(); + }); + yield* Effect.forkScoped( - withoutAmbientParentSpan( - pollTick().pipe( - Effect.repeat(Schedule.spaced(POLL_INTERVAL)), - Effect.withTracerEnabled(false), - ), - ), + withoutAmbientParentSpan(pollTick.pipe(Effect.repeat(Schedule.spaced(POLL_INTERVAL)))), ); const acquireRetention = Effect.fn("PortDiscovery.retain")(function* () { const wasIdle = yield* Ref.modify(stateRef, (state) => [ @@ -326,7 +325,7 @@ export const make = Effect.gen(function* PortDiscoveryMake() { if (wasIdle) { // Run an immediate scan + broadcast so the new retainer doesn't have // to wait up to POLL_INTERVAL for the first emission. - yield* pollTick(); + yield* pollTick; } }); diff --git a/apps/server/src/serverActivation.test.ts b/apps/server/src/serverActivation.test.ts index 83c8017838f..f07cfff319c 100644 --- a/apps/server/src/serverActivation.test.ts +++ b/apps/server/src/serverActivation.test.ts @@ -48,6 +48,6 @@ it.effect("withoutAmbientParentSpan drops inherited ParentSpan for long-lived ro expect(stillHasParent).toBe(true); const stripped = Context.omit(Tracer.ParentSpan)(Context.make(Tracer.ParentSpan, ambient)); - expect(Context.getOption(stripped, Tracer.ParentSpan)._tag).toBe("None"); + expect(Option.isNone(Context.getOption(stripped, Tracer.ParentSpan))).toBe(true); }), ); diff --git a/apps/server/src/serverActivation.ts b/apps/server/src/serverActivation.ts index 0b60f9e82f3..edabe2f7714 100644 --- a/apps/server/src/serverActivation.ts +++ b/apps/server/src/serverActivation.ts @@ -12,10 +12,13 @@ export class ServerActivation extends Context.Reference | un // Clear inherited ParentSpan so long-lived forks don't pin short-lived startup spans (#5410). export const withoutAmbientParentSpan = ( effect: Effect.Effect, -): Effect.Effect => +): Effect.Effect> => Effect.updateContext( effect, - (context) => Context.omit(Tracer.ParentSpan)(context) as Context.Context>, + (context) => + Context.omit(Tracer.ParentSpan)(context) as Context.Context< + Exclude, Tracer.ParentSpan> + >, ); /** Forks a long-running root before commit and proves it is parked at the activation boundary. */