From 42477f5f8f1a16b1c19818fcc8ed0989c1d03f29 Mon Sep 17 00:00:00 2001 From: T3 Code PR Stack <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:06:39 +0200 Subject: [PATCH] feat(client-runtime): richer reconnect detail and 12h diagnostics log Surface WebSocket close codes and ping timeouts in connection banner text, and retain short-lived disconnect events in localStorage for later inspection. --- ...dEnvironmentConnectionPresentation.test.ts | 5 +- .../src/connection/diagnosticsLog.test.ts | 52 +++++ .../src/connection/diagnosticsLog.ts | 177 ++++++++++++++++++ .../src/connection/disconnectDetail.test.ts | 68 +++++++ .../src/connection/disconnectDetail.ts | 129 +++++++++++++ .../client-runtime/src/connection/index.ts | 16 ++ .../client-runtime/src/connection/layer.ts | 10 +- .../src/connection/presentation.test.ts | 6 +- .../src/connection/presentation.ts | 16 +- .../src/connection/supervisor.test.ts | 2 + .../src/connection/supervisor.ts | 42 ++++- .../client-runtime/src/rpc/session.test.ts | 4 +- packages/client-runtime/src/rpc/session.ts | 87 +++++++-- 13 files changed, 584 insertions(+), 30 deletions(-) create mode 100644 packages/client-runtime/src/connection/diagnosticsLog.test.ts create mode 100644 packages/client-runtime/src/connection/diagnosticsLog.ts create mode 100644 packages/client-runtime/src/connection/disconnectDetail.test.ts create mode 100644 packages/client-runtime/src/connection/disconnectDetail.ts diff --git a/apps/web/src/components/cloud/cloudEnvironmentConnectionPresentation.test.ts b/apps/web/src/components/cloud/cloudEnvironmentConnectionPresentation.test.ts index 654c2462a6f..7fda1a2889d 100644 --- a/apps/web/src/components/cloud/cloudEnvironmentConnectionPresentation.test.ts +++ b/apps/web/src/components/cloud/cloudEnvironmentConnectionPresentation.test.ts @@ -32,14 +32,13 @@ describe("saved cloud environment connection presentation", () => { ), ).toEqual({ buttonLabel: "Reconnecting…", - statusText: - "Failed to connect. Reconnecting... Reason: Relay environment endpoint is unavailable.", + statusText: "Reconnecting… · Relay environment endpoint is unavailable", tone: "connecting", }); }); it.each([ - ["error", "Connection failed", "Connection failed. Reason: Access denied.", "error"], + ["error", "Connection failed", "Connection failed · Access denied", "error"], ["offline", "Offline", "Offline", "idle"], ["available", "Not connected", "Available", "idle"], ] as const)( diff --git a/packages/client-runtime/src/connection/diagnosticsLog.test.ts b/packages/client-runtime/src/connection/diagnosticsLog.test.ts new file mode 100644 index 00000000000..6ce095e769b --- /dev/null +++ b/packages/client-runtime/src/connection/diagnosticsLog.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; + +import { + CONNECTION_DIAGNOSTICS_STORAGE_KEY, + clearConnectionDiagnosticsForTests, + layer, + ConnectionDiagnosticsLog, +} from "./diagnosticsLog.ts"; + +describe("ConnectionDiagnosticsLog", () => { + it.effect("records events and prunes entries older than the retention window", () => + Effect.gen(function* () { + clearConnectionDiagnosticsForTests(); + const log = yield* ConnectionDiagnosticsLog; + const now = yield* DateTime.now; + const staleAt = DateTime.formatIso(DateTime.subtract(now, { hours: 13 })); + const freshAt = DateTime.formatIso(now); + + yield* log.record({ + at: staleAt, + environmentId: "env-old", + label: "old", + kind: "disconnect", + reason: "transport", + detail: "old disconnect", + }); + yield* log.record({ + at: freshAt, + environmentId: "env-new", + label: "t3vm", + kind: "disconnect", + reason: "transport", + detail: "t3vm closed (1006 abnormal).", + closeCode: 1006, + socketHost: "198.18.83.2:3773", + }); + + const events = yield* log.list; + expect(events.map((event) => event.environmentId)).toEqual(["env-new"]); + expect(events[0]?.detail).toContain("1006"); + expect(events[0]?.socketHost).toBe("198.18.83.2:3773"); + + if (typeof globalThis.localStorage !== "undefined") { + const raw = globalThis.localStorage.getItem(CONNECTION_DIAGNOSTICS_STORAGE_KEY); + expect(raw).toContain("env-new"); + expect(raw).not.toContain("env-old"); + } + }).pipe(Effect.provide(layer)), + ); +}); diff --git a/packages/client-runtime/src/connection/diagnosticsLog.ts b/packages/client-runtime/src/connection/diagnosticsLog.ts new file mode 100644 index 00000000000..6de0aa7485e --- /dev/null +++ b/packages/client-runtime/src/connection/diagnosticsLog.ts @@ -0,0 +1,177 @@ +/** + * Short-lived connection diagnostics for post-hoc debugging of reconnect storms. + * + * Events are stored as NDJSON-ish JSON records with a hard 12-hour retention window. + * Default sink uses localStorage when available, otherwise an in-memory ring. + * Always also emits Effect.logWarning so traces/console still see the event. + */ +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +export const CONNECTION_DIAGNOSTICS_RETENTION_MS = 12 * 60 * 60 * 1000; +export const CONNECTION_DIAGNOSTICS_STORAGE_KEY = "t3code:connection-diagnostics:v1"; +const MAX_EVENTS = 400; + +export const ConnectionDiagnosticKind = Schema.Literals([ + "disconnect", + "connect_failed", + "backoff", + "blocked", + "probe_failed", +]); +export type ConnectionDiagnosticKind = typeof ConnectionDiagnosticKind.Type; + +export class ConnectionDiagnosticEvent extends Schema.Class( + "ConnectionDiagnosticEvent", +)({ + at: Schema.String, + environmentId: Schema.String, + label: Schema.String, + kind: ConnectionDiagnosticKind, + reason: Schema.String, + detail: Schema.String, + traceId: Schema.optionalKey(Schema.String), + closeCode: Schema.optionalKey(Schema.Number), + closeReason: Schema.optionalKey(Schema.String), + /** Hostname only — never full socket URLs (tickets). */ + socketHost: Schema.optionalKey(Schema.String), + attempt: Schema.optionalKey(Schema.Number), +}) {} + +export type ConnectionDiagnosticEventInput = { + readonly environmentId: string; + readonly label: string; + readonly kind: ConnectionDiagnosticKind; + readonly reason: string; + readonly detail: string; + readonly traceId?: string | undefined; + readonly closeCode?: number | undefined; + readonly closeReason?: string | undefined; + readonly socketHost?: string | undefined; + readonly attempt?: number | undefined; + readonly at?: string | undefined; +}; + +export class ConnectionDiagnosticsLog extends Context.Service< + ConnectionDiagnosticsLog, + { + readonly record: (event: ConnectionDiagnosticEventInput) => Effect.Effect; + readonly list: Effect.Effect>; + } +>()("@t3tools/client-runtime/connection/diagnosticsLog/ConnectionDiagnosticsLog") {} + +function pruneEvents( + events: ReadonlyArray, + nowMs: number, +): ConnectionDiagnosticEvent[] { + const cutoff = nowMs - CONNECTION_DIAGNOSTICS_RETENTION_MS; + return events + .filter((event) => { + const atMs = Date.parse(event.at); + return Number.isFinite(atMs) && atMs >= cutoff; + }) + .slice(-MAX_EVENTS); +} + +function readStorage(): ConnectionDiagnosticEvent[] { + if (typeof globalThis.localStorage === "undefined") return []; + try { + const raw = globalThis.localStorage.getItem(CONNECTION_DIAGNOSTICS_STORAGE_KEY); + if (!raw) return []; + const parsed = JSON.parse(raw) as unknown; + if (!Array.isArray(parsed)) return []; + return parsed.flatMap((item) => { + try { + return [Schema.decodeSync(ConnectionDiagnosticEvent)(item)]; + } catch { + return []; + } + }); + } catch { + return []; + } +} + +function writeStorage(events: ReadonlyArray): void { + if (typeof globalThis.localStorage === "undefined") return; + try { + globalThis.localStorage.setItem(CONNECTION_DIAGNOSTICS_STORAGE_KEY, JSON.stringify(events)); + } catch { + // Quota / private mode — drop silently; Effect.log still recorded. + } +} + +let memoryEvents: ConnectionDiagnosticEvent[] = []; + +export const make = Effect.sync(() => { + const record = (input: ConnectionDiagnosticEventInput): Effect.Effect => + Effect.gen(function* () { + const nowMs = yield* Clock.currentTimeMillis; + const event = new ConnectionDiagnosticEvent({ + at: input.at ?? DateTime.formatIso(yield* DateTime.now), + environmentId: input.environmentId, + label: input.label, + kind: input.kind, + reason: input.reason, + detail: input.detail, + ...(input.traceId !== undefined ? { traceId: input.traceId } : {}), + ...(input.closeCode !== undefined ? { closeCode: input.closeCode } : {}), + ...(input.closeReason !== undefined ? { closeReason: input.closeReason } : {}), + ...(input.socketHost !== undefined ? { socketHost: input.socketHost } : {}), + ...(input.attempt !== undefined ? { attempt: input.attempt } : {}), + }); + + yield* Effect.logWarning("connection diagnostics", { + kind: event.kind, + environmentId: event.environmentId, + label: event.label, + reason: event.reason, + detail: event.detail, + ...(event.traceId !== undefined ? { traceId: event.traceId } : {}), + ...(event.closeCode !== undefined ? { closeCode: event.closeCode } : {}), + ...(event.socketHost !== undefined ? { socketHost: event.socketHost } : {}), + ...(event.attempt !== undefined ? { attempt: event.attempt } : {}), + }); + + const previous = + typeof globalThis.localStorage !== "undefined" ? readStorage() : memoryEvents; + const next = pruneEvents([...previous, event], nowMs); + if (typeof globalThis.localStorage !== "undefined") { + writeStorage(next); + } else { + memoryEvents = next; + } + }).pipe(Effect.asVoid, Effect.ignore); + + const list = Effect.gen(function* () { + const nowMs = yield* Clock.currentTimeMillis; + const previous = typeof globalThis.localStorage !== "undefined" ? readStorage() : memoryEvents; + const next = pruneEvents(previous, nowMs); + if (typeof globalThis.localStorage !== "undefined") { + writeStorage(next); + } else { + memoryEvents = next; + } + return next; + }); + + return ConnectionDiagnosticsLog.of({ record, list }); +}); + +export const layer = Layer.effect(ConnectionDiagnosticsLog, make); + +/** Test helper: clear in-memory / localStorage diagnostics. */ +export function clearConnectionDiagnosticsForTests(): void { + memoryEvents = []; + if (typeof globalThis.localStorage !== "undefined") { + try { + globalThis.localStorage.removeItem(CONNECTION_DIAGNOSTICS_STORAGE_KEY); + } catch { + // ignore + } + } +} diff --git a/packages/client-runtime/src/connection/disconnectDetail.test.ts b/packages/client-runtime/src/connection/disconnectDetail.test.ts new file mode 100644 index 00000000000..efbda05480d --- /dev/null +++ b/packages/client-runtime/src/connection/disconnectDetail.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { + describeWebSocketCloseCode, + formatDisconnectDetail, + formatDisconnectStatusFragment, +} from "./disconnectDetail.ts"; + +describe("disconnectDetail", () => { + it("maps common close codes", () => { + expect(describeWebSocketCloseCode(1000)).toBe("clean"); + expect(describeWebSocketCloseCode(1006)).toBe("abnormal"); + expect(describeWebSocketCloseCode(1012)).toBe("service restart"); + expect(describeWebSocketCloseCode(42)).toBeNull(); + }); + + it("formats a connected disconnect with close code", () => { + expect( + formatDisconnectDetail({ + label: "t3vm", + wasConnected: true, + close: { code: 1006 }, + }), + ).toBe("t3vm closed (1006 abnormal)."); + }); + + it("includes a short close reason when it adds information", () => { + expect( + formatDisconnectDetail({ + label: "t3vm", + wasConnected: true, + close: { code: 1012, reason: "service restart" }, + }), + ).toBe("t3vm closed (1012 service restart)."); + expect( + formatDisconnectDetail({ + label: "t3vm", + wasConnected: true, + close: { code: 1000, reason: "deploy rolling" }, + }), + ).toBe("t3vm closed (1000 clean: deploy rolling)."); + }); + + it("prefers ping timeout over a bare disconnect", () => { + expect( + formatDisconnectDetail({ + label: "t3vm", + wasConnected: true, + causeMessage: "ping timeout", + }), + ).toBe("t3vm ping timeout."); + }); + + it("formats open failures without claiming a prior session", () => { + expect( + formatDisconnectDetail({ + label: "t3vm", + wasConnected: false, + }), + ).toBe("t3vm could not open WebSocket."); + }); + + it("strips trailing periods for status fragments", () => { + expect(formatDisconnectStatusFragment("t3vm closed (1006 abnormal).")).toBe( + "t3vm closed (1006 abnormal)", + ); + }); +}); diff --git a/packages/client-runtime/src/connection/disconnectDetail.ts b/packages/client-runtime/src/connection/disconnectDetail.ts new file mode 100644 index 00000000000..6da377826b7 --- /dev/null +++ b/packages/client-runtime/src/connection/disconnectDetail.ts @@ -0,0 +1,129 @@ +/** + * Short, user-safe connection failure text for UI + diagnostics. + * Prefer a close code or known cause over the bare "disconnected." message. + */ + +export interface SocketCloseCapture { + readonly code?: number | undefined; + readonly reason?: string | undefined; +} + +export interface FormatDisconnectDetailInput { + readonly label: string; + readonly wasConnected: boolean; + readonly close?: SocketCloseCapture | undefined; + /** Underlying transport message when available (e.g. "ping timeout"). */ + readonly causeMessage?: string | undefined; +} + +const MAX_REASON_CHARS = 48; + +/** Well-known WebSocket close codes we surface by short name. */ +export function describeWebSocketCloseCode(code: number): string | null { + switch (code) { + case 1000: + return "clean"; + case 1001: + return "going away"; + case 1002: + return "protocol error"; + case 1003: + return "unsupported data"; + case 1005: + return "no status"; + case 1006: + return "abnormal"; + case 1007: + return "bad data"; + case 1008: + return "policy violation"; + case 1009: + return "too large"; + case 1011: + return "server error"; + case 1012: + return "service restart"; + case 1013: + return "try again later"; + case 1014: + return "bad gateway"; + case 1015: + return "TLS failed"; + default: + return null; + } +} + +function sanitizeCloseReason(reason: string | undefined): string | null { + if (typeof reason !== "string") return null; + const trimmed = reason.replace(/\s+/g, " ").trim(); + if (trimmed.length === 0) return null; + if (trimmed.length <= MAX_REASON_CHARS) return trimmed; + return `${trimmed.slice(0, MAX_REASON_CHARS - 1)}…`; +} + +function normalizeCauseMessage(message: string | undefined): string | null { + if (typeof message !== "string") return null; + const trimmed = message.replace(/\s+/g, " ").trim(); + if (trimmed.length === 0) return null; + // Drop generic Effect wrappers; keep the useful tail. + const lower = trimmed.toLowerCase(); + if (lower.includes("ping timeout")) return "ping timeout"; + if (lower.includes("socketcloseerror")) { + const match = trimmed.match(/SocketCloseError[:\s]*([^\n]+)/i); + return match?.[1]?.trim() ?? "socket closed"; + } + if (lower.includes("socketopenerror")) return "socket open failed"; + if (lower === "socket is not connected") return "socket not connected"; + return null; +} + +/** + * Full detail string stored on ConnectionTransientError / shown as secondary UI text. + */ +export function formatDisconnectDetail(input: FormatDisconnectDetailInput): string { + const label = input.label.trim() || "Environment"; + const cause = normalizeCauseMessage(input.causeMessage); + const code = input.close?.code; + const codeName = + typeof code === "number" && Number.isFinite(code) ? describeWebSocketCloseCode(code) : null; + const closeReason = sanitizeCloseReason(input.close?.reason); + + if (!input.wasConnected) { + if (cause) return `${label} could not open WebSocket (${cause}).`; + // Our open-timeout path closes with 1000; that is not useful "clean" signal. + const usefulOpenClose = + typeof code === "number" && !(code === 1000 && (closeReason === null || closeReason === "")); + if (usefulOpenClose) { + const bits = [`${code}${codeName ? ` ${codeName}` : ""}`, closeReason].filter(Boolean); + return `${label} could not open WebSocket (${bits.join(": ")}).`; + } + return `${label} could not open WebSocket.`; + } + + if (cause === "ping timeout") { + return `${label} ping timeout.`; + } + + if (typeof code === "number") { + const head = `${code}${codeName ? ` ${codeName}` : ""}`; + if ( + closeReason && + closeReason.toLowerCase() !== (codeName ?? "").toLowerCase() && + !head.toLowerCase().includes(closeReason.toLowerCase()) + ) { + return `${label} closed (${head}: ${closeReason}).`; + } + return `${label} closed (${head}).`; + } + + if (cause) return `${label} disconnected (${cause}).`; + return `${label} disconnected.`; +} + +/** + * Compact fragment for inline status lines (no trailing period). + */ +export function formatDisconnectStatusFragment(detail: string): string { + return detail.replace(/\.$/, "").trim(); +} diff --git a/packages/client-runtime/src/connection/index.ts b/packages/client-runtime/src/connection/index.ts index 53a041bbf30..7c420d2b234 100644 --- a/packages/client-runtime/src/connection/index.ts +++ b/packages/client-runtime/src/connection/index.ts @@ -31,3 +31,19 @@ export { export { ConnectionResolver } from "./resolver.ts"; export { EnvironmentSupervisor, type EnvironmentSupervisorOptions } from "./supervisor.ts"; export * as Wakeups from "./wakeups.ts"; +export { + CONNECTION_DIAGNOSTICS_RETENTION_MS, + CONNECTION_DIAGNOSTICS_STORAGE_KEY, + ConnectionDiagnosticEvent, + ConnectionDiagnosticsLog, + type ConnectionDiagnosticEventInput, + type ConnectionDiagnosticKind, + clearConnectionDiagnosticsForTests, +} from "./diagnosticsLog.ts"; +export { + describeWebSocketCloseCode, + formatDisconnectDetail, + formatDisconnectStatusFragment, + type FormatDisconnectDetailInput, + type SocketCloseCapture, +} from "./disconnectDetail.ts"; diff --git a/packages/client-runtime/src/connection/layer.ts b/packages/client-runtime/src/connection/layer.ts index 798ec01e2f0..476197d5bb7 100644 --- a/packages/client-runtime/src/connection/layer.ts +++ b/packages/client-runtime/src/connection/layer.ts @@ -6,20 +6,25 @@ import * as ConnectionResolver from "./resolver.ts"; import * as ConnectionDriver from "./driver.ts"; import * as EnvironmentRegistry from "./registry.ts"; import * as ConnectionOnboarding from "./onboarding.ts"; +import * as ConnectionDiagnosticsLog from "./diagnosticsLog.ts"; import * as PlatformConnectionSource from "../platform/source.ts"; import * as RelayEnvironmentDiscovery from "../relay/discovery.ts"; import * as RemoteEnvironmentAuthorization from "../authorization/service.ts"; import * as RpcSession from "../rpc/session.ts"; +const diagnosticsLogLayer = ConnectionDiagnosticsLog.layer; + const resolverLayer = ConnectionResolver.layer.pipe( Layer.provide(RemoteEnvironmentAuthorization.layer), ); const driverLayer = ConnectionDriver.layer.pipe( - Layer.provide(Layer.mergeAll(resolverLayer, RpcSession.layer)), + Layer.provide(Layer.mergeAll(resolverLayer, RpcSession.layer, diagnosticsLogLayer)), ); -const registryLayer = EnvironmentRegistry.layer.pipe(Layer.provide(driverLayer)); +const registryLayer = EnvironmentRegistry.layer.pipe( + Layer.provide(Layer.mergeAll(driverLayer, diagnosticsLogLayer)), +); const onboardingLayer = ConnectionOnboarding.layer.pipe(Layer.provide(registryLayer)); @@ -27,6 +32,7 @@ const connectionServicesLayer = Layer.mergeAll( registryLayer, RelayEnvironmentDiscovery.layer, onboardingLayer, + diagnosticsLogLayer, ); const connectionStartupLayer = Layer.effectDiscard( diff --git a/packages/client-runtime/src/connection/presentation.test.ts b/packages/client-runtime/src/connection/presentation.test.ts index e13638a2b41..44bc833eb0d 100644 --- a/packages/client-runtime/src/connection/presentation.test.ts +++ b/packages/client-runtime/src/connection/presentation.test.ts @@ -129,10 +129,8 @@ describe("connection presentation", () => { error: "Relay request timed out.", traceId: "trace-retry", } as const; - expect(connectionStatusText(connection)).toBe( - "Failed to connect. Reconnecting... Reason: Relay request timed out.", - ); - expect(connectionStatusTitle(connection)).toBe("Failed to connect. Reconnecting..."); + expect(connectionStatusText(connection)).toBe("Reconnecting… · Relay request timed out"); + expect(connectionStatusTitle(connection)).toBe("Reconnecting..."); }); it("presents the supervisor's offline state without consulting shell state", () => { diff --git a/packages/client-runtime/src/connection/presentation.ts b/packages/client-runtime/src/connection/presentation.ts index 168443deceb..edbb094b9a6 100644 --- a/packages/client-runtime/src/connection/presentation.ts +++ b/packages/client-runtime/src/connection/presentation.ts @@ -55,6 +55,10 @@ export function presentConnectionState( } } +function compactConnectionError(error: string): string { + return error.replace(/\.$/, "").trim(); +} + export function connectionStatusText(connection: EnvironmentConnectionPresentation): string { switch (connection.phase) { case "available": @@ -64,21 +68,25 @@ export function connectionStatusText(connection: EnvironmentConnectionPresentati case "connecting": return "Connecting..."; case "reconnecting": + // Keep the primary line short; put the useful bit after a middle dot. return connection.error - ? `Failed to connect. Reconnecting... Reason: ${connection.error}` + ? `Reconnecting… · ${compactConnectionError(connection.error)}` : "Reconnecting..."; case "connected": return "Connected"; case "error": return connection.error - ? `Connection failed. Reason: ${connection.error}` + ? `Connection failed · ${compactConnectionError(connection.error)}` : "Connection failed"; } } export function connectionStatusTitle(connection: EnvironmentConnectionPresentation): string { - if (connection.phase === "reconnecting" && connection.error) { - return "Failed to connect. Reconnecting..."; + if (connection.phase === "reconnecting") { + return "Reconnecting..."; + } + if (connection.phase === "error") { + return "Connection failed"; } return connectionStatusText({ ...connection, error: null }); } diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index 9c122e3ebf3..939a02354c3 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -27,6 +27,7 @@ import { type SupervisorConnectionState, } from "./model.ts"; import * as RpcSession from "../rpc/session.ts"; +import * as ConnectionDiagnosticsLog from "./diagnosticsLog.ts"; import * as EnvironmentSupervisor from "./supervisor.ts"; import * as ConnectionWakeups from "./wakeups.ts"; @@ -194,6 +195,7 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?: ConnectionDriver.ConnectionDriver, ConnectionDriver.ConnectionDriver.of({ connect }), ), + ConnectionDiagnosticsLog.layer, ); return { diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index c4d176b87a9..8d7fac9e5c8 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -27,6 +27,7 @@ import { } from "./model.ts"; import * as RpcSession from "../rpc/session.ts"; import { safeErrorLogAttributes } from "../errors/safeLog.ts"; +import * as ConnectionDiagnosticsLog from "./diagnosticsLog.ts"; import * as ConnectionWakeups from "./wakeups.ts"; const RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000] as const; @@ -223,6 +224,28 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( const connectivity = yield* Connectivity.Connectivity; const driver = yield* ConnectionDriver.ConnectionDriver; const wakeups = yield* ConnectionWakeups.ConnectionWakeups; + const diagnosticsLog = yield* Effect.serviceOption( + ConnectionDiagnosticsLog.ConnectionDiagnosticsLog, + ); + + const recordDiagnostic = (input: { + readonly kind: ConnectionDiagnosticsLog.ConnectionDiagnosticKind; + readonly error: ConnectionAttemptError; + readonly attempt: number; + }) => + Option.match(diagnosticsLog, { + onNone: () => Effect.void, + onSome: (log) => + log.record({ + environmentId: target.environmentId, + label: target.label, + kind: input.kind, + reason: input.error.reason, + detail: input.error.detail, + traceId: input.error.traceId, + attempt: input.attempt, + }), + }); const initialIntent: SupervisorIntent = { desired: options?.initiallyDesired ?? false, network: yield* connectivity.status, @@ -638,9 +661,21 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( } const attemptSpan: Option.Option = outcome.failure.attemptSpan; - const error: ConnectionAttemptError = outcome.failure.error; + let error: ConnectionAttemptError = outcome.failure.error; + // Attach the environment label to short transport messages from the RPC layer. + if ( + error._tag === "ConnectionTransientError" && + (error.detail === "ping timeout" || error.detail === "ping timeout.") + ) { + error = new ConnectionTransientError({ + reason: error.reason, + detail: `${target.label} ping timeout.`, + ...(error.traceId !== undefined ? { traceId: error.traceId } : {}), + }); + } latestFailure = error; if (error._tag === "ConnectionBlockedError") { + yield* recordDiagnostic({ kind: "blocked", error, attempt }); const blockedIntent = yield* Ref.get(intent); yield* setState({ desired: blockedIntent.desired, @@ -664,6 +699,11 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( delayMs, reason: error.reason, })); + yield* recordDiagnostic({ + kind: outcome.established ? "disconnect" : "connect_failed", + error, + attempt, + }); const failedIntent = yield* Ref.get(intent); yield* setState({ desired: failedIntent.desired, diff --git a/packages/client-runtime/src/rpc/session.test.ts b/packages/client-runtime/src/rpc/session.test.ts index 71649ad94dd..51dac902e29 100644 --- a/packages/client-runtime/src/rpc/session.test.ts +++ b/packages/client-runtime/src/rpc/session.test.ts @@ -261,7 +261,7 @@ describe("RpcSessionFactory", () => { expect(error).toBeInstanceOf(ConnectionTransientError); expect(error).toMatchObject({ reason: "transport", - message: "Test environment disconnected.", + message: "Test environment closed (1012 service restart).", }); yield* Effect.yieldNow; expect(sockets).toHaveLength(1); @@ -344,7 +344,7 @@ describe("RpcSessionFactory", () => { expect(error).toBeInstanceOf(ConnectionTransientError); expect(error).toMatchObject({ reason: "transport", - message: "Test environment could not establish a WebSocket connection.", + message: "Test environment could not open WebSocket.", }); expect(sockets[0]?.readyState).toBe(TestWebSocket.CLOSED); }).pipe(Effect.provide(TestClock.layer())), diff --git a/packages/client-runtime/src/rpc/session.ts b/packages/client-runtime/src/rpc/session.ts index 9625effa406..71dbea0f9d7 100644 --- a/packages/client-runtime/src/rpc/session.ts +++ b/packages/client-runtime/src/rpc/session.ts @@ -3,6 +3,7 @@ import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schedule from "effect/Schedule"; import type * as Scope from "effect/Scope"; import * as RpcClient from "effect/unstable/rpc/RpcClient"; @@ -19,9 +20,40 @@ import { ConnectionBlockedError, ConnectionTransientError as ConnectionTransientErrorClass, } from "../connection/model.ts"; +import { formatDisconnectDetail, type SocketCloseCapture } from "../connection/disconnectDetail.ts"; +import * as ConnectionDiagnosticsLog from "../connection/diagnosticsLog.ts"; const SOCKET_OPEN_TIMEOUT = "15 seconds"; +function socketHostFromUrl(socketUrl: string): string | undefined { + try { + return new URL(socketUrl).host; + } catch { + return undefined; + } +} + +function captureSocketClose( + webSocketConstructor: (url: string, protocols?: string | string[]) => globalThis.WebSocket, + sink: { current: SocketCloseCapture }, +): (url: string, protocols?: string | string[]) => globalThis.WebSocket { + return (url, protocols) => { + const socket = webSocketConstructor(url, protocols); + socket.addEventListener( + "close", + (event) => { + const closeEvent = event as CloseEvent; + sink.current = { + code: typeof closeEvent.code === "number" ? closeEvent.code : undefined, + reason: typeof closeEvent.reason === "string" ? closeEvent.reason : undefined, + }; + }, + { once: true }, + ); + return socket; + }; +} + export interface RpcSession { readonly client: WsRpcProtocolClient; readonly initialConfig: Effect.Effect; @@ -57,16 +89,27 @@ function mapSessionRpcError(error: InitialConfigError | ProbeError): ConnectionA reason: "remote-unavailable", detail: error.message, }); - case "RpcClientError": + case "RpcClientError": { + const lower = error.message.toLowerCase(); + if (lower.includes("ping timeout")) { + return new ConnectionTransientErrorClass({ + reason: "timeout", + detail: "ping timeout", + }); + } return new ConnectionTransientErrorClass({ reason: "transport", detail: error.message, }); + } } } export const make = Effect.gen(function* () { const webSocketConstructor = yield* Socket.WebSocketConstructor; + const diagnosticsLog = yield* Effect.serviceOption( + ConnectionDiagnosticsLog.ConnectionDiagnosticsLog, + ); const connect = Effect.fnUntraced(function* (connection: PreparedConnection) { yield* Effect.annotateCurrentSpan({ @@ -75,26 +118,42 @@ export const make = Effect.gen(function* () { const connected = yield* Deferred.make(); const disconnected = yield* Deferred.make(); + const closeCapture: { current: SocketCloseCapture } = { current: {} }; + const trackedConstructor = captureSocketClose(webSocketConstructor, closeCapture); const hooks = RpcClient.ConnectionHooks.of({ onConnect: Deferred.succeed(connected, undefined).pipe(Effect.asVoid), onDisconnect: Deferred.isDone(connected).pipe( - Effect.flatMap((wasConnected) => - Deferred.fail( - disconnected, - new ConnectionTransientErrorClass({ - reason: "transport", - detail: wasConnected - ? `${connection.label} disconnected.` - : `${connection.label} could not establish a WebSocket connection.`, - }), - ), - ), - Effect.asVoid, + Effect.flatMap((wasConnected) => { + const detail = formatDisconnectDetail({ + label: connection.label, + wasConnected, + close: closeCapture.current, + }); + const error = new ConnectionTransientErrorClass({ + reason: "transport", + detail, + }); + const record = Option.match(diagnosticsLog, { + onNone: () => Effect.void, + onSome: (log) => + log.record({ + environmentId: connection.environmentId, + label: connection.label, + kind: wasConnected ? "disconnect" : "connect_failed", + reason: error.reason, + detail: error.detail, + closeCode: closeCapture.current.code, + closeReason: closeCapture.current.reason, + socketHost: socketHostFromUrl(connection.socketUrl), + }), + }); + return record.pipe(Effect.andThen(Deferred.fail(disconnected, error)), Effect.asVoid); + }), ), }); const socketLayer = Socket.layerWebSocket(connection.socketUrl, { openTimeout: SOCKET_OPEN_TIMEOUT, - }).pipe(Layer.provide(Layer.succeed(Socket.WebSocketConstructor, webSocketConstructor))); + }).pipe(Layer.provide(Layer.succeed(Socket.WebSocketConstructor, trackedConstructor))); const protocolLayer = Layer.effect( RpcClient.Protocol, RpcClient.makeProtocolSocket({