From 7941caae72cb7cd70bc67248754b4a3ff5691be0 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 25 Jul 2026 10:12:16 -0700 Subject: [PATCH 1/4] feat(connect): release the Cloudflare tunnel when the environment shuts down MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloudflare bills per provisioned tunnel, so an environment that goes offline should not leave its tunnel idling. Per Cloudflare's guidance, delete the tunnel when the server stops — while keeping reconnects on the exact same URL. Relay: a new ManagedEndpointProvider.release deletes only the Cloudflare tunnel (404-tolerant) and keeps the allocation row and DNS record. The hostname is a pure function of (stage, userId, environmentId) and the tunnel-name reservation persists, so the next provision recreates the tunnel under the same name and repoints the CNAME — same endpoint URL. Exposed as DELETE /v1/client/environment-links/:environmentId/tunnel. Server: a shutdown finalizer stops the connector, calls the release endpoint best-effort (10s timeout), and drops the cached runtime config since the connector token dies with the tunnel. If the release request fails, the config is kept — the tunnel still exists, so the stored token keeps working across the restart. The existing startup reconcile then re-links and re-provisions, so reconnect stays seamless. Full unlink / publish-only downgrade already deprovisioned the tunnel; this covers the remaining server-stop case. Co-Authored-By: Claude Fable 5 --- apps/server/src/cloud/http.test.ts | 165 +++++++++++++++++- apps/server/src/cloud/http.ts | 40 +++++ apps/server/src/server.ts | 26 ++- .../environments/EnvironmentLinker.test.ts | 1 + .../ManagedEndpointProvider.test.ts | 119 +++++++++++++ .../environments/ManagedEndpointProvider.ts | 48 +++++ infra/relay/src/http/Api.ts | 14 ++ packages/contracts/src/relay.ts | 15 ++ 8 files changed, 426 insertions(+), 2 deletions(-) diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 96abf2e4b4a..cc71395f5b2 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -1,22 +1,31 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; +import * as ConfigProvider from "effect/ConfigProvider"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; import * as Tracer from "effect/Tracer"; -import { HttpClient, HttpServerRequest } from "effect/unstable/http"; +import { + HttpClient, + HttpClientResponse, + HttpServerRequest, + type HttpClientRequest, +} from "effect/unstable/http"; +import { EnvironmentId } from "@t3tools/contracts"; import { RelayClientTracer } from "@t3tools/shared/relayTracing"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; import type { RelayLinkProofRequest } from "@t3tools/contracts/relay"; +import { CLOUD_ENDPOINT_RUNTIME_CONFIG } from "./config.ts"; import { consumeCloudReplayGuards, isSupportedLinkProviderKind, linkProofScopes, reconcileDesiredCloudLink, + releaseManagedTunnelOnShutdown, } from "./http.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import { traceAuthenticatedRelayRequest, traceRelayRequest } from "./traceRelayRequest.ts"; @@ -213,6 +222,160 @@ describe("reconcileDesiredCloudLink", () => { ); }); +describe("releaseManagedTunnelOnShutdown", () => { + const relayConfigLayer = ConfigProvider.layer( + ConfigProvider.fromEnv({ env: { T3CODE_RELAY_URL: "https://relay.example.test" } }), + ); + + const cliToken: CliTokenManager.PersistedToken = { + accessToken: "cli-access-token", + refreshToken: "cli-refresh-token", + expiresAtEpochMs: Number.MAX_SAFE_INTEGER, + }; + + function makeMemorySecretStore(initial: Iterable = []) { + const values = new Map( + Array.from(initial, ([name, value]) => [name, new TextEncoder().encode(value)] as const), + ); + const store: ServerSecretStore.ServerSecretStore["Service"] = { + get: (name) => Effect.sync(() => Option.fromNullishOr(values.get(name))), + set: (name, value) => + Effect.sync(() => { + values.set(name, value); + }), + create: unusedSecretStoreOperation, + getOrCreateRandom: unusedSecretStoreOperation, + remove: (name) => + Effect.sync(() => { + values.delete(name); + }), + }; + return { store, values }; + } + + interface ReleaseHarness { + readonly store: ServerSecretStore.ServerSecretStore["Service"]; + readonly applyConfigCalls: Array; + readonly requests: Array; + readonly respond?: () => Response; + } + + const provideReleaseHarness = + (harness: ReleaseHarness) => + (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(ServerSecretStore.ServerSecretStore, harness.store), + Effect.provideService( + ServerEnvironment.ServerEnvironment, + ServerEnvironment.ServerEnvironment.of({ + getEnvironmentId: Effect.succeed(EnvironmentId.make("env_123")), + getDescriptor: Effect.die("unused"), + }), + ), + Effect.provideService( + ManagedEndpointRuntime.CloudManagedEndpointRuntime, + ManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ + applyConfig: (config) => + Effect.sync(() => { + harness.applyConfigCalls.push(config); + return { + status: "disabled", + } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus; + }), + }), + ), + Effect.provideService( + EnvironmentAuth.EnvironmentAuth, + EnvironmentAuth.EnvironmentAuth.of({} as EnvironmentAuth.EnvironmentAuth["Service"]), + ), + Effect.provideService( + CliTokenManager.CloudCliTokenManager, + CliTokenManager.CloudCliTokenManager.of({ + get: unusedSecretStoreOperation(), + getExisting: Effect.succeed(Option.some(cliToken)), + hasCredential: unusedSecretStoreOperation(), + store: () => unusedSecretStoreOperation(), + clear: unusedSecretStoreOperation(), + }), + ), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.sync(() => { + harness.requests.push(request); + return HttpClientResponse.fromWeb( + request, + (harness.respond ?? (() => Response.json({ ok: true })))(), + ); + }), + ), + ), + Effect.provide(relayConfigLayer), + ); + + it.effect("stops the connector, releases the relay tunnel, and drops the dead token", () => { + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + const released = yield* releaseManagedTunnelOnShutdown(); + + expect(released).toBe(true); + expect(applyConfigCalls).toEqual([null]); + expect(requests).toHaveLength(1); + const request = requests[0]!; + expect(request.method).toBe("DELETE"); + expect(request.url).toBe( + "https://relay.example.test/v1/client/environment-links/env_123/tunnel", + ); + expect(request.headers.authorization).toBe("Bearer cli-access-token"); + expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("does nothing for links without a stored managed tunnel runtime config", () => { + const { store } = makeMemorySecretStore(); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + const released = yield* releaseManagedTunnelOnShutdown(); + + expect(released).toBe(false); + expect(applyConfigCalls).toEqual([]); + expect(requests).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("keeps the stored connector token when the relay release request fails", () => { + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + const result = yield* Effect.result(releaseManagedTunnelOnShutdown()); + + expect(result._tag).toBe("Failure"); + expect(requests).toHaveLength(1); + // The tunnel still exists, so the stored token stays valid across the + // restart and the next boot can bring the connector back immediately. + expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => Response.json({ ok: false }, { status: 503 }), + }), + ); + }); +}); + describe("link proof provider kinds", () => { const proofRequest = ( providerKind: RelayLinkProofRequest["endpoint"]["providerKind"], diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 92423bb215e..e4925bc5830 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -28,6 +28,7 @@ import { RelayEnvironmentLinkProofPayload, RelayLinkProofRequest, RelayManagedEndpointOrigin, + RelayOkResponse, } from "@t3tools/contracts/relay"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import { @@ -622,6 +623,45 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD }, ); +// Cloudflare bills per provisioned tunnel, so an environment that goes offline +// must not leave its tunnel behind. Releasing deletes only the tunnel — the +// relay keeps the link and its hostname reservation, and the next startup's +// link reconcile provisions a replacement tunnel under the same URL. +export const releaseManagedTunnelOnShutdown = Effect.fn( + "environment.cloud.releaseManagedTunnelOnShutdown", +)(function* () { + const dependencies = yield* cloudHttpDependencies; + // Only a managed link stores a runtime config; publish-only links have no + // tunnel to release. + const runtimeConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); + if (Option.isNone(runtimeConfig)) { + return false; + } + // Stop the local connector before the relay deletes the tunnel it serves. + yield* dependencies.endpointRuntime.applyConfig(null); + const token = yield* dependencies.cliTokenManager.getExisting; + if (Option.isNone(token)) { + return false; + } + const relayUrl = yield* requireRelayUrl; + const environmentId = yield* dependencies.environment.getEnvironmentId; + yield* HttpClientRequest.delete( + `${relayUrl}/v1/client/environment-links/${encodeURIComponent(environmentId)}/tunnel`, + ).pipe( + HttpClientRequest.bearerToken(token.value.accessToken), + dependencies.httpClient.execute, + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)), + withRelayClientTracing, + ); + // The connector token died with the tunnel. Drop the stored config so the + // next start waits for the link reconcile instead of respawning the relay + // client with a dead token. Kept when the release request fails: the tunnel + // still exists, so the stored token keeps working across the restart. + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + return true; +}); + const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function* ( dependencies: CloudHttpDependencies, ) { diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 66b9823afb3..0412d910b8c 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -75,7 +75,11 @@ import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; -import { connectHttpApiLayer, reconcileDesiredCloudLink } from "./cloud/http.ts"; +import { + connectHttpApiLayer, + reconcileDesiredCloudLink, + releaseManagedTunnelOnShutdown, +} from "./cloud/http.ts"; import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; @@ -458,6 +462,26 @@ export const makeServerLayer = Layer.unwrap( const cloudDesiredLinkReconcileLayer = Layer.effectDiscard( Effect.gen(function* () { if (!hasCloudPublicConfig) return; + // Idle Cloudflare tunnels are billed, so a stopping server releases its + // tunnel; the persisted desired link brings one back — same hostname, + // fresh tunnel — when the environment starts again. Registered even + // when no link is desired yet: a client can link a running server, and + // that tunnel needs the same disposal on shutdown. + yield* Effect.addFinalizer(() => + releaseManagedTunnelOnShutdown().pipe( + Effect.timeout("10 seconds"), + Effect.tap((released) => + released ? Effect.logInfo("Released the managed tunnel on shutdown") : Effect.void, + ), + Effect.catchCause((cause) => + Effect.logWarning( + "Failed to release the managed tunnel on shutdown; the next link reuses it", + { cause }, + ), + ), + Effect.asVoid, + ), + ); if (!(yield* CloudCliState.readCliDesiredCloudLink)) return; const server = yield* HttpServer.HttpServer; const address = server.address; diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index 18a362f5247..deb48acd2a9 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -137,6 +137,7 @@ function testLayer(input?: { }), Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, { deprovision: input?.deprovision ?? (() => Effect.void), + release: () => Effect.void, provision: () => Effect.succeed({ endpoint: { diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 56bf6319d0d..e87e0a93e55 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -629,6 +629,125 @@ describe("ManagedEndpointProvider", () => { }, ); + it.effect("releases the tunnel while keeping the allocation, DNS record, and hostname", () => { + const tunnelCalls: TunnelCall[] = []; + const dnsCalls: DnsCall[] = []; + const allocationCalls: AllocationCall[] = []; + const layer = providerLayer( + makePersistentTunnelClient(tunnelCalls), + makeDnsClient(dnsCalls), + makeAllocations(allocationCalls), + ); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const; + const first = yield* provider.provision({ ...key, origin }); + yield* provider.release(key); + const second = yield* provider.provision({ ...key, origin }); + + expect(second.endpoint).toEqual(first.endpoint); + expect(tunnelCalls.map((call) => call.operation)).toEqual([ + // first provision + "list", + "create", + "putConfiguration", + "getToken", + // release deletes only the tunnel... + "delete", + // ...and the next provision recreates it under the same name + "list", + "create", + "putConfiguration", + "getToken", + ]); + // The DNS record survives the release and is repointed, never deleted. + expect(dnsCalls.map((call) => call.operation)).toEqual([ + "listRecords", + "createRecord", + "updateRecord", + ]); + expect(allocationCalls.map((call) => call.operation)).not.toContain("remove"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("treats an environment without a recorded tunnel as already released", () => { + const tunnelCalls: TunnelCall[] = []; + const dnsCalls: DnsCall[] = []; + const layer = providerLayer( + makePersistentTunnelClient(tunnelCalls), + makeDnsClient(dnsCalls), + makeAllocations(), + ); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + yield* provider.release({ userId: "user_ABC", environmentId: "env_ABC" }); + + expect(tunnelCalls).toEqual([]); + expect(dnsCalls).toEqual([]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("treats an already deleted tunnel as successfully released", () => { + const notFound = { _tag: "NotFound" } as const; + const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + ...makeTunnelClient(), + delete: (tunnelId) => + Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "delete", + tunnelId, + cause: notFound, + }), + ), + }); + const layer = providerLayer(tunnelClient, makeDnsClient(), makeAllocations()); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + yield* provider.release(key); + }).pipe(Effect.provide(layer)); + }); + + it.effect("surfaces non-not-found tunnel deletion failures when releasing", () => { + const failure = new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "delete", + tunnelId: "tunnel-id", + cause: "Cloudflare tunnel deletion failed", + }); + const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + ...makeTunnelClient(), + delete: () => Effect.fail(failure), + }); + const layer = providerLayer(tunnelClient, makeDnsClient(), makeAllocations()); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + const error = yield* Effect.flip(provider.release(key)); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointDeprovisioningFailed", + stage: "delete-tunnel", + userId: key.userId, + environmentId: key.environmentId, + tunnelId: "tunnel-id", + }); + expect(error.cause).toBe(failure); + }).pipe(Effect.provide(layer)); + }); + it.effect("treats an absent allocation as already deprovisioned", () => { const tunnelCalls: TunnelCall[] = []; const dnsCalls: DnsCall[] = []; diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 68e93f8b17c..02955bb53e3 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -131,6 +131,17 @@ export class ManagedEndpointProvider extends Context.Service< readonly userId: string; readonly environmentId: string; }) => Effect.Effect; + /** + * Deletes the provisioned Cloudflare tunnel while keeping the allocation + * (hostname + tunnel name reservation) and DNS record. Cloudflare bills per + * provisioned tunnel, so environments release the tunnel when they shut + * down; the next `provision` recreates it under the same name and repoints + * the CNAME, preserving the endpoint URL. + */ + readonly release: (input: { + readonly userId: string; + readonly environmentId: string; + }) => Effect.Effect; } >()("t3code-relay/environments/ManagedEndpointProvider") {} @@ -463,6 +474,43 @@ export const make = Effect.gen(function* () { ), ); }), + release: Effect.fn("relay.managed_endpoint_provider.release")(function* (input) { + yield* Effect.annotateCurrentSpan({ + "relay.user_id": input.userId, + "relay.environment_id": input.environmentId, + }); + const allocation = yield* allocations.get(input).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "load-allocation", + cause, + }), + ), + ); + const tunnelId = allocation?.tunnelId ?? null; + if (tunnelId === null) { + return; + } + yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "delete-tunnel", + tunnelId, + cause, + }), + ), + ); + // The recorded tunnelId is now stale, but the allocation row is left + // untouched deliberately: connect/status authorization requires a fully + // recorded allocation, and an offline environment must keep reporting + // "offline" (health probe fails) rather than "not authorized". The next + // provision lists tunnels by name, finds none, creates a replacement and + // re-records the fresh id. + }), provision: Effect.fn("relay.managed_endpoint_provider.provision")(function* (input) { yield* Effect.annotateCurrentSpan({ "relay.user_id": input.userId, diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 386f119802a..bddab486fcf 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -610,6 +610,20 @@ export const clientApi = HttpApiBuilder.group( } return { ok: unlinked }; }, mapRelayCommonApiErrors("not_authorized")), + ) + .handle( + "releaseEnvironmentTunnel", + Effect.fn("relay.api.client.releaseEnvironmentTunnel")(function* (args) { + const { params } = args; + const { userId } = yield* RelayClientPrincipal; + yield* managedEndpointProvider + .release({ + userId, + environmentId: params.environmentId, + }) + .pipe(Effect.catch(() => relayInternalErrorResponse("upstream_unavailable"))); + return { ok: true }; + }, mapRelayCommonApiErrors("not_authorized")), ); }), ); diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 881f922ae7e..c84f89f8f1f 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -946,6 +946,21 @@ export const RelayClientGroup = HttpApiGroup.make("client") success: RelayOkResponse, error: RelayAuthAndInternalErrors, }).annotate(OpenApi.Summary, "Unlink an environment"), + HttpApiEndpoint.delete( + "releaseEnvironmentTunnel", + "/v1/client/environment-links/:environmentId/tunnel", + { + headers: RelayBearerRequestHeaders, + params: RelayEnvironmentUnlinkParams, + success: RelayOkResponse, + error: RelayAuthAndInternalErrors, + }, + ) + .annotate(OpenApi.Summary, "Release an environment's managed tunnel") + .annotate( + OpenApi.Description, + "Deletes the provisioned Cloudflare tunnel while keeping the environment link and its hostname reservation, so a later link re-provisions the tunnel under the same URL. Environments call this when they shut down; Cloudflare bills per provisioned tunnel, so idle tunnels should not outlive their environment.", + ), ) .annotate(OpenApi.Description, "Cloud-user environment links and registered devices.") .middleware(RelayClientAuth); From 15ffc2b66bd2b2bcf411aec1349ab739016ec08d Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 25 Jul 2026 12:03:30 -0700 Subject: [PATCH 2/4] fix(connect): address tunnel-release review findings - Guard the release/provision race: release now claims the allocation's generation (updatedAt CAS) before deleting the tunnel. A provision racing a slow shutdown DELETE bumps the generation when it records its tunnel, so a stale claim fails and the freshly issued tunnel stays alive; a provision starting after the claim fails loudly on the deleted tunnel and the startup reconcile retry provisions a replacement. - Scope shutdown release to CLI-desired managed links. Only those are re-provisioned by the startup reconcile; a link installed by a web/mobile client comes back by reapplying the stored connector token and has no boot-time re-provision path, so its tunnel must survive the restart. All guards now run before the connector is stopped. - Send the release request to the persisted RELAY_URL_SECRET of the installed link instead of the current T3CODE_RELAY_URL config, which may have changed since the link was made. Co-Authored-By: Claude Fable 5 --- apps/server/src/cloud/http.test.ts | 60 ++++++++++--- apps/server/src/cloud/http.ts | 27 ++++-- .../environments/EnvironmentConnector.test.ts | 3 + .../ManagedEndpointAllocations.ts | 54 ++++++++++++ .../ManagedEndpointProvider.test.ts | 88 ++++++++++++++++--- .../environments/ManagedEndpointProvider.ts | 31 ++++++- 6 files changed, 232 insertions(+), 31 deletions(-) diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index cc71395f5b2..17485a66d5c 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -1,6 +1,5 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; -import * as ConfigProvider from "effect/ConfigProvider"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; @@ -17,9 +16,10 @@ import { RelayClientTracer } from "@t3tools/shared/relayTracing"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; import type { RelayLinkProofRequest } from "@t3tools/contracts/relay"; -import { CLOUD_ENDPOINT_RUNTIME_CONFIG } from "./config.ts"; +import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./config.ts"; import { consumeCloudReplayGuards, isSupportedLinkProviderKind, @@ -223,10 +223,6 @@ describe("reconcileDesiredCloudLink", () => { }); describe("releaseManagedTunnelOnShutdown", () => { - const relayConfigLayer = ConfigProvider.layer( - ConfigProvider.fromEnv({ env: { T3CODE_RELAY_URL: "https://relay.example.test" } }), - ); - const cliToken: CliTokenManager.PersistedToken = { accessToken: "cli-access-token", refreshToken: "cli-refresh-token", @@ -310,13 +306,17 @@ describe("releaseManagedTunnelOnShutdown", () => { }), ), ), - Effect.provide(relayConfigLayer), ); + // The persisted state of a CLI-managed link whose tunnel is releasable. + const managedLinkSecrets = [ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"], + ] as const; + it.effect("stops the connector, releases the relay tunnel, and drops the dead token", () => { - const { store, values } = makeMemorySecretStore([ - [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"], - ]); + const { store, values } = makeMemorySecretStore(managedLinkSecrets); const applyConfigCalls: Array = []; const requests: Array = []; @@ -350,13 +350,51 @@ describe("releaseManagedTunnelOnShutdown", () => { }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); }); - it.effect("keeps the stored connector token when the relay release request fails", () => { + it.effect("leaves the tunnel of a web/mobile-installed link untouched", () => { + // A managed runtime config without a CLI-desired link: the environment was + // linked by a web/mobile client, and nothing re-provisions the tunnel on + // the next boot, so shutdown must not release it. const { store, values } = makeMemorySecretStore([ [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], ]); const applyConfigCalls: Array = []; const requests: Array = []; + return Effect.gen(function* () { + const released = yield* releaseManagedTunnelOnShutdown(); + + expect(released).toBe(false); + expect(applyConfigCalls).toEqual([]); + expect(requests).toEqual([]); + expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("leaves the tunnel of a publish-only desired link untouched", () => { + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_CLI_DESIRED_LINK_SECRET, "publish_only"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + const released = yield* releaseManagedTunnelOnShutdown(); + + expect(released).toBe(false); + expect(applyConfigCalls).toEqual([]); + expect(requests).toEqual([]); + expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("keeps the stored connector token when the relay release request fails", () => { + const { store, values } = makeMemorySecretStore(managedLinkSecrets); + const applyConfigCalls: Array = []; + const requests: Array = []; + return Effect.gen(function* () { const result = yield* Effect.result(releaseManagedTunnelOnShutdown()); diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index e4925bc5830..40850a44653 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -69,7 +69,11 @@ import { RELAY_URL_SECRET, } from "./config.ts"; import { relayUrlConfig } from "./publicConfig.ts"; -import { readCliDesiredLinkMode, setCliDesiredCloudLink } from "./CliState.ts"; +import { + readCliDesiredCloudLink, + readCliDesiredLinkMode, + setCliDesiredCloudLink, +} from "./CliState.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts"; import { traceRelayRequest } from "./traceRelayRequest.ts"; @@ -637,16 +641,29 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( if (Option.isNone(runtimeConfig)) { return false; } - // Stop the local connector before the relay deletes the tunnel it serves. - yield* dependencies.endpointRuntime.applyConfig(null); + // Only CLI-desired managed links release on shutdown, because the startup + // reconcile that provisions the replacement tunnel only runs for them. A + // link installed by a web/mobile client comes back after a restart by + // reapplying the stored connector token — it has no boot-time re-provision + // path — so its tunnel must survive the restart. (Unlink still deletes it.) + if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") { + return false; + } const token = yield* dependencies.cliTokenManager.getExisting; if (Option.isNone(token)) { return false; } - const relayUrl = yield* requireRelayUrl; + // The link belongs to the relay it was installed against, so target the + // persisted URL: T3CODE_RELAY_URL may have changed since the link was made. + const relayUrl = yield* dependencies.secrets.get(RELAY_URL_SECRET); + if (Option.isNone(relayUrl)) { + return false; + } const environmentId = yield* dependencies.environment.getEnvironmentId; + // Stop the local connector before the relay deletes the tunnel it serves. + yield* dependencies.endpointRuntime.applyConfig(null); yield* HttpClientRequest.delete( - `${relayUrl}/v1/client/environment-links/${encodeURIComponent(environmentId)}/tunnel`, + `${bytesToString(relayUrl.value)}/v1/client/environment-links/${encodeURIComponent(environmentId)}/tunnel`, ).pipe( HttpClientRequest.bearerToken(token.value.accessToken), dependencies.httpClient.execute, diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index 63f12379870..6e3388e3977 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -188,6 +188,7 @@ function makeAllocations( tunnelName: "tunnel-name", dnsRecordId: "dns-record-id", readyAt: "2026-05-25T00:00:00.000Z", + updatedAt: "2026-05-25T00:00:00.000Z", }, ): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] { return { @@ -196,6 +197,7 @@ function makeAllocations( recordTunnel: () => Effect.die("unused"), recordDns: () => Effect.die("unused"), markReady: () => Effect.die("unused"), + claimRelease: () => Effect.die("unused"), remove: () => Effect.die("unused"), }; } @@ -467,6 +469,7 @@ describe("EnvironmentConnector", () => { tunnelName: "tunnel-name", dnsRecordId: "dns-record-id", readyAt: null, + updatedAt: "2026-05-25T00:00:00.000Z", }), }), ), diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index f6cefa69071..f0349de99e1 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -18,6 +18,11 @@ export interface ManagedEndpointAllocation { readonly tunnelName: string; readonly dnsRecordId: string | null; readonly readyAt: string | null; + /** + * Doubles as the allocation's generation marker: every mutation rewrites it, + * so `claimRelease` can detect a provision that raced a release. + */ + readonly updatedAt: string; } export function resolveReadyManagedEndpoint(input: { @@ -45,6 +50,7 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro "record-tunnel", "record-dns", "mark-ready", + "claim-release", "remove", ]), stage: Schema.Literals(["database-request", "resolve-reservation"]), @@ -80,6 +86,11 @@ interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey { readonly dnsRecordId: string; } +interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey { + readonly tunnelId: string; + readonly updatedAt: string; +} + export class ManagedEndpointAllocations extends Context.Service< ManagedEndpointAllocations, { @@ -98,6 +109,16 @@ export class ManagedEndpointAllocations extends Context.Service< readonly markReady: ( input: ManagedEndpointAllocationKey, ) => Effect.Effect; + /** + * Atomically claims the right to delete the allocation's tunnel: succeeds + * only while the recorded tunnel and generation still match what the + * caller loaded. A concurrent provision rewrites `updatedAt` when it + * records its tunnel, which makes a stale claim fail and keeps the freshly + * issued tunnel alive. + */ + readonly claimRelease: ( + input: ClaimManagedEndpointReleaseInput, + ) => Effect.Effect; readonly remove: ( input: ManagedEndpointAllocationKey, ) => Effect.Effect; @@ -112,6 +133,7 @@ const allocationSelection = { tunnelName: relayManagedEndpointAllocations.tunnelName, dnsRecordId: relayManagedEndpointAllocations.dnsRecordId, readyAt: relayManagedEndpointAllocations.readyAt, + updatedAt: relayManagedEndpointAllocations.updatedAt, }; const whereAllocation = (input: ManagedEndpointAllocationKey) => @@ -267,6 +289,38 @@ export const make = Effect.gen(function* () { ), ); }), + claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* ( + input: ClaimManagedEndpointReleaseInput, + ) { + const claimed = yield* db + .update(relayManagedEndpointAllocations) + .set({ + updatedAt: DateTime.formatIso(yield* DateTime.now), + }) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + ), + ) + .returning({ userId: relayManagedEndpointAllocations.userId }) + .pipe( + Effect.map((rows) => rows.length > 0), + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "claim-release", + stage: "database-request", + userId: input.userId, + environmentId: input.environmentId, + tunnelId: input.tunnelId, + cause, + }), + ), + ); + return claimed; + }), remove: Effect.fn("relay.managed_endpoint_allocations.remove")(function* ( input: ManagedEndpointAllocationKey, ) { diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index e87e0a93e55..aeb45c2ffc4 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -40,7 +40,14 @@ interface DnsCall { } interface AllocationCall { - readonly operation: "get" | "reserve" | "recordTunnel" | "recordDns" | "markReady" | "remove"; + readonly operation: + | "get" + | "reserve" + | "recordTunnel" + | "recordDns" + | "markReady" + | "claimRelease" + | "remove"; readonly input: unknown; } @@ -147,6 +154,18 @@ function makeDnsClient( function makeAllocations(calls: AllocationCall[] = []) { const allocations = new Map(); + let generation = 0; + const mutate = ( + key: string, + change: ( + allocation: ManagedEndpointAllocations.ManagedEndpointAllocation, + ) => ManagedEndpointAllocations.ManagedEndpointAllocation, + ) => { + const allocation = allocations.get(key); + if (allocation !== undefined) { + allocations.set(key, { ...change(allocation), updatedAt: `generation-${++generation}` }); + } + }; return ManagedEndpointAllocations.ManagedEndpointAllocations.of({ get: (input) => Effect.sync(() => { @@ -161,6 +180,7 @@ function makeAllocations(calls: AllocationCall[] = []) { tunnelId: null, dnsRecordId: null, readyAt: null, + updatedAt: `generation-${++generation}`, }; allocations.set(allocationKey(input), allocation); return allocation; @@ -168,29 +188,40 @@ function makeAllocations(calls: AllocationCall[] = []) { recordTunnel: (input) => Effect.sync(() => { calls.push({ operation: "recordTunnel", input }); - const allocation = allocations.get(allocationKey(input)); - if (allocation !== undefined) { - allocations.set(allocationKey(input), { ...allocation, tunnelId: input.tunnelId }); - } + mutate(allocationKey(input), (allocation) => ({ + ...allocation, + tunnelId: input.tunnelId, + })); }), recordDns: (input) => Effect.sync(() => { calls.push({ operation: "recordDns", input }); - const allocation = allocations.get(allocationKey(input)); - if (allocation !== undefined) { - allocations.set(allocationKey(input), { ...allocation, dnsRecordId: input.dnsRecordId }); - } + mutate(allocationKey(input), (allocation) => ({ + ...allocation, + dnsRecordId: input.dnsRecordId, + })); }), markReady: (input) => Effect.sync(() => { calls.push({ operation: "markReady", input }); + mutate(allocationKey(input), (allocation) => ({ + ...allocation, + readyAt: "2026-06-02T00:00:00.000Z", + })); + }), + claimRelease: (input) => + Effect.sync(() => { + calls.push({ operation: "claimRelease", input }); const allocation = allocations.get(allocationKey(input)); - if (allocation !== undefined) { - allocations.set(allocationKey(input), { - ...allocation, - readyAt: "2026-06-02T00:00:00.000Z", - }); + if ( + allocation === undefined || + allocation.tunnelId !== input.tunnelId || + allocation.updatedAt !== input.updatedAt + ) { + return false; } + mutate(allocationKey(input), (current) => current); + return true; }), remove: (input) => Effect.sync(() => { @@ -690,6 +721,35 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("keeps the tunnel alive when a concurrent provision outdates the release claim", () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + // Simulates a provision racing the release: the allocation generation no + // longer matches what the release loaded, so the claim fails. + const outdated = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + claimRelease: () => Effect.succeed(false), + }); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), outdated); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + yield* provider.release(key); + + expect(tunnelCalls.map((call) => call.operation)).toEqual([ + "list", + "create", + "putConfiguration", + "getToken", + ]); + }).pipe(Effect.provide(layer)); + }); + it.effect("treats an already deleted tunnel as successfully released", () => { const notFound = { _tag: "NotFound" } as const; const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 02955bb53e3..1caa09314f6 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -74,6 +74,7 @@ export class ManagedEndpointProvisioningFailed extends Schema.TaggedErrorClass + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-release", + tunnelId, + cause, + }), + ), + ); + if (!claimed) { return; } yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( From a0e5a730f7e4bcde35b5a91f8544c3446e1844f1 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 25 Jul 2026 12:09:00 -0700 Subject: [PATCH 3/4] fix(connect): only drop the runtime config when the tunnel was actually released When claimRelease loses the race to a concurrent provision, the relay skips the deletion but still answered { ok: true }, and the environment discarded its stored connector config for a tunnel that is still alive. release now reports whether the connector token is dead, the endpoint returns ok accordingly, and the environment keeps its runtime config on ok:false. Co-Authored-By: Claude Fable 5 --- apps/server/src/cloud/http.test.ts | 24 +++++++++++++++++++ apps/server/src/cloud/http.ts | 7 +++++- .../environments/EnvironmentLinker.test.ts | 2 +- .../ManagedEndpointProvider.test.ts | 11 ++++++--- .../environments/ManagedEndpointProvider.ts | 12 +++++++--- infra/relay/src/http/Api.ts | 7 ++++-- 6 files changed, 53 insertions(+), 10 deletions(-) diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 17485a66d5c..1d1ef9c9d4c 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -390,6 +390,30 @@ describe("releaseManagedTunnelOnShutdown", () => { }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); }); + it.effect("keeps the stored connector token when the relay skipped the release", () => { + // ok:false means a concurrent provision owns the recorded tunnel, so the + // stored runtime config (possibly freshly written by that provision) must + // survive. + const { store, values } = makeMemorySecretStore(managedLinkSecrets); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + const released = yield* releaseManagedTunnelOnShutdown(); + + expect(released).toBe(false); + expect(requests).toHaveLength(1); + expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => Response.json({ ok: false }), + }), + ); + }); + it.effect("keeps the stored connector token when the relay release request fails", () => { const { store, values } = makeMemorySecretStore(managedLinkSecrets); const applyConfigCalls: Array = []; diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 40850a44653..cc350bb70a8 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -662,7 +662,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( const environmentId = yield* dependencies.environment.getEnvironmentId; // Stop the local connector before the relay deletes the tunnel it serves. yield* dependencies.endpointRuntime.applyConfig(null); - yield* HttpClientRequest.delete( + const response = yield* HttpClientRequest.delete( `${bytesToString(relayUrl.value)}/v1/client/environment-links/${encodeURIComponent(environmentId)}/tunnel`, ).pipe( HttpClientRequest.bearerToken(token.value.accessToken), @@ -671,6 +671,11 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)), withRelayClientTracing, ); + // ok:false means the relay skipped deletion because a concurrent provision + // owns the recorded tunnel now — leave the stored config alone. + if (!response.ok) { + return false; + } // The connector token died with the tunnel. Drop the stored config so the // next start waits for the link reconcile instead of respawning the relay // client with a dead token. Kept when the release request fails: the tunnel diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index deb48acd2a9..7488939f0f9 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -137,7 +137,7 @@ function testLayer(input?: { }), Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, { deprovision: input?.deprovision ?? (() => Effect.void), - release: () => Effect.void, + release: () => Effect.succeed(true), provision: () => Effect.succeed({ endpoint: { diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index aeb45c2ffc4..d8a156ed7a2 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -675,9 +675,10 @@ describe("ManagedEndpointProvider", () => { const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const; const first = yield* provider.provision({ ...key, origin }); - yield* provider.release(key); + const released = yield* provider.release(key); const second = yield* provider.provision({ ...key, origin }); + expect(released).toBe(true); expect(second.endpoint).toEqual(first.endpoint); expect(tunnelCalls.map((call) => call.operation)).toEqual([ // first provision @@ -714,8 +715,9 @@ describe("ManagedEndpointProvider", () => { return Effect.gen(function* () { const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; - yield* provider.release({ userId: "user_ABC", environmentId: "env_ABC" }); + const released = yield* provider.release({ userId: "user_ABC", environmentId: "env_ABC" }); + expect(released).toBe(true); expect(tunnelCalls).toEqual([]); expect(dnsCalls).toEqual([]); }).pipe(Effect.provide(layer)); @@ -739,8 +741,11 @@ describe("ManagedEndpointProvider", () => { ...key, origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, }); - yield* provider.release(key); + const released = yield* provider.release(key); + // false tells the caller its connector token is still live, so it must + // keep its runtime config. + expect(released).toBe(false); expect(tunnelCalls.map((call) => call.operation)).toEqual([ "list", "create", diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 1caa09314f6..d620b452e56 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -138,11 +138,16 @@ export class ManagedEndpointProvider extends Context.Service< * provisioned tunnel, so environments release the tunnel when they shut * down; the next `provision` recreates it under the same name and repoints * the CNAME, preserving the endpoint URL. + * + * Resolves to whether the caller's connector token is now dead: true when + * the tunnel was deleted (or none was recorded to begin with), false when + * a concurrent provision outbid the release claim and the recorded tunnel + * — and any token issued for it — stays live. */ readonly release: (input: { readonly userId: string; readonly environmentId: string; - }) => Effect.Effect; + }) => Effect.Effect; } >()("t3code-relay/environments/ManagedEndpointProvider") {} @@ -492,7 +497,7 @@ export const make = Effect.gen(function* () { ); const tunnelId = allocation?.tunnelId ?? null; if (allocation === null || tunnelId === null) { - return; + return true; } // Claim the release against the allocation's current generation before // touching Cloudflare. A provision racing this release (fast environment @@ -520,7 +525,7 @@ export const make = Effect.gen(function* () { ), ); if (!claimed) { - return; + return false; } yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( Effect.mapError( @@ -539,6 +544,7 @@ export const make = Effect.gen(function* () { // "offline" (health probe fails) rather than "not authorized". The next // provision lists tunnels by name, finds none, creates a replacement and // re-records the fresh id. + return true; }), provision: Effect.fn("relay.managed_endpoint_provider.provision")(function* (input) { yield* Effect.annotateCurrentSpan({ diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index bddab486fcf..1a2a0521312 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -616,13 +616,16 @@ export const clientApi = HttpApiBuilder.group( Effect.fn("relay.api.client.releaseEnvironmentTunnel")(function* (args) { const { params } = args; const { userId } = yield* RelayClientPrincipal; - yield* managedEndpointProvider + // ok mirrors whether the connector token is now dead: false means a + // concurrent provision kept the recorded tunnel alive, so the caller + // must not discard its runtime config. + const released = yield* managedEndpointProvider .release({ userId, environmentId: params.environmentId, }) .pipe(Effect.catch(() => relayInternalErrorResponse("upstream_unavailable"))); - return { ok: true }; + return { ok: released }; }, mapRelayCommonApiErrors("not_authorized")), ); }), From 1a59cc14d76ebee443a140f40eab0ee829253a36 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 27 Jul 2026 07:44:11 -0700 Subject: [PATCH 4/4] fix(connect): never wipe a runtime config written by a fast restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shutdown finalizer removed CLOUD_ENDPOINT_RUNTIME_CONFIG whenever the relay confirmed the release, but a restarted process can reconcile and store a fresh connector config while the release request is still in flight — and the old finalizer would delete it. The finalizer now only removes the config when it still matches the one it released. Co-Authored-By: Claude Fable 5 --- apps/server/src/cloud/http.test.ts | 28 ++++++++++++++++++++++++++++ apps/server/src/cloud/http.ts | 11 ++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 1d1ef9c9d4c..c779c13aaf4 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -390,6 +390,34 @@ describe("releaseManagedTunnelOnShutdown", () => { }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); }); + it.effect("keeps a runtime config that a fast restart replaced mid-release", () => { + const { store, values } = makeMemorySecretStore(managedLinkSecrets); + const applyConfigCalls: Array = []; + const requests: Array = []; + const freshConfig = new TextEncoder().encode("fresh-runtime-config"); + + return Effect.gen(function* () { + const released = yield* releaseManagedTunnelOnShutdown(); + + expect(released).toBe(true); + // The finalizer only drops the config it released; the one written by + // the restarted process while the DELETE was in flight stays. + expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => { + // A restarted process reconciled and stored a fresh connector config + // while this shutdown's release request was in flight. + values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig); + return Response.json({ ok: true }); + }, + }), + ); + }); + it.effect("keeps the stored connector token when the relay skipped the release", () => { // ok:false means a concurrent provision owns the recorded tunnel, so the // stored runtime config (possibly freshly written by that provision) must diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index cc350bb70a8..eaf31a37f27 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -680,7 +680,16 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( // next start waits for the link reconcile instead of respawning the relay // client with a dead token. Kept when the release request fails: the tunnel // still exists, so the stored token keeps working across the restart. - yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + // Only dropped while it is still the config this shutdown released — a fast + // restart may already have reconciled and stored a fresh config for its + // replacement tunnel, and that one must survive this finalizer. + const storedConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); + if ( + Option.isSome(storedConfig) && + bytesToString(storedConfig.value) === bytesToString(runtimeConfig.value) + ) { + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + } return true; });