From 5ce17a2596dd92ec71889cfd62c98f534f5210fb Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Thu, 30 Jul 2026 06:52:35 -0700 Subject: [PATCH 1/2] fix(connect): survive boot-time network races and name the 403 reason Co-Authored-By: Claude Fable 5 --- .../src/features/cloud/linkEnvironment.ts | 9 ++++++++- apps/server/src/server.ts | 19 +++++++++++++++++- apps/web/src/cloud/linkEnvironment.ts | 9 ++++++++- .../src/environments/EnvironmentConnector.ts | 14 ++++--------- infra/relay/src/http/Api.ts | 6 ++++-- packages/contracts/src/relay.ts | 20 ++++++++++++++++++- 6 files changed, 61 insertions(+), 16 deletions(-) diff --git a/apps/mobile/src/features/cloud/linkEnvironment.ts b/apps/mobile/src/features/cloud/linkEnvironment.ts index 5d619c688f1..958827ee492 100644 --- a/apps/mobile/src/features/cloud/linkEnvironment.ts +++ b/apps/mobile/src/features/cloud/linkEnvironment.ts @@ -134,7 +134,14 @@ function relayProtectedErrorMessage(error: RelayProtectedErrorType): string { case "RelayEnvironmentLinkProofInvalidError": return `Relay rejected the environment link proof (${error.reason}).`; case "RelayEnvironmentConnectNotAuthorizedError": - return "Relay rejected the environment connection request."; + // "Not authorized" covers non-auth causes too; surface the reason so a + // missing link doesn't read as a credential problem. + if (error.reason === "environment_link_not_found") { + return "Relay has no active link for this environment. The environment server may not have re-established its link yet."; + } + return error.reason + ? `Relay rejected the environment connection request (${error.reason}).` + : "Relay rejected the environment connection request."; case "RelayEnvironmentEndpointUnavailableError": return `Relay could not reach the environment endpoint (${error.reason}).`; case "RelayEnvironmentEndpointTimedOutError": diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index e0d36e99bc9..4f9b598abb6 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -1,6 +1,8 @@ import { EnvironmentHttpApi } from "@t3tools/contracts"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Schedule from "effect/Schedule"; import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http"; import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; @@ -535,7 +537,22 @@ export const makeServerLayer = Layer.unwrap( yield* Effect.forkScoped( Effect.sleep("250 millis").pipe( Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)), - Effect.retry({ times: 4 }), + // On reboot this races NIC/DNS bring-up, so back off exponentially + // (capped at 30s) instead of burning all retries in a second. + // Bounded overall so a permanently broken setup still surfaces the + // warning below. Bad-request/unauthorized are deterministic local + // failures that no amount of retrying converges. + Effect.retry({ + while: (error) => + error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentHttpUnauthorizedError", + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.upTo({ duration: "10 minutes" }), + ), + }), Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), Effect.catch((cause) => Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { diff --git a/apps/web/src/cloud/linkEnvironment.ts b/apps/web/src/cloud/linkEnvironment.ts index 22ff986afbd..a245cbc54db 100644 --- a/apps/web/src/cloud/linkEnvironment.ts +++ b/apps/web/src/cloud/linkEnvironment.ts @@ -145,7 +145,14 @@ function relayProtectedErrorMessage(error: RelayProtectedErrorType): string { case "RelayEnvironmentLinkProofInvalidError": return `Relay rejected the environment link proof (${error.reason}).`; case "RelayEnvironmentConnectNotAuthorizedError": - return "Relay rejected the environment connection request."; + // "Not authorized" covers non-auth causes too; surface the reason so a + // missing link doesn't read as a credential problem. + if (error.reason === "environment_link_not_found") { + return "Relay has no active link for this environment. The environment server may not have re-established its link yet."; + } + return error.reason + ? `Relay rejected the environment connection request (${error.reason}).` + : "Relay rejected the environment connection request."; case "RelayEnvironmentEndpointUnavailableError": return `Relay could not reach the environment endpoint (${error.reason}).`; case "RelayEnvironmentEndpointTimedOutError": diff --git a/infra/relay/src/environments/EnvironmentConnector.ts b/infra/relay/src/environments/EnvironmentConnector.ts index db662aee94d..b25ec86881f 100644 --- a/infra/relay/src/environments/EnvironmentConnector.ts +++ b/infra/relay/src/environments/EnvironmentConnector.ts @@ -13,6 +13,7 @@ import { RelayEnvironmentMintResponse, RelayEnvironmentMintResponseProofPayload, RelayCloudMintCredentialProofPayload, + RelayEnvironmentConnectNotAuthorizedReason, type RelayEnvironmentConnectResponse, type RelayEnvironmentStatusResponse, } from "@t3tools/contracts/relay"; @@ -44,16 +45,9 @@ import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; import * as RelayConfiguration from "../Config.ts"; import { isManagedEndpointHostname } from "../deploymentConfig.ts"; -export const EnvironmentConnectNotAuthorizedReason = Schema.Literals([ - "client_proof_key_thumbprint_missing", - "environment_link_not_found", - "endpoint_provider_not_managed", - "managed_endpoint_allocation_not_found", - "managed_endpoint_base_domain_not_configured", - "managed_endpoint_allocation_not_ready", - "managed_endpoint_hostname_invalid", - "managed_endpoint_mismatch", -]); +// The reason set is part of the public relay contract so clients can render +// specific diagnostics (e.g. "your server never linked" vs. a real auth issue). +export const EnvironmentConnectNotAuthorizedReason = RelayEnvironmentConnectNotAuthorizedReason; export type EnvironmentConnectNotAuthorizedReason = typeof EnvironmentConnectNotAuthorizedReason.Type; diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 7510b4a8d41..50bcff665a9 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -778,9 +778,10 @@ export const dpopClientApi = HttpApiBuilder.group( }, mapRelayCommonApiErrors("invalid_dpop"), mapErrorTags({ - EnvironmentConnectNotAuthorized: (_error, traceId) => + EnvironmentConnectNotAuthorized: (error, traceId) => new RelayEnvironmentConnectNotAuthorizedError({ code: "environment_connect_not_authorized", + reason: error.reason, traceId, }), EnvironmentMintRequestFailed: (_error, traceId) => @@ -820,9 +821,10 @@ export const dpopClientApi = HttpApiBuilder.group( }, mapRelayCommonApiErrors("invalid_dpop"), mapErrorTags({ - EnvironmentConnectNotAuthorized: (_error, traceId) => + EnvironmentConnectNotAuthorized: (error, traceId) => new RelayEnvironmentConnectNotAuthorizedError({ code: "environment_connect_not_authorized", + reason: error.reason, traceId, }), EnvironmentMintRequestFailed: (_error, traceId) => diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index ff9a9e3ac61..52f7d7d4355 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -371,16 +371,34 @@ export class RelayEnvironmentLinkProofInvalidError extends Schema.TaggedErrorCla } } +export const RelayEnvironmentConnectNotAuthorizedReason = Schema.Literals([ + "client_proof_key_thumbprint_missing", + "environment_link_not_found", + "endpoint_provider_not_managed", + "managed_endpoint_allocation_not_found", + "managed_endpoint_base_domain_not_configured", + "managed_endpoint_allocation_not_ready", + "managed_endpoint_hostname_invalid", + "managed_endpoint_mismatch", +]); +export type RelayEnvironmentConnectNotAuthorizedReason = + typeof RelayEnvironmentConnectNotAuthorizedReason.Type; + export class RelayEnvironmentConnectNotAuthorizedError extends Schema.TaggedErrorClass()( "RelayEnvironmentConnectNotAuthorizedError", { code: Schema.Literal("environment_connect_not_authorized"), + // Optional so responses from relays deployed before the reason was + // threaded through still decode. + reason: Schema.optional(RelayEnvironmentConnectNotAuthorizedReason), traceId: TrimmedNonEmptyString, }, { httpApiStatus: 403 }, ) { override get message(): string { - return "Relay environment connection is not authorized"; + return this.reason + ? `Relay environment connection is not authorized: ${this.reason}` + : "Relay environment connection is not authorized"; } } From 2b9b2eafcfcc8dfd5f002057cde600eb0723e2d2 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Thu, 30 Jul 2026 07:09:39 -0700 Subject: [PATCH 2/2] fix(connect): address review - stop retrying conflict errors, drop reason alias Co-Authored-By: Claude Fable 5 --- apps/server/src/server.ts | 9 ++++++--- infra/relay/src/environments/EnvironmentConnector.ts | 10 ++-------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 4f9b598abb6..853bb0b1101 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -540,12 +540,15 @@ export const makeServerLayer = Layer.unwrap( // On reboot this races NIC/DNS bring-up, so back off exponentially // (capped at 30s) instead of burning all retries in a second. // Bounded overall so a permanently broken setup still surfaces the - // warning below. Bad-request/unauthorized are deterministic local - // failures that no amount of retrying converges. + // warning below. Bad-request/unauthorized/conflict are + // deterministic failures (malformed origin, not linked yet, linked + // to a different cloud account) that no amount of retrying + // converges. Effect.retry({ while: (error) => error._tag !== "EnvironmentHttpBadRequestError" && - error._tag !== "EnvironmentHttpUnauthorizedError", + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError", schedule: Schedule.exponential("1 second").pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(30))), diff --git a/infra/relay/src/environments/EnvironmentConnector.ts b/infra/relay/src/environments/EnvironmentConnector.ts index b25ec86881f..d840f809e5a 100644 --- a/infra/relay/src/environments/EnvironmentConnector.ts +++ b/infra/relay/src/environments/EnvironmentConnector.ts @@ -45,14 +45,8 @@ import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; import * as RelayConfiguration from "../Config.ts"; import { isManagedEndpointHostname } from "../deploymentConfig.ts"; -// The reason set is part of the public relay contract so clients can render -// specific diagnostics (e.g. "your server never linked" vs. a real auth issue). -export const EnvironmentConnectNotAuthorizedReason = RelayEnvironmentConnectNotAuthorizedReason; -export type EnvironmentConnectNotAuthorizedReason = - typeof EnvironmentConnectNotAuthorizedReason.Type; - function environmentConnectNotAuthorizedReasonMessage( - reason: EnvironmentConnectNotAuthorizedReason, + reason: RelayEnvironmentConnectNotAuthorizedReason, ): string { switch (reason) { case "client_proof_key_thumbprint_missing": @@ -79,7 +73,7 @@ export class EnvironmentConnectNotAuthorized extends Schema.TaggedErrorClass