From 5f2de907f4ef7630c2938520c6b2c17ea3ac2a4a Mon Sep 17 00:00:00 2001 From: Ranjesh Jaganathan Date: Wed, 6 May 2026 07:35:36 -0700 Subject: [PATCH 1/2] =?UTF-8?q?fix:=20connection=20stability=20=E2=80=94?= =?UTF-8?q?=20stop=20node=20reconnect=20storms,=20fix=20bootstrap=20token?= =?UTF-8?q?=20handling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical fixes for connection management bugs introduced in PR #272: 1. Node reconnect storm during pairing (WindowsNodeClient) - Added ShouldAutoReconnect() override with _pairingBlocked flag - Flag survives OnDisconnected() (which clears _isPendingApproval) - Added rate-limit detection for terminal auth errors - Marked _pairingBlocked/_rateLimited as volatile for thread safety - Clear _rateLimited on successful hello-ok (transient, not permanent) 2. Backoff jitter (WebSocketClientBase) - Added 0-25% random jitter to prevent thundering herd when operator + node clients reconnect simultaneously 3. Client leak on reinitialize (App.xaml.cs) - Added _gatewayClient?.Dispose() before creating new client - Old clients were keeping reconnect loops alive as zombies 4. Bootstrap token not saved as Settings.Token - Setup code decoder no longer persists bootstrap to Settings.Token - Prevents reconnect storms on app restart with stale bootstrap token - TestConnection skips writing bootstrap value to Settings.Token - InitializeGatewayClient falls back to BootstrapToken for bootstrap flow 5. Token PasswordBox → TextBox - Users can see what they pasted (SetupWizardWindow + ConnectionPage) 6. Clear stale tray data on disconnect - Sessions/channels/nodes/models cleared when disconnected/error - Tray menu no longer shows old data alongside 'Disconnected' 7. Onboarding UX fixes - Removed disruptive auto-paste-on-focus from setup code field - Setup code state only updates on valid decode (prevents focus loss) - Added 'Relaunch First-Run Setup' button to Debug page Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/OpenClaw.Shared/WebSocketClientBase.cs | 4 ++ src/OpenClaw.Shared/WindowsNodeClient.cs | 35 +++++++++++++ src/OpenClaw.Tray.WinUI/App.xaml.cs | 33 +++++++++++-- .../Onboarding/Pages/ConnectionPage.cs | 49 ++++++------------- .../Pages/ConnectionPage.xaml | 2 +- .../Pages/ConnectionPage.xaml.cs | 9 ++-- src/OpenClaw.Tray.WinUI/Pages/DebugPage.xaml | 1 + .../Pages/DebugPage.xaml.cs | 5 ++ .../Windows/HubWindow.xaml.cs | 1 + .../Windows/SetupWizardWindow.cs | 14 +++--- .../WebSocketClientBaseTests.cs | 2 +- 11 files changed, 104 insertions(+), 51 deletions(-) diff --git a/src/OpenClaw.Shared/WebSocketClientBase.cs b/src/OpenClaw.Shared/WebSocketClientBase.cs index aecde0a02..97a581332 100644 --- a/src/OpenClaw.Shared/WebSocketClientBase.cs +++ b/src/OpenClaw.Shared/WebSocketClientBase.cs @@ -251,6 +251,10 @@ protected async Task ReconnectWithBackoffAsync() while (!_disposed && !_cts.Token.IsCancellationRequested && ShouldAutoReconnect()) { var delay = BackoffMs[Math.Min(_reconnectAttempts, BackoffMs.Length - 1)]; + // Add 0-25% jitter to prevent thundering herd when multiple clients + // (operator + node) reconnect on the same schedule + var jitter = Random.Shared.Next(0, delay / 4); + delay += jitter; _reconnectAttempts++; _logger.Warn($"{ClientRole} reconnecting in {delay}ms (attempt {_reconnectAttempts})"); RaiseStatusChanged(ConnectionStatus.Connecting); diff --git a/src/OpenClaw.Shared/WindowsNodeClient.cs b/src/OpenClaw.Shared/WindowsNodeClient.cs index a9d4acdc1..c88b82427 100644 --- a/src/OpenClaw.Shared/WindowsNodeClient.cs +++ b/src/OpenClaw.Shared/WindowsNodeClient.cs @@ -30,6 +30,10 @@ public class WindowsNodeClient : WebSocketClientBase private bool _isPaired; // Bridges the gap between an approval event and the next hello-ok when the gateway omits auth.deviceToken. private bool _pairingApprovedAwaitingReconnect; + // Persists across disconnect/error so ShouldAutoReconnect can block reconnect + // even after OnDisconnected clears _isPendingApproval. + private volatile bool _pairingBlocked; + private volatile bool _rateLimited; private readonly string _gatewayToken; private readonly string? _bootstrapToken; @@ -277,6 +281,7 @@ private void HandlePairingRequestedEvent(JsonElement root, string? eventType) _isPendingApproval = true; _isPaired = false; + _pairingBlocked = true; _pairingApprovedAwaitingReconnect = false; _logger.Info($"[NODE] Pairing requested for this device via {eventType}"); @@ -310,6 +315,7 @@ private async Task HandlePairingResolvedEventAsync(JsonElement root, string? eve { _isPendingApproval = false; _isPaired = true; + _pairingBlocked = false; // Allow reconnect after approval _pairingApprovedAwaitingReconnect = true; PairingStatusChanged?.Invoke(this, new PairingStatusEventArgs( @@ -603,6 +609,7 @@ private void HandleResponse(JsonElement root) PublishGatewaySelf(GatewaySelfInfo.FromHelloOk(payload)); var reconnectingAfterApproval = _pairingApprovedAwaitingReconnect; _isConnected = true; + _rateLimited = false; // Clear transient rate-limit on successful connect ResetReconnectAttempts(); // Extract node ID if returned @@ -654,6 +661,7 @@ private void HandleResponse(JsonElement root) { _isPendingApproval = true; _isPaired = false; + _pairingBlocked = true; _logger.Info("Not yet paired - check 'openclaw devices list' for pending approval"); _logger.Info($"To approve, run: openclaw devices approve {_deviceIdentity.DeviceId}"); PairingStatusChanged?.Invoke(this, new PairingStatusEventArgs( @@ -717,6 +725,7 @@ private void HandleRequestError(JsonElement root) _isPendingApproval = true; _isPaired = false; + _pairingBlocked = true; _pairingApprovedAwaitingReconnect = false; var detail = !string.IsNullOrWhiteSpace(pairingRequestId) @@ -731,6 +740,18 @@ private void HandleRequestError(JsonElement root) return; } + // Rate-limit / terminal auth errors — stop reconnecting + if (error.Contains("too many failed", StringComparison.OrdinalIgnoreCase) || + error.Contains("rate limit", StringComparison.OrdinalIgnoreCase) || + error.Contains("origin not allowed", StringComparison.OrdinalIgnoreCase) || + error.Contains("token mismatch", StringComparison.OrdinalIgnoreCase)) + { + _rateLimited = true; + _logger.Warn($"[NODE] Terminal auth error; stopping reconnect. Error: {error}"); + RaiseStatusChanged(ConnectionStatus.Error); + return; + } + _logger.Error($"Node registration failed: {error} (code: {errorCode})"); RaiseStatusChanged(ConnectionStatus.Error); } @@ -997,6 +1018,20 @@ private void PublishGatewaySelf(GatewaySelfInfo info) GatewaySelfUpdated?.Invoke(this, info); } + protected override bool ShouldAutoReconnect() + { + // Don't reconnect while awaiting pairing approval — each reconnect + // generates a new pairing request on the gateway, causing a storm. + // _pairingBlocked survives OnDisconnected (which clears _isPendingApproval). + if (_pairingBlocked) + return false; + + if (_rateLimited) + return false; + + return true; + } + protected override void OnDisconnected() { _isConnected = false; diff --git a/src/OpenClaw.Tray.WinUI/App.xaml.cs b/src/OpenClaw.Tray.WinUI/App.xaml.cs index 0b3968d30..c6fa8f868 100644 --- a/src/OpenClaw.Tray.WinUI/App.xaml.cs +++ b/src/OpenClaw.Tray.WinUI/App.xaml.cs @@ -1491,19 +1491,30 @@ private void InitializeGatewayClient(bool useBootstrapHandoffAuth = false) return; } - if (string.IsNullOrWhiteSpace(_settings.Token)) + // Need either a regular token or a bootstrap token to connect + var effectiveToken = _settings.Token; + if (string.IsNullOrWhiteSpace(effectiveToken)) { - Logger.Info("Gateway token not configured — skipping operator client initialization"); - return; + if (useBootstrapHandoffAuth && !string.IsNullOrWhiteSpace(_settings.BootstrapToken)) + { + // Bootstrap-only flow (setup code / QR): use bootstrap token for initial pairing + effectiveToken = _settings.BootstrapToken; + } + else + { + Logger.Info("Gateway token not configured — skipping operator client initialization"); + return; + } } // Unsubscribe from old client if exists UnsubscribeGatewayEvents(); + _gatewayClient?.Dispose(); _lastGatewaySelf = null; _gatewayClient = new OpenClawGatewayClient( gatewayUrl, - _settings.Token, + effectiveToken, new AppLogger(), useBootstrapHandoffAuth); _gatewayClient.SetUserRules(_settings.UserRules.Count > 0 ? _settings.UserRules : null); @@ -1944,6 +1955,19 @@ private void OnConnectionStatusChanged(object? sender, ConnectionStatus status) if (_hubWindow != null && !_hubWindow.IsClosed) _hubWindow.LastAuthError = null; } + + // Clear stale data when disconnected so tray menu doesn't show old sessions/nodes + if (status == ConnectionStatus.Disconnected || status == ConnectionStatus.Error) + { + _lastSessions = Array.Empty(); + _lastChannels = Array.Empty(); + _lastNodes = Array.Empty(); + _lastNodePairList = null; + _lastDevicePairList = null; + _lastModelsList = null; + _lastGatewaySelf = null; + } + UpdateTrayIcon(); _dispatcherQueue?.TryEnqueue(UpdateStatusDetailWindow); @@ -2497,6 +2521,7 @@ private void ShowHub(string? navigateTo = null) _hubWindow.OpenDashboardAction = OpenDashboard; _hubWindow.CheckForUpdatesAction = () => _ = CheckForUpdatesUserInitiatedAsync(); _hubWindow.QuickSendAction = () => ShowQuickSend(); + _hubWindow.OpenSetupAction = () => _ = ShowOnboardingAsync(); _hubWindow.ConnectAction = () => { InitializeGatewayClient(); diff --git a/src/OpenClaw.Tray.WinUI/Onboarding/Pages/ConnectionPage.cs b/src/OpenClaw.Tray.WinUI/Onboarding/Pages/ConnectionPage.cs index 4355a8f7e..1d59a766b 100644 --- a/src/OpenClaw.Tray.WinUI/Onboarding/Pages/ConnectionPage.cs +++ b/src/OpenClaw.Tray.WinUI/Onboarding/Pages/ConnectionPage.cs @@ -136,14 +136,14 @@ void SelectMode(ConnectionMode m) void OnSetupCodeChanged(string code) { - setSetupCode(code); if (string.IsNullOrWhiteSpace(code)) return; var result = SetupCodeDecoder.Decode(code); if (!result.Success) { - // Not a valid setup code — user might be still typing + // Not a valid setup code — user might be still typing. + // Don't call setSetupCode here to avoid re-render that steals focus. if (code.Length > 2048) Logger.Warn("[Connection] Setup code rejected: exceeds 2048 character limit"); else @@ -151,6 +151,8 @@ void OnSetupCodeChanged(string code) return; } + // Valid setup code decoded — now update state (will re-render) + setSetupCode(code); if (result.Url != null) { setUrl(result.Url); @@ -159,7 +161,8 @@ void OnSetupCodeChanged(string code) if (result.Token != null) { setToken(result.Token); - Props.Settings.Token = result.Token; + // Bootstrap token goes to BootstrapToken only — it's single-use for pairing. + // Don't save as Settings.Token (causes reconnect storms on restart). Props.Settings.BootstrapToken = result.Token; } setStatusMsg($"✅ {LocalizationHelper.GetString("Onboarding_Connection_StatusDecoded")}"); @@ -205,7 +208,13 @@ bool TryCopyPairingCommand(string command) async void TestConnection() { Props.Settings.GatewayUrl = url; - Props.Settings.Token = token; + // Only save to Settings.Token if the user entered a manual token, + // not a decoded bootstrap token (which belongs in BootstrapToken only). + if (string.IsNullOrWhiteSpace(Props.Settings.BootstrapToken) || + !string.Equals(token, Props.Settings.BootstrapToken, StringComparison.Ordinal)) + { + Props.Settings.Token = token; + } // When SSH mode, start the managed tunnel before health-checking the local URL. if (mode == ConnectionMode.Ssh) @@ -473,40 +482,14 @@ void PasteSetupCode() catch { /* clipboard unavailable — ignore */ } } - // Setup code row: TextField + Paste + QR buttons (Grid keeps the field expanding) + // Setup code row: TextField + Paste + QR buttons cardChildren.Add( Grid(["1*", "Auto", "Auto"], ["Auto"], TextField(setupCode, OnSetupCodeChanged, placeholder: LocalizationHelper.GetString("Onboarding_Connection_SetupCodePlaceholder"), header: LocalizationHelper.GetString("Onboarding_Connection_SetupCode")) - .OnGotFocus((sender, _) => - { - if (sender is Microsoft.UI.Xaml.Controls.TextBox tb && string.IsNullOrEmpty(tb.Text)) - { - try - { - var content = global::Windows.ApplicationModel.DataTransfer.Clipboard.GetContent(); - if (content.Contains(global::Windows.ApplicationModel.DataTransfer.StandardDataFormats.Text)) - { - var task = content.GetTextAsync(); - task.Completed = (op, status) => - { - if (status == global::Windows.Foundation.AsyncStatus.Completed) - { - var text = op.GetResults(); - tb.DispatcherQueue.TryEnqueue(() => - { - tb.Text = text; - OnSetupCodeChanged(text); - }); - } - }; - } - } - catch { } - } - }) - .Grid(row: 0, column: 0), + .Grid(row: 0, column: 0) + .Set(tb => Microsoft.UI.Xaml.Automation.AutomationProperties.SetAutomationId(tb, "OnboardingSetupCode")), Button(LocalizationHelper.GetString("Onboarding_Connection_PasteSetup"), PasteSetupCode) .VAlign(VerticalAlignment.Bottom) .Margin(6, 0, 0, 0) diff --git a/src/OpenClaw.Tray.WinUI/Pages/ConnectionPage.xaml b/src/OpenClaw.Tray.WinUI/Pages/ConnectionPage.xaml index 44c610ff8..99ab53eba 100644 --- a/src/OpenClaw.Tray.WinUI/Pages/ConnectionPage.xaml +++ b/src/OpenClaw.Tray.WinUI/Pages/ConnectionPage.xaml @@ -100,7 +100,7 @@ -