feat: add operator and node gateway lifecycle telemetry

docs/TELEMETRY.md

@@ -78,6 +78,68 @@ The OpenTelemetry log pipeline should not export:
 
 If a new log category should be exported, add it deliberately and review the structured fields it can emit.
 
+## Gateway connection lifecycle
+
+The tray exports gateway lifecycle diagnostics when an endpoint is configured:
+
+- operator connect traces: `openclaw.connection.operator.connect` and
+  `openclaw.connection.operator.reconnect`
+- Windows node connect traces: `openclaw.connection.node.connect` and
+  `openclaw.connection.node.reconnect`
+- coarse operator phase spans:
+  `openclaw.connection.operator.prepare`,
+  `openclaw.connection.operator.transport`, and
+  `openclaw.connection.operator.handshake`
+- coarse Windows node phase spans:
+  `openclaw.connection.node.prepare`,
+  `openclaw.connection.node.transport`, and
+  `openclaw.connection.node.handshake`
+- metrics: `openclaw.connection.attempts`,
+  `openclaw.connection.attempt.duration`, and
+  `openclaw.connection.state.transitions`
+- structured state logs in the `OpenClaw.Telemetry.Connection` category
+
+Lifecycle attributes are limited to role, operation, outcome, coarse error
+category, and finite operator/node/overall states. Gateway URLs, IDs, device
+IDs, pairing request IDs, credentials, error messages, and diagnostic-ring
+text are not exported.
+
+The operator phase spans distinguish local credential/client/tunnel preparation,
+WebSocket transport establishment, and the gateway challenge/hello handshake.
+The Windows node initiates its gateway connection: its prepare span includes
+credential resolution, client creation, and synchronous capability registration;
+its transport span covers the outbound WebSocket; and its handshake span covers
+the gateway's `connect.challenge`, the signed connect request, and `hello-ok`.
+
+A node attempt succeeds only after `hello-ok` yields connected and paired
+readiness. Pending approval completes the attempt as `pairing_required`; human
+approval wait time is not included in an open span. If the existing node client
+later begins automatic transport recovery, the actual retry is recorded as
+`openclaw.connection.node.reconnect` beginning with the transport phase.
+Manager-driven starts, including the fresh connection after approval, remain
+`openclaw.connection.node.connect`.
+
+An attempt with outcome `superseded` was replaced by a newer local lifecycle
+request before it completed. This is not a gateway or authentication failure.
+It exists to make overlapping connection orchestration visible instead of
+silently dropping work that had already started. A short `superseded` span
+followed by a normal connection span commonly means an automatic or previously
+queued start raced with a newer explicit start; the replacement attempt owns the
+eventual connection result.
+
+Pairing and classified gateway failures complete from their specific events
+before generic connection status handling. If an active attempt instead ends
+with an unclassified `Disconnected` status, telemetry uses `server_close` as a
+finite, reasonless fallback because that status carries no close cause.
+`Disconnected` covers both orderly remote closes and premature transport loss,
+so `server_close` does not prove that the gateway intentionally closed the
+connection. Other network failures report `Error` and use
+`network_unreachable`; this fallback can therefore be less specific without
+changing connection behavior.
+
+The phase spans intentionally do not trace signing, serialization, response
+parsing, capability details, or token persistence as separate operations.
+
 ## Endpoint handling
 
 The endpoint setting is a collector endpoint, not a credential or request-parameter store. Accept plain `http` and `https` collector URLs with optional path prefixes. Reject URLs with embedded user info, query strings, or fragments.

src/OpenClaw.Connection/INodeConnector.cs

@@ -50,6 +50,15 @@ Task ConnectAsync(
     Task DisconnectAsync();
 }
 
+/// <summary>
+/// Optional telemetry milestones exposed by production node connectors.
+/// </summary>
+public interface INodeConnectorTelemetryEvents
+{
+    event EventHandler TransportConnected;
+    event EventHandler<GatewayErrorKind> ConnectionFailure;
+}
+
 public sealed class NodeClientCreatedEventArgs : EventArgs
 {
     public NodeClientCreatedEventArgs(WindowsNodeClient client, string? bearerToken)

src/OpenClaw.Connection/NodeConnector.cs

@@ -7,7 +7,7 @@ namespace OpenClaw.Connection;
 /// Capability setup (canvas, screen capture, etc.) is handled by NodeService,
 /// which has WinUI dependencies and remains in App.xaml.cs for now.
 /// </summary>
-public sealed class NodeConnector : INodeConnector
+public sealed class NodeConnector : INodeConnector, INodeConnectorTelemetryEvents
 {
     private readonly IOpenClawLogger _logger;
     private readonly ConnectionDiagnostics? _diagnostics;
@@ -21,6 +21,8 @@ public sealed class NodeConnector : INodeConnector
     public event EventHandler<PairingStatusEventArgs>? PairingStatusChanged;
     public event EventHandler<DeviceTokenReceivedEventArgs>? DeviceTokenReceived;
     public event EventHandler<NodeClientCreatedEventArgs>? ClientCreated;
+    public event EventHandler? TransportConnected;
+    public event EventHandler<GatewayErrorKind>? ConnectionFailure;
 
     public NodeConnector(IOpenClawLogger logger, ConnectionDiagnostics? diagnostics = null)
     {
@@ -163,20 +165,15 @@ private async Task ConnectCoreAsync(
         }
 
         client.StatusChanged += (s, e) =>
-        {
-            if (IsCurrentClient(s, generation))
-                StatusChanged?.Invoke(this, e);
-        };
+            ForwardIfCurrent(s, generation, e, StatusChanged);
+        client.TransportConnected += (s, _) =>
+            ForwardIfCurrent(s, generation, EventArgs.Empty, TransportConnected);
+        client.ConnectionFailure += (s, e) =>
+            ForwardIfCurrent(s, generation, e, ConnectionFailure);
         client.PairingStatusChanged += (s, e) =>
-        {
-            if (IsCurrentClient(s, generation))
-                PairingStatusChanged?.Invoke(this, e);
-        };
+            ForwardIfCurrent(s, generation, e, PairingStatusChanged);
         client.DeviceTokenReceived += (s, e) =>
-        {
-            if (IsCurrentClient(s, generation))
-                DeviceTokenReceived?.Invoke(this, e);
-        };
+            ForwardIfCurrent(s, generation, e, DeviceTokenReceived);
 
         try
         {
@@ -221,6 +218,41 @@ private bool IsCurrentClient(object? sender, long generation)
         }
     }
 
+    // Validation and dispatch stay atomic so a retired client cannot publish after its
+    // replacement. Subscribers must remain synchronous and must not block on connector
+    // lifecycle work while this lock is held.
+    private void ForwardIfCurrent<T>(
+        object? sender,
+        long generation,
+        T args,
+        EventHandler<T>? handler)
+    {
+        lock (_clientLifecycleLock)
+        {
+            if (Interlocked.Read(ref _clientGeneration) == generation &&
+                ReferenceEquals(sender, _client))
+            {
+                handler?.Invoke(this, args);
+            }
+        }
+    }
+
+    private void ForwardIfCurrent(
+        object? sender,
+        long generation,
+        EventArgs args,
+        EventHandler? handler)
+    {
+        lock (_clientLifecycleLock)
+        {
+            if (Interlocked.Read(ref _clientGeneration) == generation &&
+                ReferenceEquals(sender, _client))
+            {
+                handler?.Invoke(this, args);
+            }
+        }
+    }
+
     private void DisconnectIfCurrent(long generation)
     {
         lock (_clientLifecycleLock)

src/OpenClaw.Shared/OpenClawGatewayClient.cs

@@ -164,9 +164,13 @@ protected override Task ProcessMessageAsync(string json)
     protected override Task OnConnectedAsync()
     {
         ResetUnsupportedMethodFlags();
+        RaiseTransportConnected();
         return Task.CompletedTask;
     }
 
+    protected void RaiseTransportConnected() =>
+        TransportConnected?.Invoke(this, EventArgs.Empty);
+
     protected override bool ShouldAutoReconnect()
     {
         // PairingRequired must stay visible, but approval only takes effect on a fresh socket.
@@ -232,6 +236,8 @@ protected override void OnDisposing()
     public event EventHandler<DeviceTokenReceivedEventArgs>? DeviceTokenReceived;
     /// <summary>Raised when the hello-ok handshake completes successfully.</summary>
     public event EventHandler? HandshakeSucceeded;
+    /// <summary>Raised after the WebSocket transport connects, before the gateway handshake begins.</summary>
+    public event EventHandler? TransportConnected;
     /// <summary>Raised when the gateway requires pairing approval for this device.</summary>
     public event EventHandler<string?>? PairingRequired;
     /// <summary>Raised when v3 signature was rejected and client fell back to v2.</summary>

src/OpenClaw.Shared/Telemetry/OpenClawTelemetry.cs

@@ -40,6 +40,78 @@ public static class OpenClawTelemetry
         return activity;
     }
 
+    /// <summary>
+    /// Starts a manually-controlled span without leaving it as the ambient activity.
+    /// </summary>
+    /// <remarks>
+    /// Use this for operations that begin in one asynchronous callback and finish in another.
+    /// The caller owns the returned activity and must finish it with
+    /// <see cref="StopDetachedActivity(Activity?)"/> so stopping it cannot replace a newer
+    /// ambient activity with the context captured when this span started.
+    /// </remarks>
+    public static Activity? StartDetachedActivity(
+        string spanName,
+        IEnumerable<OpenClawTelemetryTag>? tags = null,
+        System.Diagnostics.ActivityKind kind = System.Diagnostics.ActivityKind.Internal,
+        OpenClawActivitySourceName source = OpenClawActivitySourceName.OpenClaw)
+    {
+        var previous = Activity.Current;
+        try
+        {
+            return StartActivity(spanName, tags, kind, source);
+        }
+        finally
+        {
+            Activity.Current = previous;
+        }
+    }
+
+    /// <summary>
+    /// Starts a manually-controlled child span without leaving it as the ambient activity.
+    /// </summary>
+    public static Activity? StartDetachedActivity(
+        string spanName,
+        ActivityContext parentContext,
+        IEnumerable<OpenClawTelemetryTag>? tags = null,
+        System.Diagnostics.ActivityKind kind = System.Diagnostics.ActivityKind.Internal,
+        OpenClawActivitySourceName source = OpenClawActivitySourceName.OpenClaw)
+    {
+        if (string.IsNullOrWhiteSpace(spanName))
+            throw new ArgumentException("Span name cannot be empty.", nameof(spanName));
+
+        var previous = Activity.Current;
+        try
+        {
+            var activity = source.ToActivitySource().StartActivity(spanName, kind, parentContext);
+            ApplyTags(activity, tags);
+            return activity;
+        }
+        finally
+        {
+            Activity.Current = previous;
+        }
+    }
+
+    /// <summary>
+    /// Stops and disposes a detached activity without changing the caller's ambient activity.
+    /// </summary>
+    public static void StopDetachedActivity(Activity? activity)
+    {
+        if (activity == null)
+            return;
+
+        var current = Activity.Current;
+        try
+        {
+            activity.Stop();
+            activity.Dispose();
+        }
+        finally
+        {
+            Activity.Current = current;
+        }
+    }
+
     /// <summary>
     /// Runs a synchronous action inside a span and automatically marks success, cancellation, or failure.
     /// </summary>

src/OpenClaw.Shared/WindowsNodeClient.cs

@@ -71,6 +71,10 @@ public class WindowsNodeClient : WebSocketClientBase
     public event EventHandler<DeviceTokenReceivedEventArgs>? DeviceTokenReceived;
     /// <summary>Raised when the hello-ok handshake completes successfully.</summary>
     public event EventHandler? HandshakeSucceeded;
+    /// <summary>Raised after the WebSocket transport connects, before the gateway challenge arrives.</summary>
+    public event EventHandler? TransportConnected;
+    /// <summary>Raised with a finite classification before a terminal handshake error is published.</summary>
+    public event EventHandler<GatewayErrorKind>? ConnectionFailure;
 
     public new bool IsConnected => _isConnected;
     public string? NodeId => _nodeId;
@@ -108,6 +112,12 @@ public class WindowsNodeClient : WebSocketClientBase
 
     protected override int ReceiveBufferSize => 65536;
     protected override string ClientRole => "node";
+
+    protected override Task OnConnectedAsync()
+    {
+        TransportConnected?.Invoke(this, EventArgs.Empty);
+        return Task.CompletedTask;
+    }
 
     public WindowsNodeClient(string gatewayUrl, string token, string dataPath, IOpenClawLogger? logger = null, string? bootstrapToken = null)
         : base(gatewayUrl, ResolveRequiredCredential(token, bootstrapToken, dataPath, logger), logger)
@@ -704,7 +714,7 @@ private string BuildNodeConnectMessage(string? nonce, long ts)
         return (new Dictionary<string, string> { ["token"] = _gatewayToken }, _gatewayToken);
     }
 
-    private void HandleResponse(JsonElement root)
+    internal void HandleResponse(JsonElement root)
     {
         if (root.TryGetProperty("ok", out var okProp) &&
             okProp.ValueKind == JsonValueKind.False)
@@ -884,6 +894,7 @@ private void HandleRequestError(JsonElement root)
         {
             _rateLimited = true;
             _logger.Warn($"[NODE] Terminal auth error; stopping reconnect. Error: {TokenSanitizer.Sanitize(error)}");
+            ConnectionFailure?.Invoke(this, ClassifyConnectionFailure(error, errorCode));
             RaiseStatusChanged(ConnectionStatus.Error);
             return;
         }
@@ -900,9 +911,20 @@ private void HandleRequestError(JsonElement root)
         }
 
         _logger.Error($"Node registration failed: {TokenSanitizer.Sanitize(error)} (code: {errorCode})");
+        ConnectionFailure?.Invoke(this, ClassifyConnectionFailure(error, errorCode));
         RaiseStatusChanged(ConnectionStatus.Error);
     }
 
+    private static GatewayErrorKind ClassifyConnectionFailure(string error, string errorCode)
+    {
+        if (error.Contains("too many failed", StringComparison.OrdinalIgnoreCase))
+            return GatewayErrorKind.RateLimited;
+        if (error.Contains("origin not allowed", StringComparison.OrdinalIgnoreCase))
+            return GatewayErrorKind.Auth;
+
+        return GatewayErrorClassifier.Classify($"{errorCode} {error}");
+    }
+
     private bool PayloadTargetsCurrentDevice(JsonElement payload)
     {
         if (TryGetString(payload, "deviceId", out var deviceId) &&

src/OpenClaw.Tray.WinUI/App.xaml.cs

@@ -496,7 +496,8 @@ private async Task OnLaunchedAsync(LaunchActivatedEventArgs args)
         _settings = new SettingsManager();
         _previousSettingsSnapshot = _settings.ToSettingsData().ToConnectionSnapshot();
         _openTelemetryConnection = new OpenTelemetryEndpointConnection();
-        ApplyOpenTelemetryEndpointSettings();
+        await _openTelemetryConnection.ApplyAsync(
+            OpenTelemetryEndpointOptions.FromSettings(_settings));
         _chatCoordinator = new OpenClawTray.Chat.OpenClawChatCoordinator(
             _settings,
             () => _nodeService,
@@ -1934,6 +1935,7 @@ private void RaiseChatProviderChanged()
     /// </summary>
     private void OnManagerStateChanged(object? sender, GatewayConnectionSnapshot snap)
     {
+        _openTelemetryConnection?.SendConnectionState(snap);
         var mapped = ConnectionStatusPresenter.ToLegacyStatus(snap);
         var connectedSideEffectsKey = snap.OperatorState == RoleConnectionState.Connected
             ? $"{snap.GatewayId ?? snap.GatewayUrl ?? "unknown"}|{snap.OperatorDeviceId ?? "unknown"}"
@@ -4461,12 +4463,6 @@ private async Task ExitApplicationAsync()
             _chatCoordinator = null;
         });
 
-        SafeShutdownStep("OpenTelemetry endpoint", () =>
-        {
-            _openTelemetryConnection?.Dispose();
-            _openTelemetryConnection = null;
-        });
-
         // Dispose runtime services
         var connectionManager = _connectionManager;
         if (connectionManager != null)
@@ -4478,6 +4474,12 @@ await SafeShutdownStepAsync("gateway client", async () =>
             _connectionManager = null;
         }
 
+        SafeShutdownStep("OpenTelemetry endpoint", () =>
+        {
+            _openTelemetryConnection?.Dispose();
+            _openTelemetryConnection = null;
+        });
+
         var nodeService = _nodeService;
         if (nodeService != null)
         {