Add OpenTelemetry trace id and span id onto the logging context

api/pom.xml

@@ -71,6 +71,10 @@
             <artifactId>cloud-framework-direct-download</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-api</artifactId>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

api/src/main/java/org/apache/cloudstack/api/filter/ApiTraceFilter.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.api.filter;
+
+import org.apache.cloudstack.context.LogContext;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.IOException;
+import java.util.UUID;
+import javax.servlet.FilterConfig;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+
+public class ApiTraceFilter implements Filter {
+    @Override
+    public void init(FilterConfig filterConfig) throws ServletException {
+    }
+
+    @Override
+    public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
+            throws IOException, ServletException {
+        try {
+            HttpServletRequest httpReq = (HttpServletRequest) request;
+            String traceId = httpReq.getHeader(LogContext.X_B3_TRACEID_KEY);
+            if (StringUtils.isBlank(traceId)) {
+                traceId = UUID.randomUUID().toString();
+            }
+
+            LogContext.current().putContextParameter(LogContext.X_B3_TRACEID_KEY, traceId);
+            chain.doFilter(request, response);
+        } finally {
+            LogContext.current().removeContextParameter(LogContext.X_B3_TRACEID_KEY);
+        }
+    }
+
+    @Override
+    public void destroy() {
+    }
+}

api/src/main/java/org/apache/cloudstack/context/LogContext.java

@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.context;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
@@ -53,6 +54,10 @@ public class LogContext {
     private long userId;
     private final Map<String, String> context = new HashMap<String, String>();
 
+    public final static String X_B3_TRACEID_KEY = "traceid";
+    public final static String MOSAIC_TRACE_ID_KEY = "mosaic_trace_id";
+    public final static String MOSAIC_SPAN_ID_KEY = "mosaic_span_id";
+
     static EntityManager s_entityMgr;
 
     public static void init(EntityManager entityMgr) {
@@ -78,6 +83,20 @@ protected LogContext(User user, Account account, String logContextId) {
 
     public void putContextParameter(String key, String value) {
         context.put(key, value);
+        MDC.put(key, value);
+    }
+
+    public void removeContextParameter(String key) {
+        context.remove(key);
+        MDC.remove(key);
+    }
+
+    public void removeContextParameters() {
+        // Iterate over a copy of the keys: removeContextParameter mutates the context
+        // map, so iterating the live keySet/entrySet would throw ConcurrentModificationException.
+        for (String key : new ArrayList<>(context.keySet())) {
+            removeContextParameter(key);
+        }
     }
 
     public String getContextParameter(String key) {

api/src/main/java/org/apache/cloudstack/context/TraceContextMdcWrapper.java

@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.context;
+
+import org.apache.log4j.MDC;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.ContextStorage;
+import io.opentelemetry.context.Scope;
+
+/**
+ * Mirrors the active OpenTelemetry span onto the Log4j MDC so management-server log
+ * lines carry mosaic_trace_id and mosaic_span_id on every thread that has an active
+ * span (API requests, agent-command dispatch, async jobs), not just the servlet path.
+ *
+ * The OpenTelemetry agent populates the log MDC automatically for Log4j2 and Logback,
+ * but not for Log4j 1.2 (reload4j), which the management server uses. This wrapper
+ * fills that gap by hooking the OpenTelemetry context lifecycle: whenever a span
+ * becomes current on a thread it copies the ids into the MDC, and restores the
+ * previous values when that scope closes. Install once at startup via {@link #register()}.
+ */
+public class TraceContextMdcWrapper implements ContextStorage {
+
+    private final ContextStorage delegate;
+
+    TraceContextMdcWrapper(ContextStorage delegate) {
+        this.delegate = delegate;
+    }
+
+    /**
+     * Install the wrapper. Must be called before the first OpenTelemetry context is
+     * used, i.e. at management-server startup, before the server accepts requests.
+     */
+    public static void register() {
+        ContextStorage.addWrapper(TraceContextMdcWrapper::new);
+    }
+
+    @Override
+    public Scope attach(Context toAttach) {
+        Object previousTraceId = MDC.get(LogContext.MOSAIC_TRACE_ID_KEY);
+        Object previousSpanId = MDC.get(LogContext.MOSAIC_SPAN_ID_KEY);
+        SpanContext spanContext = Span.fromContext(toAttach).getSpanContext();
+        if (spanContext.isValid()) {
+            MDC.put(LogContext.MOSAIC_TRACE_ID_KEY, spanContext.getTraceId());
+            MDC.put(LogContext.MOSAIC_SPAN_ID_KEY, spanContext.getSpanId());
+        } else {
+            MDC.remove(LogContext.MOSAIC_TRACE_ID_KEY);
+            MDC.remove(LogContext.MOSAIC_SPAN_ID_KEY);
+        }
+        Scope delegateScope = delegate.attach(toAttach);
+        return () -> {
+            delegateScope.close();
+            restore(LogContext.MOSAIC_TRACE_ID_KEY, previousTraceId);
+            restore(LogContext.MOSAIC_SPAN_ID_KEY, previousSpanId);
+        };
+    }
+
+    private static void restore(String key, Object previous) {
+        if (previous != null) {
+            MDC.put(key, previous);
+        } else {
+            MDC.remove(key);
+        }
+    }
+
+    @Override
+    public Context current() {
+        return delegate.current();
+    }
+}

api/src/test/java/org/apache/cloudstack/context/TraceContextMdcWrapperTest.java

@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.context;
+
+import org.apache.log4j.MDC;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.api.trace.TraceFlags;
+import io.opentelemetry.api.trace.TraceState;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.ContextStorage;
+import io.opentelemetry.context.Scope;
+
+public class TraceContextMdcWrapperTest {
+
+    private static final String TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736";
+    private static final String SPAN_ID = "00f067aa0ba902b7";
+    private static final String OTHER_TRACE_ID = "d75597dcda4f6e7b9c1a2b3c4d5e6f70";
+    private static final String OTHER_SPAN_ID = "aabbccddeeff0011";
+
+    // Minimal delegate so we test the wrapper in isolation, no real context storage.
+    private final ContextStorage noopDelegate = new ContextStorage() {
+        @Override
+        public Scope attach(Context toAttach) {
+            return () -> { };
+        }
+
+        @Override
+        public Context current() {
+            return Context.root();
+        }
+    };
+
+    private final TraceContextMdcWrapper wrapper = new TraceContextMdcWrapper(noopDelegate);
+
+    @After
+    public void tearDown() {
+        MDC.remove(LogContext.MOSAIC_TRACE_ID_KEY);
+        MDC.remove(LogContext.MOSAIC_SPAN_ID_KEY);
+    }
+
+    private static Context contextWithSpan(String traceId, String spanId) {
+        return Context.root().with(Span.wrap(
+                SpanContext.create(traceId, spanId, TraceFlags.getSampled(), TraceState.getDefault())));
+    }
+
+    @Test
+    public void putsTraceContextOnMdcWhileScopeOpenAndRestoresOnClose() {
+        Scope scope = wrapper.attach(contextWithSpan(TRACE_ID, SPAN_ID));
+        Assert.assertEquals(TRACE_ID, MDC.get(LogContext.MOSAIC_TRACE_ID_KEY));
+        Assert.assertEquals(SPAN_ID, MDC.get(LogContext.MOSAIC_SPAN_ID_KEY));
+
+        scope.close();
+        Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY));
+        Assert.assertNull(MDC.get(LogContext.MOSAIC_SPAN_ID_KEY));
+    }
+
+    @Test
+    public void leavesMdcUnsetWhenNoActiveSpan() {
+        Scope scope = wrapper.attach(Context.root());
+        Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY));
+        Assert.assertNull(MDC.get(LogContext.MOSAIC_SPAN_ID_KEY));
+        scope.close();
+    }
+
+    @Test
+    public void restoresOuterSpanWhenNestedScopeCloses() {
+        Scope outer = wrapper.attach(contextWithSpan(TRACE_ID, SPAN_ID));
+        Scope inner = wrapper.attach(contextWithSpan(OTHER_TRACE_ID, OTHER_SPAN_ID));
+        Assert.assertEquals(OTHER_TRACE_ID, MDC.get(LogContext.MOSAIC_TRACE_ID_KEY));
+
+        inner.close();
+        Assert.assertEquals(TRACE_ID, MDC.get(LogContext.MOSAIC_TRACE_ID_KEY));
+
+        outer.close();
+        Assert.assertNull(MDC.get(LogContext.MOSAIC_TRACE_ID_KEY));
+    }
+}

client/src/main/java/org/apache/cloudstack/ServerDaemon.java

@@ -55,6 +55,8 @@
 import com.cloud.utils.PropertiesUtil;
 import com.cloud.utils.server.ServerProperties;
 
+import org.apache.cloudstack.context.TraceContextMdcWrapper;
+
 /***
  * The ServerDaemon class implements the embedded server, it can be started either
  * using JSVC or directly from the JAR along with additional jars not shaded in the uber-jar.
@@ -108,6 +110,9 @@ public class ServerDaemon implements Daemon {
     //////////////////////////////////////////////////
 
     public static void main(final String... anArgs) throws Exception {
+        // Install the trace-context to MDC hook before the server starts, so every
+        // thread with an active OpenTelemetry span carries mosaic_trace_id in its logs.
+        TraceContextMdcWrapper.register();
         final ServerDaemon daemon = new ServerDaemon();
         daemon.init(null);
         daemon.start();

client/src/main/webapp/WEB-INF/web.xml

@@ -36,6 +36,16 @@
         <param-value>classpath:META-INF/cloudstack/webApplicationContext.xml</param-value>
     </context-param>
 
+    <filter>
+        <filter-name>apiTraceFilter</filter-name>
+        <filter-class>org.apache.cloudstack.api.filter.ApiTraceFilter</filter-class>
+    </filter>
+
+    <filter-mapping>
+        <filter-name>apiTraceFilter</filter-name>
+        <url-pattern>/api/*</url-pattern>
+    </filter-mapping>
+
     <servlet>
         <servlet-name>cloudStartupServlet</servlet-name>
         <servlet-class>com.cloud.servlet.CloudStartupServlet</servlet-class>