diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index ca3c30bcd8..622822f22a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -40,6 +40,7 @@ Use subheadings with the "=====" level for adding notes for unreleased changes: [float] ===== Features * Added option to make routing-key part of RabbitMQ transaction/span names - {pull}3636[#3636] +* Added internal option for capturing request bodies for apache httpclient v4 - {pull}3692[#3692] [[release-notes-1.x]] === Java Agent version 1.x diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/BodyCaptureImpl.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/BodyCaptureImpl.java new file mode 100644 index 0000000000..289a5f6fd2 --- /dev/null +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/BodyCaptureImpl.java @@ -0,0 +1,144 @@ +package co.elastic.apm.agent.impl.context; + +import co.elastic.apm.agent.objectpool.Resetter; +import co.elastic.apm.agent.objectpool.impl.QueueBasedObjectPool; +import co.elastic.apm.agent.tracer.configuration.WebConfiguration; +import co.elastic.apm.agent.tracer.metadata.BodyCapture; +import co.elastic.apm.agent.tracer.pooling.Allocator; +import co.elastic.apm.agent.tracer.pooling.ObjectPool; +import co.elastic.apm.agent.tracer.pooling.Recyclable; +import org.jctools.queues.atomic.MpmcAtomicArrayQueue; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +public class BodyCaptureImpl implements BodyCapture, Recyclable { + private static final ObjectPool BYTE_BUFFER_POOL = QueueBasedObjectPool.of(new MpmcAtomicArrayQueue(128), false, + new Allocator() { + @Override + public ByteBuffer createInstance() { + return ByteBuffer.allocate(WebConfiguration.MAX_BODY_CAPTURE_BYTES); + } + }, + new Resetter() { + @Override + public void recycle(ByteBuffer object) { + object.clear(); + } + }); + + private enum CaptureState { + NOT_ELIGIBLE, + ELIGIBLE, + STARTED + } + + private volatile CaptureState state; + + private final StringBuilder charset; + + /** + * The maximum number of bytes to capture, if the body is longer remaining bytes will be dropped. + */ + private int numBytesToCapture; + + @Nullable + private ByteBuffer bodyBuffer; + + BodyCaptureImpl() { + charset = new StringBuilder(); + resetState(); + } + + @Override + public void resetState() { + state = CaptureState.NOT_ELIGIBLE; + charset.setLength(0); + if (bodyBuffer != null) { + BYTE_BUFFER_POOL.recycle(bodyBuffer); + } + } + + @Override + public void markEligibleForCapturing() { + if (state == CaptureState.NOT_ELIGIBLE) { + synchronized (this) { + if (state == CaptureState.NOT_ELIGIBLE) { + state = CaptureState.ELIGIBLE; + } + } + } + } + + @Override + public boolean isEligibleForCapturing() { + return state != CaptureState.NOT_ELIGIBLE; + } + + @Override + public boolean startCapture(@Nullable String requestCharset, int numBytesToCapture) { + if (numBytesToCapture > WebConfiguration.MAX_BODY_CAPTURE_BYTES) { + throw new IllegalArgumentException("Capturing " + numBytesToCapture + " bytes is not supported, maximum is " + WebConfiguration.MAX_BODY_CAPTURE_BYTES + " bytes"); + } + if (state == CaptureState.ELIGIBLE) { + synchronized (this) { + if (state == CaptureState.ELIGIBLE) { + if (requestCharset != null) { + this.charset.append(requestCharset); + } + this.numBytesToCapture = numBytesToCapture; + state = CaptureState.STARTED; + return true; + } + } + } + return false; + } + + private void acquireBodyBufferIfRequired() { + if (state != CaptureState.STARTED) { + throw new IllegalStateException("Capturing has not been started!"); + } + if (bodyBuffer == null) { + bodyBuffer = BYTE_BUFFER_POOL.createInstance(); + } + } + + @Override + public void append(byte b) { + acquireBodyBufferIfRequired(); + if (!isFull()) { + bodyBuffer.put(b); + } + } + + @Override + public void append(byte[] b, int offset, int len) { + acquireBodyBufferIfRequired(); + int remaining = numBytesToCapture - bodyBuffer.position(); + if (remaining > 0) { + bodyBuffer.put(b, offset, Math.min(len, remaining)); + } + } + + @Override + public boolean isFull() { + if (bodyBuffer == null) { + return false; + } + return bodyBuffer.position() >= numBytesToCapture; + } + + @Nullable + public CharSequence getCharset() { + if (charset.length() == 0) { + return null; + } + return charset; + } + + @Nullable + public ByteBuffer getBody() { + return bodyBuffer; + } +} diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/HttpImpl.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/HttpImpl.java index 4995e3ef51..67a7169682 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/HttpImpl.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/HttpImpl.java @@ -30,6 +30,8 @@ public class HttpImpl implements Recyclable, Http { */ private final UrlImpl url = new UrlImpl(); + private final BodyCaptureImpl requestBody = new BodyCaptureImpl(); + /** * HTTP method used by this HTTP outgoing span */ @@ -65,6 +67,11 @@ public int getStatusCode() { return statusCode; } + @Override + public BodyCaptureImpl getRequestBody() { + return requestBody; + } + @Override public HttpImpl withUrl(@Nullable String url) { if (url != null) { @@ -88,6 +95,7 @@ public HttpImpl withStatusCode(int statusCode) { @Override public void resetState() { url.resetState(); + requestBody.resetState(); method = null; statusCode = 0; } @@ -98,9 +106,4 @@ public boolean hasContent() { statusCode > 0; } - public void copyFrom(HttpImpl other) { - url.copyFrom(other.url); - method = other.method; - statusCode = other.statusCode; - } } diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/DslJsonSerializer.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/DslJsonSerializer.java index 89759bed42..141a987f6f 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/DslJsonSerializer.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/DslJsonSerializer.java @@ -18,8 +18,23 @@ */ package co.elastic.apm.agent.report.serialize; -import co.elastic.apm.agent.impl.context.*; +import co.elastic.apm.agent.impl.context.AbstractContextImpl; +import co.elastic.apm.agent.impl.context.BodyCaptureImpl; +import co.elastic.apm.agent.impl.context.CloudOriginImpl; +import co.elastic.apm.agent.impl.context.DbImpl; +import co.elastic.apm.agent.impl.context.DestinationImpl; +import co.elastic.apm.agent.impl.context.Headers; import co.elastic.apm.agent.impl.context.HttpImpl; +import co.elastic.apm.agent.impl.context.MessageImpl; +import co.elastic.apm.agent.impl.context.RequestImpl; +import co.elastic.apm.agent.impl.context.ResponseImpl; +import co.elastic.apm.agent.impl.context.ServiceOriginImpl; +import co.elastic.apm.agent.impl.context.ServiceTargetImpl; +import co.elastic.apm.agent.impl.context.SocketImpl; +import co.elastic.apm.agent.impl.context.SpanContextImpl; +import co.elastic.apm.agent.impl.context.TransactionContextImpl; +import co.elastic.apm.agent.impl.context.UrlImpl; +import co.elastic.apm.agent.impl.context.UserImpl; import co.elastic.apm.agent.impl.error.ErrorCaptureImpl; import co.elastic.apm.agent.impl.metadata.Agent; import co.elastic.apm.agent.impl.metadata.CloudProviderInfo; @@ -33,16 +48,26 @@ import co.elastic.apm.agent.impl.metadata.ServiceImpl; import co.elastic.apm.agent.impl.metadata.SystemInfo; import co.elastic.apm.agent.impl.stacktrace.StacktraceConfigurationImpl; -import co.elastic.apm.agent.impl.transaction.*; +import co.elastic.apm.agent.impl.transaction.AbstractSpanImpl; +import co.elastic.apm.agent.impl.transaction.Composite; +import co.elastic.apm.agent.impl.transaction.DroppedSpanStats; +import co.elastic.apm.agent.impl.transaction.FaasImpl; +import co.elastic.apm.agent.impl.transaction.FaasTriggerImpl; +import co.elastic.apm.agent.impl.transaction.IdImpl; +import co.elastic.apm.agent.impl.transaction.OTelSpanKind; +import co.elastic.apm.agent.impl.transaction.SpanCount; import co.elastic.apm.agent.impl.transaction.SpanImpl; -import co.elastic.apm.agent.tracer.metrics.Labels; +import co.elastic.apm.agent.impl.transaction.StackFrame; +import co.elastic.apm.agent.impl.transaction.TraceContextImpl; +import co.elastic.apm.agent.impl.transaction.TransactionImpl; import co.elastic.apm.agent.report.ApmServerClient; import co.elastic.apm.agent.sdk.internal.collections.LongList; import co.elastic.apm.agent.sdk.logging.Logger; import co.elastic.apm.agent.sdk.logging.LoggerFactory; import co.elastic.apm.agent.tracer.metadata.PotentiallyMultiValuedMap; -import co.elastic.apm.agent.tracer.pooling.Recyclable; import co.elastic.apm.agent.tracer.metrics.DslJsonUtil; +import co.elastic.apm.agent.tracer.metrics.Labels; +import co.elastic.apm.agent.tracer.pooling.Recyclable; import com.dslplatform.json.BoolConverter; import com.dslplatform.json.DslJson; import com.dslplatform.json.JsonWriter; @@ -54,6 +79,7 @@ import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.nio.CharBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -1028,21 +1054,36 @@ private void serializeSpanLinks(List spanLinks) { } private void serializeOTel(SpanImpl span) { - serializeOtel(span, Collections.emptyList()); + serializeOtel(span, Collections.emptyList(), requestBodyToString(span.getContext().getHttp().getRequestBody())); + } + + @Nullable + private CharSequence requestBodyToString(BodyCaptureImpl requestBody) { + //TODO: perform proper, charset aware conversion to string + ByteBuffer buffer = requestBody.getBody(); + if (buffer == null || buffer.position() == 0) { + return null; + } + buffer.flip(); + StringBuilder result = new StringBuilder(); + while (buffer.hasRemaining()) { + result.append((char) buffer.get()); + } + return result; } private void serializeOTel(TransactionImpl transaction) { List profilingCorrelationStackTraceIds = transaction.getProfilingCorrelationStackTraceIds(); synchronized (profilingCorrelationStackTraceIds) { - serializeOtel(transaction, profilingCorrelationStackTraceIds); + serializeOtel(transaction, profilingCorrelationStackTraceIds, null); } } - private void serializeOtel(AbstractSpanImpl span, List profilingStackTraceIds) { + private void serializeOtel(AbstractSpanImpl span, List profilingStackTraceIds, @Nullable CharSequence httpRequestBody) { OTelSpanKind kind = span.getOtelKind(); Map attributes = span.getOtelAttributes(); - boolean hasAttributes = !attributes.isEmpty() || !profilingStackTraceIds.isEmpty(); + boolean hasAttributes = !attributes.isEmpty() || !profilingStackTraceIds.isEmpty() || httpRequestBody != null; boolean hasKind = kind != null; if (hasKind || hasAttributes) { writeFieldName("otel"); @@ -1092,6 +1133,13 @@ private void serializeOtel(AbstractSpanImpl span, List profilingStack } jw.writeByte(ARRAY_END); } + if (httpRequestBody != null) { + if (!isFirstAttrib) { + jw.writeByte(COMMA); + } + writeFieldName("http.request.body.content"); + jw.writeString(httpRequestBody); + } jw.writeByte(OBJECT_END); } diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/impl/context/BodyCaptureTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/impl/context/BodyCaptureTest.java new file mode 100644 index 0000000000..2c53ef665b --- /dev/null +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/impl/context/BodyCaptureTest.java @@ -0,0 +1,56 @@ +package co.elastic.apm.agent.impl.context; + +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +import static co.elastic.apm.agent.testutils.assertions.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class BodyCaptureTest { + + @Test + public void testAppendTruncation() { + BodyCaptureImpl capture = new BodyCaptureImpl(); + capture.markEligibleForCapturing(); + capture.startCapture("foobar", 10); + assertThat(capture.isFull()).isFalse(); + + capture.append("123Hello World!".getBytes(StandardCharsets.UTF_8), 3, 5); + assertThat(capture.isFull()).isFalse(); + + capture.append(" from the other side".getBytes(StandardCharsets.UTF_8), 0, 20); + assertThat(capture.isFull()).isTrue(); + + ByteBuffer content = capture.getBody(); + int size = content.position(); + byte[] contentBytes = new byte[size]; + content.position(0); + content.get(contentBytes); + + assertThat(contentBytes).isEqualTo("Hello from".getBytes(StandardCharsets.UTF_8)); + } + + @Test + public void testLifecycle() { + BodyCaptureImpl capture = new BodyCaptureImpl(); + + assertThat(capture.isEligibleForCapturing()).isFalse(); + assertThat(capture.startCapture("foobar", 42)) + .isFalse(); + assertThatThrownBy(() -> capture.append((byte) 42)).isInstanceOf(IllegalStateException.class); + + capture.markEligibleForCapturing(); + assertThat(capture.isEligibleForCapturing()).isTrue(); + assertThatThrownBy(() -> capture.append((byte) 42)).isInstanceOf(IllegalStateException.class); + + assertThat(capture.startCapture("foobar", 42)) + .isTrue(); + capture.append((byte) 42); //ensure no exception thrown + + // startCapture should return true only once + assertThat(capture.startCapture("foobar", 42)) + .isFalse(); + } +} diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/report/serialize/DslJsonSerializerTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/report/serialize/DslJsonSerializerTest.java index 34caa3ba05..7006fd0fa8 100644 --- a/apm-agent-core/src/test/java/co/elastic/apm/agent/report/serialize/DslJsonSerializerTest.java +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/report/serialize/DslJsonSerializerTest.java @@ -27,6 +27,7 @@ import co.elastic.apm.agent.impl.TextHeaderMapAccessor; import co.elastic.apm.agent.impl.baggage.BaggageImpl; import co.elastic.apm.agent.impl.context.AbstractContextImpl; +import co.elastic.apm.agent.impl.context.BodyCaptureImpl; import co.elastic.apm.agent.impl.context.DestinationImpl; import co.elastic.apm.agent.impl.context.Headers; import co.elastic.apm.agent.impl.context.RequestImpl; @@ -46,8 +47,13 @@ import co.elastic.apm.agent.impl.sampling.ConstantSampler; import co.elastic.apm.agent.impl.sampling.Sampler; import co.elastic.apm.agent.impl.stacktrace.StacktraceConfigurationImpl; -import co.elastic.apm.agent.impl.transaction.*; +import co.elastic.apm.agent.impl.transaction.AbstractSpanImpl; +import co.elastic.apm.agent.impl.transaction.IdImpl; +import co.elastic.apm.agent.impl.transaction.OTelSpanKind; import co.elastic.apm.agent.impl.transaction.SpanImpl; +import co.elastic.apm.agent.impl.transaction.StackFrame; +import co.elastic.apm.agent.impl.transaction.TraceContextImpl; +import co.elastic.apm.agent.impl.transaction.TransactionImpl; import co.elastic.apm.agent.report.ApmServerClient; import co.elastic.apm.agent.sdk.internal.collections.LongList; import co.elastic.apm.agent.sdk.internal.util.IOUtils; @@ -425,6 +431,21 @@ void testSpanHttpContextSerialization() { assertThat(http.get("status_code").intValue()).isEqualTo(523); } + @Test + void testSpanHttpRequestBodySerialization() { + SpanImpl span = new SpanImpl(tracer); + + BodyCaptureImpl bodyCapture = span.getContext().getHttp().getRequestBody(); + bodyCapture.markEligibleForCapturing(); + bodyCapture.startCapture("utf-8", 50); + bodyCapture.append("foobar".getBytes(StandardCharsets.UTF_8), 0, 6); + + JsonNode spanJson = readJsonString(writer.toJsonString(span)); + JsonNode otel = spanJson.get("otel"); + JsonNode attribs = otel.get("attributes"); + assertThat(attribs.get("http.request.body.content").textValue()).isEqualTo("foobar"); + } + public static boolean[][] getContentCombinations() { return new boolean[][]{ {true, true, true, true}, diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient-common/src/main/java/co/elastic/apm/agent/httpclient/common/AbstractApacheHttpClientAdvice.java b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient-common/src/main/java/co/elastic/apm/agent/httpclient/common/AbstractApacheHttpClientAdvice.java index 3a866b8aa9..1998180c3b 100644 --- a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient-common/src/main/java/co/elastic/apm/agent/httpclient/common/AbstractApacheHttpClientAdvice.java +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient-common/src/main/java/co/elastic/apm/agent/httpclient/common/AbstractApacheHttpClientAdvice.java @@ -34,7 +34,7 @@ public abstract class AbstractApacheHttpClientAdvice { public static & - TextHeaderGetter> Object startSpan(final Tracer tracer, + TextHeaderGetter> Span startSpan(final Tracer tracer, final ApacheHttpClientApiAdapter adapter, final WRAPPER request, @Nullable final HTTPHOST httpHost, diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient-common/src/main/java/co/elastic/apm/agent/httpclient/common/AbstractApacheHttpClientAsyncAdvice.java b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient-common/src/main/java/co/elastic/apm/agent/httpclient/common/AbstractApacheHttpClientAsyncAdvice.java index f3b0f234f8..a4fd57dbff 100644 --- a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient-common/src/main/java/co/elastic/apm/agent/httpclient/common/AbstractApacheHttpClientAsyncAdvice.java +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient-common/src/main/java/co/elastic/apm/agent/httpclient/common/AbstractApacheHttpClientAsyncAdvice.java @@ -20,8 +20,8 @@ import co.elastic.apm.agent.httpclient.HttpClientHelper; -import co.elastic.apm.agent.tracer.TraceState; import co.elastic.apm.agent.tracer.Span; +import co.elastic.apm.agent.tracer.TraceState; import co.elastic.apm.agent.tracer.Tracer; public abstract class AbstractApacheHttpClientAsyncAdvice { @@ -40,6 +40,7 @@ public static activeContext = tracer.currentContext(); Span span = activeContext.createExitSpan(); if (span != null) { + span.getContext().getHttp().getRequestBody().markEligibleForCapturing(); span.withType(HttpClientHelper.EXTERNAL_TYPE) .withSubtype(HttpClientHelper.HTTP_SUBTYPE) .withSync(false) diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpClientInstrumentation.java b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpClientInstrumentation.java index 1d60900984..2bc725c95b 100644 --- a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpClientInstrumentation.java +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpClientInstrumentation.java @@ -20,7 +20,11 @@ import co.elastic.apm.agent.httpclient.common.AbstractApacheHttpClientAdvice; import co.elastic.apm.agent.httpclient.v4.helper.ApacheHttpClient4ApiAdapter; +import co.elastic.apm.agent.httpclient.v4.helper.RequestBodyCaptureRegistry; import co.elastic.apm.agent.httpclient.v4.helper.RequestHeaderAccessor; +import co.elastic.apm.agent.sdk.logging.Logger; +import co.elastic.apm.agent.sdk.logging.LoggerFactory; +import co.elastic.apm.agent.tracer.Span; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.NamedElement; import net.bytebuddy.description.method.MethodDescription; @@ -47,13 +51,17 @@ public class ApacheHttpClientInstrumentation extends BaseApacheHttpClientInstrumentation { public static class ApacheHttpClient4Advice extends AbstractApacheHttpClientAdvice { + private static final Logger logger = LoggerFactory.getLogger(ApacheHttpClient4Advice.class); + private static final ApacheHttpClient4ApiAdapter adapter = ApacheHttpClient4ApiAdapter.get(); @Nullable @Advice.OnMethodEnter(suppress = Throwable.class, inline = false) public static Object onBeforeExecute(@Advice.Argument(0) HttpRoute route, @Advice.Argument(1) HttpRequestWrapper request) throws URISyntaxException { - return startSpan(tracer, adapter, request, route.getTargetHost(), RequestHeaderAccessor.INSTANCE); + Span span = startSpan(tracer, adapter, request, route.getTargetHost(), RequestHeaderAccessor.INSTANCE); + RequestBodyCaptureRegistry.potentiallyCaptureRequestBody(request, tracer.getActive()); + return span; } @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class, inline = false) diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpEntityGetContentInstrumentation.java b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpEntityGetContentInstrumentation.java new file mode 100644 index 0000000000..b08ea36da9 --- /dev/null +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpEntityGetContentInstrumentation.java @@ -0,0 +1,94 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package co.elastic.apm.agent.httpclient.v4; + +import co.elastic.apm.agent.httpclient.RequestBodyRecordingInputStream; +import co.elastic.apm.agent.httpclient.RequestBodyRecordingOutputStream; +import co.elastic.apm.agent.httpclient.v4.helper.RequestBodyCaptureRegistry; +import co.elastic.apm.agent.sdk.logging.Logger; +import co.elastic.apm.agent.sdk.logging.LoggerFactory; +import co.elastic.apm.agent.tracer.Span; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.NamedElement; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; +import org.apache.http.HttpEntity; + +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URISyntaxException; + +import static co.elastic.apm.agent.sdk.bytebuddy.CustomElementMatchers.classLoaderCanLoadClass; +import static net.bytebuddy.matcher.ElementMatchers.hasSuperType; +import static net.bytebuddy.matcher.ElementMatchers.isBootstrapClassLoader; +import static net.bytebuddy.matcher.ElementMatchers.nameContains; +import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith; +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.not; +import static net.bytebuddy.matcher.ElementMatchers.returns; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +public class ApacheHttpEntityGetContentInstrumentation extends BaseApacheHttpClientInstrumentation { + + public static class ApacheHttpEntityGetContentAdvice { + + private static final Logger logger = LoggerFactory.getLogger(ApacheHttpEntityGetContentAdvice.class); + + @Advice.OnMethodExit(suppress = Throwable.class, inline = false) + @Advice.AssignReturned.ToReturned + public static InputStream onExit(@Advice.This HttpEntity thiz, @Advice.Return InputStream content) throws URISyntaxException { + Span clientSpan = RequestBodyCaptureRegistry.removeSpanFor(thiz); + if (clientSpan != null) { + logger.debug("Wrapping input stream for request body capture for HttpEntity {} ({}) for span {}", thiz.getClass().getName(), System.identityHashCode(thiz), clientSpan); + return new RequestBodyRecordingInputStream(content, clientSpan); + } + return content; + } + } + + @Override + public String getAdviceClassName() { + return "co.elastic.apm.agent.httpclient.v4.ApacheHttpEntityGetContentInstrumentation$ApacheHttpEntityGetContentAdvice"; + } + + @Override + public ElementMatcher.Junction getClassLoaderMatcher() { + return not(isBootstrapClassLoader()) + .and(classLoaderCanLoadClass("org.apache.http.HttpEntity")); + } + + @Override + public ElementMatcher getTypeMatcherPreFilter() { + return nameStartsWith("org.apache.http").and(nameContains("Entity")); + } + + @Override + public ElementMatcher getTypeMatcher() { + return hasSuperType(named("org.apache.http.HttpEntity")); + } + + @Override + public ElementMatcher getMethodMatcher() { + return named("getContent") + .and(takesArguments(0)); + } + +} diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpEntityWriteToInstrumentation.java b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpEntityWriteToInstrumentation.java new file mode 100644 index 0000000000..0b06c8b9fc --- /dev/null +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpEntityWriteToInstrumentation.java @@ -0,0 +1,99 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package co.elastic.apm.agent.httpclient.v4; + +import co.elastic.apm.agent.httpclient.RequestBodyRecordingOutputStream; +import co.elastic.apm.agent.httpclient.v4.helper.RequestBodyCaptureRegistry; +import co.elastic.apm.agent.sdk.logging.Logger; +import co.elastic.apm.agent.sdk.logging.LoggerFactory; +import co.elastic.apm.agent.tracer.Span; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.NamedElement; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; +import org.apache.http.HttpEntity; + +import java.io.OutputStream; +import java.net.URISyntaxException; + +import static co.elastic.apm.agent.sdk.bytebuddy.CustomElementMatchers.classLoaderCanLoadClass; +import static net.bytebuddy.matcher.ElementMatchers.hasSuperType; +import static net.bytebuddy.matcher.ElementMatchers.isBootstrapClassLoader; +import static net.bytebuddy.matcher.ElementMatchers.nameContains; +import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith; +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.not; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +public class ApacheHttpEntityWriteToInstrumentation extends BaseApacheHttpClientInstrumentation { + + public static class ApacheHttpEntityWriteToAdvice { + + private static final Logger logger = LoggerFactory.getLogger(ApacheHttpEntityWriteToAdvice.class); + + @Advice.OnMethodEnter(suppress = Throwable.class, inline = false) + @Advice.AssignReturned.ToArguments(@Advice.AssignReturned.ToArguments.ToArgument(0)) + public static OutputStream onEnter(@Advice.This HttpEntity thiz, @Advice.Argument(0) OutputStream drain) throws URISyntaxException { + Span clientSpan = RequestBodyCaptureRegistry.removeSpanFor(thiz); + if (clientSpan != null) { + logger.debug("Wrapping output stream for request body capture for HttpEntity {} ({}) for span {}", thiz.getClass().getName(), System.identityHashCode(thiz), clientSpan); + return new RequestBodyRecordingOutputStream(drain, clientSpan); + } + return drain; + } + + @Advice.OnMethodExit(suppress = Throwable.class, inline = false) + public static void onExit(@Advice.Enter OutputStream potentiallyWrappedStream) throws URISyntaxException { + if (potentiallyWrappedStream instanceof RequestBodyRecordingOutputStream) { + ((RequestBodyRecordingOutputStream) potentiallyWrappedStream).releaseSpan(); + } + } + } + + @Override + public String getAdviceClassName() { + return "co.elastic.apm.agent.httpclient.v4.ApacheHttpEntityWriteToInstrumentation$ApacheHttpEntityWriteToAdvice"; + } + + @Override + public ElementMatcher.Junction getClassLoaderMatcher() { + return not(isBootstrapClassLoader()) + .and(classLoaderCanLoadClass("org.apache.http.HttpEntity")); + } + + @Override + public ElementMatcher getTypeMatcherPreFilter() { + return nameStartsWith("org.apache.http").and(nameContains("Entity")); + } + + @Override + public ElementMatcher getTypeMatcher() { + return hasSuperType(named("org.apache.http.HttpEntity")); + } + + @Override + public ElementMatcher getMethodMatcher() { + return named("writeTo") + .and(takesArguments(1)) + .and(takesArgument(0, OutputStream.class)); + } + +} diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/LegacyApacheHttpClientInstrumentation.java b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/LegacyApacheHttpClientInstrumentation.java index 8097b84d1c..21fe4ad545 100644 --- a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/LegacyApacheHttpClientInstrumentation.java +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/LegacyApacheHttpClientInstrumentation.java @@ -19,10 +19,11 @@ package co.elastic.apm.agent.httpclient.v4; import co.elastic.apm.agent.httpclient.HttpClientHelper; +import co.elastic.apm.agent.httpclient.v4.helper.RequestBodyCaptureRegistry; import co.elastic.apm.agent.httpclient.v4.helper.RequestHeaderAccessor; -import co.elastic.apm.agent.tracer.TraceState; import co.elastic.apm.agent.tracer.Outcome; import co.elastic.apm.agent.tracer.Span; +import co.elastic.apm.agent.tracer.TraceState; import co.elastic.apm.agent.tracer.configuration.CoreConfiguration; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.NamedElement; @@ -105,6 +106,7 @@ public static Object onBeforeExecute(@Advice.Argument(0) @Nullable HttpHost host } } + RequestBodyCaptureRegistry.potentiallyCaptureRequestBody(request, tracer.getActive()); tracer.currentContext().propagateContext(request, RequestHeaderAccessor.INSTANCE, RequestHeaderAccessor.INSTANCE); return span; } diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/helper/HttpAsyncRequestProducerWrapper.java b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/helper/HttpAsyncRequestProducerWrapper.java index 6d874c13fc..a8648894ba 100644 --- a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/helper/HttpAsyncRequestProducerWrapper.java +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/helper/HttpAsyncRequestProducerWrapper.java @@ -85,6 +85,7 @@ public HttpRequest generateRequest() throws IOException, HttpException { // trace context propagation if (request != null) { if (span != null) { + RequestBodyCaptureRegistry.potentiallyCaptureRequestBody(request, span); RequestLine requestLine = request.getRequestLine(); if (requestLine != null) { String method = requestLine.getMethod(); diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/helper/RequestBodyCaptureRegistry.java b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/helper/RequestBodyCaptureRegistry.java new file mode 100644 index 0000000000..009cc1fe9b --- /dev/null +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/java/co/elastic/apm/agent/httpclient/v4/helper/RequestBodyCaptureRegistry.java @@ -0,0 +1,76 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package co.elastic.apm.agent.httpclient.v4.helper; + +import co.elastic.apm.agent.httpclient.HttpClientHelper; +import co.elastic.apm.agent.sdk.logging.Logger; +import co.elastic.apm.agent.sdk.logging.LoggerFactory; +import co.elastic.apm.agent.sdk.state.GlobalState; +import co.elastic.apm.agent.tracer.AbstractSpan; +import co.elastic.apm.agent.tracer.GlobalTracer; +import co.elastic.apm.agent.tracer.Span; +import co.elastic.apm.agent.tracer.reference.ReferenceCountedMap; +import org.apache.http.HttpEntity; +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpRequest; + +import javax.annotation.Nullable; + +public class RequestBodyCaptureRegistry { + + private static final Logger logger = LoggerFactory.getLogger(RequestBodyCaptureRegistry.class); + + @GlobalState + public static class MapHolder { + private static final ReferenceCountedMap> entityToClientSpan = GlobalTracer.get().newReferenceCountedMap(); + + + public static void captureBodyFor(Object entity, Span httpClientSpan) { + entityToClientSpan.put(entity, httpClientSpan); + } + + @Nullable + public static Span removeSpanFor(Object entity) { + return entityToClientSpan.remove(entity); + } + } + + + public static void potentiallyCaptureRequestBody(HttpRequest request, @Nullable AbstractSpan span) { + if (HttpClientHelper.startRequestBodyCapture(span, request, RequestHeaderAccessor.INSTANCE)) { + if (request instanceof HttpEntityEnclosingRequest) { + HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity(); + if (entity != null) { + logger.debug("Enabling request capture for entity {}() for span {}", entity.getClass().getName(), System.identityHashCode(entity), span); + MapHolder.captureBodyFor(entity, (Span) span); + } else { + logger.debug("HttpEntity is null for span {}", span); + } + } else { + logger.debug("Not capturing request body because {} is not an HttpEntityEnclosingRequest", request.getClass().getName()); + } + } + } + + @Nullable + public static Span removeSpanFor(HttpEntity entity) { + return MapHolder.removeSpanFor(entity); + } + +} diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/resources/META-INF/services/co.elastic.apm.agent.sdk.ElasticApmInstrumentation b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/resources/META-INF/services/co.elastic.apm.agent.sdk.ElasticApmInstrumentation index 753c7c4605..8d04c1a373 100644 --- a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/resources/META-INF/services/co.elastic.apm.agent.sdk.ElasticApmInstrumentation +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/main/resources/META-INF/services/co.elastic.apm.agent.sdk.ElasticApmInstrumentation @@ -2,3 +2,5 @@ co.elastic.apm.agent.httpclient.v4.ApacheHttpClientInstrumentation co.elastic.apm.agent.httpclient.v4.ApacheHttpAsyncClientInstrumentation co.elastic.apm.agent.httpclient.v4.ApacheHttpAsyncClientRedirectInstrumentation co.elastic.apm.agent.httpclient.v4.LegacyApacheHttpClientInstrumentation +co.elastic.apm.agent.httpclient.v4.ApacheHttpEntityWriteToInstrumentation +co.elastic.apm.agent.httpclient.v4.ApacheHttpEntityGetContentInstrumentation diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpAsyncClientInstrumentationTest.java b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpAsyncClientInstrumentationTest.java index 20543c1073..b2698ebc2e 100644 --- a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpAsyncClientInstrumentationTest.java +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpAsyncClientInstrumentationTest.java @@ -24,9 +24,11 @@ import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.concurrent.FutureCallback; import org.apache.http.conn.UnsupportedSchemeException; +import org.apache.http.entity.InputStreamEntity; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClients; import org.assertj.core.api.Assertions; @@ -34,6 +36,7 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.concurrent.CompletableFuture; @@ -89,6 +92,39 @@ public void cancelled() { responseFuture.get(); } + @Override + protected boolean isBodyCapturingSupported() { + return true; + } + + @Override + protected void performPost(String path, byte[] data, String contentTypeHeader) throws Exception { + final CompletableFuture responseFuture = new CompletableFuture<>(); + + HttpClientContext httpClientContext = HttpClientContext.create(); + HttpPost request = new HttpPost(path); + request.setEntity(new InputStreamEntity(new ByteArrayInputStream(data))); + request.setHeader("Content-Type", contentTypeHeader); + client.execute(request, httpClientContext, new FutureCallback<>() { + @Override + public void completed(HttpResponse result) { + responseFuture.complete(result); + } + + @Override + public void failed(Exception ex) { + responseFuture.completeExceptionally(ex); + } + + @Override + public void cancelled() { + responseFuture.cancel(true); + } + }); + + responseFuture.get(); + } + @Test public void testSpanFinishOnEarlyException() throws Exception { @@ -125,5 +161,4 @@ public void testSpanWithIllegalProtocol() throws Exception { Assertions.assertThat(reporter.getSpans()).hasSize(1); } - } diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpClientInstrumentationTest.java b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpClientInstrumentationTest.java index 91189c295d..cfa20bd6a0 100644 --- a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpClientInstrumentationTest.java +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/ApacheHttpClientInstrumentationTest.java @@ -21,11 +21,14 @@ import co.elastic.apm.agent.httpclient.AbstractHttpClientInstrumentationTest; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.InputStreamEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.junit.AfterClass; import org.junit.BeforeClass; +import java.io.ByteArrayInputStream; import java.io.IOException; public class ApacheHttpClientInstrumentationTest extends AbstractHttpClientInstrumentationTest { @@ -49,4 +52,18 @@ protected void performGet(String path) throws Exception { response.close(); } + @Override + protected boolean isBodyCapturingSupported() { + return true; + } + + @Override + protected void performPost(String path, byte[] content, String contentTypeHeader) throws Exception { + HttpPost request = new HttpPost(path); + request.setEntity(new InputStreamEntity(new ByteArrayInputStream(content))); + request.setHeader("Content-Type", contentTypeHeader); + + client.execute(request); + } + } diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/LegacyApacheHttpClientBasicHttpRequestInstrumentationTest.java b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/LegacyApacheHttpClientBasicHttpRequestInstrumentationTest.java index 72a3828ac3..62e8088ce7 100644 --- a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/LegacyApacheHttpClientBasicHttpRequestInstrumentationTest.java +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/LegacyApacheHttpClientBasicHttpRequestInstrumentationTest.java @@ -59,4 +59,6 @@ protected void performGet(String path) throws Exception { throw (Exception) e.getTargetException(); } } + + } diff --git a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/LegacyApacheHttpClientHttpUriRequestInstrumentationTest.java b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/LegacyApacheHttpClientHttpUriRequestInstrumentationTest.java index 499f1d9032..16c56da974 100644 --- a/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/LegacyApacheHttpClientHttpUriRequestInstrumentationTest.java +++ b/apm-agent-plugins/apm-apache-httpclient/apm-apache-httpclient4-plugin/src/test/java/co/elastic/apm/agent/httpclient/v4/LegacyApacheHttpClientHttpUriRequestInstrumentationTest.java @@ -19,14 +19,18 @@ package co.elastic.apm.agent.httpclient.v4; import co.elastic.apm.agent.httpclient.AbstractHttpClientInstrumentationTest; +import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.entity.InputStreamEntity; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.util.EntityUtils; import org.junit.AfterClass; import org.junit.BeforeClass; +import java.io.ByteArrayInputStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -48,12 +52,30 @@ public static void close() { @Override protected void performGet(String path) throws Exception { + HttpGet request = new HttpGet(path); + performRequest(request); + } + + private static void performRequest(HttpRequest request) throws Exception { Method execute = client.getClass().getMethod("execute", HttpUriRequest.class); try { - HttpResponse response = (HttpResponse) execute.invoke(client, new HttpGet(path)); + HttpResponse response = (HttpResponse) execute.invoke(client, request); EntityUtils.consume(response.getEntity()); } catch (InvocationTargetException e) { throw (Exception) e.getTargetException(); } } + + @Override + protected boolean isBodyCapturingSupported() { + return true; + } + + @Override + protected void performPost(String path, byte[] content, String contentTypeHeader) throws Exception { + HttpPost request = new HttpPost(path); + request.setEntity(new InputStreamEntity(new ByteArrayInputStream(content))); + request.setHeader("Content-Type", contentTypeHeader); + performRequest(request); + } } diff --git a/apm-agent-plugins/apm-httpclient-core/src/main/java/co/elastic/apm/agent/httpclient/HttpClientHelper.java b/apm-agent-plugins/apm-httpclient-core/src/main/java/co/elastic/apm/agent/httpclient/HttpClientHelper.java index 2d62702437..61bb7e7e66 100644 --- a/apm-agent-plugins/apm-httpclient-core/src/main/java/co/elastic/apm/agent/httpclient/HttpClientHelper.java +++ b/apm-agent-plugins/apm-httpclient-core/src/main/java/co/elastic/apm/agent/httpclient/HttpClientHelper.java @@ -18,16 +18,26 @@ */ package co.elastic.apm.agent.httpclient; +import co.elastic.apm.agent.common.util.WildcardMatcher; import co.elastic.apm.agent.sdk.logging.Logger; import co.elastic.apm.agent.sdk.logging.LoggerFactory; -import co.elastic.apm.agent.tracer.TraceState; +import co.elastic.apm.agent.tracer.AbstractSpan; +import co.elastic.apm.agent.tracer.GlobalTracer; import co.elastic.apm.agent.tracer.Span; +import co.elastic.apm.agent.tracer.TraceState; +import co.elastic.apm.agent.tracer.configuration.WebConfiguration; +import co.elastic.apm.agent.tracer.dispatch.TextHeaderGetter; import javax.annotation.Nullable; import java.net.URI; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class HttpClientHelper { + private static final Pattern CHARSET_EXTRACTOR = Pattern.compile(";\\s*charset\\s*=\\s*((\"[^\"]+)|([^;\\s]+))"); + private static final Logger logger = LoggerFactory.getLogger(HttpClientHelper.class); public static final String EXTERNAL_TYPE = "external"; @@ -54,6 +64,9 @@ public static Span startHttpClientSpan(TraceState activeContext, String me @Nullable String scheme, @Nullable CharSequence hostName, int port) { Span span = activeContext.createExitSpan(); if (span != null) { + if (span.isSampled()) { + span.getContext().getHttp().getRequestBody().markEligibleForCapturing(); + } updateHttpSpanNameAndContext(span, method, uri, scheme, hostName, port); } if (logger.isTraceEnabled()) { @@ -62,6 +75,42 @@ public static Span startHttpClientSpan(TraceState activeContext, String me return span; } + public static boolean startRequestBodyCapture(@Nullable AbstractSpan abstractSpan, R request, TextHeaderGetter headerGetter) { + if (!(abstractSpan instanceof Span)) { + return false; + } + Span span = (Span) abstractSpan; + if (!span.getContext().getHttp().getRequestBody().isEligibleForCapturing()) { + return false; + } + WebConfiguration webConfig = GlobalTracer.get().getConfig(WebConfiguration.class); + int byteCount = webConfig.getCaptureClientRequestBytes(); + if (byteCount == 0) { + return false; + } + List contentTypes = webConfig.getCaptureContentTypes(); + String contentTypeHeader = headerGetter.getFirstHeader("Content-Type", request); + if (contentTypeHeader == null) { + contentTypeHeader = ""; + } + if (WildcardMatcher.anyMatch(contentTypes, contentTypeHeader) == null) { + return false; + } + String charset = extractCharsetFromContentType(contentTypeHeader); + return span.getContext().getHttp().getRequestBody().startCapture(charset, byteCount); + } + + //Visible for testing + @Nullable + static String extractCharsetFromContentType(String contentTypeHeader) { + Matcher matcher = CHARSET_EXTRACTOR.matcher(contentTypeHeader); + if (matcher.find()) { + String potentiallyQuotedCharset = matcher.group(1); + return potentiallyQuotedCharset.replace("\"", ""); + } + return null; + } + public static void updateHttpSpanNameAndContext(Span span, String method, @Nullable String uri, String scheme, CharSequence hostName, int port) { span.withType(EXTERNAL_TYPE) .withSubtype(HTTP_SUBTYPE) diff --git a/apm-agent-plugins/apm-httpclient-core/src/main/java/co/elastic/apm/agent/httpclient/RequestBodyRecordingInputStream.java b/apm-agent-plugins/apm-httpclient-core/src/main/java/co/elastic/apm/agent/httpclient/RequestBodyRecordingInputStream.java new file mode 100644 index 0000000000..f8e228eaf8 --- /dev/null +++ b/apm-agent-plugins/apm-httpclient-core/src/main/java/co/elastic/apm/agent/httpclient/RequestBodyRecordingInputStream.java @@ -0,0 +1,104 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package co.elastic.apm.agent.httpclient; + +import co.elastic.apm.agent.tracer.Span; +import co.elastic.apm.agent.tracer.metadata.BodyCapture; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.InputStream; + +public class RequestBodyRecordingInputStream extends InputStream { + + private final InputStream delegate; + + @Nullable + private Span clientSpan; + + public RequestBodyRecordingInputStream(InputStream delegate, Span clientSpan) { + this.delegate = delegate; + clientSpan.incrementReferences(); + this.clientSpan = clientSpan; + } + + + protected void appendToBody(byte b) { + if (clientSpan != null) { + BodyCapture requestBody = clientSpan.getContext().getHttp().getRequestBody(); + requestBody.append(b); + if (requestBody.isFull()) { + releaseSpan(); + } + } + } + + protected void appendToBody(byte[] b, int off, int len) { + if (clientSpan != null) { + BodyCapture requestBody = clientSpan.getContext().getHttp().getRequestBody(); + requestBody.append(b, off, len); + if (requestBody.isFull()) { + releaseSpan(); + } + } + } + + public void releaseSpan() { + if (clientSpan != null) { + clientSpan.decrementReferences(); + clientSpan = null; + } + } + + @Override + public int read() throws IOException { + int character = delegate.read(); + if (character == -1) { + releaseSpan(); + } else { + appendToBody((byte) character); + } + return character; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int readBytes = delegate.read(b, off, len); + if (readBytes == -1) { + releaseSpan(); + } else { + appendToBody(b, off, readBytes); + } + return readBytes; + } + + @Override + public int available() throws IOException { + return delegate.available(); + } + + @Override + public void close() throws IOException { + try { + releaseSpan(); + } finally { + delegate.close(); + } + } +} diff --git a/apm-agent-plugins/apm-httpclient-core/src/main/java/co/elastic/apm/agent/httpclient/RequestBodyRecordingOutputStream.java b/apm-agent-plugins/apm-httpclient-core/src/main/java/co/elastic/apm/agent/httpclient/RequestBodyRecordingOutputStream.java new file mode 100644 index 0000000000..aaf640c1d3 --- /dev/null +++ b/apm-agent-plugins/apm-httpclient-core/src/main/java/co/elastic/apm/agent/httpclient/RequestBodyRecordingOutputStream.java @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package co.elastic.apm.agent.httpclient; + +import co.elastic.apm.agent.tracer.Span; +import co.elastic.apm.agent.tracer.metadata.BodyCapture; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.OutputStream; + +public class RequestBodyRecordingOutputStream extends OutputStream { + + private final OutputStream delegate; + + @Nullable + private Span clientSpan; + + public RequestBodyRecordingOutputStream(OutputStream delegate, Span clientSpan) { + this.delegate = delegate; + clientSpan.incrementReferences(); + this.clientSpan = clientSpan; + } + + @Override + public void write(int b) throws IOException { + try { + appendToBody((byte) b); + } finally { + delegate.write(b); + } + } + + protected void appendToBody(byte b) { + if (clientSpan != null) { + BodyCapture body = clientSpan.getContext().getHttp().getRequestBody(); + body.append(b); + if (body.isFull()) { + releaseSpan(); + } + } + } + + @Override + public void write(byte[] b) throws IOException { + try { + appendToBody(b, 0, b.length); + } finally { + delegate.write(b); + } + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + try { + appendToBody(b, off, len); + } finally { + delegate.write(b, off, len); + } + } + + protected void appendToBody(byte[] b, int off, int len) { + if (clientSpan != null) { + BodyCapture body = clientSpan.getContext().getHttp().getRequestBody(); + body.append(b, off, len); + if (body.isFull()) { + releaseSpan(); + } + } + } + + public void releaseSpan() { + if (clientSpan != null) { + clientSpan.decrementReferences(); + clientSpan = null; + } + } + + @Override + public void close() throws IOException { + try { + releaseSpan(); + } finally { + delegate.close(); + } + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } +} diff --git a/apm-agent-plugins/apm-httpclient-core/src/test/java/co/elastic/apm/agent/httpclient/AbstractHttpClientInstrumentationTest.java b/apm-agent-plugins/apm-httpclient-core/src/test/java/co/elastic/apm/agent/httpclient/AbstractHttpClientInstrumentationTest.java index 7d0f33372f..788757662f 100644 --- a/apm-agent-plugins/apm-httpclient-core/src/test/java/co/elastic/apm/agent/httpclient/AbstractHttpClientInstrumentationTest.java +++ b/apm-agent-plugins/apm-httpclient-core/src/test/java/co/elastic/apm/agent/httpclient/AbstractHttpClientInstrumentationTest.java @@ -21,16 +21,18 @@ import co.elastic.apm.agent.AbstractInstrumentationTest; import co.elastic.apm.agent.configuration.CoreConfigurationImpl; import co.elastic.apm.agent.impl.TextHeaderMapAccessor; +import co.elastic.apm.agent.impl.context.BodyCaptureImpl; import co.elastic.apm.agent.impl.context.DestinationImpl; import co.elastic.apm.agent.impl.context.HttpImpl; import co.elastic.apm.agent.impl.transaction.SpanImpl; import co.elastic.apm.agent.impl.transaction.TraceContextImpl; import co.elastic.apm.agent.impl.transaction.TransactionImpl; -import co.elastic.apm.agent.tracer.util.ResultUtil; import co.elastic.apm.agent.tracer.Outcome; -import co.elastic.apm.agent.tracer.TraceState; import co.elastic.apm.agent.tracer.Scope; +import co.elastic.apm.agent.tracer.TraceState; +import co.elastic.apm.agent.tracer.configuration.WebConfiguration; import co.elastic.apm.agent.tracer.dispatch.TextHeaderGetter; +import co.elastic.apm.agent.tracer.util.ResultUtil; import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder; import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import com.github.tomakehurst.wiremock.http.HttpHeader; @@ -45,6 +47,8 @@ import org.junit.Test; import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -52,6 +56,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import static co.elastic.apm.agent.impl.transaction.TraceContextImpl.W3C_TRACE_PARENT_TEXTUAL_HEADER_NAME; import static co.elastic.apm.agent.testutils.assertions.Assertions.assertThat; @@ -114,6 +119,29 @@ public void testHttpCall() { expectSpan(path).verify(); } + @Test + public void testPostBodyCapture() throws Exception { + if (!isBodyCapturingSupported()) { + return; + } + doReturn(5).when(config.getConfig(WebConfiguration.class)).getCaptureClientRequestBytes(); + byte[] content = "Hello World!".getBytes(StandardCharsets.UTF_8); + String path = "/"; + performPost(getBaseUrl() + path, content, "text/plain; charset=utf-8"); + expectSpan(path) + .withRequestBodySatisfying(body -> { + ByteBuffer buffer = body.getBody(); + assertThat(buffer).isNotNull(); + int numBytes = buffer.position(); + buffer.position(0); + byte[] contentBytes = new byte[numBytes]; + buffer.get(contentBytes); + assertThat(contentBytes).isEqualTo("Hello".getBytes(StandardCharsets.UTF_8)); + assertThat(Objects.toString(body.getCharset())).isEqualTo("utf-8"); + }) + .verify(); + } + @Test public void testDisabledOutgoingHeaders() { doReturn(true).when(config.getConfig(CoreConfigurationImpl.class)).isOutgoingTraceContextHeadersInjectionDisabled(); @@ -247,6 +275,11 @@ protected boolean isRedirectFollowingSupported() { return true; } + // assumption + protected boolean isBodyCapturingSupported() { + return false; + } + // assumption protected boolean isErrorOnCircularRedirectSupported() { return isRedirectFollowingSupported(); @@ -280,6 +313,10 @@ public boolean isAsync() { protected abstract void performGet(String path) throws Exception; + protected void performPost(String path, byte[] content, String contentTypeHeader) throws Exception { + throw new UnsupportedOperationException(); + } + protected VerifyBuilder expectSpan(String path) { return new VerifyBuilder(path); } @@ -292,6 +329,8 @@ protected class VerifyBuilder { private boolean traceContextHeaders = true; private boolean requestExecuted = true; + private Consumer requestBodyVerification = null; + private VerifyBuilder(String path) { this.path = path; } @@ -323,6 +362,11 @@ public VerifyBuilder withoutRequestExecuted() { return this; } + public VerifyBuilder withRequestBodySatisfying(Consumer requestBodyVerification) { + this.requestBodyVerification = requestBodyVerification; + return this; + } + public SpanImpl verify() { assertThat(reporter.getFirstSpan(500)).isNotNull(); assertThat(reporter.getSpans()).hasSize(1); @@ -351,6 +395,11 @@ public SpanImpl verify() { assertThat(span).hasOutcome(Outcome.FAILURE); } + BodyCaptureImpl reqBody = span.getContext().getHttp().getRequestBody(); + if (requestBodyVerification != null) { + requestBodyVerification.accept(reqBody); + } + if (isAsync()) { assertThat(span).isAsync(); } diff --git a/apm-agent-plugins/apm-httpclient-core/src/test/java/co/elastic/apm/agent/httpclient/HttpClientHelperTest.java b/apm-agent-plugins/apm-httpclient-core/src/test/java/co/elastic/apm/agent/httpclient/HttpClientHelperTest.java index e40199c27f..9242d167ed 100644 --- a/apm-agent-plugins/apm-httpclient-core/src/test/java/co/elastic/apm/agent/httpclient/HttpClientHelperTest.java +++ b/apm-agent-plugins/apm-httpclient-core/src/test/java/co/elastic/apm/agent/httpclient/HttpClientHelperTest.java @@ -123,4 +123,17 @@ private void createSpanWithUrl(String s) throws URISyntaxException { HttpClientHelper.startHttpClientSpan(tracer.getActive(), "GET", new URI(s), null) .end(); } + + @Test + void testContentTypeCharsetExtraction() { + assertThat(HttpClientHelper.extractCharsetFromContentType("multipart/form-data; boundary=---------------------------974767299852498929531610575")) + .isNull(); + assertThat(HttpClientHelper.extractCharsetFromContentType("Content-Type: text/html; charset=utf-8")) + .isEqualTo("utf-8"); + assertThat(HttpClientHelper.extractCharsetFromContentType("Content-Type: text/html; charset = foobar;baz")) + .isEqualTo("foobar"); + assertThat(HttpClientHelper.extractCharsetFromContentType("Content-Type: application/json; charset = \"foo bar\";baz")) + .isEqualTo("foo bar"); + + } } diff --git a/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/configuration/WebConfiguration.java b/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/configuration/WebConfiguration.java index d7fdb00f91..2a35facaf5 100644 --- a/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/configuration/WebConfiguration.java +++ b/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/configuration/WebConfiguration.java @@ -27,8 +27,12 @@ import java.util.Collections; import java.util.List; +import static co.elastic.apm.agent.tracer.configuration.RangeValidator.isInRange; + public class WebConfiguration extends ConfigurationOptionProvider { + public static final int MAX_BODY_CAPTURE_BYTES = 1024; + private static final String HTTP_CATEGORY = "HTTP"; private final ConfigurationOption> captureContentTypes = ConfigurationOption @@ -121,6 +125,17 @@ public class WebConfiguration extends ConfigurationOptionProvider { .dynamic(true) .buildWithDefault(Collections.emptyList()); + private final ConfigurationOption captureClientRequestBytes = ConfigurationOption.integerOption() + .addValidator(isInRange(0, MAX_BODY_CAPTURE_BYTES)) + .key("capture_http_client_request_body_size") + .configurationCategory(HTTP_CATEGORY) + .tags("added[1.50.0]", "internal") + .description("Configures how many bytes of http-client request bodies shall be captured. " + + "Note that only request bodies will be captured for content types matching the capture_body_content_types configuration. " + + " The maximum allowed value is " + MAX_BODY_CAPTURE_BYTES + " , a value of 0 disables body capturing") + .dynamic(true) + .buildWithDefault(0); + public List getIgnoreUrls() { return ignoreUrls.get(); } @@ -141,4 +156,8 @@ public List getCaptureContentTypes() { return captureContentTypes.get(); } + public int getCaptureClientRequestBytes() { + return captureClientRequestBytes.get(); + } + } diff --git a/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/metadata/BodyCapture.java b/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/metadata/BodyCapture.java new file mode 100644 index 0000000000..ef8df98de0 --- /dev/null +++ b/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/metadata/BodyCapture.java @@ -0,0 +1,47 @@ +package co.elastic.apm.agent.tracer.metadata; + +import javax.annotation.Nullable; + +public interface BodyCapture { + + /** + * Requests that the body for this span may be captured. + * Whether it is actually captured may depend on further details not known yet when this method is called + * (e.g. the Content-Type header). + */ + void markEligibleForCapturing(); + + /** + * @return true, if {@link #markEligibleForCapturing()} was called for this span. + */ + boolean isEligibleForCapturing(); + + /** + * This method acts as a protection mechanism so that only one instrumentation tries to capture the body. + * It returns true, if the calling instrumentation shall start adding body byte via {@link #append(byte)}. + *

+ * For this to happen, {@link #markEligibleForCapturing()} must have been called first. + *

+ * After {@link #startCapture(String, int)} has returned true once, subsequent calls will return false. + * So for example if instrumentation A and B are active for the same span, only the first one will actually be capturing the body, + * because {@link #startCapture(String, int)} only returns true once. + * + * @param charset the charset (if available) with which the request-body is encoded. + * @param numBytesToCapture the number of bytes to capture, to configure the limit of the internal buffer + * + * @return true, if the calling instrumentation should be capturing the body (by calling {@link #append(byte)} + */ + boolean startCapture(@Nullable String charset, int numBytesToCapture); + + void append(byte b); + + void append(byte[] b, int offset, int len); + + /** + * Checks if the limit number of bytes to capture has been reached. In this case future append calls would be a no-op. + * If this is the case, the caller can consider releasing the reference to the span to prevent potential memory leaks. + * + * @return true, if the maximum number of bytes supported has already been captured + */ + boolean isFull(); +} diff --git a/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/metadata/Http.java b/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/metadata/Http.java index d4d614c039..aad5ad3d85 100644 --- a/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/metadata/Http.java +++ b/apm-agent-tracer/src/main/java/co/elastic/apm/agent/tracer/metadata/Http.java @@ -39,4 +39,6 @@ public interface Http { @Nullable String getMethod(); + + BodyCapture getRequestBody(); }