diff --git a/changelog.d/13786.feature b/changelog.d/13786.feature new file mode 100644 index 000000000000..e43d6a83afe1 --- /dev/null +++ b/changelog.d/13786.feature @@ -0,0 +1 @@ +Add `opentracing.request_headers_to_tag` config to specify which request headers extract and tag traces with in order to correlate timeouts in reverse-proxy layers in front of Synapse with traces. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index cd546041b2d4..8823727024bb 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3612,6 +3612,11 @@ Sub-options include: * `jaeger_config`: Jaeger can be configured to sample traces at different rates. All configuration options provided by Jaeger can be set here. Jaeger's configuration is mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/). +* `request_headers_to_tag`: A list of headers to extract from the request and + add to to the top-level servlet tracing span as tags. Useful when you're using + a reverse proxy service like Cloudflare to protect your Synapse instance in + order to correlate and match up requests that timed out at the Cloudflare + layer to the Synapse traces. Example configuration: ```yaml @@ -3629,6 +3634,9 @@ opentracing: param: 1 logging: false + + request_headers_to_tag: + - "cf-ray" ``` --- ## Workers ## diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 4a75eb6b21da..09ebd847a528 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -173,6 +173,21 @@ async def get_user_by_req( parent_span.set_tag("device_id", requester.device_id) if requester.app_service is not None: parent_span.set_tag("appservice_id", requester.app_service.id) + + # Tag any headers that we need to extract from the request. This + # is useful to specify any headers that a reverse-proxy in front + # of Synapse may be sending to correlate and match up something + # in that layer to a Synapse trace. ex. when Cloudflare times + # out it gives a `cf-ray` header which we can also tag here to + # find the trace. + for header_name in self.hs.config.tracing.request_headers_to_tag: + headers = request.requestHeaders.getRawHeaders(header_name) + if headers is not None: + parent_span.set_tag( + SynapseTags.REQUEST_HEADER_PREFIX + header_name, + str(headers[0]), + ) + return requester @cancellable diff --git a/synapse/config/tracer.py b/synapse/config/tracer.py index c19270c6c56c..4bcb16ab02ef 100644 --- a/synapse/config/tracer.py +++ b/synapse/config/tracer.py @@ -37,6 +37,13 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.force_tracing_for_users: Set[str] = set() + # A list of headers to extract from the request and add to to the + # top-level servlet tracing span as tags. Useful when you're using a + # reverse proxy service like Cloudflare to protect your Synapse instance + # in order to correlate and match up requests that timed out at the + # Cloudflare layer to the Synapse traces. + self.request_headers_to_tag = [] + if not self.opentracer_enabled: return @@ -62,3 +69,13 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: ("opentracing", "force_tracing_for_users", f"index {i}"), ) self.force_tracing_for_users.add(u) + + request_headers_to_tag = opentracing_config.get( + "request_headers_to_tag", + [], + ) + if not isinstance(request_headers_to_tag, list): + raise ConfigError( + "Expected a list", ("opentracing", "request_headers_to_tag") + ) + self.request_headers_to_tag = request_headers_to_tag diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index ca2735dd6dc0..0319ea72fcf6 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -303,6 +303,10 @@ class SynapseTags: # incoming HTTP request ID (as written in the logs) REQUEST_ID = "request_id" + # Tag a header from the incoming request. The name of the header should be + # appended to this prefix. + REQUEST_HEADER_PREFIX = "request_header." + # HTTP request tag (used to distinguish full vs incremental syncs, etc) REQUEST_TAG = "request_tag"