From dd66e8fd15737c15c4d6d3a1bc86092a2b10be71 Mon Sep 17 00:00:00 2001 From: Mayank Pande Date: Fri, 17 Jan 2025 14:32:41 +0530 Subject: [PATCH 1/6] feat: fix for trace id using headers --- containers/container.go | 7 +----- ebpftracer/l7/http.go | 6 +++-- ebpftracer/l7/l7_test.go | 14 ++++++------ flags/flags.go | 1 + tracing/tracing.go | 48 ++++++++++++++++++++++++++++------------ 5 files changed, 47 insertions(+), 29 deletions(-) diff --git a/containers/container.go b/containers/container.go index b8cfacef..b0f1fe66 100644 --- a/containers/container.go +++ b/containers/container.go @@ -743,7 +743,6 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R case l7.ProtocolHTTP: stats.observe(r.Status.Http(), "", r.Duration) payload := "" - headers := "" method := "" uri := "" response := "" @@ -759,10 +758,6 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R base64String := base64.StdEncoding.EncodeToString(body) payload = string(base64String) } - if req != nil && req.Header != nil { - headersStr := l7.ConvertHeadersToString(req.Header) - headers = base64.StdEncoding.EncodeToString([]byte(headersStr)) - } method = req.Method if utf8.ValidString(req.URL.Path) { uri = req.URL.Path @@ -776,7 +771,7 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R if r.Response != nil { response = base64.StdEncoding.EncodeToString(r.Response) } - trace.HttpRequest(method, uri, r.Status, r.Duration, r.PayloadSize, payload, headers, response, host) + trace.HttpRequest(method, uri, r.Status, r.Duration, r.PayloadSize, payload, req.Header, response, host) case l7.ProtocolHTTP2: if conn.http2Parser == nil { conn.http2Parser = l7.NewHttp2Parser() diff --git a/ebpftracer/l7/http.go b/ebpftracer/l7/http.go index 01a8a4a6..1d5fce4a 100644 --- a/ebpftracer/l7/http.go +++ b/ebpftracer/l7/http.go @@ -2,6 +2,7 @@ package l7 import ( "bytes" + "encoding/base64" "encoding/json" "errors" "io" @@ -129,7 +130,7 @@ func ParseHostFromHttpRequest(input string) (string, error) { return host, nil } -func ConvertHeadersToString(headers http.Header) string { +func ConvertHeadersToBase64String(headers http.Header) string { headerMap := make(map[string][]string) for key, values := range headers { headerMap[key] = values @@ -138,7 +139,8 @@ func ConvertHeadersToString(headers http.Header) string { if err != nil { return "" } - return string(jsonString) + header64 := base64.StdEncoding.EncodeToString([]byte(jsonString)) + return string(header64) } func SanitizeString(value string) string { diff --git a/ebpftracer/l7/l7_test.go b/ebpftracer/l7/l7_test.go index ad4194b0..6437d1df 100644 --- a/ebpftracer/l7/l7_test.go +++ b/ebpftracer/l7/l7_test.go @@ -108,7 +108,7 @@ func TestParseHost(t *testing.T) { assert.Nil(t, nil, err) assert.Equal(t, "xzxxxxxxzx", string(body)) - headers := ConvertHeadersToString(req.Header) + headers := ConvertHeadersToBase64String(req.Header) assert.NotNil(t, headers) requestString = "POST / HTTP/1.1\r\nHost: ec2.us-east-1.amazonaws.com\r\nUser-Agent: aws-sdk-go/1.50.10 (go1.21.6; linux; amd64) karpenter.sh-v0.34.0\r\nContent-Length: 422\r\nAuthorization: AWS4-HMAC-SHA256 Credential=ASIAUCTZOIG67J6IJFOY/20240408/us-east-1/ec2/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-security-token, Signature=3d7eadceec976dd17fbdcc07ae43a75264baf179fec70c6aa2be7411fd552e70\r\nContent-Type: application/x-www-form-urlencoded; charset=utf-8\r\nX-Amz-Date: 20240408T094330Z\r\nX-Amz-Security-Token: IQoJb3JpZ2luX2VjEML//////////wEaCXVzLWVhc3QtMSJHMEUCIB8pKvU76WFQ99xqZH8LQXhOBfCP/lhnMSkmIp1LcvLEAiEAk3XkJCPMFGQC4KqMk92boxaeNHiLvbinmW9AQgEGnBcq/gQI6///////////ARABGgwyODA1MDEzMDU3ODkiDJA/sBIYp1t0z9szDirSBPDDQ+mKiBwDvCwqZ/HH9wSh9U6WKYjh0EPX9i3cHE46oee4CAUoQwEmDy9xZp02UZcpOjiDF6iaKyOvi93+LoiwJVBNi53o/PnMoStfI4OmG4Rc61YYcPc6t6SSXeBEGCzeeesWAGBDJ6GE2PWYYQAKS3YrpxYE+UKdb4hxEBO5t8RgEO/ooiHehFC/5VSmjMeMfT/XMdmAxBplX/wCMoz2vMcpkCi3fCsX0RNjxT7bAO/D2jOUrIX8UjM38h2VGNliSfCl8B3HbCETxYiz66AhMGabknMpJ8Dcotm\x00" @@ -118,7 +118,7 @@ func TestParseHost(t *testing.T) { assert.Equal(t, "ec2.us-east-1.amazonaws.com", req.Host) assert.Equal(t, "POST", req.Method) - headers = ConvertHeadersToString(req.Header) + headers = ConvertHeadersToBase64String(req.Header) assert.NotNil(t, headers) requestString = "POST / HTTP/1.1\r\nHost: ec2.us-east-1.amazonaws.com\r\nUser-Agent: aws-sdk-go/1.50.10 (go1.21.6; linux; amd64) karpenter.sh-v0.34.0\r\nContent-Length: 422\r\nAuthorization: AWS4-HMAC-SHA256 Credential=ASIAUCTZOIG67J6IJFOY/20240408/us-east-1/ec2/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-security-token, Signature=3d7eadceec976dd17fbdcc07ae43a75264baf179fec70c6aa2be7411fd552e70\r\nContent-Type: application/x-www-form-urlencoded; charset=utf-8\r\nX-Amz-Date: 20240408T094330Z\r\nX-Amz-Security-Token: IQoJb3JpZ2luX2VjEML//////////wEaCXVzLWVhc3QtMSJHMEUCIB8pKvU76WFQ99xqZH8LQXhOBfCP/lhnMSkmIp1LcvLEAiEAk3XkJCPMFGQC4KqMk92boxaeNHiLvbinmW9AQgEGnBcq/gQI6///////////ARABGgwyODA1MDEzMDU3ODkiDJA/sBIYp1t0z9szDirSBPDDQ+mKiBwDvCwqZ/HH9wSh9U6WKYjh0EPX9i3cHE46oee4CAUoQwEmDy9xZp02UZcpOjiDF6iaKyOvi93+LoiwJVBNi53o/PnMoStfI4OmG4Rc61YYcPc6t6SSXeBEGCzeeesWAGBDJ6GE2PWYYQAKS3YrpxYE+UKdb4hxEBO5t8RgEO/ooiHehFC/5VSmjMeMfT/XMdmAxBplX/wCMoz2vMcpkCi3fCsX0RNjxT7bAO/D2jOUrIX8UjM38h2VGNliSfCl8B3HbCETxYiz66AhMGabknMpJ8Dcotm\x00" @@ -128,7 +128,7 @@ func TestParseHost(t *testing.T) { assert.Equal(t, "ec2.us-east-1.amazonaws.com", req.Host) assert.Equal(t, "POST", req.Method) - headers = ConvertHeadersToString(req.Header) + headers = ConvertHeadersToBase64String(req.Header) assert.NotNil(t, headers) requestString = "GET /api/v1/namespaces/nudgebee-agent/pods/nudgebee-agent-pgnxh/log?container=node-agent&previous=False&tailLines=1000×tamps=True HTTP/1.1\r\nHost: 10.100.0.1\r\nAccept-Encoding: identity\r\nAccept: application/json\r\nUser-Agent: OpenAPI-Generator/26.1.0/python\r\nauthorization: bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjNhZTFlNjlkN2I1ZmFkNzhhNTg2YTVjN2JmMDNjOGU3YjUzNGI4N2MifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjIl0sImV4cCI6MTc0NDEwMjY0OSwiaWF0IjoxNzEyNTY2NjQ5LCJpc3MiOiJodHRwczovL29pZGMuZWtzLnVzLWVhc3QtMS5hbWF6b25hd3MuY29tL2lkL0I4RkU4OUQ4MjdEN0Q2RTE0REIyMUMzRkIwQTk1Q0E3Iiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJudWRnZWJlZS1hZ2VudCIsInBvZCI6eyJuYW1lIjoibnVkZ2ViZWUtYWdlbnQtcnVubmVyLTZiZmY2NmQ3YjgtcWttbmwiLCJ1aWQiOiJiN2M4ZGM2NC04Y2Q0LTRiMWUtODg1Zi00YWMzMWE4NDYwMWIifSwic2VydmljZWFjY291bnQiOnsibmFtZSI6Im51ZGdlYmVlLWFnZW50LXJ1bm5lci1zZXJ2aWNlLWFjY291bnQiLCJ1aWQiOiJjODUzMDc2YS1hODVkLTQzYzItOWU3NS00MTk5ODIzMTVjOGIifSwid2FybmFmdGVyIjoxNzEyNTcwMjU2fSwibmJmIjoxNzEyNTY2NjQ5LCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6b\x00" @@ -138,7 +138,7 @@ func TestParseHost(t *testing.T) { assert.Equal(t, "10.100.0.1", req.Host) assert.Equal(t, "GET", req.Method) - headers = ConvertHeadersToString(req.Header) + headers = ConvertHeadersToBase64String(req.Header) assert.NotNil(t, headers) requestString = "PUT /nudgebee-dev-loki-logs/fake/ece64b2028d30d33/18ebcc10d27%3A18ebcfd01d3%3A3377d9aa HTTP/1.1\r\nHost: s3.amazonaws.com\r\nUser-Agent: aws-sdk-go/1.44.315 (go1.21.3; linux; amd64)\r\nContent-Length: 14183\r\nAuthorization: AWS4-HMAC-SHA256 Credential=ASIAUCTZOIG62SVE2F5J/20240408/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-storage-class, Signature=a4db153ba7127721c3a5efbbab1a0e194e473d23147a472c92ff672d9dff160c\r\nContent-Md5: VBSxwLVjCvMeC+z4V86k8A==\r\nX-Amz-Content-Sha256: fe19171172133fadb236229e71a8f60169fc5268da749b1eeeeb3396e7d90f64\r\nX-Amz-Date: 20240408T094454Z\r\nX-Amz-Security-Token: IQoJb3JpZ2luX2VjEL3//////////wEaCXVzLWVhc3QtMSJGMEQCIAvNhx/LC+nIkqUC+rRD3MOFuxr39bc+qmRwVPljywxuAiA/8bmpr+kmRy7zXBzBQ7eyayLftqLutyGrzu8TIZWVbyrGBQjm//////////8BEAEaDDI4MDUwMTMwNTc4OSIMRsng8zavN9ilo6+0KpoFc/pdelkZBjWQle/CNG4XKcnd9WFbu87JAB3xa1P97vfIBjhqpmHoPk4V38r21Vepl0NjeUnWahJA6NnOWW2mdhRv51+WGUmKSHz1j6NCRcC8axDXs65jJzF6WgqCGZGNa0aiUQP7oIGGV8\x00" @@ -148,7 +148,7 @@ func TestParseHost(t *testing.T) { assert.Equal(t, "s3.amazonaws.com", req.Host) assert.Equal(t, "PUT", req.Method) - headers = ConvertHeadersToString(req.Header) + headers = ConvertHeadersToBase64String(req.Header) assert.NotNil(t, headers) requestString = "POST /api/v1/write HTTP/1.1\r\nHost: vmsingle-victoria-victoria-metrics-k8s-stack.victoria.svc:8429\r\nUser-Agent: vmagent\r\nContent-Length: 282715\r\nContent-Encoding: snappy\r\nContent-Type: application/x-protobuf\r\nX-Prometheus-Remote-Write-Version: 0.1.0\r\nAccept-Encoding: gzip\r\n\r\n\x91\x9b\xdc\x01\xb0\n\xe4\f\n$\n\b__name__\x12\x18container_memory_failcnt\n$\n\t\x15\x1c\xd0\x12\x17nudgebee-agent-opencost\n;\n\x05image\x122quay.io/kubecost1\x15\n\x00-\x01*X-model:prod-1.108.0\n\x1b\n\t\x01\x87\x18space\x12\x0e6c\x00 \n/\n\x03pod\x12(6\x17\x00\x15z\xd8-55b677fb47-87mz4\n \n\x17beta_kubernetes_io_arch\x12\x05amd64\n.\n J\"\x00pinstance_type\x12\nm5ad.large\n\x1e\n\x15J0\x00\xf0Cos\x12\x05linux\n&\n\x1eeks_amazonaws_com_capacityType\x12\x04SPOT\n5\n(failure_domain_JW\x00Pregion\x12\tus-east-1\n4\n&\x867\x00\x14zone\x12\n\x155\x10a\n(\n\b\x11Ҙ\x12\x1cip-172-31-0-236.ec2.internal\n\x0e\n\x03job\x12\a!1 let\n=\n\x19k8%5\xf0= 400, + traceId, semconv.HTTPURL(fmt.Sprintf("http://%s%s", requestHost, requestPath)), semconv.HTTPMethod(method), semconv.HTTPStatusCode(int(status)), @@ -225,7 +245,7 @@ func (t *Trace) Http2Request(method, path, scheme string, status l7.Status, dura requestPayload = payload } - t.createSpan(method, duration, status > 400, + t.createSpan(method, duration, status > 400, "", semconv.HTTPURL(fmt.Sprintf("%s://%s%s", scheme, t.destination.String(), path)), semconv.HTTPMethod(method), semconv.HTTPStatusCode(int(status)), @@ -237,7 +257,7 @@ func (t *Trace) PostgresQuery(query string, error bool, duration time.Duration) if t == nil || query == "" { return } - t.createSpan("query", duration, error, + t.createSpan("query", duration, error, "", semconv.DBSystemPostgreSQL, semconv.DBStatement(query), ) @@ -247,7 +267,7 @@ func (t *Trace) MysqlQuery(query string, error bool, duration time.Duration) { if t == nil || query == "" { return } - t.createSpan("query", duration, error, + t.createSpan("query", duration, error, "", semconv.DBSystemMySQL, semconv.DBStatement(query), ) @@ -257,7 +277,7 @@ func (t *Trace) MongoQuery(query string, error bool, duration time.Duration) { if t == nil || query == "" { return } - t.createSpan("query", duration, error, + t.createSpan("query", duration, error, "", semconv.DBSystemMongoDB, semconv.DBStatement(query), ) @@ -276,7 +296,7 @@ func (t *Trace) MemcachedQuery(cmd string, items []string, error bool, duration } else if len(items) > 1 { attrs = append(attrs, MemcacheDBItemKeyName.StringSlice(items)) } - t.createSpan(cmd, duration, error, attrs...) + t.createSpan(cmd, duration, error, "", attrs...) } func (t *Trace) RedisQuery(cmd, args string, error bool, duration time.Duration) { @@ -287,7 +307,7 @@ func (t *Trace) RedisQuery(cmd, args string, error bool, duration time.Duration) if args != "" { statement += " " + args } - t.createSpan(cmd, duration, error, + t.createSpan(cmd, duration, error, "", semconv.DBSystemRedis, semconv.DBOperation(cmd), semconv.DBStatement(statement), @@ -298,7 +318,7 @@ func (t *Trace) ClickhouseQuery(query string, error bool, duration time.Duration if t == nil { return } - t.createSpan("query", duration, error, + t.createSpan("query", duration, error, "", semconv.DBSystemClickhouse, semconv.DBStatement(query), ) @@ -315,7 +335,7 @@ func (t *Trace) ZookeeperRequest(op string, args string, status l7.Status, durat if args != "" { statement += " " + args } - t.createSpan(op, duration, status.Zookeeper() != "ok", + t.createSpan(op, duration, status.Zookeeper() != "ok", "", semconv.DBSystemKey.String("zookeeper"), semconv.DBOperation(op), semconv.DBStatementKey.String(statement), From 395f0ffaa6dcd2d8380cd1fca2859930c9954fb0 Mon Sep 17 00:00:00 2001 From: Mayank Pande Date: Fri, 17 Jan 2025 15:35:04 +0530 Subject: [PATCH 2/6] fix: fix for headers failure --- containers/container.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/containers/container.go b/containers/container.go index b0f1fe66..d9c53390 100644 --- a/containers/container.go +++ b/containers/container.go @@ -3,6 +3,7 @@ package containers import ( "io" "log" + "net/http" "os" "sort" "strings" @@ -747,11 +748,13 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R uri := "" response := "" host := "" + headers := http.Header{} req, err := l7.ParseHTTPRequest(r.Payload) if err != nil { log.Printf("Failed to parse payload %s, %q", err, string(r.Payload)) method, uri = l7.ParseHttp(r.Payload) host, _ = l7.ParseHostFromHttpRequest(string(r.Payload)) + headers = req.Header } else { if req != nil && req.Body != nil { body, _ := io.ReadAll(req.Body) @@ -771,7 +774,7 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R if r.Response != nil { response = base64.StdEncoding.EncodeToString(r.Response) } - trace.HttpRequest(method, uri, r.Status, r.Duration, r.PayloadSize, payload, req.Header, response, host) + trace.HttpRequest(method, uri, r.Status, r.Duration, r.PayloadSize, payload, headers, response, host) case l7.ProtocolHTTP2: if conn.http2Parser == nil { conn.http2Parser = l7.NewHttp2Parser() From 271895805e7b0524b02de332322806902845e486 Mon Sep 17 00:00:00 2001 From: Mayank Pande Date: Fri, 17 Jan 2025 16:15:48 +0530 Subject: [PATCH 3/6] fix: fix for traceparent header parsing: --- tracing/tracing.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tracing/tracing.go b/tracing/tracing.go index 47535057..014e815e 100644 --- a/tracing/tracing.go +++ b/tracing/tracing.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "fmt" "net/http" + "strings" "time" "unicode/utf8" @@ -161,7 +162,7 @@ func (t *Trace) createSpan(name string, duration time.Duration, error bool, trac start := end.Add(-duration) ctx := context.Background() if traceId != "" { - // Convert traceId to a TraceID and create a SpanContext + traceId = ParseTraceIdHeaders(traceId) traceID, err := trace.TraceIDFromHex(traceId) if err != nil { context.Background() @@ -181,6 +182,19 @@ func (t *Trace) createSpan(name string, duration time.Duration, error bool, trac span.End(trace.WithTimestamp(end)) } +func ParseTraceIdHeaders(traceId string) string { + if traceId == "" { + return "" + } + // if its a traceparent header, extract the traceId + parts := strings.Split(traceId, "-") + if len(parts) == 4 { + traceId = parts[1] + } + + return traceId +} + func (t *Trace) HttpRequest(method, path string, status l7.Status, duration time.Duration, requestSize uint64, payload string, headers http.Header, response string, host string) { if t == nil || method == "" { return From 156775a12368b355c13ad014b5d11609eb00223a Mon Sep 17 00:00:00 2001 From: Mayank Pande Date: Fri, 17 Jan 2025 16:33:35 +0530 Subject: [PATCH 4/6] fix: fix for trace parsing failure --- containers/container.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/containers/container.go b/containers/container.go index d9c53390..75fbe5eb 100644 --- a/containers/container.go +++ b/containers/container.go @@ -754,7 +754,9 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R log.Printf("Failed to parse payload %s, %q", err, string(r.Payload)) method, uri = l7.ParseHttp(r.Payload) host, _ = l7.ParseHostFromHttpRequest(string(r.Payload)) - headers = req.Header + if req.Header != nil { + headers = req.Header + } } else { if req != nil && req.Body != nil { body, _ := io.ReadAll(req.Body) From 5a195f477e14d7d62af1971f620d35da63246220 Mon Sep 17 00:00:00 2001 From: Mayank Pande Date: Fri, 17 Jan 2025 16:40:45 +0530 Subject: [PATCH 5/6] fix: fix for trace id header parsing --- containers/container.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/containers/container.go b/containers/container.go index 75fbe5eb..7e236f50 100644 --- a/containers/container.go +++ b/containers/container.go @@ -754,9 +754,6 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R log.Printf("Failed to parse payload %s, %q", err, string(r.Payload)) method, uri = l7.ParseHttp(r.Payload) host, _ = l7.ParseHostFromHttpRequest(string(r.Payload)) - if req.Header != nil { - headers = req.Header - } } else { if req != nil && req.Body != nil { body, _ := io.ReadAll(req.Body) @@ -772,6 +769,9 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R uri = string([]rune(req.URL.Path)) } host = req.Host + if req.Header != nil { + headers = req.Header + } } if r.Response != nil { response = base64.StdEncoding.EncodeToString(r.Response) From fcfe83b95e1d612efdb844531e638e763f827b67 Mon Sep 17 00:00:00 2001 From: Mayank Pande Date: Fri, 17 Jan 2025 17:32:42 +0530 Subject: [PATCH 6/6] fix: fix for multiple trace headers --- containers/l7.go | 2 +- flags/flags.go | 2 +- tracing/tracing.go | 12 ++++++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/containers/l7.go b/containers/l7.go index b925888a..3da4c98c 100644 --- a/containers/l7.go +++ b/containers/l7.go @@ -69,7 +69,7 @@ func (s L7Stats) get(protocol l7.Protocol, key common.DestinationKey, r *l7.Requ labels = append(labels, "method") case l7.ProtocolHTTP: method, path := l7.ParseHttp(r.Payload) - if ValidUtf8(r.Payload) { + if ValidUtf8([]byte(path)) { constLabels["path"] = path } else { log.Printf("Failed to parse path %s", path) diff --git a/flags/flags.go b/flags/flags.go index 168c1fac..df61a87a 100644 --- a/flags/flags.go +++ b/flags/flags.go @@ -46,7 +46,7 @@ var ( SensitiveHeader = kingpin.Flag("sensitive-headers", "sanitize headers using patterns").Default("Authorization, Cookie, X-Action-Token").Envar("SENSITIVE_HEADERS").String() DisableKubeProbe = kingpin.Flag("disable-kube-probe", "disable kube probe trace").Default("true").Envar("DISABLE_KUBE_PROBE").Bool() DisableSensitiveLogParsing = kingpin.Flag("disable-sensitive-log-parsing", "disable sensitive log parsing").Default("false").Envar("DISABLE_SENSITIVE_LOG_PARSING").Bool() - TraceIdHeaders = kingpin.Flag("trace-id-headers", "trace id headers").Default("X-Request-Id").Envar("TRACE_ID_HEADERS").String() + TraceIdHeaders = kingpin.Flag("trace-id-headers", "trace id headers").Default("Traceparent,X-Request-Id").Envar("TRACE_ID_HEADERS").String() ) func GetString(fl *string) string { diff --git a/tracing/tracing.go b/tracing/tracing.go index 014e815e..2fe68f31 100644 --- a/tracing/tracing.go +++ b/tracing/tracing.go @@ -223,8 +223,16 @@ func (t *Trace) HttpRequest(method, path string, status l7.Status, duration time traceId := "" if headers != nil { requestHeaders = l7.ConvertHeadersToBase64String(headers) - if id := headers.Get(*flags.TraceIdHeaders); id != "" { - traceId = id + traceIdHeaders := strings.Split(*flags.TraceIdHeaders, ",") + for _, header := range traceIdHeaders { + if id := headers.Get(header); id != "" { + // check for lowercase header + traceId = id + break + } else if id := headers.Get(strings.ToLower(header)); id != "" { + traceId = id + break + } } }