From 27b9c5c7c2cec35f35e5d85f263747b0801aca93 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Sun, 12 Apr 2026 16:38:02 +0530 Subject: [PATCH 1/2] fix: HTTP2 nil pointer panic and broken GOMEMLIMIT under hostPID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes found during production health check across 3 clusters: 1. HTTP2 nil pointer dereference (ebpftracer/l7/http2.go): When maxActiveRequests limit is hit, the break exits the switch but decoder.Write() still runs with a stale emit func from a previous call, dereferencing a nil request. Set a no-op emit func before breaking so HPACK dynamic table stays in sync without the panic. This was causing 100K+ recovered panics per 6h across KP and DEV, wasting CPU and inflating page cache by ~150Mi per affected pod. 2. GOMEMLIMIT never set under hostPID (main.go): readCgroupMemoryLimit() reads /sys/fs/cgroup/memory.max which doesn't exist under the agent's init.scope cgroup (hostPID places PID 1 at the host root). The pod's actual 1Gi limit lives at /sys/fs/cgroup/kubepods.slice/.../cri-containerd-xxx.scope/memory.max. Fix: read /proc/self/cgroup to resolve the container's own cgroup path. Without this, GOMEMLIMIT was dead code on all clusters — GC didn't trigger during ELF parsing spikes, letting RSS hit 1Gi and causing OOM kills. --- ebpftracer/l7/http2.go | 5 ++++- main.go | 23 ++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/ebpftracer/l7/http2.go b/ebpftracer/l7/http2.go index 7c484d5..13df64e 100644 --- a/ebpftracer/l7/http2.go +++ b/ebpftracer/l7/http2.go @@ -263,7 +263,10 @@ func (p *Http2Parser) decodeHeaderBlock( req := p.activeRequests[streamId] if req == nil { if len(p.activeRequests) >= maxActiveRequests { - // Too many active streams; skip to prevent unbounded growth + // Too many active streams; set no-op emit so decoder.Write still + // processes the HPACK block (keeps dynamic table in sync) without + // dereferencing a nil request. + decoder.SetEmitFunc(func(hf hpack.HeaderField) {}) break } req = &Http2Request{ diff --git a/main.go b/main.go index b200bfc..96a0c11 100644 --- a/main.go +++ b/main.go @@ -297,8 +297,29 @@ func (o *RateLimitedLogOutput) Write(data []byte) (int, error) { } // readCgroupMemoryLimit reads the memory limit from cgroup v2 or v1. +// The agent runs with hostPID, so /proc/1/cgroup points to the host root (init.scope) +// which has no memory limit. We use /proc/self/cgroup to find the container's own +// cgroup path where kubelet enforces the pod memory limit. func readCgroupMemoryLimit() (int64, error) { - // Try cgroup v2 first + // Try cgroup v2: read our own cgroup path and look up memory.max there. + if data, err := os.ReadFile("/proc/self/cgroup"); err == nil { + // cgroupv2 format: "0::/kubepods.slice/.../cri-containerd-xxx.scope" + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + parts := strings.SplitN(line, ":", 3) + if len(parts) == 3 && parts[0] == "0" { + cgPath := "/sys/fs/cgroup" + parts[2] + "/memory.max" + if mem, err := os.ReadFile(cgPath); err == nil { + s := strings.TrimSpace(string(mem)) + if s != "max" { + if limit, err := strconv.ParseInt(s, 10, 64); err == nil { + return limit, nil + } + } + } + } + } + } + // Fallback: try cgroup v2 at root (non-hostPID containers) if data, err := os.ReadFile("/sys/fs/cgroup/memory.max"); err == nil { s := strings.TrimSpace(string(data)) if s != "max" { From b0d75f68054b7f5ac7bf7e745aa0d5dcc92f26a0 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Mon, 13 Apr 2026 11:06:53 +0530 Subject: [PATCH 2/2] fix: handle cgroup v1 under hostPID in readCgroupMemoryLimit Parse /proc/self/cgroup for both v2 (hierarchy-id "0") and v1 ("memory" controller) to resolve the container's own cgroup path. Previously only v2 was handled via /proc/self/cgroup; v1 clusters with hostPID would still fall through to the root path which reports the host limit, not the pod limit. --- main.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 96a0c11..dfbe008 100644 --- a/main.go +++ b/main.go @@ -301,17 +301,26 @@ func (o *RateLimitedLogOutput) Write(data []byte) (int, error) { // which has no memory limit. We use /proc/self/cgroup to find the container's own // cgroup path where kubelet enforces the pod memory limit. func readCgroupMemoryLimit() (int64, error) { - // Try cgroup v2: read our own cgroup path and look up memory.max there. + // Parse /proc/self/cgroup to find the container's own cgroup path. + // cgroupv2: "0::/kubepods.slice/.../cri-containerd-xxx.scope" + // cgroupv1: "N:memory:/kubepods/burstable/pod-xxx/container-yyy" if data, err := os.ReadFile("/proc/self/cgroup"); err == nil { - // cgroupv2 format: "0::/kubepods.slice/.../cri-containerd-xxx.scope" for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { parts := strings.SplitN(line, ":", 3) - if len(parts) == 3 && parts[0] == "0" { - cgPath := "/sys/fs/cgroup" + parts[2] + "/memory.max" + if len(parts) < 3 { + continue + } + var cgPath string + if parts[0] == "0" { // cgroup v2 + cgPath = "/sys/fs/cgroup" + parts[2] + "/memory.max" + } else if strings.Contains(parts[1], "memory") { // cgroup v1 + cgPath = "/sys/fs/cgroup/memory" + parts[2] + "/memory.limit_in_bytes" + } + if cgPath != "" { if mem, err := os.ReadFile(cgPath); err == nil { s := strings.TrimSpace(string(mem)) if s != "max" { - if limit, err := strconv.ParseInt(s, 10, 64); err == nil { + if limit, err := strconv.ParseInt(s, 10, 64); err == nil && limit > 0 && limit < 1<<62 { return limit, nil } } @@ -328,11 +337,10 @@ func readCgroupMemoryLimit() (int64, error) { } } } - // Try cgroup v1 + // Fallback: try cgroup v1 at root (non-hostPID containers) if data, err := os.ReadFile("/sys/fs/cgroup/memory/memory.limit_in_bytes"); err == nil { s := strings.TrimSpace(string(data)) if limit, err := strconv.ParseInt(s, 10, 64); err == nil { - // cgroup v1 reports a very large number when unlimited if limit < 1<<62 { return limit, nil }