Skip to content

fix: reduce memory footprint by removing unused labels and shrinking eBPF maps #217

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mayankpande88 merged 2 commits into from
Mar 7, 2026
Merged

fix: reduce memory footprint by removing unused labels and shrinking eBPF maps #217

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4d11a47
fix: reduce memory footprint by removing unused labels and shrinking …
mayankpande88 Mar 7, 2026
51c543d
fix: sync MaxPayloadSize Go constant with eBPF MAX_PAYLOAD_SIZE (4096)
mayankpande88 Mar 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions containers/container.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ var (
)

const (
connectionStatsCacheSize = 4096 // LRU cache size for connection stats
connectionStatsCacheSize = 2048 // LRU cache size for connection stats
)

type ContainerID string
Expand Down Expand Up @@ -437,13 +437,13 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
workload_dest := enrichedKey.GetDestinationWorkload()
actualDestWorkload := enrichedKey.GetActualDestinationWorkload()

ch <- counter(metrics.NetConnectionsSuccessful, float64(stats.Count), enrichedKey.DestinationLabelValue(), enrichedKey.ActualDestinationLabelValue(), workload_src.Name, workload_src.Namespace, workload_src.Kind, workload_dest.Name, workload_dest.Namespace, workload_dest.Kind, actualDestWorkload.Name, actualDestWorkload.Namespace, actualDestWorkload.Kind, workload_src.Region, workload_src.Zone, workload_dest.Region, workload_dest.Zone, actualDestWorkload.Region, actualDestWorkload.Zone, actualDestWorkload.Instance)
ch <- counter(metrics.NetConnectionsTotalTime, stats.TotalTime.Seconds(), enrichedKey.DestinationLabelValue(), enrichedKey.ActualDestinationLabelValue(), workload_src.Name, workload_src.Namespace, workload_src.Kind, workload_dest.Name, workload_dest.Namespace, workload_dest.Kind, actualDestWorkload.Name, actualDestWorkload.Namespace, actualDestWorkload.Kind, workload_src.Region, workload_src.Zone, workload_dest.Region, workload_dest.Zone, actualDestWorkload.Region, actualDestWorkload.Zone, actualDestWorkload.Instance)
ch <- counter(metrics.NetConnectionsSuccessful, float64(stats.Count), enrichedKey.DestinationLabelValue(), enrichedKey.ActualDestinationLabelValue(), workload_src.Name, workload_src.Namespace, workload_src.Kind, workload_dest.Name, workload_dest.Namespace, workload_dest.Kind, actualDestWorkload.Name, actualDestWorkload.Namespace, actualDestWorkload.Kind)
ch <- counter(metrics.NetConnectionsTotalTime, stats.TotalTime.Seconds(), enrichedKey.DestinationLabelValue(), enrichedKey.ActualDestinationLabelValue(), workload_src.Name, workload_src.Namespace, workload_src.Kind, workload_dest.Name, workload_dest.Namespace, workload_dest.Kind, actualDestWorkload.Name, actualDestWorkload.Namespace, actualDestWorkload.Kind)
if stats.Retransmissions > 0 {
ch <- counter(metrics.NetRetransmits, float64(stats.Retransmissions), enrichedKey.DestinationLabelValue(), enrichedKey.ActualDestinationLabelValue(), workload_src.Name, workload_src.Namespace, workload_src.Kind, workload_dest.Name, workload_dest.Namespace, workload_dest.Kind, actualDestWorkload.Name, actualDestWorkload.Namespace, actualDestWorkload.Kind, workload_src.Region, workload_src.Zone, workload_dest.Region, workload_dest.Zone, actualDestWorkload.Region, actualDestWorkload.Zone, actualDestWorkload.Instance)
ch <- counter(metrics.NetRetransmits, float64(stats.Retransmissions), enrichedKey.DestinationLabelValue(), enrichedKey.ActualDestinationLabelValue(), workload_src.Name, workload_src.Namespace, workload_src.Kind, workload_dest.Name, workload_dest.Namespace, workload_dest.Kind, actualDestWorkload.Name, actualDestWorkload.Namespace, actualDestWorkload.Kind)
}
ch <- counter(metrics.NetBytesSent, float64(stats.BytesSent), enrichedKey.DestinationLabelValue(), enrichedKey.ActualDestinationLabelValue(), workload_src.Name, workload_src.Namespace, workload_src.Kind, workload_dest.Name, workload_dest.Namespace, workload_dest.Kind, actualDestWorkload.Name, actualDestWorkload.Namespace, actualDestWorkload.Kind, workload_src.Region, workload_src.Zone, workload_dest.Region, workload_dest.Zone, actualDestWorkload.Region, actualDestWorkload.Zone, actualDestWorkload.Instance)
ch <- counter(metrics.NetBytesReceived, float64(stats.BytesReceived), enrichedKey.DestinationLabelValue(), enrichedKey.ActualDestinationLabelValue(), workload_src.Name, workload_src.Namespace, workload_src.Kind, workload_dest.Name, workload_dest.Namespace, workload_dest.Kind, actualDestWorkload.Name, actualDestWorkload.Namespace, actualDestWorkload.Kind, workload_src.Region, workload_src.Zone, workload_dest.Region, workload_dest.Zone, actualDestWorkload.Region, actualDestWorkload.Zone, actualDestWorkload.Instance)
ch <- counter(metrics.NetBytesSent, float64(stats.BytesSent), enrichedKey.DestinationLabelValue(), enrichedKey.ActualDestinationLabelValue(), workload_src.Name, workload_src.Namespace, workload_src.Kind, workload_dest.Name, workload_dest.Namespace, workload_dest.Kind, actualDestWorkload.Name, actualDestWorkload.Namespace, actualDestWorkload.Kind)
ch <- counter(metrics.NetBytesReceived, float64(stats.BytesReceived), enrichedKey.DestinationLabelValue(), enrichedKey.ActualDestinationLabelValue(), workload_src.Name, workload_src.Namespace, workload_src.Kind, workload_dest.Name, workload_dest.Namespace, workload_dest.Kind, actualDestWorkload.Name, actualDestWorkload.Namespace, actualDestWorkload.Kind)
Comment thread
mayankpande88 marked this conversation as resolved.
}
// Copy failedConnectionAttempts under read lock to avoid concurrent map access
c.lock.RLock()
Expand All @@ -455,7 +455,7 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {

for dst, count := range failedConnCopy {
workload := c.ip_resolver.ResolveIP(dst.IP().String())
ch <- counter(metrics.NetConnectionsFailed, float64(count), dst.String(), workload.Name, workload.Namespace, workload.Kind, workload.Name, workload.Namespace, workload.Kind, c.srcWorkload.Region, c.srcWorkload.Zone, workload.Region, workload.Zone, workload.Region, workload.Zone, workload.Instance)
ch <- counter(metrics.NetConnectionsFailed, float64(count), dst.String(), workload.Name, workload.Namespace, workload.Kind, workload.Name, workload.Namespace, workload.Kind)
}

connections := map[common.DestinationKey]int{}
Expand All @@ -470,7 +470,7 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
for enrichedKey, count := range connections {
actualDestWorkload := enrichedKey.GetActualDestinationWorkload()
destWorkload := enrichedKey.GetDestinationWorkload()
ch <- gauge(metrics.NetConnectionsActive, float64(count), enrichedKey.DestinationLabelValue(), enrichedKey.ActualDestinationLabelValue(), c.srcWorkload.Name, c.srcWorkload.Namespace, c.srcWorkload.Kind, destWorkload.Name, destWorkload.Namespace, destWorkload.Kind, actualDestWorkload.Name, actualDestWorkload.Namespace, actualDestWorkload.Kind, c.srcWorkload.Region, c.srcWorkload.Zone, destWorkload.Region, destWorkload.Zone, actualDestWorkload.Region, actualDestWorkload.Zone, actualDestWorkload.Instance)
ch <- gauge(metrics.NetConnectionsActive, float64(count), enrichedKey.DestinationLabelValue(), enrichedKey.ActualDestinationLabelValue(), c.srcWorkload.Name, c.srcWorkload.Namespace, c.srcWorkload.Kind, destWorkload.Name, destWorkload.Namespace, destWorkload.Kind, actualDestWorkload.Name, actualDestWorkload.Namespace, actualDestWorkload.Kind)
}

// Copy logParsers under read lock to avoid concurrent map access
Expand Down Expand Up @@ -592,7 +592,7 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
if !*flags.DisablePinger {
for ip, rtt := range c.ping() {
destination_workload := c.ip_resolver.ResolveIP(ip.String())
ch <- gauge(metrics.NetLatency, rtt, ip.String(), destination_workload.Name, destination_workload.Namespace, destination_workload.Kind, destination_workload.Instance)
ch <- gauge(metrics.NetLatency, rtt, ip.String(), destination_workload.Name, destination_workload.Namespace, destination_workload.Kind)
}
}

Expand Down
14 changes: 0 additions & 14 deletions containers/l7.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,13 +142,6 @@ func (s *L7Stats) observe(protocol l7.Protocol, status, method, path string, dur
labelInterner.intern(actualDestWorkload.Kind),
labelInterner.intern(actualDestWorkload.Name),
labelInterner.intern(actualDestWorkload.Namespace),
labelInterner.intern(srcWorkload.Region),
labelInterner.intern(srcWorkload.Zone),
labelInterner.intern(destWorkload.Region),
labelInterner.intern(destWorkload.Zone),
labelInterner.intern(actualDestWorkload.Region),
labelInterner.intern(actualDestWorkload.Zone),
labelInterner.intern(actualDestWorkload.Instance),
}

// Protocol-specific labels for counters (keep all labels including path for HTTP)
Expand Down Expand Up @@ -242,13 +235,6 @@ func (s *L7Stats) ensureInitialized(protocol l7.Protocol) {
"actual_destination_workload_kind",
"actual_destination_workload_name",
"actual_destination_workload_namespace",
"src_region",
"src_az",
"destination_workload_region",
"destination_workload_az",
"actual_destination_region",
"actual_destination_az",
"actual_destination_instance",
}

// Initialize request counter
Expand Down
16 changes: 8 additions & 8 deletions containers/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,14 +90,14 @@ var metrics = struct {
DiskWriteBytes: metric("container_resources_disk_written_bytes_total", "Total number of bytes written to the disk by the container", "mount_point", "device", "volume"),

NetListenInfo: metric("container_net_tcp_listen_info", "Listen address of the container", "listen_addr", "proxy"),
NetConnectionsSuccessful: metric("container_net_tcp_successful_connects_total", "Total number of successful TCP connects", "destination", "actual_destination", "src_workload_name", "src_workload_namespace", "src_workload_kind", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind", "src_region", "src_az", "destination_workload_region", "destination_workload_az", "actual_destination_region", "actual_destination_az", "actual_destination_instance"),
NetConnectionsTotalTime: metric("container_net_tcp_connection_time_seconds_total", "Time spent on TCP connections", "destination", "actual_destination", "src_workload_name", "src_workload_namespace", "src_workload_kind", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind", "src_region", "src_az", "destination_workload_region", "destination_workload_az", "actual_destination_region", "actual_destination_az", "actual_destination_instance"),
NetConnectionsFailed: metric("container_net_tcp_failed_connects_total", "Total number of failed TCP connects", "destination", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind", "src_region", "src_az", "destination_workload_region", "destination_workload_az", "actual_destination_region", "actual_destination_az", "actual_destination_instance"),
NetConnectionsActive: metric("container_net_tcp_active_connections", "Number of active outbound connections used by the container", "destination", "actual_destination", "src_workload_name", "src_workload_namespace", "src_workload_kind", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind", "src_region", "src_az", "destination_workload_region", "destination_workload_az", "actual_destination_region", "actual_destination_az", "actual_destination_instance"),
NetRetransmits: metric("container_net_tcp_retransmits_total", "Total number of retransmitted TCP segments", "destination", "actual_destination", "src_workload_name", "src_workload_namespace", "src_workload_kind", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind", "src_region", "src_az", "destination_workload_region", "destination_workload_az", "actual_destination_region", "actual_destination_az", "actual_destination_instance"),
NetLatency: metric("container_net_latency_seconds", "Round-trip time between the container and a remote IP", "destination_ip", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "destination_instance"),
NetBytesSent: metric("container_net_tcp_bytes_sent_total", "Total number of bytes sent to the peer", "destination", "actual_destination", "src_workload_name", "src_workload_namespace", "src_workload_kind", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind", "src_region", "src_az", "destination_workload_region", "destination_workload_az", "actual_destination_region", "actual_destination_az", "actual_destination_instance"),
NetBytesReceived: metric("container_net_tcp_bytes_received_total", "Total number of bytes received from the peer", "destination", "actual_destination", "src_workload_name", "src_workload_namespace", "src_workload_kind", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind", "src_region", "src_az", "destination_workload_region", "destination_workload_az", "actual_destination_region", "actual_destination_az", "actual_destination_instance"),
NetConnectionsSuccessful: metric("container_net_tcp_successful_connects_total", "Total number of successful TCP connects", "destination", "actual_destination", "src_workload_name", "src_workload_namespace", "src_workload_kind", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind"),
NetConnectionsTotalTime: metric("container_net_tcp_connection_time_seconds_total", "Time spent on TCP connections", "destination", "actual_destination", "src_workload_name", "src_workload_namespace", "src_workload_kind", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind"),
NetConnectionsFailed: metric("container_net_tcp_failed_connects_total", "Total number of failed TCP connects", "destination", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind"),
NetConnectionsActive: metric("container_net_tcp_active_connections", "Number of active outbound connections used by the container", "destination", "actual_destination", "src_workload_name", "src_workload_namespace", "src_workload_kind", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind"),
NetRetransmits: metric("container_net_tcp_retransmits_total", "Total number of retransmitted TCP segments", "destination", "actual_destination", "src_workload_name", "src_workload_namespace", "src_workload_kind", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind"),
NetLatency: metric("container_net_latency_seconds", "Round-trip time between the container and a remote IP", "destination_ip", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind"),
NetBytesSent: metric("container_net_tcp_bytes_sent_total", "Total number of bytes sent to the peer", "destination", "actual_destination", "src_workload_name", "src_workload_namespace", "src_workload_kind", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind"),
NetBytesReceived: metric("container_net_tcp_bytes_received_total", "Total number of bytes received from the peer", "destination", "actual_destination", "src_workload_name", "src_workload_namespace", "src_workload_kind", "destination_workload_name", "destination_workload_namespace", "destination_workload_kind", "actual_destination_workload_name", "actual_destination_workload_namespace", "actual_destination_workload_kind"),

LogMessages: metric("container_log_messages_total", "Number of messages grouped by the automatically extracted repeated pattern", "source", "level", "pattern_hash", "sample"),
SensitiveLogMessages: metric("container_sensitive_log_messages_total", "Number of messages that contain sensitive information", "source", "pattern", "sample", "regex", "name", "pattern_hash"),
Expand Down
20 changes: 10 additions & 10 deletions ebpftracer/ebpf.go
View file
Open in desktop

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions ebpftracer/ebpf/tcp/state.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#define MAX_CONNECTIONS 1000000
#define MAX_PAYLOAD_SIZE 8192 // must be power of 2, increased for LLM HTTP/2 payloads
#define MAX_CONNECTIONS 100000
#define MAX_PAYLOAD_SIZE 4096 // must be power of 2
Comment thread
mayankpande88 marked this conversation as resolved.

struct tcp_event {
__u64 fd;
Expand Down Expand Up @@ -106,7 +106,7 @@ struct {
__uint(type, BPF_MAP_TYPE_LRU_HASH);
__uint(key_size, sizeof(struct l7_request_key));
__uint(value_size, sizeof(struct l7_request));
__uint(max_entries, 8192);
__uint(max_entries, 4096);
} active_l7_requests SEC(".maps");

SEC("tracepoint/sock/inet_sock_set_state")
Expand Down
2 changes: 1 addition & 1 deletion ebpftracer/tracer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
)

const (
MaxPayloadSize = 8192 // Must match MAX_PAYLOAD_SIZE in eBPF
MaxPayloadSize = 4096 // Must match MAX_PAYLOAD_SIZE in eBPF

l7EventHeaderSize = 96 // 56 bytes base + 40 bytes socket tuple fields
tcpEventSize = 104
Expand Down
Loading