chore: upstream rebase

CONTRIBUTING.md

@@ -5,7 +5,7 @@ Below are some basic guidelines.
 
 
 ## Requirements
-* Linux ≥v4.16 (amd64, arm64)
+* Linux ≥v5.1 (amd64, arm64)
 * Go v1.23
 
 

README.md

@@ -5,7 +5,7 @@
 
 The agent gathers metrics related to a node and the containers running on it, and it exposes them in the Prometheus format.
 
-It uses eBPF to track container related events such as TCP connects, so the minimum supported Linux kernel version is 4.16.
+It uses eBPF to track container related events such as TCP connects, so the minimum supported Linux kernel version is 5.1.
 
 <img src="https://coroot.com/static/img/blog/ebpf.svg" width="800" />
 

cgroup/cgroup.go

@@ -25,6 +25,7 @@ var (
 	lxcIdRegexp         = regexp.MustCompile(`/lxc/([^/]+)`)
 	systemSliceIdRegexp = regexp.MustCompile(`(/(system|runtime|reserved)\.slice/([^/]+))`)
 	talosIdRegexp       = regexp.MustCompile(`/(system|podruntime)/([^/]+)`)
+	lxcPayloadRegexp    = regexp.MustCompile(`/lxc\.payload\.([^/]+)`)
 )
 
 type ContainerType uint8
@@ -129,7 +130,14 @@ func NewFromProcessCgroupFile(filePath string) (*Cgroup, error) {
 			continue
 		}
 		for _, cgType := range strings.Split(parts[1], ",") {
-			p := path.Join(baseCgroupPath, parts[2])
+			cgPath := parts[2]
+			if strings.HasPrefix(parts[2], "/lxc.payload.") {
+				pp := strings.Split(cgPath, "/")
+				if len(parts) > 2 {
+					cgPath = "/" + pp[1]
+				}
+			}
+			p := path.Join(baseCgroupPath, cgPath)
 			switch p {
 			case "/", "/init.scope":
 				continue
@@ -147,27 +155,25 @@ func NewFromProcessCgroupFile(filePath string) (*Cgroup, error) {
 
 func containerByCgroup(cgroupPath string) (ContainerType, string, error) {
 	parts := strings.Split(strings.TrimLeft(cgroupPath, "/"), "/")
-	if cgroupPath == "/init" {
-		return ContainerTypeTalosRuntime, "/talos/init", nil
-	}
-	if len(parts) < 2 {
+	if len(parts) == 0 {
 		return ContainerTypeStandaloneProcess, "", nil
 	}
-	if *flags.DisableKubeProbe && parts[1] == "kubelet.service" {
+	if *flags.DisableKubeProbe && len(parts) >= 2 && parts[1] == "kubelet.service" {
 		return ContainerTypeStandaloneProcess, "", nil
 	}
 	prefix := parts[0]
-	if prefix == "user.slice" || prefix == "init.scope" {
+	switch {
+	case cgroupPath == "/init":
+		return ContainerTypeTalosRuntime, "/talos/init", nil
+	case prefix == "user.slice" || prefix == "init.scope":
 		return ContainerTypeStandaloneProcess, "", nil
-	}
-	if prefix == "docker" || (prefix == "system.slice" && strings.HasPrefix(parts[1], "docker-")) {
+	case prefix == "docker" || (prefix == "system.slice" && len(parts) > 1 && strings.HasPrefix(parts[1], "docker-")):
 		matches := dockerIdRegexp.FindStringSubmatch(cgroupPath)
 		if matches == nil {
 			return ContainerTypeUnknown, "", fmt.Errorf("invalid docker cgroup %s", cgroupPath)
 		}
 		return ContainerTypeDocker, matches[1], nil
-	}
-	if strings.Contains(cgroupPath, "kubepods") {
+	case strings.Contains(cgroupPath, "kubepods"):
 		crioMatches := crioIdRegexp.FindStringSubmatch(cgroupPath)
 		if crioMatches != nil {
 			return ContainerTypeCrio, crioMatches[1], nil
@@ -184,27 +190,33 @@ func containerByCgroup(cgroupPath string) (ContainerType, string, error) {
 			return ContainerTypeSandbox, "", nil
 		}
 		return ContainerTypeDocker, matches[1], nil
-	}
-	if prefix == "lxc" {
-		matches := lxcIdRegexp.FindStringSubmatch(cgroupPath)
-		if matches == nil {
-			return ContainerTypeUnknown, "", fmt.Errorf("invalid lxc cgroup %s", cgroupPath)
-		}
-		return ContainerTypeLxc, matches[1], nil
-	}
-	if prefix == "system" || prefix == "podruntime" {
+	case prefix == "system" || prefix == "podruntime":
 		matches := talosIdRegexp.FindStringSubmatch(cgroupPath)
 		if matches == nil {
 			return ContainerTypeUnknown, "", fmt.Errorf("invalid talos runtime cgroup %s", cgroupPath)
 		}
 		return ContainerTypeTalosRuntime, path.Join("/talos/", matches[2]), nil
-	}
-	if prefix == "system.slice" || prefix == "runtime.slice" || prefix == "reserved.slice" {
+	case prefix == "system.slice" || prefix == "runtime.slice" || prefix == "reserved.slice":
 		matches := systemSliceIdRegexp.FindStringSubmatch(cgroupPath)
 		if matches == nil {
 			return ContainerTypeUnknown, "", fmt.Errorf("invalid systemd cgroup %s", cgroupPath)
 		}
 		return ContainerTypeSystemdService, strings.Replace(matches[1], "\\x2d", "-", -1), nil
+	case prefix == "lxc":
+		matches := lxcIdRegexp.FindStringSubmatch(cgroupPath)
+		if matches == nil {
+			return ContainerTypeUnknown, "", fmt.Errorf("invalid lxc cgroup %s", cgroupPath)
+		}
+		return ContainerTypeLxc, matches[1], nil
+	case strings.HasPrefix(prefix, "lxc.payload."):
+		matches := lxcPayloadRegexp.FindStringSubmatch(cgroupPath)
+		if matches == nil {
+			return ContainerTypeUnknown, "", fmt.Errorf("invalid lxc payload cgroup %s", cgroupPath)
+		}
+		return ContainerTypeLxc, "/lxc/" + matches[1], nil
+	case len(parts) < 2:
+		return ContainerTypeStandaloneProcess, "", nil
 	}
+
 	return ContainerTypeUnknown, "", fmt.Errorf("unknown container: %s", cgroupPath)
 }

cgroup/cgroup_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestNewFromProcessCgroupFile(t *testing.T) {
@@ -75,6 +76,12 @@ func TestNewFromProcessCgroupFile(t *testing.T) {
 	assert.Equal(t, "95cbe853416f52d927dec41f1406dd75015ea131244a1ca875a7cd4ebe927ac8", cg.ContainerId)
 	assert.Equal(t, ContainerTypeDocker, cg.ContainerType)
 
+	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/3000/cgroup"))
+	require.Nil(t, err)
+	assert.Equal(t, "/lxc.payload.first", cg.Id)
+	assert.Equal(t, "/lxc/first", cg.ContainerId)
+	assert.Equal(t, ContainerTypeLxc, cg.ContainerType)
+
 	baseCgroupPath = "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podc83d0428_58af_41eb_8dba_b9e6eddffe7b.slice/docker-0e612005fd07e7f47e2cd07df99a2b4e909446814d71d0b5e4efc7159dd51252.scope"
 	defer func() {
 		baseCgroupPath = ""
@@ -178,4 +185,14 @@ func TestContainerByCgroup(t *testing.T) {
 	as.Equal(typ, ContainerTypeTalosRuntime)
 	as.Equal("/talos/init", id)
 	as.Nil(err)
+
+	typ, id, err = containerByCgroup("/lxc.payload.first")
+	as.Equal(typ, ContainerTypeLxc)
+	as.Equal("/lxc/first", id)
+	as.Nil(err)
+
+	typ, id, err = containerByCgroup("/lxc.monitor.first")
+	as.Equal(ContainerTypeStandaloneProcess, typ)
+	as.Equal("", id)
+	as.Nil(err)
 }

cgroup/fixtures/proc/3000/cgroup

@@ -0,0 +1 @@
+0::/lxc.payload.first/system.slice/systemd-logind.service

cgroup/memory.go

@@ -48,8 +48,8 @@ func (cg *Cgroup) memoryStatV1() (*MemoryStat, error) {
 	//	 mapped_file is accounted only when the memory cgroup is owner of page
 	//	 cache.)
 	return &MemoryStat{
-		RSS:   vars["rss"] + vars["mapped_file"],
-		Cache: vars["cache"],
+		RSS:   vars["total_rss"] + vars["total_mapped_file"],
+		Cache: vars["total_cache"],
 		Limit: limit,
 	}, nil
 }

containers/app.go

@@ -122,6 +122,8 @@ func guessApplicationTypeByCmdline(cmdline []byte) string {
 		return "java"
 	case bytes.HasSuffix(cmd, []byte("ollama")):
 		return "ollama"
+	case bytes.HasSuffix(cmd, []byte("fdbserver")):
+		return "foundationdb"
 	case bytes.Contains(cmd, []byte("victoria-metrics")) ||
 		bytes.Contains(cmd, []byte("vmstorage")) ||
 		bytes.Contains(cmd, []byte("vminsert")) ||

containers/container.go

@@ -298,12 +298,12 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 	if disks, err := node.GetDisks(); err == nil {
 		ioStat := c.cgroup.IOStat()
 		for majorMinor, mounts := range c.getMounts() {
-			dev := disks.GetParentBlockDevice(majorMinor)
-			if dev == nil {
-				continue
+			var device string
+			if dev := disks.GetParentBlockDevice(majorMinor); dev != nil {
+				device = dev.Name
 			}
 			for mountPoint, fsStat := range mounts {
-				dls := []string{mountPoint, dev.Name, c.metadata.volumes[mountPoint]}
+				dls := []string{mountPoint, device, c.metadata.volumes[mountPoint]}
 				ch <- gauge(metrics.DiskSize, float64(fsStat.CapacityBytes), dls...)
 				ch <- gauge(metrics.DiskUsed, float64(fsStat.UsedBytes), dls...)
 				ch <- gauge(metrics.DiskReserved, float64(fsStat.ReservedBytes), dls...)
@@ -790,10 +790,20 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R
 		}
 	}
 
-	trace := c.tracer.NewTrace(conn.DestinationKey.ActualDestinationIfKnown(), conn.srcWorkload, conn.DestinationKey.GetDestinationWorkload(), conn.DestinationKey.GetActualDestinationWorkload())
+	ebpfTracesDisabled := false
+	for _, p := range c.processes {
+		if p.Flags.EbpfTracesDisabled {
+			ebpfTracesDisabled = true
+			break
+		}
+	}
+	var trace *tracing.Trace
 	traceId := ""
-	if headers != nil {
-		traceId = trace.ExtractTraceId(headers)
+	if !ebpfTracesDisabled {
+		trace := c.tracer.NewTrace(conn.DestinationKey.ActualDestinationIfKnown(), conn.srcWorkload, conn.DestinationKey.GetDestinationWorkload(), conn.DestinationKey.GetActualDestinationWorkload())
+		if headers != nil {
+			traceId = trace.ExtractTraceId(headers)
+		}
 	}
 	stats := c.l7Stats.get(r.Protocol, conn.DestinationKey, r, conn.srcWorkload, traceId)
 	switch r.Protocol {
@@ -883,6 +893,8 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R
 		stats.observe(r.Status.Zookeeper(), "", r.Duration)
 		op, arg := l7.ParseZookeeper(r.Payload)
 		trace.ZookeeperRequest(op, arg, r.Status, r.Duration)
+	case l7.ProtocolFoundationDB:
+		stats.observe(r.Status.String(), "", r.Duration)
 	}
 	return nil
 }
@@ -1086,6 +1098,13 @@ func (c *Container) runLogParser(logPath string) {
 		return
 	}
 
+	for _, p := range c.processes {
+		if p.Flags.LogMonitoringDisabled {
+			klog.InfoS("skipping log monitoring due to COROOT_LOG_MONITORING=disabled", "cg", c.cgroup.Id)
+			return
+		}
+	}
+
 	containerId := string(c.id)
 
 	if logPath != "" {

containers/metrics.go

@@ -112,34 +112,36 @@ var metrics = struct {
 
 var (
 	L7Requests = map[l7.Protocol]prometheus.CounterOpts{
-		l7.ProtocolHTTP:       {Name: "container_http_requests_total", Help: "Total number of outbound HTTP requests"},
-		l7.ProtocolPostgres:   {Name: "container_postgres_queries_total", Help: "Total number of outbound Postgres queries"},
-		l7.ProtocolRedis:      {Name: "container_redis_queries_total", Help: "Total number of outbound Redis queries"},
-		l7.ProtocolMemcached:  {Name: "container_memcached_queries_total", Help: "Total number of outbound Memcached queries"},
-		l7.ProtocolMysql:      {Name: "container_mysql_queries_total", Help: "Total number of outbound Mysql queries"},
-		l7.ProtocolMongo:      {Name: "container_mongo_queries_total", Help: "Total number of outbound Mongo queries"},
-		l7.ProtocolKafka:      {Name: "container_kafka_requests_total", Help: "Total number of outbound Kafka requests"},
-		l7.ProtocolCassandra:  {Name: "container_cassandra_queries_total", Help: "Total number of outbound Cassandra requests"},
-		l7.ProtocolRabbitmq:   {Name: "container_rabbitmq_messages_total", Help: "Total number of Rabbitmq messages produced or consumed by the container"},
-		l7.ProtocolNats:       {Name: "container_nats_messages_total", Help: "Total number of NATS messages produced or consumed by the container"},
-		l7.ProtocolDubbo2:     {Name: "container_dubbo_requests_total", Help: "Total number of outbound DUBBO requests"},
-		l7.ProtocolDNS:        {Name: "container_dns_requests_total", Help: "Total number of outbound DNS requests"},
-		l7.ProtocolClickhouse: {Name: "container_clickhouse_queries_total", Help: "Total number of outbound ClickHouse queries"},
-		l7.ProtocolZookeeper:  {Name: "container_zookeeper_requests_total", Help: "Total number of outbound Zookeeper requests"},
+		l7.ProtocolHTTP:         {Name: "container_http_requests_total", Help: "Total number of outbound HTTP requests"},
+		l7.ProtocolPostgres:     {Name: "container_postgres_queries_total", Help: "Total number of outbound Postgres queries"},
+		l7.ProtocolRedis:        {Name: "container_redis_queries_total", Help: "Total number of outbound Redis queries"},
+		l7.ProtocolMemcached:    {Name: "container_memcached_queries_total", Help: "Total number of outbound Memcached queries"},
+		l7.ProtocolMysql:        {Name: "container_mysql_queries_total", Help: "Total number of outbound Mysql queries"},
+		l7.ProtocolMongo:        {Name: "container_mongo_queries_total", Help: "Total number of outbound Mongo queries"},
+		l7.ProtocolKafka:        {Name: "container_kafka_requests_total", Help: "Total number of outbound Kafka requests"},
+		l7.ProtocolCassandra:    {Name: "container_cassandra_queries_total", Help: "Total number of outbound Cassandra requests"},
+		l7.ProtocolRabbitmq:     {Name: "container_rabbitmq_messages_total", Help: "Total number of Rabbitmq messages produced or consumed by the container"},
+		l7.ProtocolNats:         {Name: "container_nats_messages_total", Help: "Total number of NATS messages produced or consumed by the container"},
+		l7.ProtocolDubbo2:       {Name: "container_dubbo_requests_total", Help: "Total number of outbound DUBBO requests"},
+		l7.ProtocolDNS:          {Name: "container_dns_requests_total", Help: "Total number of outbound DNS requests"},
+		l7.ProtocolClickhouse:   {Name: "container_clickhouse_queries_total", Help: "Total number of outbound ClickHouse queries"},
+		l7.ProtocolZookeeper:    {Name: "container_zookeeper_requests_total", Help: "Total number of outbound Zookeeper requests"},
+		l7.ProtocolFoundationDB: {Name: "container_foundationdb_requests_total", Help: "Total number of outbound FoundationDB requests"},
 	}
 	L7Latency = map[l7.Protocol]prometheus.HistogramOpts{
-		l7.ProtocolHTTP:       {Name: "container_http_requests_duration_seconds_total", Help: "Histogram of the response time for each outbound HTTP request"},
-		l7.ProtocolPostgres:   {Name: "container_postgres_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound Postgres query"},
-		l7.ProtocolRedis:      {Name: "container_redis_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound Redis query"},
-		l7.ProtocolMemcached:  {Name: "container_memcached_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound Memcached query"},
-		l7.ProtocolMysql:      {Name: "container_mysql_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound Mysql query"},
-		l7.ProtocolMongo:      {Name: "container_mongo_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound Mongo query"},
-		l7.ProtocolKafka:      {Name: "container_kafka_requests_duration_seconds_total", Help: "Histogram of the execution time for each outbound Kafka request"},
-		l7.ProtocolCassandra:  {Name: "container_cassandra_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound Cassandra request"},
-		l7.ProtocolDubbo2:     {Name: "container_dubbo_requests_duration_seconds_total", Help: "Histogram of the response time for each outbound DUBBO request"},
-		l7.ProtocolDNS:        {Name: "container_dns_requests_duration_seconds_total", Help: "Histogram of the response time for each outbound DNS request"},
-		l7.ProtocolClickhouse: {Name: "container_clickhouse_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound ClickHouse query"},
-		l7.ProtocolZookeeper:  {Name: "container_zookeeper_requests_duration_seconds_total", Help: "Histogram of the execution time for each outbound Zookeeper request"},
+		l7.ProtocolHTTP:         {Name: "container_http_requests_duration_seconds_total", Help: "Histogram of the response time for each outbound HTTP request"},
+		l7.ProtocolPostgres:     {Name: "container_postgres_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound Postgres query"},
+		l7.ProtocolRedis:        {Name: "container_redis_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound Redis query"},
+		l7.ProtocolMemcached:    {Name: "container_memcached_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound Memcached query"},
+		l7.ProtocolMysql:        {Name: "container_mysql_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound Mysql query"},
+		l7.ProtocolMongo:        {Name: "container_mongo_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound Mongo query"},
+		l7.ProtocolKafka:        {Name: "container_kafka_requests_duration_seconds_total", Help: "Histogram of the execution time for each outbound Kafka request"},
+		l7.ProtocolCassandra:    {Name: "container_cassandra_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound Cassandra request"},
+		l7.ProtocolDubbo2:       {Name: "container_dubbo_requests_duration_seconds_total", Help: "Histogram of the response time for each outbound DUBBO request"},
+		l7.ProtocolDNS:          {Name: "container_dns_requests_duration_seconds_total", Help: "Histogram of the response time for each outbound DNS request"},
+		l7.ProtocolClickhouse:   {Name: "container_clickhouse_queries_duration_seconds_total", Help: "Histogram of the execution time for each outbound ClickHouse query"},
+		l7.ProtocolZookeeper:    {Name: "container_zookeeper_requests_duration_seconds_total", Help: "Histogram of the execution time for each outbound Zookeeper request"},
+		l7.ProtocolFoundationDB: {Name: "container_foundationdb_requests_duration_seconds_total", Help: "Histogram of the execution time for each outbound FoundationDB request"},
 	}
 )
 

containers/process.go

@@ -28,6 +28,8 @@ type Process struct {
 	Pid       uint32
 	StartedAt time.Time
 
+	Flags proc.Flags
+
 	netNsId string
 
 	ctx        context.Context
@@ -46,6 +48,7 @@ type Process struct {
 
 func NewProcess(pid uint32, stats *taskstats.Stats, tracer *ebpftracer.Tracer) *Process {
 	p := &Process{Pid: pid, StartedAt: stats.BeginTime}
+	p.Flags, _ = proc.GetFlags(pid)
 	p.ctx, p.cancelFunc = context.WithCancel(context.Background())
 	go p.instrument(tracer)
 	return p

containers/registry.go

@@ -40,6 +40,7 @@ type ProcessInfo struct {
 	Pid         uint32
 	ContainerId ContainerID
 	StartedAt   time.Time
+	Flags       proc.Flags
 }
 
 type IPResolver interface {
@@ -245,7 +246,7 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 				if c := r.getOrCreateContainer(e.Pid); c != nil {
 					p := c.onProcessStart(e.Pid)
 					if r.processInfoCh != nil && p != nil {
-						r.processInfoCh <- ProcessInfo{Pid: p.Pid, ContainerId: c.id, StartedAt: p.StartedAt}
+						r.processInfoCh <- ProcessInfo{Pid: p.Pid, ContainerId: c.id, StartedAt: p.StartedAt, Flags: p.Flags}
 					}
 				}
 			case ebpftracer.EventTypeProcessExit: