chore: test to prod

.github/workflows/dev-ci.yaml

@@ -35,7 +35,7 @@ jobs:
       - name: Build and Push Agent Image
         uses: docker/build-push-action@v3
         with:
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           context: .
           file: 'Dockerfile'
           push: true

CONTRIBUTING.md

@@ -5,7 +5,7 @@ Below are some basic guidelines.
 
 
 ## Requirements
-* Linux ≥v4.16 (amd64, arm64)
+* Linux ≥v5.1 (amd64, arm64)
 * Go v1.23
 
 

Dockerfile

@@ -1,9 +1,5 @@
-FROM golang:1.23.10-bullseye AS builder
-
-RUN apt-get update \
- && apt-get install -y --no-install-recommends libsystemd-dev \
- && rm -rf /var/lib/apt/lists/*
-
+FROM golang:1.24.7-trixie AS builder
+RUN apt update && apt install -y libsystemd-dev
 WORKDIR /tmp/src
 COPY go.mod .
 COPY go.sum .
@@ -16,6 +12,9 @@ FROM registry.access.redhat.com/ubi9/ubi-minimal AS runtime
 
 ARG VERSION=unknown
 
+# Install SSL/TLS libraries for HTTPS LLM API tracing
+RUN microdnf install -y openssl-libs
+
 COPY --from=builder /tmp/src/coroot-node-agent /usr/bin/coroot-node-agent
 
 ENTRYPOINT ["coroot-node-agent"]

Dockerfile.alpine

@@ -22,12 +22,15 @@ RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-extldflags='-Wl,-z,lazy' -X
 # Runtime stage
 FROM alpine:3.22 AS release-stage
 
+# Install SSL/TLS libraries for HTTPS LLM API tracing
+RUN apk add --no-cache openssl libssl3 ca-certificates
+
 WORKDIR /app
 
 # Copy the statically linked binary
 COPY --from=builder /app/coroot-node-agent /usr/bin/coroot-node-agent
 
-# Ensure it’s executable
+# Ensure it's executable
 RUN chmod +x /usr/bin/coroot-node-agent
 
 # Run it

README.md

@@ -5,7 +5,7 @@
 
 The agent gathers metrics related to a node and the containers running on it, and it exposes them in the Prometheus format.
 
-It uses eBPF to track container related events such as TCP connects, so the minimum supported Linux kernel version is 4.16.
+It uses eBPF to track container related events such as TCP connects, so the minimum supported Linux kernel version is 5.1.
 
 <img src="https://coroot.com/static/img/blog/ebpf.svg" width="800" />
 

cgroup/cgroup.go

@@ -25,6 +25,7 @@ var (
 	lxcIdRegexp         = regexp.MustCompile(`/lxc/([^/]+)`)
 	systemSliceIdRegexp = regexp.MustCompile(`(/(system|runtime|reserved)\.slice/([^/]+))`)
 	talosIdRegexp       = regexp.MustCompile(`/(system|podruntime)/([^/]+)`)
+	lxcPayloadRegexp    = regexp.MustCompile(`/lxc\.payload\.([^/]+)`)
 )
 
 type ContainerType uint8
@@ -129,7 +130,14 @@ func NewFromProcessCgroupFile(filePath string) (*Cgroup, error) {
 			continue
 		}
 		for _, cgType := range strings.Split(parts[1], ",") {
-			p := path.Join(baseCgroupPath, parts[2])
+			cgPath := parts[2]
+			if strings.HasPrefix(parts[2], "/lxc.payload.") {
+				pp := strings.Split(cgPath, "/")
+				if len(parts) > 2 {
+					cgPath = "/" + pp[1]
+				}
+			}
+			p := path.Join(baseCgroupPath, cgPath)
 			switch p {
 			case "/", "/init.scope":
 				continue
@@ -147,27 +155,25 @@ func NewFromProcessCgroupFile(filePath string) (*Cgroup, error) {
 
 func containerByCgroup(cgroupPath string) (ContainerType, string, error) {
 	parts := strings.Split(strings.TrimLeft(cgroupPath, "/"), "/")
-	if cgroupPath == "/init" {
-		return ContainerTypeTalosRuntime, "/talos/init", nil
-	}
-	if len(parts) < 2 {
+	if len(parts) == 0 {
 		return ContainerTypeStandaloneProcess, "", nil
 	}
-	if *flags.DisableKubeProbe && parts[1] == "kubelet.service" {
+	if *flags.DisableKubeProbe && len(parts) > 1 && parts[1] == "kubelet.service" {
 		return ContainerTypeStandaloneProcess, "", nil
 	}
 	prefix := parts[0]
-	if prefix == "user.slice" || prefix == "init.scope" {
+	switch {
+	case cgroupPath == "/init":
+		return ContainerTypeTalosRuntime, "/talos/init", nil
+	case prefix == "user.slice" || prefix == "init.scope":
 		return ContainerTypeStandaloneProcess, "", nil
-	}
-	if prefix == "docker" || (prefix == "system.slice" && strings.HasPrefix(parts[1], "docker-")) {
+	case prefix == "docker" || (prefix == "system.slice" && len(parts) > 1 && strings.HasPrefix(parts[1], "docker-")):
 		matches := dockerIdRegexp.FindStringSubmatch(cgroupPath)
 		if matches == nil {
 			return ContainerTypeUnknown, "", fmt.Errorf("invalid docker cgroup %s", cgroupPath)
 		}
 		return ContainerTypeDocker, matches[1], nil
-	}
-	if strings.Contains(cgroupPath, "kubepods") {
+	case strings.Contains(cgroupPath, "kubepods"):
 		crioMatches := crioIdRegexp.FindStringSubmatch(cgroupPath)
 		if crioMatches != nil {
 			return ContainerTypeCrio, crioMatches[1], nil
@@ -184,27 +190,33 @@ func containerByCgroup(cgroupPath string) (ContainerType, string, error) {
 			return ContainerTypeSandbox, "", nil
 		}
 		return ContainerTypeDocker, matches[1], nil
-	}
-	if prefix == "lxc" {
-		matches := lxcIdRegexp.FindStringSubmatch(cgroupPath)
-		if matches == nil {
-			return ContainerTypeUnknown, "", fmt.Errorf("invalid lxc cgroup %s", cgroupPath)
-		}
-		return ContainerTypeLxc, matches[1], nil
-	}
-	if prefix == "system" || prefix == "podruntime" {
+	case prefix == "system" || prefix == "podruntime":
 		matches := talosIdRegexp.FindStringSubmatch(cgroupPath)
 		if matches == nil {
 			return ContainerTypeUnknown, "", fmt.Errorf("invalid talos runtime cgroup %s", cgroupPath)
 		}
 		return ContainerTypeTalosRuntime, path.Join("/talos/", matches[2]), nil
-	}
-	if prefix == "system.slice" || prefix == "runtime.slice" || prefix == "reserved.slice" {
+	case prefix == "system.slice" || prefix == "runtime.slice" || prefix == "reserved.slice":
 		matches := systemSliceIdRegexp.FindStringSubmatch(cgroupPath)
 		if matches == nil {
 			return ContainerTypeUnknown, "", fmt.Errorf("invalid systemd cgroup %s", cgroupPath)
 		}
 		return ContainerTypeSystemdService, strings.Replace(matches[1], "\\x2d", "-", -1), nil
+	case prefix == "lxc":
+		matches := lxcIdRegexp.FindStringSubmatch(cgroupPath)
+		if matches == nil {
+			return ContainerTypeUnknown, "", fmt.Errorf("invalid lxc cgroup %s", cgroupPath)
+		}
+		return ContainerTypeLxc, matches[1], nil
+	case strings.HasPrefix(prefix, "lxc.payload."):
+		matches := lxcPayloadRegexp.FindStringSubmatch(cgroupPath)
+		if matches == nil {
+			return ContainerTypeUnknown, "", fmt.Errorf("invalid lxc payload cgroup %s", cgroupPath)
+		}
+		return ContainerTypeLxc, "/lxc/" + matches[1], nil
+	case len(parts) < 2:
+		return ContainerTypeStandaloneProcess, "", nil
 	}
+
 	return ContainerTypeUnknown, "", fmt.Errorf("unknown container: %s", cgroupPath)
 }

cgroup/cgroup_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestNewFromProcessCgroupFile(t *testing.T) {
@@ -75,6 +76,12 @@ func TestNewFromProcessCgroupFile(t *testing.T) {
 	assert.Equal(t, "95cbe853416f52d927dec41f1406dd75015ea131244a1ca875a7cd4ebe927ac8", cg.ContainerId)
 	assert.Equal(t, ContainerTypeDocker, cg.ContainerType)
 
+	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/3000/cgroup"))
+	require.Nil(t, err)
+	assert.Equal(t, "/lxc.payload.first", cg.Id)
+	assert.Equal(t, "/lxc/first", cg.ContainerId)
+	assert.Equal(t, ContainerTypeLxc, cg.ContainerType)
+
 	baseCgroupPath = "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podc83d0428_58af_41eb_8dba_b9e6eddffe7b.slice/docker-0e612005fd07e7f47e2cd07df99a2b4e909446814d71d0b5e4efc7159dd51252.scope"
 	defer func() {
 		baseCgroupPath = ""
@@ -178,4 +185,14 @@ func TestContainerByCgroup(t *testing.T) {
 	as.Equal(typ, ContainerTypeTalosRuntime)
 	as.Equal("/talos/init", id)
 	as.Nil(err)
+
+	typ, id, err = containerByCgroup("/lxc.payload.first")
+	as.Equal(typ, ContainerTypeLxc)
+	as.Equal("/lxc/first", id)
+	as.Nil(err)
+
+	typ, id, err = containerByCgroup("/lxc.monitor.first")
+	as.Equal(ContainerTypeStandaloneProcess, typ)
+	as.Equal("", id)
+	as.Nil(err)
 }

cgroup/cpu.go

@@ -16,7 +16,7 @@ type CPUStat struct {
 	LimitCores           float64
 }
 
-func (cg Cgroup) CpuStat() *CPUStat {
+func (cg *Cgroup) CpuStat() *CPUStat {
 	cpu, cpuacct := cg.subsystems["cpu"], cg.subsystems["cpuacct"]
 	if cpu == "" || cpuacct == "" {
 		st, _ := cg.cpuStatV2()
@@ -26,7 +26,7 @@ func (cg Cgroup) CpuStat() *CPUStat {
 	return st
 }
 
-func (cg Cgroup) cpuStatV1() (*CPUStat, error) {
+func (cg *Cgroup) cpuStatV1() (*CPUStat, error) {
 	if cg.subsystems["cpu"] == "" || cg.subsystems["cpuacct"] == "" {
 		return nil, nil
 	}
@@ -56,7 +56,7 @@ func (cg Cgroup) cpuStatV1() (*CPUStat, error) {
 	return res, nil
 }
 
-func (cg Cgroup) cpuStatV2() (*CPUStat, error) {
+func (cg *Cgroup) cpuStatV2() (*CPUStat, error) {
 	if cg.subsystems[""] == "" {
 		return nil, nil
 	}

cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/cpu.pressure

@@ -0,0 +1,2 @@
+some avg10=0.00 avg60=0.00 avg300=0.00 total=465907442
+full avg10=0.00 avg60=0.00 avg300=0.00 total=463529433

cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/io.pressure

@@ -0,0 +1,2 @@
+some avg10=0.00 avg60=0.00 avg300=0.05 total=17657662684
+full avg10=0.00 avg60=0.00 avg300=0.05 total=17636951020

cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/memory.pressure

@@ -0,0 +1,2 @@
+some avg10=0.00 avg60=0.00 avg300=0.00 total=6937313991
+full avg10=0.00 avg60=0.00 avg300=0.00 total=6934649214

cgroup/fixtures/proc/3000/cgroup

@@ -0,0 +1 @@
+0::/lxc.payload.first/system.slice/systemd-logind.service

cgroup/memory.go

@@ -48,8 +48,8 @@ func (cg *Cgroup) memoryStatV1() (*MemoryStat, error) {
 	//	 mapped_file is accounted only when the memory cgroup is owner of page
 	//	 cache.)
 	return &MemoryStat{
-		RSS:   vars["rss"] + vars["mapped_file"],
-		Cache: vars["cache"],
+		RSS:   vars["total_rss"] + vars["total_mapped_file"],
+		Cache: vars["total_cache"],
 		Limit: limit,
 	}, nil
 }

cgroup/psi.go

@@ -0,0 +1,85 @@
+package cgroup
+
+import (
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/coroot/coroot-node-agent/common"
+	"k8s.io/klog/v2"
+)
+
+type PSIStats struct {
+	CPUSecondsSome    float64
+	CPUSecondsFull    float64
+	MemorySecondsSome float64
+	MemorySecondsFull float64
+	IOSecondsSome     float64
+	IOSecondsFull     float64
+}
+
+type PressureTotals struct {
+	SomeSecondsTotal float64
+	FullSecondsTotal float64
+}
+
+func (cg *Cgroup) PSI() *PSIStats {
+	if cg.subsystems[""] == "" {
+		return nil
+	}
+	stats := &PSIStats{}
+	for _, controller := range []string{"cpu", "memory", "io"} {
+		p, err := cg.readPressure(controller)
+		if err != nil {
+			if !common.IsNotExist(err) {
+				klog.Warningln(err)
+			}
+			return nil
+		}
+		switch controller {
+		case "cpu":
+			stats.CPUSecondsSome = p.SomeSecondsTotal
+			stats.CPUSecondsFull = p.FullSecondsTotal
+		case "memory":
+			stats.MemorySecondsSome = p.SomeSecondsTotal
+			stats.MemorySecondsFull = p.FullSecondsTotal
+		case "io":
+			stats.IOSecondsSome = p.SomeSecondsTotal
+			stats.IOSecondsFull = p.FullSecondsTotal
+		}
+	}
+	return stats
+}
+
+func (cg *Cgroup) readPressure(controller string) (*PressureTotals, error) {
+	data, err := os.ReadFile(path.Join(cg2Root, cg.subsystems[""], controller+".pressure"))
+	if err != nil {
+		return nil, err
+	}
+	pressure := &PressureTotals{}
+	for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
+		parts := strings.Fields(line)
+		if len(parts) == 0 {
+			continue
+		}
+		kind := parts[0]
+		for _, p := range parts[1:] {
+			if strings.HasPrefix(p, "total=") {
+				vStr := strings.TrimPrefix(p, "total=")
+				v, err := strconv.ParseUint(vStr, 10, 64)
+				if err != nil {
+					return nil, err
+				}
+				switch kind {
+				case "some":
+					pressure.SomeSecondsTotal = float64(v) / 1e6 // microseconds to seconds
+				case "full":
+					pressure.FullSecondsTotal = float64(v) / 1e6
+				}
+				break
+			}
+		}
+	}
+	return pressure, nil
+}

cgroup/psi_test.go

@@ -0,0 +1,27 @@
+package cgroup
+
+import (
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCgroupPSI(t *testing.T) {
+	cgRoot = "fixtures/cgroup"
+	cg2Root = "fixtures/cgroup"
+
+	cg, _ := NewFromProcessCgroupFile(path.Join("fixtures/proc/400/cgroup"))
+	stat := cg.PSI()
+	require.NotNil(t, stat)
+	assert.Equal(t, float64(465907442)/1e6, stat.CPUSecondsSome)
+	assert.Equal(t, float64(463529433)/1e6, stat.CPUSecondsFull)
+	assert.Equal(t, float64(6937313991)/1e6, stat.MemorySecondsSome)
+	assert.Equal(t, float64(6934649214)/1e6, stat.MemorySecondsFull)
+	assert.Equal(t, float64(17657662684)/1e6, stat.IOSecondsSome)
+	assert.Equal(t, float64(17636951020)/1e6, stat.IOSecondsFull)
+
+	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/100/cgroup"))
+	assert.Nil(t, cg.PSI())
+}