chore: rebase with upstream

.github/workflows/dev-ci.yaml

@@ -37,7 +37,7 @@ jobs:
         with:
           platforms: linux/amd64,linux/arm64
           context: .
-          file: 'Dockerfile.alpine'
+          file: 'Dockerfile'
           push: true
           tags: |
             864186153326.dkr.ecr.us-east-1.amazonaws.com/nudgebee-node-agent:${{ env.tag }}

.github/workflows/prod-ci.yaml

@@ -38,7 +38,7 @@ jobs:
         with:
           context: .
           platforms: linux/amd64,linux/arm64
-          file: 'Dockerfile.alpine'
+          file: 'Dockerfile'
           push: true
           tags: |
             740395098545.dkr.ecr.us-east-1.amazonaws.com/nudgebee-node-agent:${{ env.tag }}

Dockerfile

@@ -6,11 +6,11 @@ COPY go.sum .
 RUN go mod download
 COPY . .
 ARG VERSION=unknown
-RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-X 'github.com/coroot/coroot-node-agent/flags.Version=${VERSION}'" -o coroot-node-agent .
+RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-extldflags='-Wl,-z,lazy' -X 'github.com/coroot/coroot-node-agent/flags.Version=${VERSION}'" -o coroot-node-agent .
 
 FROM registry.access.redhat.com/ubi9/ubi
 
 ARG VERSION=unknown
 
 COPY --from=builder /tmp/src/coroot-node-agent /usr/bin/coroot-node-agent
-ENTRYPOINT ["coroot-node-agent"]
+ENTRYPOINT ["coroot-node-agent"]

Dockerfile.alpine

@@ -1,32 +1,34 @@
-FROM golang:1.23-alpine as build-stage
-# Set destination for COPY
-WORKDIR /app
-RUN apk add --no-cache --update go gcc g++ bash git
-RUN echo "unicode=\"YES\"" >> /etc/rc.conf && \
-    apk add --no-cache --virtual .build_deps \
-        autoconf file libc-dev make pkgconf python3 py3-pip ninja \
-        util-linux pciutils usbutils coreutils binutils findutils grep \
-        build-base abuild binutils binutils-doc gcc-doc gperf libcap libcap-dev \
-        valgrind-dev libmount && pip3 install meson --break-system-packages 
-
-RUN cd /tmp && git clone https://github.com/systemd/systemd
+# Build stage
+FROM golang:1.23.8-alpine AS builder
 
-# RUN cd /tmp/systemd && \
-#     meson build && \
-#     ninja build
+# Install build tools
+RUN apk add --no-cache build-base
 
-RUN apk add elogind-dev
+# Set up workspace
+WORKDIR /app
 
-COPY go.mod go.sum ./
+# Copy go mod files and download dependencies
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
 
-RUN  go mod download 
+# Copy the rest of the code
+COPY . .
 
-COPY ./ ./
+# Build statically linked binary
 ARG VERSION=unknown
-RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-X main.version=$VERSION" -o coroot-node-agent .
+RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-extldflags='-Wl,-z,lazy' -X 'github.com/coroot/coroot-node-agent/flags.Version=${VERSION}'" -o coroot-node-agent .
+
+# Runtime stage
+FROM alpine:3.22 AS release-stage
 
-FROM alpine:3.19 AS release-stage
 WORKDIR /app
-COPY --from=build-stage /app/coroot-node-agent /usr/bin/coroot-node-agent
+
+# Copy the statically linked binary
+COPY --from=builder /app/coroot-node-agent /usr/bin/coroot-node-agent
+
+# Ensure it’s executable
 RUN chmod +x /usr/bin/coroot-node-agent
-CMD ["coroot-node-agent"]
+
+# Run it
+CMD ["/usr/bin/coroot-node-agent"]

containers/app.go

@@ -120,6 +120,8 @@ func guessApplicationTypeByCmdline(cmdline []byte) string {
 		return "nats"
 	case bytes.HasSuffix(cmd, []byte("java")):
 		return "java"
+	case bytes.HasSuffix(cmd, []byte("ollama")):
+		return "ollama"
 	case bytes.Contains(cmd, []byte("victoria-metrics")) ||
 		bytes.Contains(cmd, []byte("vmstorage")) ||
 		bytes.Contains(cmd, []byte("vminsert")) ||

containers/container.go

@@ -37,6 +37,7 @@ var (
 	pingTimeout               = 300 * time.Millisecond
 	multilineCollectorTimeout = time.Second
 	payloadThreshold          = 1024 * 1024
+	gpuStatsWindow            = 15 * time.Second
 )
 
 type ContainerID string
@@ -146,6 +147,8 @@ type Container struct {
 	l7Stats  L7Stats
 	dnsStats *L7Metrics
 
+	gpuStats map[string]*GpuUsage
+
 	oomKills                 int
 	pythonThreadLockWaitTime time.Duration
 
@@ -206,6 +209,8 @@ func NewContainer(id ContainerID, cg *cgroup.Cgroup, md *ContainerMetadata, pid
 		l7Stats:                  L7Stats{},
 		dnsStats:                 &L7Metrics{},
 
+		gpuStats: map[string]*GpuUsage{},
+
 		mounts:     map[string]proc.MountInfo{},
 		seenMounts: map[uint64]struct{}{},
 
@@ -407,7 +412,27 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 				process.dotNetMonitor.Collect(ch)
 			}
 		}
+
+		for _, usage := range c.gpuStats {
+			usage.Reset()
+		}
+		if usage := process.getGPUUsage(); usage != nil {
+			for uuid, u := range usage {
+				tu := c.gpuStats[uuid]
+				if tu == nil {
+					tu = &GpuUsage{}
+					c.gpuStats[uuid] = tu
+				}
+				tu.GPU += u.GPU
+				tu.Memory += u.Memory
+			}
+		}
 	}
+	for uuid, usage := range c.gpuStats {
+		ch <- gauge(metrics.GpuUsagePercent, usage.GPU, uuid)
+		ch <- gauge(metrics.GpuMemoryUsagePercent, usage.Memory, uuid)
+	}
+
 	for appType := range appTypes {
 		ch <- gauge(metrics.ApplicationType, 1, appType)
 	}

containers/metrics.go

@@ -52,6 +52,9 @@ var metrics = struct {
 
 	PythonThreadLockWaitTime *prometheus.Desc
 
+	GpuUsagePercent       *prometheus.Desc
+	GpuMemoryUsagePercent *prometheus.Desc
+
 	Ip2Fqdn *prometheus.Desc
 }{
 	ContainerInfo: metric("container_info", "Meta information about the container", "image", "systemd_triggered_by"),
@@ -102,6 +105,9 @@ var metrics = struct {
 	Ip2Fqdn: metric("ip_to_fqdn", "Mapping IP addresses to FQDNs based on DNS requests initiated by containers", "ip", "fqdn"),
 
 	PythonThreadLockWaitTime: metric("container_python_thread_lock_wait_time_seconds", "Time spent waiting acquiring GIL in seconds"),
+
+	GpuUsagePercent:       metric("container_resources_gpu_usage_percent", "Percent of GPU compute resources used by the container", "gpu_uuid"),
+	GpuMemoryUsagePercent: metric("container_resources_gpu_memory_usage_percent", "Percent of GPU memory used by the container", "gpu_uuid"),
 }
 
 var (

containers/process.go

@@ -8,11 +8,22 @@ import (
 
 	"github.com/cilium/ebpf/link"
 	"github.com/coroot/coroot-node-agent/ebpftracer"
+	"github.com/coroot/coroot-node-agent/gpu"
 	"github.com/coroot/coroot-node-agent/proc"
 	"github.com/jpillora/backoff"
 	"github.com/mdlayher/taskstats"
 )
 
+type GpuUsage struct {
+	GPU    float64
+	Memory float64
+}
+
+func (gu *GpuUsage) Reset() {
+	gu.Memory = 0
+	gu.GPU = 0
+}
+
 type Process struct {
 	Pid       uint32
 	StartedAt time.Time
@@ -29,6 +40,8 @@ type Process struct {
 	goTlsUprobesChecked   bool
 	openSslUprobesChecked bool
 	pythonGilChecked      bool
+
+	gpuUsageSamples []gpu.ProcessUsageSample
 }
 
 func NewProcess(pid uint32, stats *taskstats.Stats, tracer *ebpftracer.Tracer) *Process {
@@ -97,6 +110,43 @@ func (p *Process) instrumentPython(cmdline []byte, tracer *ebpftracer.Tracer) {
 	p.uprobes = append(p.uprobes, tracer.AttachPythonThreadLockProbes(p.Pid)...)
 }
 
+func (p *Process) addGpuUsageSample(sample gpu.ProcessUsageSample) {
+	p.removeOldGpuUsageSamples(sample.Timestamp.Add(-gpuStatsWindow))
+	p.gpuUsageSamples = append(p.gpuUsageSamples, sample)
+}
+
+func (p *Process) getGPUUsage() map[string]*GpuUsage {
+	p.removeOldGpuUsageSamples(time.Now().Add(-gpuStatsWindow))
+	if len(p.gpuUsageSamples) == 0 {
+		return nil
+	}
+	gpuStatsWindowSeconds := gpuStatsWindow.Seconds()
+	res := make(map[string]*GpuUsage)
+	for _, sample := range p.gpuUsageSamples {
+		u := res[sample.UUID]
+		if u == nil {
+			u = &GpuUsage{}
+			res[sample.UUID] = u
+		}
+		u.GPU += float64(sample.GPUPercent) / gpuStatsWindowSeconds
+		u.Memory += float64(sample.MemoryPercent) / gpuStatsWindowSeconds
+	}
+	return res
+}
+
+func (p *Process) removeOldGpuUsageSamples(cutoff time.Time) {
+	i := 0
+	for ; i < len(p.gpuUsageSamples); i++ {
+		if p.gpuUsageSamples[i].Timestamp.After(cutoff) {
+			break
+		}
+	}
+	if i > 0 {
+		copy(p.gpuUsageSamples, p.gpuUsageSamples[i:])
+		p.gpuUsageSamples = p.gpuUsageSamples[:len(p.gpuUsageSamples)-i]
+	}
+}
+
 func (p *Process) Close() {
 	p.cancelFunc()
 	for _, u := range p.uprobes {

containers/registry.go

@@ -14,6 +14,7 @@ import (
 	"github.com/coroot/coroot-node-agent/common"
 	"github.com/coroot/coroot-node-agent/ebpftracer"
 	"github.com/coroot/coroot-node-agent/flags"
+	"github.com/coroot/coroot-node-agent/gpu"
 	"github.com/coroot/coroot-node-agent/proc"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/vishvananda/netns"
@@ -69,9 +70,11 @@ type Registry struct {
 	trafficStatsLastUpdated time.Time
 	trafficStatsLock        sync.Mutex
 	trafficStatsUpdateCh    chan *TrafficStatsUpdate
+
+	gpuProcessUsageSampleChan chan gpu.ProcessUsageSample
 }
 
-func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo, ip_resolver *common.K8sIPResolver) (*Registry, error) {
+func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo, ip_resolver *common.K8sIPResolver, gpuProcessUsageSampleChan chan gpu.ProcessUsageSample) (*Registry, error) {
 	ns, err := proc.GetSelfNetNs()
 	if err != nil {
 		return nil, err
@@ -122,6 +125,8 @@ func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo, ip
 		ip_resolver:          ip_resolver,
 		tracer:               ebpftracer.NewTracer(hostNetNs, selfNetNs, *flags.DisableL7Tracing),
 		trafficStatsUpdateCh: make(chan *TrafficStatsUpdate),
+
+		gpuProcessUsageSampleChan: gpuProcessUsageSampleChan,
 	}
 	if err = reg.Register(r); err != nil {
 		return nil, err
@@ -214,6 +219,12 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 			if c := r.containersByPid[u.Pid]; c != nil {
 				c.updateTrafficStats(u)
 			}
+		case sample := <-r.gpuProcessUsageSampleChan:
+			if c := r.containersByPid[sample.Pid]; c != nil {
+				if p := c.processes[sample.Pid]; p != nil {
+					p.addGpuUsageSample(sample)
+				}
+			}
 		case e, more := <-ch:
 			if !more {
 				return

flags/flags.go

@@ -48,15 +48,17 @@ var (
 
 	ScrapeInterval             = kingpin.Flag("scrape-interval", "How often to gather metrics from the agent").Default("15s").Envar("SCRAPE_INTERVAL").Duration()
 	WalDir                     = kingpin.Flag("wal-dir", "Path to where the agent stores data (e.g. the metrics Write-Ahead Log)").Default("/tmp/coroot-node-agent").Envar("WAL_DIR").String()
+	MaxSpoolSize               = kingpin.Flag("max-spool-size", "Maximum size of the on-disk spool used to buffer data when it cannot be sent to collector. Supports size suffixes like KB, MB, or GB.").Default("500MB").Envar("MAX_SPOOL_SIZE").Bytes()
 	ResolveDns                 = kingpin.Flag("resolve-dns", "should resolve DNS").Default("false").Envar("RESOLVE_DNS").Bool()
 	IgnoreControlPlane         = kingpin.Flag("ignore-control-plane", "ignore control plane like loki").Default("karpenter,loki,prometheus,grafana,kubelet,etcd,apiserver,victoria,nudgebee-agent,kube-system").Envar("IGNORE_CONTROL_PLANE").String()
 	SanitizeHeaders            = kingpin.Flag("sanitize-headers", "should sanitize headers").Default("true").Envar("SANITIZE_HEADERS").Bool()
 	SensitiveHeader            = kingpin.Flag("sensitive-headers", "sanitize headers using patterns").Default("Authorization, Cookie, X-Action-Token").Envar("SENSITIVE_HEADERS").String()
 	DisableKubeProbe           = kingpin.Flag("disable-kube-probe", "disable kube probe trace").Default("true").Envar("DISABLE_KUBE_PROBE").Bool()
 	DisableSensitiveLogParsing = kingpin.Flag("disable-sensitive-log-parsing", "disable sensitive log parsing").Default("false").Envar("DISABLE_SENSITIVE_LOG_PARSING").Bool()
 	TraceIdHeaders             = kingpin.Flag("trace-id-headers", "trace id headers").Default("Traceparent,X-Request-Id").Envar("TRACE_ID_HEADERS").String()
-	agentVersion               = kingpin.Flag("version", "Print version and exit").Default("false").Bool()
-	Version                    = "unknown"
+
+	agentVersion = kingpin.Flag("version", "Print version and exit").Default("false").Bool()
+	Version      = "unknown"
 )
 
 func GetString(fl *string) string {