diff --git a/.github/workflows/dev-ci.yaml b/.github/workflows/dev-ci.yaml
index 1fc33a90..756f2bcf 100644
--- a/.github/workflows/dev-ci.yaml
+++ b/.github/workflows/dev-ci.yaml
@@ -37,7 +37,7 @@ jobs:
         with:
           platforms: linux/amd64,linux/arm64
           context: .
-          file: 'Dockerfile.alpine'
+          file: 'Dockerfile'
           push: true
           tags: |
             864186153326.dkr.ecr.us-east-1.amazonaws.com/nudgebee-node-agent:${{ env.tag }}
diff --git a/.github/workflows/prod-ci.yaml b/.github/workflows/prod-ci.yaml
index d901721c..d3e71c9e 100644
--- a/.github/workflows/prod-ci.yaml
+++ b/.github/workflows/prod-ci.yaml
@@ -38,7 +38,7 @@ jobs:
         with:
           context: .
           platforms: linux/amd64,linux/arm64
-          file: 'Dockerfile.alpine'
+          file: 'Dockerfile'
           push: true
           tags: |
             740395098545.dkr.ecr.us-east-1.amazonaws.com/nudgebee-node-agent:${{ env.tag }}
diff --git a/Dockerfile b/Dockerfile
index e91ff290..57f76faf 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,11 +6,11 @@ COPY go.sum .
 RUN go mod download
 COPY . .
 ARG VERSION=unknown
-RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-X 'github.com/coroot/coroot-node-agent/flags.Version=${VERSION}'" -o coroot-node-agent .
+RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-extldflags='-Wl,-z,lazy' -X 'github.com/coroot/coroot-node-agent/flags.Version=${VERSION}'" -o coroot-node-agent .
 
 FROM registry.access.redhat.com/ubi9/ubi
 
 ARG VERSION=unknown
 
 COPY --from=builder /tmp/src/coroot-node-agent /usr/bin/coroot-node-agent
-ENTRYPOINT ["coroot-node-agent"]
+ENTRYPOINT ["coroot-node-agent"]
\ No newline at end of file
diff --git a/Dockerfile.alpine b/Dockerfile.alpine
index 49be3740..d6af7aa5 100644
--- a/Dockerfile.alpine
+++ b/Dockerfile.alpine
@@ -1,32 +1,34 @@
-FROM golang:1.23-alpine as build-stage
-# Set destination for COPY
-WORKDIR /app
-RUN apk add --no-cache --update go gcc g++ bash git
-RUN echo "unicode=\"YES\"" >> /etc/rc.conf && \
-    apk add --no-cache --virtual .build_deps \
-        autoconf file libc-dev make pkgconf python3 py3-pip ninja \
-        util-linux pciutils usbutils coreutils binutils findutils grep \
-        build-base abuild binutils binutils-doc gcc-doc gperf libcap libcap-dev \
-        valgrind-dev libmount && pip3 install meson --break-system-packages 
-
-RUN cd /tmp && git clone https://github.com/systemd/systemd
+# Build stage
+FROM golang:1.23.8-alpine AS builder
 
-# RUN cd /tmp/systemd && \
-#     meson build && \
-#     ninja build
+# Install build tools
+RUN apk add --no-cache build-base
 
-RUN apk add elogind-dev
+# Set up workspace
+WORKDIR /app
 
-COPY go.mod go.sum ./
+# Copy go mod files and download dependencies
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
 
-RUN  go mod download 
+# Copy the rest of the code
+COPY . .
 
-COPY ./ ./
+# Build statically linked binary
 ARG VERSION=unknown
-RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-X main.version=$VERSION" -o coroot-node-agent .
+RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-extldflags='-Wl,-z,lazy' -X 'github.com/coroot/coroot-node-agent/flags.Version=${VERSION}'" -o coroot-node-agent .
+
+# Runtime stage
+FROM alpine:3.22 AS release-stage
 
-FROM alpine:3.19 AS release-stage
 WORKDIR /app
-COPY --from=build-stage /app/coroot-node-agent /usr/bin/coroot-node-agent
+
+# Copy the statically linked binary
+COPY --from=builder /app/coroot-node-agent /usr/bin/coroot-node-agent
+
+# Ensure it’s executable
 RUN chmod +x /usr/bin/coroot-node-agent
-CMD ["coroot-node-agent"]
\ No newline at end of file
+
+# Run it
+CMD ["/usr/bin/coroot-node-agent"]
diff --git a/containers/app.go b/containers/app.go
index c6fa3090..9e252f13 100644
--- a/containers/app.go
+++ b/containers/app.go
@@ -120,6 +120,8 @@ func guessApplicationTypeByCmdline(cmdline []byte) string {
 		return "nats"
 	case bytes.HasSuffix(cmd, []byte("java")):
 		return "java"
+	case bytes.HasSuffix(cmd, []byte("ollama")):
+		return "ollama"
 	case bytes.Contains(cmd, []byte("victoria-metrics")) ||
 		bytes.Contains(cmd, []byte("vmstorage")) ||
 		bytes.Contains(cmd, []byte("vminsert")) ||
diff --git a/containers/container.go b/containers/container.go
index 69bc25fe..1f05458f 100644
--- a/containers/container.go
+++ b/containers/container.go
@@ -37,6 +37,7 @@ var (
 	pingTimeout               = 300 * time.Millisecond
 	multilineCollectorTimeout = time.Second
 	payloadThreshold          = 1024 * 1024
+	gpuStatsWindow            = 15 * time.Second
 )
 
 type ContainerID string
@@ -146,6 +147,8 @@ type Container struct {
 	l7Stats  L7Stats
 	dnsStats *L7Metrics
 
+	gpuStats map[string]*GpuUsage
+
 	oomKills                 int
 	pythonThreadLockWaitTime time.Duration
 
@@ -206,6 +209,8 @@ func NewContainer(id ContainerID, cg *cgroup.Cgroup, md *ContainerMetadata, pid
 		l7Stats:                  L7Stats{},
 		dnsStats:                 &L7Metrics{},
 
+		gpuStats: map[string]*GpuUsage{},
+
 		mounts:     map[string]proc.MountInfo{},
 		seenMounts: map[uint64]struct{}{},
 
@@ -407,7 +412,27 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 				process.dotNetMonitor.Collect(ch)
 			}
 		}
+
+		for _, usage := range c.gpuStats {
+			usage.Reset()
+		}
+		if usage := process.getGPUUsage(); usage != nil {
+			for uuid, u := range usage {
+				tu := c.gpuStats[uuid]
+				if tu == nil {
+					tu = &GpuUsage{}
+					c.gpuStats[uuid] = tu
+				}
+				tu.GPU += u.GPU
+				tu.Memory += u.Memory
+			}
+		}
 	}
+	for uuid, usage := range c.gpuStats {
+		ch <- gauge(metrics.GpuUsagePercent, usage.GPU, uuid)
+		ch <- gauge(metrics.GpuMemoryUsagePercent, usage.Memory, uuid)
+	}
+
 	for appType := range appTypes {
 		ch <- gauge(metrics.ApplicationType, 1, appType)
 	}
diff --git a/containers/metrics.go b/containers/metrics.go
index 41483b85..79db9d7c 100644
--- a/containers/metrics.go
+++ b/containers/metrics.go
@@ -52,6 +52,9 @@ var metrics = struct {
 
 	PythonThreadLockWaitTime *prometheus.Desc
 
+	GpuUsagePercent       *prometheus.Desc
+	GpuMemoryUsagePercent *prometheus.Desc
+
 	Ip2Fqdn *prometheus.Desc
 }{
 	ContainerInfo: metric("container_info", "Meta information about the container", "image", "systemd_triggered_by"),
@@ -102,6 +105,9 @@ var metrics = struct {
 	Ip2Fqdn: metric("ip_to_fqdn", "Mapping IP addresses to FQDNs based on DNS requests initiated by containers", "ip", "fqdn"),
 
 	PythonThreadLockWaitTime: metric("container_python_thread_lock_wait_time_seconds", "Time spent waiting acquiring GIL in seconds"),
+
+	GpuUsagePercent:       metric("container_resources_gpu_usage_percent", "Percent of GPU compute resources used by the container", "gpu_uuid"),
+	GpuMemoryUsagePercent: metric("container_resources_gpu_memory_usage_percent", "Percent of GPU memory used by the container", "gpu_uuid"),
 }
 
 var (
diff --git a/containers/process.go b/containers/process.go
index ddf5ac27..9ee7242d 100644
--- a/containers/process.go
+++ b/containers/process.go
@@ -8,11 +8,22 @@ import (
 
 	"github.com/cilium/ebpf/link"
 	"github.com/coroot/coroot-node-agent/ebpftracer"
+	"github.com/coroot/coroot-node-agent/gpu"
 	"github.com/coroot/coroot-node-agent/proc"
 	"github.com/jpillora/backoff"
 	"github.com/mdlayher/taskstats"
 )
 
+type GpuUsage struct {
+	GPU    float64
+	Memory float64
+}
+
+func (gu *GpuUsage) Reset() {
+	gu.Memory = 0
+	gu.GPU = 0
+}
+
 type Process struct {
 	Pid       uint32
 	StartedAt time.Time
@@ -29,6 +40,8 @@ type Process struct {
 	goTlsUprobesChecked   bool
 	openSslUprobesChecked bool
 	pythonGilChecked      bool
+
+	gpuUsageSamples []gpu.ProcessUsageSample
 }
 
 func NewProcess(pid uint32, stats *taskstats.Stats, tracer *ebpftracer.Tracer) *Process {
@@ -97,6 +110,43 @@ func (p *Process) instrumentPython(cmdline []byte, tracer *ebpftracer.Tracer) {
 	p.uprobes = append(p.uprobes, tracer.AttachPythonThreadLockProbes(p.Pid)...)
 }
 
+func (p *Process) addGpuUsageSample(sample gpu.ProcessUsageSample) {
+	p.removeOldGpuUsageSamples(sample.Timestamp.Add(-gpuStatsWindow))
+	p.gpuUsageSamples = append(p.gpuUsageSamples, sample)
+}
+
+func (p *Process) getGPUUsage() map[string]*GpuUsage {
+	p.removeOldGpuUsageSamples(time.Now().Add(-gpuStatsWindow))
+	if len(p.gpuUsageSamples) == 0 {
+		return nil
+	}
+	gpuStatsWindowSeconds := gpuStatsWindow.Seconds()
+	res := make(map[string]*GpuUsage)
+	for _, sample := range p.gpuUsageSamples {
+		u := res[sample.UUID]
+		if u == nil {
+			u = &GpuUsage{}
+			res[sample.UUID] = u
+		}
+		u.GPU += float64(sample.GPUPercent) / gpuStatsWindowSeconds
+		u.Memory += float64(sample.MemoryPercent) / gpuStatsWindowSeconds
+	}
+	return res
+}
+
+func (p *Process) removeOldGpuUsageSamples(cutoff time.Time) {
+	i := 0
+	for ; i < len(p.gpuUsageSamples); i++ {
+		if p.gpuUsageSamples[i].Timestamp.After(cutoff) {
+			break
+		}
+	}
+	if i > 0 {
+		copy(p.gpuUsageSamples, p.gpuUsageSamples[i:])
+		p.gpuUsageSamples = p.gpuUsageSamples[:len(p.gpuUsageSamples)-i]
+	}
+}
+
 func (p *Process) Close() {
 	p.cancelFunc()
 	for _, u := range p.uprobes {
diff --git a/containers/registry.go b/containers/registry.go
index 193a7949..15be6902 100644
--- a/containers/registry.go
+++ b/containers/registry.go
@@ -14,6 +14,7 @@ import (
 	"github.com/coroot/coroot-node-agent/common"
 	"github.com/coroot/coroot-node-agent/ebpftracer"
 	"github.com/coroot/coroot-node-agent/flags"
+	"github.com/coroot/coroot-node-agent/gpu"
 	"github.com/coroot/coroot-node-agent/proc"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/vishvananda/netns"
@@ -69,9 +70,11 @@ type Registry struct {
 	trafficStatsLastUpdated time.Time
 	trafficStatsLock        sync.Mutex
 	trafficStatsUpdateCh    chan *TrafficStatsUpdate
+
+	gpuProcessUsageSampleChan chan gpu.ProcessUsageSample
 }
 
-func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo, ip_resolver *common.K8sIPResolver) (*Registry, error) {
+func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo, ip_resolver *common.K8sIPResolver, gpuProcessUsageSampleChan chan gpu.ProcessUsageSample) (*Registry, error) {
 	ns, err := proc.GetSelfNetNs()
 	if err != nil {
 		return nil, err
@@ -122,6 +125,8 @@ func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo, ip
 		ip_resolver:          ip_resolver,
 		tracer:               ebpftracer.NewTracer(hostNetNs, selfNetNs, *flags.DisableL7Tracing),
 		trafficStatsUpdateCh: make(chan *TrafficStatsUpdate),
+
+		gpuProcessUsageSampleChan: gpuProcessUsageSampleChan,
 	}
 	if err = reg.Register(r); err != nil {
 		return nil, err
@@ -214,6 +219,12 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 			if c := r.containersByPid[u.Pid]; c != nil {
 				c.updateTrafficStats(u)
 			}
+		case sample := <-r.gpuProcessUsageSampleChan:
+			if c := r.containersByPid[sample.Pid]; c != nil {
+				if p := c.processes[sample.Pid]; p != nil {
+					p.addGpuUsageSample(sample)
+				}
+			}
 		case e, more := <-ch:
 			if !more {
 				return
diff --git a/flags/flags.go b/flags/flags.go
index 600d9261..f4f2cf75 100644
--- a/flags/flags.go
+++ b/flags/flags.go
@@ -48,6 +48,7 @@ var (
 
 	ScrapeInterval             = kingpin.Flag("scrape-interval", "How often to gather metrics from the agent").Default("15s").Envar("SCRAPE_INTERVAL").Duration()
 	WalDir                     = kingpin.Flag("wal-dir", "Path to where the agent stores data (e.g. the metrics Write-Ahead Log)").Default("/tmp/coroot-node-agent").Envar("WAL_DIR").String()
+	MaxSpoolSize               = kingpin.Flag("max-spool-size", "Maximum size of the on-disk spool used to buffer data when it cannot be sent to collector. Supports size suffixes like KB, MB, or GB.").Default("500MB").Envar("MAX_SPOOL_SIZE").Bytes()
 	ResolveDns                 = kingpin.Flag("resolve-dns", "should resolve DNS").Default("false").Envar("RESOLVE_DNS").Bool()
 	IgnoreControlPlane         = kingpin.Flag("ignore-control-plane", "ignore control plane like loki").Default("karpenter,loki,prometheus,grafana,kubelet,etcd,apiserver,victoria,nudgebee-agent,kube-system").Envar("IGNORE_CONTROL_PLANE").String()
 	SanitizeHeaders            = kingpin.Flag("sanitize-headers", "should sanitize headers").Default("true").Envar("SANITIZE_HEADERS").Bool()
@@ -55,8 +56,9 @@ var (
 	DisableKubeProbe           = kingpin.Flag("disable-kube-probe", "disable kube probe trace").Default("true").Envar("DISABLE_KUBE_PROBE").Bool()
 	DisableSensitiveLogParsing = kingpin.Flag("disable-sensitive-log-parsing", "disable sensitive log parsing").Default("false").Envar("DISABLE_SENSITIVE_LOG_PARSING").Bool()
 	TraceIdHeaders             = kingpin.Flag("trace-id-headers", "trace id headers").Default("Traceparent,X-Request-Id").Envar("TRACE_ID_HEADERS").String()
-	agentVersion               = kingpin.Flag("version", "Print version and exit").Default("false").Bool()
-	Version                    = "unknown"
+
+	agentVersion = kingpin.Flag("version", "Print version and exit").Default("false").Bool()
+	Version      = "unknown"
 )
 
 func GetString(fl *string) string {
diff --git a/go.mod b/go.mod
index 1d045620..d3b8b459 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.23.8
 require (
 	cloud.google.com/go/compute/metadata v0.6.0
 	github.com/ClickHouse/ch-go v0.65.0
+	github.com/NVIDIA/go-nvml v0.12.4-1
 	github.com/agoda-com/opentelemetry-logs-go v0.4.1
 	github.com/cilium/cilium v1.17.3
 	github.com/cilium/ebpf v0.17.3
@@ -15,14 +16,16 @@ require (
 	github.com/florianl/go-conntrack v0.3.0
 	github.com/go-kit/log v0.2.1
 	github.com/godbus/dbus/v5 v5.1.0
+	github.com/golang/snappy v0.0.4
 	github.com/google/uuid v1.6.0
-	github.com/grafana/pyroscope/ebpf v0.4.8
+	github.com/grafana/pyroscope/ebpf v0.4.9
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/jpillora/backoff v1.0.0
 	github.com/mdlayher/taskstats v0.0.0-20230712191918-387b3d561d14
 	github.com/nudgebee/logparser v0.0.0-20250410071541-48857637acb3
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
 	github.com/prometheus/client_golang v1.20.5
+	github.com/prometheus/client_model v0.6.1
 	github.com/prometheus/common v0.61.0
 	github.com/prometheus/prometheus v0.51.2
 	github.com/pyroscope-io/dotnetdiag v1.2.1
@@ -50,7 +53,23 @@ require (
 	k8s.io/klog/v2 v2.130.1
 )
 
-require github.com/gobwas/glob v0.2.3
+require (
+	github.com/aws/aws-sdk-go v1.50.32 // indirect
+	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
+	github.com/dennwc/varint v1.0.0 // indirect
+	github.com/gobwas/glob v0.2.3
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
+	github.com/hashicorp/go-version v1.7.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
+	github.com/prometheus/common/sigv4 v0.1.0 // indirect
+	go.opentelemetry.io/collector/featuregate v1.3.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.3.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.96.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/goleak v1.3.0 // indirect
+)
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect
@@ -63,8 +82,6 @@ require (
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270 // indirect
-	github.com/aws/aws-sdk-go v1.50.32 // indirect
-	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@@ -82,7 +99,6 @@ require (
 	github.com/containerd/typeurl v1.0.2 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dennwc/varint v1.0.0 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
@@ -108,10 +124,8 @@ require (
 	github.com/go-openapi/validate v0.24.0 // indirect
 	github.com/gogo/googleapis v1.4.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
@@ -119,16 +133,13 @@ require (
 	github.com/gopacket/gopacket v1.3.1 // indirect
 	github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect
-	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/hashicorp/hcl v1.0.1-vault-5 // indirect
 	github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
-	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mackerelio/go-osstat v0.2.5 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
@@ -146,7 +157,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
@@ -158,8 +168,6 @@ require (
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common/sigv4 v0.1.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
@@ -181,15 +189,10 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.17 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/collector/featuregate v1.3.0 // indirect
-	go.opentelemetry.io/collector/pdata v1.3.0 // indirect
-	go.opentelemetry.io/collector/semconv v0.96.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
 	go.opentelemetry.io/otel/metric v1.34.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.5.0 // indirect
-	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/dig v1.17.1 // indirect
-	go.uber.org/goleak v1.3.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
@@ -209,6 +212,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gotest.tools v2.2.0+incompatible
 	k8s.io/apiextensions-apiserver v0.32.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
 	k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
diff --git a/go.sum b/go.sum
index c3de9b92..7c391f09 100644
--- a/go.sum
+++ b/go.sum
@@ -94,6 +94,8 @@ github.com/Microsoft/hcsshim v0.9.12 h1:0Wgl1fRF4WmBuqP6EnHk2w3m7CCCumD/KUumZxp7
 github.com/Microsoft/hcsshim v0.9.12/go.mod h1:qAiPvMgZoM0wpkVg6qMdSEu+1VtI6/qHOOPkTGt8ftQ=
 github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
 github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
+github.com/NVIDIA/go-nvml v0.12.4-1 h1:WKUvqshhWSNTfm47ETRhv0A0zJyr1ncCuHiXwoTrBEc=
+github.com/NVIDIA/go-nvml v0.12.4-1/go.mod h1:8Llmj+1Rr+9VGGwZuRer5N/aCjxGuR5nPb/9ebBiIEQ=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
@@ -492,8 +494,8 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
-github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
+github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
diff --git a/gpu/gpu.go b/gpu/gpu.go
new file mode 100644
index 00000000..f91d690d
--- /dev/null
+++ b/gpu/gpu.go
@@ -0,0 +1,290 @@
+package gpu
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/NVIDIA/go-nvml/pkg/nvml"
+	"github.com/coroot/coroot-node-agent/proc"
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/klog/v2"
+)
+
+var (
+	gpuInfo = prometheus.NewDesc(
+		"node_gpu_info",
+		"Meta information about the GPU",
+		[]string{"gpu_uuid", "name"}, nil,
+	)
+	gpuMemoryTotal = prometheus.NewDesc(
+		"node_resources_gpu_memory_total_bytes",
+		"Total memory available on the GPU in bytes",
+		[]string{"gpu_uuid"}, nil,
+	)
+	gpuMemoryUsed = prometheus.NewDesc(
+		"node_resources_gpu_memory_used_bytes",
+		"GPU memory currently in use in bytes",
+		[]string{"gpu_uuid"}, nil,
+	)
+	gpuMemoryUsageAvg = prometheus.NewDesc(
+		"node_resources_gpu_memory_utilization_percent_avg",
+		"Average GPU memory utilization (percentage) over the collection interval",
+		[]string{"gpu_uuid"}, nil,
+	)
+	gpuTemperature = prometheus.NewDesc(
+		"node_resources_gpu_temperature_celsius",
+		"Current temperature of the GPU in Celsius",
+		[]string{"gpu_uuid"}, nil,
+	)
+	gpuPowerWatts = prometheus.NewDesc(
+		"node_resources_gpu_power_usage_watts",
+		"Current power usage of the GPU in watts",
+		[]string{"gpu_uuid"}, nil,
+	)
+	gpuMemoryUsagePeak = prometheus.NewDesc(
+		"node_resources_gpu_memory_utilization_percent_peak",
+		"Peak GPU memory utilization (percentage) over the collection interval",
+		[]string{"gpu_uuid"}, nil,
+	)
+	gpuUsageAvg = prometheus.NewDesc(
+		"node_resources_gpu_utilization_percent_avg",
+		"Average GPU core utilization (percentage) over the collection interval",
+		[]string{"gpu_uuid"}, nil,
+	)
+	gpuUsagePeak = prometheus.NewDesc(
+		"node_resources_gpu_utilization_percent_peak",
+		"Peak GPU core utilization (percentage) over the collection interval",
+		[]string{"gpu_uuid"}, nil,
+	)
+)
+
+type Collector struct {
+	ProcessUsageSampleCh chan ProcessUsageSample
+	iface                nvml.Interface
+	devices              []*Device
+	lock                 sync.Mutex
+}
+
+type Device struct {
+	UUID           string
+	Name           string
+	device         nvml.Device
+	lastSampleTime map[nvml.SamplingType]uint64
+}
+
+type ProcessUsageSample struct {
+	UUID          string
+	Pid           uint32
+	Timestamp     time.Time
+	GPUPercent    uint32
+	MemoryPercent uint32
+}
+
+func NewCollector() (*Collector, error) {
+	c := &Collector{
+		ProcessUsageSampleCh: make(chan ProcessUsageSample, 100),
+	}
+
+	libPath, err := findNvidiaMLLib()
+	if err != nil {
+		klog.Infoln(err)
+		return c, nil
+	}
+	klog.Infof("found NVML lib at %s", libPath)
+
+	c.iface = nvml.New(nvml.WithLibraryPath(libPath))
+	if ret := c.iface.Init(); ret != nvml.SUCCESS {
+		return c, fmt.Errorf("unable to initialize NVML: %s", nvml.ErrorString(ret))
+	}
+	count, ret := c.iface.DeviceGetCount()
+	if ret != nvml.SUCCESS {
+		return c, fmt.Errorf("unable to get device count: %s", nvml.ErrorString(ret))
+	}
+	var names []string
+	for i := 0; i < count; i++ {
+		device, ret := c.iface.DeviceGetHandleByIndex(i)
+		if ret != nvml.SUCCESS {
+			return c, errors.New(nvml.ErrorString(ret))
+		}
+		dev := Device{
+			lastSampleTime: map[nvml.SamplingType]uint64{},
+			device:         device,
+		}
+		if dev.UUID, ret = device.GetUUID(); ret != nvml.SUCCESS {
+			return c, errors.New(nvml.ErrorString(ret))
+		}
+		if dev.Name, ret = device.GetName(); ret != nvml.SUCCESS {
+			return c, errors.New(nvml.ErrorString(ret))
+		}
+		names = append(names, dev.Name)
+		c.devices = append(c.devices, &dev)
+	}
+	if len(names) > 0 {
+		klog.Infof("found %d GPU: %s", len(names), strings.Join(names, ", "))
+	}
+	go c.processUtilizationPoller()
+	return c, nil
+}
+
+func (c *Collector) processUtilizationPoller() {
+	ticker := time.NewTicker(1 * time.Second)
+	lastTs := uint64(time.Now().UnixMicro())
+	for range ticker.C {
+		for _, dev := range c.devices {
+			samples, _ := dev.device.GetProcessUtilization(lastTs)
+			for _, sample := range samples {
+				if sample.TimeStamp <= lastTs {
+					continue
+				}
+				if sample.SmUtil > 0 {
+					c.ProcessUsageSampleCh <- ProcessUsageSample{
+						UUID:          dev.UUID,
+						Pid:           sample.Pid,
+						GPUPercent:    sample.SmUtil,
+						MemoryPercent: sample.MemUtil,
+						Timestamp:     time.UnixMicro(int64(sample.TimeStamp)),
+					}
+				}
+				lastTs = sample.TimeStamp
+			}
+		}
+	}
+}
+
+func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- gpuInfo
+	ch <- gpuMemoryTotal
+	ch <- gpuMemoryUsed
+	ch <- gpuMemoryUsageAvg
+	ch <- gpuMemoryUsagePeak
+	ch <- gpuUsageAvg
+	ch <- gpuUsagePeak
+	ch <- gpuTemperature
+	ch <- gpuPowerWatts
+}
+
+func (c *Collector) Collect(ch chan<- prometheus.Metric) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	for _, dev := range c.devices {
+		ch <- gauge(gpuInfo, 1, dev.UUID, dev.Name)
+
+		mi, ret := dev.device.GetMemoryInfo()
+		if ret == nvml.SUCCESS {
+			ch <- gauge(gpuMemoryTotal, float64(mi.Total), dev.UUID)
+			ch <- gauge(gpuMemoryUsed, float64(mi.Used), dev.UUID)
+		}
+		if t, ret := dev.device.GetTemperature(nvml.TEMPERATURE_GPU); ret == nvml.SUCCESS {
+			ch <- gauge(gpuTemperature, float64(t), dev.UUID)
+		}
+		if mw, ret := dev.device.GetPowerUsage(); ret == nvml.SUCCESS {
+			ch <- gauge(gpuPowerWatts, float64(mw)/1000., dev.UUID)
+		}
+		for _, st := range []nvml.SamplingType{nvml.GPU_UTILIZATION_SAMPLES, nvml.MEMORY_UTILIZATION_SAMPLES} {
+			lastTs := dev.lastSampleTime[st]
+			valtype, samples, ret := dev.device.GetSamples(st, lastTs)
+			if ret != nvml.SUCCESS {
+				continue
+			}
+			total := float64(0)
+			count := float64(0)
+			peak := float64(0)
+			for _, sample := range samples {
+				if sample.TimeStamp <= lastTs {
+					continue
+				}
+				value, err := valueToFloat(valtype, sample.SampleValue)
+				if err != nil {
+					continue
+				}
+				total += value
+				if value > peak {
+					peak = value
+				}
+				count++
+				lastTs = sample.TimeStamp
+			}
+			if count > 0 {
+				switch st {
+				case nvml.GPU_UTILIZATION_SAMPLES:
+					ch <- gauge(gpuUsageAvg, total/count, dev.UUID)
+					ch <- gauge(gpuUsagePeak, peak, dev.UUID)
+				case nvml.MEMORY_UTILIZATION_SAMPLES:
+					ch <- gauge(gpuMemoryUsageAvg, total/count, dev.UUID)
+					ch <- gauge(gpuMemoryUsagePeak, peak, dev.UUID)
+				}
+			}
+			dev.lastSampleTime[st] = lastTs
+		}
+	}
+}
+
+func (c *Collector) Close() {
+	c.iface.Shutdown()
+}
+
+func findNvidiaMLLib() (string, error) {
+	paths := []string{
+		// gpu-operator
+		"/run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1",
+		"/run/nvidia/driver/usr/lib64/libnvidia-ml.so.1",
+		"/home/kubernetes/bin/nvidia/lib64/libnvidia-ml.so.1", //GKE
+
+		"/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1",
+		"/usr/lib64/libnvidia-ml.so.1",
+		"/usr/local/cuda/lib64/libnvidia-ml.so.1",
+		"/usr/lib/libnvidia-ml.so.1",
+	}
+	if runtime.GOARCH == "arm64" {
+		paths = append(paths,
+			"/usr/lib/aarch64-linux-gnu/libnvidia-ml.so.1",
+			"/run/nvidia/driver/usr/lib/aarch64-linux-gnu/libnvidia-ml.so.1",
+			"/home/kubernetes/bin/nvidia/lib64-aarch64/libnvidia-ml.so.1", //GKE
+		)
+	}
+	for _, p := range paths {
+		if _, err := os.Stat(proc.HostPath(p)); err == nil {
+			return proc.HostPath(p), nil
+		}
+	}
+	return "", fmt.Errorf("libnvidia-ml.so.1 not found in known paths")
+}
+
+func valueToFloat(valueType nvml.ValueType, value [8]byte) (float64, error) {
+	r := bytes.NewReader(value[:])
+	switch valueType {
+	case nvml.VALUE_TYPE_DOUBLE:
+		var v float64
+		err := binary.Read(r, binary.LittleEndian, &v)
+		return v, err
+	case nvml.VALUE_TYPE_UNSIGNED_INT:
+		var v uint32
+		err := binary.Read(r, binary.LittleEndian, &v)
+		return float64(v), err
+	case nvml.VALUE_TYPE_UNSIGNED_LONG, nvml.VALUE_TYPE_UNSIGNED_LONG_LONG:
+		var v uint64
+		err := binary.Read(r, binary.LittleEndian, &v)
+		return float64(v), err
+	case nvml.VALUE_TYPE_SIGNED_LONG_LONG:
+		var v int64
+		err := binary.Read(r, binary.LittleEndian, &v)
+		return float64(v), err
+	case nvml.VALUE_TYPE_SIGNED_INT:
+		var v int32
+		err := binary.Read(r, binary.LittleEndian, &v)
+		return float64(v), err
+	default:
+		return 0, fmt.Errorf("unsupported value type %d", valueType)
+	}
+}
+
+func gauge(desc *prometheus.Desc, value float64, labelValues ...string) prometheus.Metric {
+	return prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, value, labelValues...)
+}
diff --git a/main.go b/main.go
index 14938c3b..7feedd16 100644
--- a/main.go
+++ b/main.go
@@ -14,6 +14,7 @@ import (
 	"github.com/coroot/coroot-node-agent/common"
 	"github.com/coroot/coroot-node-agent/containers"
 	"github.com/coroot/coroot-node-agent/flags"
+	"github.com/coroot/coroot-node-agent/gpu"
 	"github.com/coroot/coroot-node-agent/logs"
 	"github.com/coroot/coroot-node-agent/node"
 	"github.com/coroot/coroot-node-agent/proc"
@@ -184,21 +185,25 @@ func main() {
 	if err := registerer.Register(nodeCollector); err != nil {
 		klog.Exitln(err)
 	}
+
+	gpuCollector, err := gpu.NewCollector()
+	if err != nil {
+		klog.Warningln("failed to initialize GPU collector:", err)
+	}
+	if err := registerer.Register(gpuCollector); err != nil {
+		klog.Exitln(err)
+	}
 	registerer.MustRegister(info("node_agent_info", version))
 
 	if md := nodeCollector.Metadata(); md != nil {
 		region := md.Region
 		az := md.AvailabilityZone
 		if region != "" && az != "" {
-			registerer = prometheus.WrapRegistererWith(
-				prometheus.Labels{"az": md.AvailabilityZone, "region": md.Region},
-				registerer,
-			)
+			registerer = prometheus.WrapRegistererWith(prometheus.Labels{"az": az, "region": region}, registerer)
 		}
 	}
 	processInfoCh := profiling.Init(machineId, hostname)
-
-	cr, err := containers.NewRegistry(registerer, processInfoCh, resolver)
+	cr, err := containers.NewRegistry(registerer, processInfoCh, resolver, gpuCollector.ProcessUsageSampleCh)
 	if err != nil {
 		klog.Exitln(err)
 	}
@@ -207,7 +212,7 @@ func main() {
 	profiling.Start()
 	defer profiling.Stop()
 
-	if err := prom.StartAgent(machineId); err != nil {
+	if err := prom.StartAgent(registry, machineId); err != nil {
 		klog.Exitln(err)
 	}
 
diff --git a/prom/agent.go b/prom/agent.go
deleted file mode 100644
index 502916b2..00000000
--- a/prom/agent.go
+++ /dev/null
@@ -1,103 +0,0 @@
-package prom
-
-import (
-	"time"
-
-	"github.com/coroot/coroot-node-agent/common"
-	"github.com/coroot/coroot-node-agent/flags"
-	"github.com/go-kit/log/level"
-	"github.com/prometheus/client_golang/prometheus"
-	promConfig "github.com/prometheus/common/config"
-	"github.com/prometheus/common/model"
-	"github.com/prometheus/prometheus/config"
-	"github.com/prometheus/prometheus/discovery/targetgroup"
-	"github.com/prometheus/prometheus/scrape"
-	"github.com/prometheus/prometheus/storage"
-	"github.com/prometheus/prometheus/storage/remote"
-	"github.com/prometheus/prometheus/tsdb"
-	"github.com/prometheus/prometheus/tsdb/agent"
-	"k8s.io/klog/v2"
-)
-
-const (
-	RemoteFlushDeadline = time.Minute
-	jobName             = "nudgebee-node-agent"
-	RemoteWriteTimeout  = 30 * time.Second
-)
-
-func StartAgent(machineId string) error {
-	logger := level.NewFilter(Logger{}, level.AllowInfo())
-
-	if *flags.MetricsEndpoint == nil {
-		return nil
-	}
-	klog.Infoln("metrics remote write endpoint:", (*flags.MetricsEndpoint).String())
-	cfg := config.DefaultConfig
-	cfg.GlobalConfig.ScrapeInterval = model.Duration(*flags.ScrapeInterval)
-	cfg.GlobalConfig.ScrapeTimeout = model.Duration(*flags.ScrapeInterval)
-	cfg.RemoteWriteConfigs = append(cfg.RemoteWriteConfigs,
-		&config.RemoteWriteConfig{
-			URL:           &promConfig.URL{URL: *flags.MetricsEndpoint},
-			Headers:       common.AuthHeaders(),
-			RemoteTimeout: model.Duration(RemoteWriteTimeout),
-			QueueConfig:   config.DefaultQueueConfig,
-			HTTPClientConfig: promConfig.HTTPClientConfig{
-				TLSConfig: promConfig.TLSConfig{InsecureSkipVerify: *flags.InsecureSkipVerify},
-			},
-		},
-	)
-	cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, &config.ScrapeConfig{
-		JobName:                 jobName,
-		HonorLabels:             true,
-		ScrapeClassicHistograms: true,
-		MetricsPath:             "/metrics",
-		Scheme:                  "http",
-		EnableCompression:       false,
-	})
-
-	opts := agent.DefaultOptions()
-	localStorage := &readyStorage{stats: tsdb.NewDBStats()}
-	scraper := &readyScrapeManager{}
-	remoteStorage := remote.NewStorage(logger, prometheus.DefaultRegisterer, localStorage.StartTime, *flags.WalDir, RemoteFlushDeadline, scraper)
-	fanoutStorage := storage.NewFanout(logger, localStorage, remoteStorage)
-
-	if err := remoteStorage.ApplyConfig(&cfg); err != nil {
-		return err
-	}
-
-	scrapeManager, err := scrape.NewManager(nil, logger, fanoutStorage, prometheus.DefaultRegisterer)
-	if err != nil {
-		return err
-	}
-	if err = scrapeManager.ApplyConfig(&cfg); err != nil {
-		return err
-	}
-	scraper.Set(scrapeManager)
-	db, err := agent.Open(logger, prometheus.DefaultRegisterer, remoteStorage, *flags.WalDir, opts)
-	if err != nil {
-		return err
-	}
-	localStorage.Set(db, 0)
-	db.SetWriteNotified(remoteStorage)
-
-	tch := make(chan map[string][]*targetgroup.Group, 1)
-	tch <- map[string][]*targetgroup.Group{
-		jobName: {
-			&targetgroup.Group{
-				Targets: []model.LabelSet{
-					{
-						model.InstanceLabel: model.LabelValue(machineId),
-						model.AddressLabel:  model.LabelValue(*flags.ListenAddress),
-					},
-				},
-				Labels: model.LabelSet{model.JobLabel: jobName},
-			},
-		},
-	}
-	go func() {
-		if err = scrapeManager.Run(tch); err != nil {
-			klog.Errorln(err)
-		}
-	}()
-	return nil
-}
diff --git a/prom/remote_writer.go b/prom/remote_writer.go
new file mode 100644
index 00000000..7e9c5672
--- /dev/null
+++ b/prom/remote_writer.go
@@ -0,0 +1,357 @@
+package prom
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/coroot/coroot-node-agent/common"
+	"github.com/coroot/coroot-node-agent/flags"
+	"github.com/golang/snappy"
+	"github.com/jpillora/backoff"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/prompb"
+	"github.com/prometheus/prometheus/util/fmtutil"
+	"k8s.io/klog/v2"
+)
+
+const RemoteWriteTimeout = 30 * time.Second
+
+type Agent struct {
+	reg    *prometheus.Registry
+	url    *url.URL
+	labels map[string]string
+
+	httpClient http.Client
+
+	spoolDir     string
+	maxSpoolSize int64
+}
+
+func StartAgent(reg *prometheus.Registry, machineId string) error {
+	if *flags.MetricsEndpoint == nil {
+		return nil
+	}
+	klog.Infoln("metrics remote write endpoint:", (*flags.MetricsEndpoint).String())
+
+	up := prometheus.NewGauge(prometheus.GaugeOpts{Name: "up"})
+	up.Set(1)
+	reg.MustRegister(up)
+
+	a := &Agent{
+		reg: reg,
+		url: *flags.MetricsEndpoint,
+		labels: map[string]string{
+			model.InstanceLabel: machineId,
+			model.JobLabel:      "coroot-node-agent",
+		},
+		httpClient: http.Client{
+			Timeout: RemoteWriteTimeout,
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: *flags.InsecureSkipVerify},
+			},
+		},
+		spoolDir:     path.Join(*flags.WalDir, "spool"),
+		maxSpoolSize: int64(*flags.MaxSpoolSize),
+	}
+	if _, err := os.Stat(*flags.WalDir); os.IsNotExist(err) {
+		if err = os.Mkdir(*flags.WalDir, 0750); err != nil {
+			return err
+		}
+	}
+	if _, err := os.Stat(a.spoolDir); os.IsNotExist(err) {
+		if err = os.Mkdir(a.spoolDir, 0750); err != nil {
+			return err
+		}
+	}
+	go a.sendLoop()
+	go a.scrapeLoop()
+	return nil
+}
+
+func (a *Agent) scrapeLoop() {
+	if err := a.scrape(); err != nil {
+		klog.Warningln("failed to scrape metrics:", err)
+	}
+	ticker := time.NewTicker(*flags.ScrapeInterval)
+	for range ticker.C {
+		if err := a.scrape(); err != nil {
+			klog.Warningln("failed to scrape metrics:", err)
+		}
+	}
+}
+
+func (a *Agent) sendLoop() {
+	b := backoff.Backoff{Factor: 2, Min: 5 * time.Second, Max: time.Minute}
+	for {
+		fName, err := a.getOldestSpoolFile()
+		if err != nil || fName == "" {
+			if err != nil {
+				klog.Warningln("failed to get oldest spool file:", err)
+			}
+			time.Sleep(5 * time.Second)
+			continue
+		}
+		err = func() error {
+			if err := a.send(fName); err != nil {
+				return err
+			}
+			return os.Remove(fName)
+		}()
+		if err != nil {
+			dur := b.Duration()
+			klog.Warningf(
+				"failed to send metrics to %s, next attempt in %s: %s",
+				a.url, dur.String(), err,
+			)
+			time.Sleep(dur)
+			continue
+		}
+		b.Reset()
+	}
+}
+
+func (a *Agent) send(fPath string) error {
+	f, err := os.Open(fPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	req, err := http.NewRequest(http.MethodPost, a.url.String(), f)
+	if err != nil {
+		return err
+	}
+	for k, v := range common.AuthHeaders() {
+		req.Header.Set(k, v)
+	}
+	req.Header.Set("User-Agent", "coroot-node-agent")
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("Content-Encoding", "snappy")
+	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	t := time.Now()
+	resp, err := a.httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode >= 300 {
+		return errors.New(resp.Status)
+	}
+	klog.Infof("sent metrics in %s", time.Since(t).Truncate(time.Millisecond))
+	return nil
+}
+
+func (a *Agent) scrape() error {
+	timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+
+	mfs, err := a.reg.Gather()
+	if err != nil {
+		return err
+	}
+	mfsByName := make(map[string]*dto.MetricFamily)
+	for _, mf := range mfs {
+		mfsByName[mf.GetName()] = mf
+	}
+	wr := buildWriteRequest(mfs, timestamp, a.labels)
+	decompressed, err := wr.Marshal()
+	if err != nil {
+		return err
+	}
+
+	compressed := snappy.Encode(nil, decompressed)
+	err = a.writeToSpool(timestamp, compressed)
+	return err
+}
+
+func (a *Agent) writeToSpool(timestamp int64, payload []byte) error {
+	if err := a.truncateSpoolIfNeeded(); err != nil {
+		return err
+	}
+	fileName := fmt.Sprintf("spool-%d.done", timestamp)
+	f, err := os.CreateTemp(a.spoolDir, fileName)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = f.Close()
+		_ = os.Remove(f.Name())
+	}()
+	if _, err = f.Write(payload); err != nil {
+		return err
+	}
+	if err = f.Close(); err != nil {
+		return err
+	}
+	if err = os.Rename(f.Name(), path.Join(a.spoolDir, fileName)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (a *Agent) truncateSpoolIfNeeded() error {
+	files, err := a.listSpoolFiles()
+	if err != nil {
+		return err
+	}
+	if len(files) <= 1 {
+		return nil
+	}
+	totalSize := int64(0)
+	for _, f := range files {
+		st, err := os.Stat(f)
+		if err != nil {
+			return err
+		}
+		totalSize += st.Size()
+	}
+	if totalSize > a.maxSpoolSize {
+		klog.Warningln("spool size exceeded, removing the oldest file:", files[0])
+		if err = os.Remove(files[0]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (a *Agent) listSpoolFiles() ([]string, error) {
+	entities, err := os.ReadDir(a.spoolDir)
+	if err != nil {
+		return nil, err
+	}
+	files := make([]string, 0, len(entities))
+	for _, e := range entities {
+		name := e.Name()
+		if strings.HasPrefix(name, "spool-") && strings.HasSuffix(name, ".done") {
+			files = append(files, path.Join(a.spoolDir, name))
+		}
+	}
+	sort.Strings(files)
+	return files, nil
+}
+
+func (a *Agent) getOldestSpoolFile() (string, error) {
+	files, err := a.listSpoolFiles()
+	if err != nil {
+		return "", err
+	}
+	if len(files) == 0 {
+		return "", nil
+	}
+	return files[0], nil
+}
+
+func makeLabelsMap(m *dto.Metric, metricName string, extraLabels map[string]string) map[string]string {
+	labels := make(map[string]string, len(m.Label)+len(extraLabels)+2) //1 for name, 1 for possible le
+	labels[model.MetricNameLabel] = metricName
+	for key, value := range extraLabels {
+		labels[key] = value
+	}
+	for _, label := range m.Label {
+		labels[label.GetName()] = label.GetValue()
+	}
+	return labels
+}
+
+func makeLabels(labelsMap map[string]string, metricNameSuffix, bucket string) []prompb.Label {
+	l := len(labelsMap)
+	if bucket != "" {
+		l++
+	}
+	sortedLabelNames := make([]string, 0, l)
+	for label := range labelsMap {
+		sortedLabelNames = append(sortedLabelNames, label)
+	}
+	if bucket != "" {
+		sortedLabelNames = append(sortedLabelNames, model.BucketLabel)
+	}
+	sort.Strings(sortedLabelNames)
+	labels := make([]prompb.Label, len(sortedLabelNames))
+
+	var name, value string
+	var i int
+	for i, name = range sortedLabelNames {
+		value = labelsMap[name]
+		switch name {
+		case model.MetricNameLabel:
+			if metricNameSuffix != "" {
+				value += metricNameSuffix
+			}
+		case model.BucketLabel:
+			value = bucket
+		}
+		labels[i].Name = name
+		labels[i].Value = value
+	}
+	return labels
+}
+
+func buildWriteRequest(mfs []*dto.MetricFamily, timestamp int64, extraLabels map[string]string) *prompb.WriteRequest {
+	wr := &prompb.WriteRequest{}
+	for _, mf := range mfs {
+		if len(mf.Metric) == 0 {
+			continue
+		}
+		mtype := fmtutil.MetricMetadataTypeValue[mf.Type.String()]
+		mName := mf.GetName()
+		metadata := prompb.MetricMetadata{
+			MetricFamilyName: mName,
+			Type:             prompb.MetricMetadata_MetricType(mtype),
+			Help:             mf.GetHelp(),
+		}
+		wr.Metadata = append(wr.Metadata, metadata)
+
+		for _, metric := range mf.Metric {
+			addTimeseries(wr, metric, makeLabelsMap(metric, mName, extraLabels), timestamp)
+		}
+	}
+	return wr
+}
+
+func addTimeseries(wr *prompb.WriteRequest, m *dto.Metric, labels map[string]string, timestamp int64) {
+	switch {
+	case m.Gauge != nil:
+		wr.Timeseries = append(wr.Timeseries, prompb.TimeSeries{
+			Samples: []prompb.Sample{{
+				Timestamp: timestamp,
+				Value:     m.GetGauge().GetValue(),
+			}},
+			Labels: makeLabels(labels, "", ""),
+		})
+	case m.Counter != nil:
+		wr.Timeseries = append(wr.Timeseries, prompb.TimeSeries{
+			Samples: []prompb.Sample{{
+				Timestamp: timestamp,
+				Value:     m.GetCounter().GetValue(),
+			}},
+			Labels: makeLabels(labels, "", ""),
+		})
+	case m.Histogram != nil:
+		for _, b := range m.GetHistogram().Bucket {
+			wr.Timeseries = append(wr.Timeseries, prompb.TimeSeries{
+				Samples: []prompb.Sample{{Timestamp: timestamp, Value: float64(b.GetCumulativeCount())}},
+				Labels:  makeLabels(labels, "_bucket", fmt.Sprint(b.GetUpperBound())),
+			})
+		}
+		wr.Timeseries = append(wr.Timeseries, prompb.TimeSeries{
+			Samples: []prompb.Sample{{Timestamp: timestamp, Value: float64(m.Histogram.GetSampleCount())}},
+			Labels:  makeLabels(labels, "_bucket", "+Inf"),
+		})
+		wr.Timeseries = append(wr.Timeseries, prompb.TimeSeries{
+			Samples: []prompb.Sample{{Timestamp: timestamp, Value: m.GetHistogram().GetSampleSum()}},
+			Labels:  makeLabels(labels, "_sum", ""),
+		})
+		wr.Timeseries = append(wr.Timeseries, prompb.TimeSeries{
+			Samples: []prompb.Sample{{Timestamp: timestamp, Value: float64(m.GetHistogram().GetSampleCount())}},
+			Labels:  makeLabels(labels, "_count", ""),
+		})
+	}
+}
diff --git a/prom/wrappers.go b/prom/wrappers.go
deleted file mode 100644
index 301d1853..00000000
--- a/prom/wrappers.go
+++ /dev/null
@@ -1,220 +0,0 @@
-package prom
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"math"
-	"sync"
-
-	"k8s.io/klog/v2"
-
-	"github.com/prometheus/prometheus/config"
-	"github.com/prometheus/prometheus/model/exemplar"
-	"github.com/prometheus/prometheus/model/histogram"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/prometheus/prometheus/model/metadata"
-	"github.com/prometheus/prometheus/scrape"
-	"github.com/prometheus/prometheus/storage"
-	"github.com/prometheus/prometheus/tsdb"
-	"github.com/prometheus/prometheus/tsdb/agent"
-)
-
-// copy-pasted from Prometheus' main.go
-
-type readyStorage struct {
-	mtx             sync.RWMutex
-	db              storage.Storage
-	startTimeMargin int64
-	stats           *tsdb.DBStats
-}
-
-func (s *readyStorage) ApplyConfig(conf *config.Config) error {
-	return nil
-}
-
-func (s *readyStorage) Set(db storage.Storage, startTimeMargin int64) {
-	s.mtx.Lock()
-	defer s.mtx.Unlock()
-	s.db = db
-	s.startTimeMargin = startTimeMargin
-}
-
-func (s *readyStorage) get() storage.Storage {
-	s.mtx.RLock()
-	x := s.db
-	s.mtx.RUnlock()
-	return x
-}
-
-func (s *readyStorage) getStats() *tsdb.DBStats {
-	s.mtx.RLock()
-	x := s.stats
-	s.mtx.RUnlock()
-	return x
-}
-
-func (s *readyStorage) StartTime() (int64, error) {
-	if x := s.get(); x != nil {
-		switch db := x.(type) {
-		case *agent.DB:
-			return db.StartTime()
-		default:
-			panic(fmt.Sprintf("unknown storage type %T", db))
-		}
-	}
-	return math.MaxInt64, tsdb.ErrNotReady
-}
-
-func (s *readyStorage) Querier(mint, maxt int64) (storage.Querier, error) {
-	if x := s.get(); x != nil {
-		return x.Querier(mint, maxt)
-	}
-	return nil, tsdb.ErrNotReady
-}
-
-func (s *readyStorage) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) {
-	if x := s.get(); x != nil {
-		return x.ChunkQuerier(mint, maxt)
-	}
-	return nil, tsdb.ErrNotReady
-}
-
-func (s *readyStorage) ExemplarQuerier(ctx context.Context) (storage.ExemplarQuerier, error) {
-	if x := s.get(); x != nil {
-		switch db := x.(type) {
-		case *agent.DB:
-			return nil, agent.ErrUnsupported
-		default:
-			panic(fmt.Sprintf("unknown storage type %T", db))
-		}
-	}
-	return nil, tsdb.ErrNotReady
-}
-
-func (s *readyStorage) Appender(ctx context.Context) storage.Appender {
-	if x := s.get(); x != nil {
-		return x.Appender(ctx)
-	}
-	return notReadyAppender{}
-}
-
-type notReadyAppender struct{}
-
-func (n notReadyAppender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) {
-	return 0, tsdb.ErrNotReady
-}
-
-func (n notReadyAppender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) {
-	return 0, tsdb.ErrNotReady
-}
-
-func (n notReadyAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) {
-	return 0, tsdb.ErrNotReady
-}
-
-func (n notReadyAppender) AppendCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64) (storage.SeriesRef, error) {
-	return 0, tsdb.ErrNotReady
-}
-
-func (n notReadyAppender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, m metadata.Metadata) (storage.SeriesRef, error) {
-	return 0, tsdb.ErrNotReady
-}
-
-func (n notReadyAppender) Commit() error { return tsdb.ErrNotReady }
-
-func (n notReadyAppender) Rollback() error { return tsdb.ErrNotReady }
-
-func (s *readyStorage) Close() error {
-	if x := s.get(); x != nil {
-		return x.Close()
-	}
-	return nil
-}
-
-func (s *readyStorage) CleanTombstones() error {
-	if x := s.get(); x != nil {
-		switch db := x.(type) {
-		case *agent.DB:
-			return agent.ErrUnsupported
-		default:
-			panic(fmt.Sprintf("unknown storage type %T", db))
-		}
-	}
-	return tsdb.ErrNotReady
-}
-
-func (s *readyStorage) Delete(ctx context.Context, mint, maxt int64, ms ...*labels.Matcher) error {
-	if x := s.get(); x != nil {
-		switch db := x.(type) {
-		case *agent.DB:
-			return agent.ErrUnsupported
-		default:
-			panic(fmt.Sprintf("unknown storage type %T", db))
-		}
-	}
-	return tsdb.ErrNotReady
-}
-
-func (s *readyStorage) Snapshot(dir string, withHead bool) error {
-	if x := s.get(); x != nil {
-		switch db := x.(type) {
-		case *agent.DB:
-			return agent.ErrUnsupported
-		default:
-			panic(fmt.Sprintf("unknown storage type %T", db))
-		}
-	}
-	return tsdb.ErrNotReady
-}
-
-func (s *readyStorage) Stats(statsByLabelName string, limit int) (*tsdb.Stats, error) {
-	if x := s.get(); x != nil {
-		switch db := x.(type) {
-		case *agent.DB:
-			return nil, agent.ErrUnsupported
-		default:
-			panic(fmt.Sprintf("unknown storage type %T", db))
-		}
-	}
-	return nil, tsdb.ErrNotReady
-}
-
-func (s *readyStorage) WALReplayStatus() (tsdb.WALReplayStatus, error) {
-	if x := s.getStats(); x != nil {
-		return x.Head.WALReplayStatus.GetWALReplayStatus(), nil
-	}
-	return tsdb.WALReplayStatus{}, tsdb.ErrNotReady
-}
-
-var ErrNotReady = errors.New("Scrape manager not ready")
-
-type readyScrapeManager struct {
-	mtx sync.RWMutex
-	m   *scrape.Manager
-}
-
-func (rm *readyScrapeManager) Set(m *scrape.Manager) {
-	rm.mtx.Lock()
-	defer rm.mtx.Unlock()
-
-	rm.m = m
-}
-
-func (rm *readyScrapeManager) Get() (*scrape.Manager, error) {
-	rm.mtx.RLock()
-	defer rm.mtx.RUnlock()
-
-	if rm.m != nil {
-		return rm.m, nil
-	}
-
-	return nil, ErrNotReady
-}
-
-type Logger struct{}
-
-func (l Logger) Log(v ...interface{}) error {
-	klog.Infoln(v...)
-	return nil
-}