Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
771 changes: 771 additions & 0 deletions common/ip_resolver.go

Large diffs are not rendered by default.

895 changes: 895 additions & 0 deletions common/ip_resolver_test.go

Large diffs are not rendered by default.

50 changes: 31 additions & 19 deletions containers/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,17 +63,21 @@ func (p *LogParser) Stop() {
}

type AddrPair struct {
src netaddr.IPPort
dst netaddr.IPPort
src netaddr.IPPort
dst netaddr.IPPort
srcWorkload common.Workload
dstWorkload common.Workload
}

type ActiveConnection struct {
Dest netaddr.IPPort
ActualDest netaddr.IPPort
Pid uint32
Fd uint64
Timestamp uint64
Closed time.Time
Dest netaddr.IPPort
ActualDest netaddr.IPPort
srcWorkload common.Workload
dstWorkload common.Workload
Pid uint32
Fd uint64
Timestamp uint64
Closed time.Time

http2Parser *l7.Http2Parser
postgresParser *l7.PostgresParser
Expand Down Expand Up @@ -129,10 +133,11 @@ type Container struct {

lock sync.RWMutex

done chan struct{}
done chan struct{}
ip_resolver IPResolver
}

func NewContainer(id ContainerID, cg *cgroup.Cgroup, md *ContainerMetadata, hostConntrack *Conntrack, pid uint32) (*Container, error) {
func NewContainer(id ContainerID, cg *cgroup.Cgroup, md *ContainerMetadata, hostConntrack *Conntrack, pid uint32, ip_resolver IPResolver) (*Container, error) {
netNs, err := proc.GetNetNs(pid)
if err != nil {
return nil, err
Expand Down Expand Up @@ -164,7 +169,8 @@ func NewContainer(id ContainerID, cg *cgroup.Cgroup, md *ContainerMetadata, host

hostConntrack: hostConntrack,

done: make(chan struct{}),
done: make(chan struct{}),
ip_resolver: ip_resolver,
}

for _, n := range md.networks {
Expand Down Expand Up @@ -485,6 +491,9 @@ func (c *Container) onConnectionOpen(pid uint32, fd uint64, src, dst netaddr.IPP
if dst.IP().IsLoopback() && !p.isHostNs() {
return
}
srcWorkload := c.ip_resolver.ResolveIP(netaddr.IPPort(src).IP().String())
dstWorkload := c.ip_resolver.ResolveIP(netaddr.IPPort(dst).IP().String())

actualDst, err := c.getActualDestination(p, src, dst)
if err != nil {
if !common.IsNotExist(err) {
Expand All @@ -508,13 +517,15 @@ func (c *Container) onConnectionOpen(pid uint32, fd uint64, src, dst netaddr.IPP
} else {
c.connectsSuccessful[AddrPair{src: dst, dst: *actualDst}]++
connection := &ActiveConnection{
Dest: dst,
ActualDest: *actualDst,
Pid: pid,
Fd: fd,
Timestamp: timestamp,
}
c.connectionsActive[AddrPair{src: src, dst: dst}] = connection
Dest: dst,
ActualDest: *actualDst,
Pid: pid,
Fd: fd,
Timestamp: timestamp,
srcWorkload: srcWorkload,
dstWorkload: dstWorkload,
}
c.connectionsActive[AddrPair{src: src, dst: dst, srcWorkload: srcWorkload, dstWorkload: dstWorkload}] = connection
c.connectionsByPidFd[PidFd{Pid: pid, Fd: fd}] = connection
}
c.connectLastAttempt[dst] = time.Now()
Expand Down Expand Up @@ -572,7 +583,8 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R
if timestamp != 0 && conn.Timestamp != timestamp {
return
}
stats := c.l7Stats.get(r.Protocol, conn.Dest, conn.ActualDest, r)

stats := c.l7Stats.get(r.Protocol, conn.Dest, conn.ActualDest, r, conn.srcWorkload, conn.dstWorkload)
trace := tracing.NewTrace(string(c.id), conn.ActualDest)
switch r.Protocol {
case l7.ProtocolHTTP:
Expand Down
5 changes: 3 additions & 2 deletions containers/l7.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package containers
import (
"time"

"github.com/coroot/coroot-node-agent/common"
"github.com/coroot/coroot-node-agent/ebpftracer/l7"
"github.com/prometheus/client_golang/prometheus"
"inet.af/netaddr"
Expand Down Expand Up @@ -36,7 +37,7 @@ func (m *L7Metrics) observe(status, method string, duration time.Duration) {

type L7Stats map[l7.Protocol]map[AddrPair]*L7Metrics // protocol -> dst:actual_dst -> metrics

func (s L7Stats) get(protocol l7.Protocol, destination, actualDestination netaddr.IPPort, r *l7.RequestData) *L7Metrics {
func (s L7Stats) get(protocol l7.Protocol, destination, actualDestination netaddr.IPPort, r *l7.RequestData, srcWorkload common.Workload, dstWorkload common.Workload) *L7Metrics {
if protocol == l7.ProtocolHTTP2 {
protocol = l7.ProtocolHTTP
}
Expand All @@ -50,7 +51,7 @@ func (s L7Stats) get(protocol l7.Protocol, destination, actualDestination netadd
if m == nil {
m = &L7Metrics{}
protoStats[dest] = m
constLabels := map[string]string{"destination": destination.String(), "actual_destination": actualDestination.String()}
constLabels := map[string]string{"destination": destination.String(), "actual_destination": actualDestination.String(), "dest_kind": dstWorkload.Kind, "dest_workload_name": dstWorkload.Name, "dest_workload_namespace": dstWorkload.Namespace, "src_kind": srcWorkload.Kind, "src_workload_name": srcWorkload.Name, "src_workload_namespace": srcWorkload.Namespace}
labels := []string{"status"}
switch protocol {
case l7.ProtocolRabbitmq, l7.ProtocolNats:
Expand Down
14 changes: 11 additions & 3 deletions containers/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ type ProcessInfo struct {
StartedAt time.Time
}

type IPResolver interface {
ResolveIP(string) common.Workload
StartWatching() error
StopWatching()
}

type Registry struct {
reg prometheus.Registerer

Expand All @@ -44,9 +50,10 @@ type Registry struct {
containersByPid map[uint32]*Container

processInfoCh chan<- ProcessInfo
ip_resolver IPResolver
}

func NewRegistry(reg prometheus.Registerer, kernelVersion string, processInfoCh chan<- ProcessInfo) (*Registry, error) {
func NewRegistry(reg prometheus.Registerer, kernelVersion string, processInfoCh chan<- ProcessInfo, ip_resolver *common.K8sIPResolver) (*Registry, error) {
ns, err := proc.GetSelfNetNs()
if err != nil {
return nil, err
Expand Down Expand Up @@ -100,7 +107,8 @@ func NewRegistry(reg prometheus.Registerer, kernelVersion string, processInfoCh

processInfoCh: processInfoCh,

tracer: ebpftracer.NewTracer(kernelVersion, *flags.DisableL7Tracing),
tracer: ebpftracer.NewTracer(kernelVersion, *flags.DisableL7Tracing),
ip_resolver: ip_resolver,
}

go r.handleEvents(r.events)
Expand Down Expand Up @@ -303,7 +311,7 @@ func (r *Registry) getOrCreateContainer(pid uint32) *Container {
r.containersByCgroupId[cg.Id] = c
return c
}
c, err := NewContainer(id, cg, md, r.hostConntrack, pid)
c, err := NewContainer(id, cg, md, r.hostConntrack, pid, r.ip_resolver)
if err != nil {
klog.Warningf("failed to create container pid=%d cg=%s id=%s: %s", pid, cg.Id, id, err)
return nil
Expand Down
3 changes: 2 additions & 1 deletion flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ var (
ApiKey = kingpin.Flag("api-key", "Coroot API key").Envar("API_KEY").String()
ScrapeInterval = kingpin.Flag("scrape-interval", "How often to gather metrics from the agent").Default("15s").Envar("SCRAPE_INTERVAL").Duration()

WalDir = kingpin.Flag("wal-dir", "Path to where the agent stores data (e.g. the metrics Write-Ahead Log)").Default("/tmp/coroot-node-agent").Envar("WAL_DIR").String()
WalDir = kingpin.Flag("wal-dir", "Path to where the agent stores data (e.g. the metrics Write-Ahead Log)").Default("/tmp/coroot-node-agent").Envar("WAL_DIR").String()
ResolveDns = kingpin.Flag("resolve-dns", "should resolve DNS").Default("true").Envar("RESOLVE_DNS").Bool()
)

func GetString(fl *string) string {
Expand Down
20 changes: 16 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ require (
github.com/docker/docker v25.0.0+incompatible
github.com/florianl/go-conntrack v0.3.0
github.com/go-kit/log v0.2.1
github.com/google/uuid v1.5.0
github.com/grafana/pyroscope/ebpf v0.4.1
github.com/hashicorp/golang-lru/v2 v2.0.5
github.com/jpillora/backoff v1.0.0
github.com/mdlayher/taskstats v0.0.0-20230712191918-387b3d561d14
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
Expand All @@ -40,6 +42,9 @@ require (
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/yaml.v2 v2.4.0
inet.af/netaddr v0.0.0-20230525184311-b8eac61e914a
k8s.io/api v0.28.6
k8s.io/apimachinery v0.28.6
k8s.io/client-go v0.28.6
k8s.io/klog/v2 v2.120.1
)

Expand Down Expand Up @@ -73,6 +78,8 @@ require (
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-logfmt/logfmt v0.6.0 // indirect
Expand All @@ -96,13 +103,13 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 // indirect
github.com/google/uuid v1.5.0 // indirect
github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
Expand All @@ -125,6 +132,7 @@ require (
github.com/moby/term v0.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
Expand Down Expand Up @@ -169,18 +177,22 @@ require (
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
golang.org/x/oauth2 v0.16.0 // indirect
golang.org/x/sync v0.6.0 // indirect
golang.org/x/term v0.16.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.17.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect
google.golang.org/grpc v1.61.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apimachinery v0.28.6 // indirect
k8s.io/client-go v0.28.6 // indirect
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

replace (
Expand Down
10 changes: 9 additions & 1 deletion go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,6 @@ github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8E
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/emicklei/go-restful v2.16.0+incompatible h1:rgqiKNjTnFQA6kkhFe16D8epTksy9HQ1MyrbDXSdYhM=
github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
Expand All @@ -367,6 +366,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A=
github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM=
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
Expand Down Expand Up @@ -462,6 +463,8 @@ github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUri
github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg=
github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
Expand Down Expand Up @@ -851,13 +854,18 @@ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE=
github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM=
github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
Expand Down
30 changes: 29 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package main

import (
"bytes"
"log"
"net/http"
_ "net/http/pprof"
"os"
Expand All @@ -22,6 +23,8 @@ import (
"golang.org/x/mod/semver"
"golang.org/x/sys/unix"
"golang.org/x/time/rate"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
)

Expand Down Expand Up @@ -95,12 +98,37 @@ func whitelistNodeExternalNetworks() {
}
}

func getClientSet() (*kubernetes.Clientset, error) {
config, err := rest.InClusterConfig()
if err != nil {
return nil, err
}

clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return clientset, nil
}

func main() {
klog.LogToStderr(false)
klog.SetOutput(&RateLimitedLogOutput{limiter: rate.NewLimiter(rate.Limit(*flags.LogPerSecond), *flags.LogBurst)})

klog.Infoln("agent version:", version)

clientset, err := getClientSet()
if err != nil {
log.Fatalf("Error getting kubernetes clientset: %v", err)
}
resolver, err := common.NewK8sIPResolver(clientset, *flags.ResolveDns)
if err != nil {
log.Fatalf("Error creating resolver: %v", err)
}
err = resolver.StartWatching()
if err != nil {
log.Fatalf("Error watching cluster's state: %v", err)
}
hostname, kv, err := uname()
if err != nil {
klog.Exitln("failed to get uname:", err)
Expand Down Expand Up @@ -133,7 +161,7 @@ func main() {

processInfoCh := profiling.Init(machineId, hostname)

cr, err := containers.NewRegistry(registerer, kv, processInfoCh)
cr, err := containers.NewRegistry(registerer, kv, processInfoCh, resolver)
if err != nil {
klog.Exitln(err)
}
Expand Down