diff --git a/common/log_parser_test.go b/common/log_parser_test.go index 34924d5..af2f685 100644 --- a/common/log_parser_test.go +++ b/common/log_parser_test.go @@ -12,7 +12,11 @@ import ( func TestLogParser(t *testing.T) { const defaultPatternsPerLevel = 256 ch := make(chan logparser.LogEntry) - parser := logparser.NewParser(ch, nil, nil, 1*time.Second, defaultPatternsPerLevel, false) + parser := logparser.NewParser(ch, nil, nil, 1*time.Second, defaultPatternsPerLevel, logparser.SensitiveConfig{ + Enabled: true, + MinConfidence: "high", + MaxDetections: 100, + }) ch <- logparser.LogEntry{Timestamp: time.Now(), Content: "INFO:root:AWS access key: AKIAIOSFODNN7EXAMPLE", Level: logparser.LevelInfo} diff --git a/containers/container.go b/containers/container.go index 445a6f0..bb74553 100644 --- a/containers/container.go +++ b/containers/container.go @@ -1746,13 +1746,19 @@ func (c *Container) runLogParser(logPath string) { } containerId := string(c.id) + sensitiveCfg := logparser.SensitiveConfig{ + Enabled: !*flags.DisableSensitiveLogParsing, + SampleRate: *flags.SensitiveLogSampleRate, + MinConfidence: *flags.SensitiveLogMinConfidence, + MaxDetections: *flags.SensitiveLogMaxDetectionsPerContainer, + } if logPath != "" { if c.logParsers[logPath] != nil { return } ch := make(chan logparser.LogEntry) - parser := logparser.NewParser(ch, nil, logs.OtelLogEmitter(containerId), multilineCollectorTimeout, *flags.LogPatternsPerContainer, *flags.DisableSensitiveLogParsing) + parser := logparser.NewParser(ch, nil, logs.OtelLogEmitter(containerId), multilineCollectorTimeout, *flags.LogPatternsPerContainer, sensitiveCfg) reader, err := logs.NewTailReader(proc.HostPath(logPath), ch) if err != nil { klog.Warningln(err) @@ -1772,7 +1778,7 @@ func (c *Container) runLogParser(logPath string) { klog.Warningln(err) return } - parser := logparser.NewParser(ch, nil, logs.OtelLogEmitter(containerId), multilineCollectorTimeout, *flags.LogPatternsPerContainer, *flags.DisableSensitiveLogParsing) + parser := logparser.NewParser(ch, nil, logs.OtelLogEmitter(containerId), multilineCollectorTimeout, *flags.LogPatternsPerContainer, sensitiveCfg) stop := func() { JournaldUnsubscribe(c.metadata.systemd.Unit) } @@ -1789,7 +1795,7 @@ func (c *Container) runLogParser(logPath string) { delete(c.logParsers, "stdout/stderr") } ch := make(chan logparser.LogEntry) - parser := logparser.NewParser(ch, c.metadata.logDecoder, logs.OtelLogEmitter(containerId), multilineCollectorTimeout, *flags.LogPatternsPerContainer, *flags.DisableSensitiveLogParsing) + parser := logparser.NewParser(ch, c.metadata.logDecoder, logs.OtelLogEmitter(containerId), multilineCollectorTimeout, *flags.LogPatternsPerContainer, sensitiveCfg) reader, err := logs.NewTailReader(proc.HostPath(c.metadata.logPath), ch) if err != nil { klog.Warningln(err) diff --git a/flags/flags.go b/flags/flags.go index d460b74..9a256ca 100644 --- a/flags/flags.go +++ b/flags/flags.go @@ -52,17 +52,20 @@ var ( ProfilesEndpoint = kingpin.Flag("profiles-endpoint", "The URL of the endpoint to send profiles to").Envar("PROFILES_ENDPOINT").URL() InsecureSkipVerify = kingpin.Flag("insecure-skip-verify", "whether to skip verifying the certificate or not").Envar("INSECURE_SKIP_VERIFY").Default("false").Bool() - ScrapeInterval = kingpin.Flag("scrape-interval", "How often to gather metrics from the agent").Default("15s").Envar("SCRAPE_INTERVAL").Duration() - WalDir = kingpin.Flag("wal-dir", "Path to where the agent stores data (e.g. the metrics Write-Ahead Log)").Default("/tmp/coroot-node-agent").Envar("WAL_DIR").String() - MaxSpoolSize = kingpin.Flag("max-spool-size", "Maximum size of the on-disk spool used to buffer data when it cannot be sent to collector. Supports size suffixes like KB, MB, or GB.").Default("500MB").Envar("MAX_SPOOL_SIZE").Bytes() - ResolveDns = kingpin.Flag("resolve-dns", "should resolve DNS").Default("false").Envar("RESOLVE_DNS").Bool() - IgnoreControlPlane = kingpin.Flag("ignore-control-plane", "ignore control plane like loki").Default("karpenter,loki,prometheus,grafana,kubelet,etcd,apiserver,victoria,nudgebee-agent,kube-system").Envar("IGNORE_CONTROL_PLANE").String() - SanitizeHeaders = kingpin.Flag("sanitize-headers", "should sanitize headers").Default("true").Envar("SANITIZE_HEADERS").Bool() - SensitiveHeader = kingpin.Flag("sensitive-headers", "sanitize headers using patterns").Default("Authorization, Cookie, X-Action-Token").Envar("SENSITIVE_HEADERS").String() - DisableKubeProbe = kingpin.Flag("disable-kube-probe", "disable kube probe trace").Default("true").Envar("DISABLE_KUBE_PROBE").Bool() - EnableDynamicLogTailing = kingpin.Flag("enable-dynamic-log-tailing", "Enable automatic tailing of log files opened by container processes.").Default("false").Envar("ENABLE_DYNAMIC_LOG_TAILING").Bool() - DisableSensitiveLogParsing = kingpin.Flag("disable-sensitive-log-parsing", "disable sensitive log parsing").Default("true").Envar("DISABLE_SENSITIVE_LOG_PARSING").Bool() - TraceIdHeaders = kingpin.Flag("trace-id-headers", "trace id headers").Default("Traceparent,X-Request-Id").Envar("TRACE_ID_HEADERS").String() + ScrapeInterval = kingpin.Flag("scrape-interval", "How often to gather metrics from the agent").Default("15s").Envar("SCRAPE_INTERVAL").Duration() + WalDir = kingpin.Flag("wal-dir", "Path to where the agent stores data (e.g. the metrics Write-Ahead Log)").Default("/tmp/coroot-node-agent").Envar("WAL_DIR").String() + MaxSpoolSize = kingpin.Flag("max-spool-size", "Maximum size of the on-disk spool used to buffer data when it cannot be sent to collector. Supports size suffixes like KB, MB, or GB.").Default("500MB").Envar("MAX_SPOOL_SIZE").Bytes() + ResolveDns = kingpin.Flag("resolve-dns", "should resolve DNS").Default("false").Envar("RESOLVE_DNS").Bool() + IgnoreControlPlane = kingpin.Flag("ignore-control-plane", "ignore control plane like loki").Default("karpenter,loki,prometheus,grafana,kubelet,etcd,apiserver,victoria,nudgebee-agent,kube-system").Envar("IGNORE_CONTROL_PLANE").String() + SanitizeHeaders = kingpin.Flag("sanitize-headers", "should sanitize headers").Default("true").Envar("SANITIZE_HEADERS").Bool() + SensitiveHeader = kingpin.Flag("sensitive-headers", "sanitize headers using patterns").Default("Authorization, Cookie, X-Action-Token").Envar("SENSITIVE_HEADERS").String() + DisableKubeProbe = kingpin.Flag("disable-kube-probe", "disable kube probe trace").Default("true").Envar("DISABLE_KUBE_PROBE").Bool() + EnableDynamicLogTailing = kingpin.Flag("enable-dynamic-log-tailing", "Enable automatic tailing of log files opened by container processes.").Default("false").Envar("ENABLE_DYNAMIC_LOG_TAILING").Bool() + DisableSensitiveLogParsing = kingpin.Flag("disable-sensitive-log-parsing", "disable sensitive log parsing").Default("true").Envar("DISABLE_SENSITIVE_LOG_PARSING").Bool() + SensitiveLogSampleRate = kingpin.Flag("sensitive-log-sample-rate", "Sample rate for sensitive log detection: check 1-in-N lines (0 or 1 = every line)").Default("100").Envar("SENSITIVE_LOG_SAMPLE_RATE").Int() + SensitiveLogMinConfidence = kingpin.Flag("sensitive-log-min-confidence", "Minimum confidence level for sensitive patterns: high, medium, low").Default("medium").Envar("SENSITIVE_LOG_MIN_CONFIDENCE").String() + SensitiveLogMaxDetectionsPerContainer = kingpin.Flag("sensitive-log-max-detections-per-container", "Max unique sensitive patterns tracked per container (0 = unlimited)").Default("100").Envar("SENSITIVE_LOG_MAX_DETECTIONS_PER_CONTAINER").Int() + TraceIdHeaders = kingpin.Flag("trace-id-headers", "trace id headers").Default("Traceparent,X-Request-Id").Envar("TRACE_ID_HEADERS").String() HttpPathNormalizationRules = kingpin.Flag("http-path-normalization-rules", "Custom HTTP path normalization rules in format 'pattern1:replacement1,pattern2:replacement2'").Envar("HTTP_PATH_NORMALIZATION_RULES").String() diff --git a/go.mod b/go.mod index 735b76b..20244e4 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/jpillora/backoff v1.0.0 github.com/mdlayher/taskstats v0.0.0-20230712191918-387b3d561d14 - github.com/nudgebee/logparser v0.0.0-20260310062405-9d6258bdd771 + github.com/nudgebee/logparser v0.0.0-20260313095946-b22168b95d9c github.com/opencontainers/runtime-spec v1.1.0 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 diff --git a/go.sum b/go.sum index 4379105..160a6c5 100644 --- a/go.sum +++ b/go.sum @@ -323,8 +323,8 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nudgebee/logparser v0.0.0-20260310062405-9d6258bdd771 h1:DfyX+uJ3B1x/Bz0Ie7jtLi9UoIP5PCC5myZCk6JkfmM= -github.com/nudgebee/logparser v0.0.0-20260310062405-9d6258bdd771/go.mod h1:oFnM9D6YEjZzb1jy0kJ/NUkTbkZA+BgNYjpLO4n4szA= +github.com/nudgebee/logparser v0.0.0-20260313095946-b22168b95d9c h1:C0Jx8+61ndlWeKrS25W7hFJW/qpxUpG96tBQdjffd5I= +github.com/nudgebee/logparser v0.0.0-20260313095946-b22168b95d9c/go.mod h1:oFnM9D6YEjZzb1jy0kJ/NUkTbkZA+BgNYjpLO4n4szA= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=