Skip to content

Commit 8165dc9

Browse files
QuentinBissonclayton-cornell
QuentinBisson
and
clayton-cornell
authored
feat: add node filtering to loki.source.podlogs (#4022)
* feat: add node filtering to loki.source.podlogs * Update docs/sources/reference/components/loki/loki.source.podlogs.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Update docs/sources/reference/components/loki/loki.source.podlogs.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Update docs/sources/reference/components/loki/loki.source.podlogs.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Update docs/sources/reference/components/loki/loki.source.podlogs.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Update docs/sources/reference/components/loki/loki.source.podlogs.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Update docs/sources/reference/components/loki/loki.source.podlogs.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Fix tests * Update docs/sources/reference/components/loki/loki.source.podlogs.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Update docs/sources/reference/components/loki/loki.source.podlogs.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --------- Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>
1 parent 232e6a3 commit 8165dc9

File tree

6 files changed

+410
-2
lines changed

6 files changed

+410
-2
lines changed

‎CHANGELOG.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ Main (unreleased)
3636

3737
- Add the `otelcol.receiver.fluentforward` receiver to receive logs via Fluent Forward Protocol. (@rucciva)
3838

39+
- Add `node_filter` configuration block to `loki.source.podlogs` component to enable node-based filtering for pod discovery. When enabled, only pods running on the specified node will be discovered and monitored, significantly reducing API server load and network traffic in DaemonSet deployments. (@QuentinBisson)
40+
3941
- (_Experimental_) Additions to experimental `database_observability.mysql` component:
4042
- `query_sample` collector now supports auto-enabling the necessary `setup_consumers` settings (@cristiangreco)
4143

@@ -352,7 +354,7 @@ v1.8.3
352354

353355
- Fix `mimir.rules.kubernetes` panic on non-leader debug info retrieval (@TheoBrigitte)
354356

355-
- Fix detection of the “streams limit exceeded” error in the Loki client so that metrics are correctly labeled as `ReasonStreamLimited`. (@maratkhv)
357+
- Fix detection of the "streams limit exceeded" error in the Loki client so that metrics are correctly labeled as `ReasonStreamLimited`. (@maratkhv)
356358

357359
- Fix `loki.source.file` race condition that often lead to panic when using `decompression`. (@kalleep)
358360

‎docs/sources/reference/components/loki/loki.source.podlogs.md‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ You can use the following blocks with `loki.source.podlogs`:
129129
| [`clustering`][clustering] | Configure the component for when {{< param "PRODUCT_NAME" >}} is running in clustered mode. | no |
130130
| [`namespace_selector`][selector] | Label selector for which namespaces to discover `PodLogs` in. | no |
131131
| `namespace_selector` > [`match_expression`][match_expression] | Label selector expression for which namespaces to discover `PodLogs` in. | no |
132+
| [`node_filter`][node_filter] | Filter Pods by node to limit discovery scope. | no |
132133
| [`selector`][selector] | Label selector for which `PodLogs` to discover. | no |
133134
| `selector` > [`match_expression`][match_expression] | Label selector expression for which `PodLogs` to discover. | no |
134135

@@ -140,6 +141,7 @@ For example, `client` > `basic_auth` refers to a `basic_auth` block defined insi
140141
[basic_auth]: #basic_auth
141142
[clustering]: #clustering
142143
[match_expression]: #match_expression
144+
[node_filter]: #node_filter
143145
[oauth2]: #oauth2
144146
[selector]: #selector-and-namespace_selector
145147
[tls_config]: #tls_config
@@ -213,6 +215,37 @@ Clustering looks only at the following labels for determining the shard key:
213215

214216
[using clustering]: ../../../../get-started/clustering/
215217

218+
### `node_filter`
219+
220+
The `node_filter` block configures node-based filtering for Pod discovery.
221+
222+
The following arguments are supported:
223+
224+
| Name | Type | Description | Default | Required |
225+
| ----------- | -------- | ----------------------------------------------------------------------------------------- | ------- | -------- |
226+
| `enabled` | `bool` | Enable node-based filtering for Pod discovery. | `false` | no |
227+
| `node_name` | `string` | Node name to filter Pods by. Falls back to the `NODE_NAME` environment variable if empty. | `""` | no |
228+
229+
When you set `enabled` to `true`, `loki.source.podlogs` only discovers and collects logs from Pods running on the specified node.
230+
This is particularly useful when running {{< param "PRODUCT_NAME" >}} as a DaemonSet to avoid collecting logs from Pods on other nodes.
231+
232+
If you don't specify `node_name`, `loki.source.podlogs` attempts to use the `NODE_NAME` environment variable.
233+
This allows for easy configuration in DaemonSet deployments where you can inject the node name with the [Kubernetes downward API][].
234+
235+
[Kubernetes downward API]: https://kubernetes.io/docs/concepts/workloads/pods/downward-api/
236+
237+
Example DaemonSet configuration:
238+
239+
```yaml
240+
env:
241+
- name: NODE_NAME
242+
valueFrom:
243+
fieldRef:
244+
fieldPath: spec.nodeName
245+
```
246+
247+
Node filtering significantly reduces API server load and network traffic by limiting Pod discovery to only the local node, making it highly recommended for DaemonSet deployments in large clusters.
248+
216249
### `match_expression`
217250

218251
The `match_expression` block describes a Kubernetes label match expression for `PodLogs` or Namespace discovery.
@@ -292,6 +325,25 @@ loki.write "local" {
292325
}
293326
```
294327

328+
This example shows how to use node filtering for DaemonSet deployments to collect logs only from Pods running on the current node:
329+
330+
```alloy
331+
loki.source.podlogs "daemonset" {
332+
forward_to = [loki.write.local.receiver]
333+
334+
node_filter {
335+
enabled = true
336+
// node_name will be automatically read from NODE_NAME environment variable
337+
}
338+
}
339+
340+
loki.write "local" {
341+
endpoint {
342+
url = sys.env("LOKI_URL")
343+
}
344+
}
345+
```
346+
295347
<!-- START GENERATED COMPATIBLE COMPONENTS -->
296348

297349
## Compatible components

‎internal/component/loki/source/podlogs/podlogs.go‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,23 @@ type Arguments struct {
4949
NamespaceSelector config.LabelSelector `alloy:"namespace_selector,block,optional"`
5050
TailFromEnd bool `alloy:"tail_from_end,attr,optional"`
5151

52+
// Node filtering settings to limit pod discovery to specific nodes.
53+
NodeFilter NodeFilterConfig `alloy:"node_filter,block,optional"`
54+
5255
Clustering cluster.ComponentBlock `alloy:"clustering,block,optional"`
5356
}
5457

58+
// NodeFilterConfig configures node-based filtering for pod discovery.
59+
// When enabled, only pods running on the specified node will be discovered,
60+
// which is useful for DaemonSet deployments to avoid cross-node log collection.
61+
type NodeFilterConfig struct {
62+
// Enabled controls whether node filtering is active.
63+
Enabled bool `alloy:"enabled,attr,optional"`
64+
// NodeName specifies the node name to filter by. If empty, the component
65+
// will attempt to use the NODE_NAME environment variable.
66+
NodeName string `alloy:"node_name,attr,optional"`
67+
}
68+
5569
// DefaultArguments holds default settings for loki.source.kubernetes.
5670
var DefaultArguments = Arguments{
5771
Client: commonk8s.DefaultClientArguments,
@@ -267,8 +281,13 @@ func (c *Component) updateReconciler(args Arguments) error {
267281
var (
268282
selectorChanged = !reflect.DeepEqual(c.args.Selector, args.Selector)
269283
namespaceSelectorChanged = !reflect.DeepEqual(c.args.NamespaceSelector, args.NamespaceSelector)
284+
nodeFilterChanged = !reflect.DeepEqual(c.args.NodeFilter, args.NodeFilter)
270285
)
271-
if !selectorChanged && !namespaceSelectorChanged {
286+
287+
// Update node filter configuration
288+
c.reconciler.UpdateNodeFilter(args.NodeFilter.Enabled, args.NodeFilter.NodeName)
289+
290+
if !selectorChanged && !namespaceSelectorChanged && !nodeFilterChanged {
272291
return nil
273292
}
274293

‎internal/component/loki/source/podlogs/podlogs_test.go‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,96 @@ func TestBadAlloyConfig(t *testing.T) {
3535
err := syntax.Unmarshal([]byte(exampleAlloyConfig), &args)
3636
require.ErrorContains(t, err, "at most one of basic_auth, authorization, oauth2, bearer_token & bearer_token_file must be configured")
3737
}
38+
39+
func TestNodeFilterConfig(t *testing.T) {
40+
tests := []struct {
41+
name string
42+
config string
43+
expectedError string
44+
expectedResult Arguments
45+
}{
46+
{
47+
name: "node filter disabled by default",
48+
config: `
49+
forward_to = []
50+
`,
51+
expectedResult: Arguments{
52+
NodeFilter: NodeFilterConfig{
53+
Enabled: false,
54+
NodeName: "",
55+
},
56+
},
57+
},
58+
{
59+
name: "node filter enabled with explicit node name",
60+
config: `
61+
forward_to = []
62+
node_filter {
63+
enabled = true
64+
node_name = "worker-node-1"
65+
}
66+
`,
67+
expectedResult: Arguments{
68+
NodeFilter: NodeFilterConfig{
69+
Enabled: true,
70+
NodeName: "worker-node-1",
71+
},
72+
},
73+
},
74+
{
75+
name: "node filter enabled without node name",
76+
config: `
77+
forward_to = []
78+
node_filter {
79+
enabled = true
80+
}
81+
`,
82+
expectedResult: Arguments{
83+
NodeFilter: NodeFilterConfig{
84+
Enabled: true,
85+
NodeName: "",
86+
},
87+
},
88+
},
89+
{
90+
name: "node filter with only node name specified",
91+
config: `
92+
forward_to = []
93+
node_filter {
94+
node_name = "worker-node-2"
95+
}
96+
`,
97+
expectedResult: Arguments{
98+
NodeFilter: NodeFilterConfig{
99+
Enabled: false, // default value
100+
NodeName: "worker-node-2",
101+
},
102+
},
103+
},
104+
}
105+
106+
for _, tt := range tests {
107+
t.Run(tt.name, func(t *testing.T) {
108+
var args Arguments
109+
err := syntax.Unmarshal([]byte(tt.config), &args)
110+
111+
if tt.expectedError != "" {
112+
require.ErrorContains(t, err, tt.expectedError)
113+
return
114+
}
115+
116+
require.NoError(t, err)
117+
require.Equal(t, tt.expectedResult.NodeFilter.Enabled, args.NodeFilter.Enabled)
118+
require.Equal(t, tt.expectedResult.NodeFilter.NodeName, args.NodeFilter.NodeName)
119+
})
120+
}
121+
}
122+
123+
func TestNodeFilterConfigDefaults(t *testing.T) {
124+
var args Arguments
125+
args.SetToDefault()
126+
127+
// Verify node filter is disabled by default
128+
require.False(t, args.NodeFilter.Enabled)
129+
require.Empty(t, args.NodeFilter.NodeName)
130+
}

‎internal/component/loki/source/podlogs/reconciler.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package podlogs
33
import (
44
"context"
55
"fmt"
6+
"os"
67
"slices"
78
"sort"
89
"strings"
@@ -17,6 +18,7 @@ import (
1718
"github.com/prometheus/prometheus/util/strutil"
1819
corev1 "k8s.io/api/core/v1"
1920
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
21+
"k8s.io/apimachinery/pkg/fields"
2022
"k8s.io/apimachinery/pkg/labels"
2123
"sigs.k8s.io/controller-runtime/pkg/client"
2224

@@ -69,6 +71,8 @@ type reconciler struct {
6971
podLogsSelector labels.Selector
7072
podLogsNamespaceSelector labels.Selector
7173
shouldDistribute bool
74+
nodeFilterEnabled bool
75+
nodeFilterName string
7276

7377
debugMut sync.RWMutex
7478
debugInfo []DiscoveredPodLogs
@@ -96,6 +100,33 @@ func (r *reconciler) UpdateSelectors(podLogs, namespace labels.Selector) {
96100
r.podLogsNamespaceSelector = namespace
97101
}
98102

103+
// UpdateNodeFilter updates the node filter configuration used by the reconciler.
104+
func (r *reconciler) UpdateNodeFilter(enabled bool, nodeName string) {
105+
r.reconcileMut.Lock()
106+
defer r.reconcileMut.Unlock()
107+
108+
r.nodeFilterEnabled = enabled
109+
r.nodeFilterName = nodeName
110+
}
111+
112+
// getNodeFilterName returns the effective node name to filter by.
113+
// If enabled but no node name is provided, it falls back to the NODE_NAME environment variable.
114+
func (r *reconciler) getNodeFilterName() string {
115+
r.reconcileMut.RLock()
116+
defer r.reconcileMut.RUnlock()
117+
118+
if !r.nodeFilterEnabled {
119+
return ""
120+
}
121+
122+
if r.nodeFilterName != "" {
123+
return r.nodeFilterName
124+
}
125+
126+
// Fall back to NODE_NAME environment variable
127+
return os.Getenv("NODE_NAME")
128+
}
129+
99130
// SetDistribute configures whether targets are distributed amongst the cluster.
100131
func (r *reconciler) SetDistribute(distribute bool) {
101132
r.reconcileMut.Lock()
@@ -236,6 +267,15 @@ func (r *reconciler) reconcilePodLogs(ctx context.Context, cli client.Client, po
236267
opts := []client.ListOption{
237268
client.MatchingLabelsSelector{Selector: sel},
238269
}
270+
271+
// Add node filtering if enabled
272+
if nodeFilterName := r.getNodeFilterName(); nodeFilterName != "" {
273+
level.Debug(r.log).Log("msg", "applying node filter for pod discovery", "node", nodeFilterName, "key", key)
274+
opts = append(opts, client.MatchingFieldsSelector{
275+
Selector: fields.OneTermEqualSelector("spec.nodeName", nodeFilterName),
276+
})
277+
}
278+
239279
var podList corev1.PodList
240280
if err := cli.List(ctx, &podList, opts...); err != nil {
241281
discoveredPodLogs.ReconcileError = fmt.Sprintf("failed to list Pods: %s", err)

0 commit comments

Comments
 (0)