Encode the behavior into the promtail converter that promtail does internally to limit k8s discovery (#5046)

* Encode the behavior into the promtail converter that promtail does internally to limit k8s discovery

* Update doc

* Update static convert that uses promtail conversion

* Fix spacing in generated config

Author: dehaansa

CHANGELOG.md

@@ -61,6 +61,8 @@ Main (unreleased)
 
 - Fix Docker log corruption for multiplexed long lines. (@axd1x8a)
 
+- Fix the promtail converter behavior to mimic promtail behavior by default and limit kubernetes discovery to the same node. (@dehaansa)
+
 - Allow configuration of `force_attempt_http2` and default it to `true` for otelcol exporters with HTTP client configurations. (@dehaansa)
 
 v1.12.0

docs/sources/set-up/migrate/from-promtail.md

@@ -179,7 +179,7 @@ The following list is specific to the convert command and not {{< param "PRODUCT
   Make sure that you use the new metric names, for example, in your alerts and dashboards queries.
 * The logs produced by {{< param "PRODUCT_NAME" >}} differ from those produced by Promtail.
 * {{< param "PRODUCT_NAME" >}} exposes the {{< param "PRODUCT_NAME" >}} [UI][], which differs from the Promtail Web UI.
-* If you are converting a Promtail configuration for a Kubernetes daemonset, [modify the generated configuration][single-node-discovery] to ensure `discovery.kubernetes` only discovers Pods residing on the same node as the {{< param "PRODUCT_NAME" >}} Pod.
+* If you are converting a Promtail configuration and not deploying as a Kubernetes daemonset, [modify the generated configuration][single-node-discovery] to ensure `discovery.kubernetes` discovery behaves as expected. The converter makes the same assumption as promtail that any `pod` discovery is for a daemonset deployment.
 
 [Promtail]: https://www.grafana.com/docs/loki/latest/clients/promtail/
 [debugging]: #debugging

internal/converter/internal/promtailconvert/internal/build/service_discovery.go

@@ -12,6 +12,7 @@ import (
 	"github.com/grafana/alloy/internal/loki/promtail/scrapeconfig"
 	"github.com/prometheus/common/model"
 	prom_discover "github.com/prometheus/prometheus/discovery"
+	"github.com/prometheus/prometheus/discovery/kubernetes"
 )
 
 func (s *ScrapeConfigBuilder) AppendSDs() {
@@ -86,6 +87,13 @@ func toDiscoveryConfig(cfg *scrapeconfig.Config) prom_discover.Configs {
 	}
 
 	for _, sd := range cfg.ServiceDiscoveryConfig.KubernetesSDConfigs {
+		// See https://github.com/grafana/loki/blob/main/clients/pkg/promtail/targets/file/filetargetmanager.go#L126
+		if sd.Role == kubernetes.RolePod {
+			sd.Selectors = append(sd.Selectors, kubernetes.SelectorConfig{
+				Role:  kubernetes.RolePod,
+				Field: `"spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)`,
+			})
+		}
 		sdConfigs = append(sdConfigs, sd)
 	}
 

internal/converter/internal/promtailconvert/promtailconvert.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
+	"strings"
 
 	promtailcfg "github.com/grafana/alloy/internal/loki/promtail/config"
 	"github.com/grafana/alloy/internal/loki/promtail/file"
@@ -84,11 +85,19 @@ func Convert(in []byte, extraArgs []string) ([]byte, diag.Diagnostics) {
 		return nil, diags
 	}
 
-	prettyByte, newDiags := common.PrettyPrint(buf.Bytes())
+	prettyByte, newDiags := common.PrettyPrint([]byte(fixFunctionUse(buf.String())))
 	diags.AddAll(newDiags)
 	return prettyByte, diags
 }
 
+// There is not a way to express a function call in a struct to encode back into Alloy config, so we need to fix it
+func fixFunctionUse(buf string) string {
+	return strings.ReplaceAll(buf,
+		`"\"spec.nodeName=\" + coalesce(sys.env(\"HOSTNAME\"), constants.hostname)"`,
+		`"spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)`,
+	)
+}
+
 // AppendAll analyzes the entire promtail config in memory and transforms it
 // into Alloy components. It then appends each argument to the file builder.
 func AppendAll(f *builder.File, cfg *promtailcfg.Config, labelPrefix string, diags diag.Diagnostics) diag.Diagnostics {

internal/converter/internal/promtailconvert/promtailconvert_test.go

@@ -11,7 +11,7 @@ import (
 	_ "github.com/grafana/alloy/internal/static/metrics/instance" // Imported to override default values via the init function.
 )
 
-// Set this flag to update snapshots e.g. `go test -v ./interal/converter/internal/promtailconverter/...` -fix-tests
+// Set this flag to update snapshots e.g. `go test -v ./internal/converter/internal/promtailconvert/... -fix-tests`
 var fixTestsFlag = flag.Bool("fix-tests", false, "update the test files with the current generated content")
 
 func TestConvert(t *testing.T) {

internal/converter/internal/promtailconvert/testdata/kubernetes.alloy

@@ -15,11 +15,21 @@ discovery.kubernetes "fun" {
 		server_name          = "example.local"
 		insecure_skip_verify = true
 	}
+
+	selectors {
+		role  = "pod"
+		field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)
+	}
 }
 
 discovery.kubernetes "fun_2" {
 	role            = "pod"
 	kubeconfig_file = "/home/toby/.kube/config"
+
+	selectors {
+		role  = "pod"
+		field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)
+	}
 }
 
 discovery.kubernetes "fun_3" {
@@ -30,6 +40,11 @@ discovery.kubernetes "fun_3" {
 		type             = "Bearer"
 		credentials_file = "/home/robin/.special_token"
 	}
+
+	selectors {
+		role  = "pod"
+		field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)
+	}
 }
 
 discovery.kubernetes "fun_4" {
@@ -40,6 +55,11 @@ discovery.kubernetes "fun_4" {
 		type             = "Bearer"
 		credentials_file = "/home/toby/.token"
 	}
+
+	selectors {
+		role  = "pod"
+		field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)
+	}
 }
 
 discovery.kubernetes "fun_5" {

internal/converter/internal/promtailconvert/testdata/mixed_pipeline.alloy

@@ -1,6 +1,11 @@
 discovery.kubernetes "uber_pipeline" {
 	role            = "pod"
 	kubeconfig_file = "/home/toby/.kube/config"
+
+	selectors {
+		role  = "pod"
+		field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)
+	}
 }
 
 discovery.consulagent "uber_pipeline" {

internal/converter/internal/promtailconvert/testdata/pipeline_stages_cri_empty.alloy

@@ -1,6 +1,11 @@
 discovery.kubernetes "example" {
 	role            = "pod"
 	kubeconfig_file = "/home/toby/.kube/config"
+
+	selectors {
+		role  = "pod"
+		field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)
+	}
 }
 
 loki.process "example" {

internal/converter/internal/promtailconvert/testdata/pipeline_stages_drop.alloy

@@ -1,6 +1,11 @@
 discovery.kubernetes "example" {
 	role            = "pod"
 	kubeconfig_file = "/home/toby/.kube/config"
+
+	selectors {
+		role  = "pod"
+		field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)
+	}
 }
 
 loki.process "example" {

internal/converter/internal/promtailconvert/testdata/pipeline_stages_match_nested.alloy

@@ -1,6 +1,11 @@
 discovery.kubernetes "example" {
 	role            = "pod"
 	kubeconfig_file = "/home/toby/.kube/config"
+
+	selectors {
+		role  = "pod"
+		field = "spec.nodeName=" + coalesce(sys.env("HOSTNAME"), constants.hostname)
+	}
 }
 
 loki.process "example" {