feat: support metadata in Remote Write V2 (#5045)

* Support metadata in Remote Write V2

* Apply suggestions from code review

Co-authored-by: Kyle Eckhart <kgeckhart@users.noreply.github.com>

* Fix lint issues

* PR review fixes

* Batch the test samples into a singe remote write request as soon as possible.

This reduces flakyness and reduce test times.

---------

Co-authored-by: Kyle Eckhart <kgeckhart@users.noreply.github.com>

Author: ptodev

docs/sources/reference/components/prometheus/prometheus.remote_write.md

@@ -292,6 +292,7 @@ In those cases, exported fields are kept at their last healthy values.
 * `prometheus_remote_storage_shards` (gauge): The number of shards used for concurrent delivery of metrics to an endpoint.
 * `prometheus_remote_write_wal_exemplars_appended_total` (counter): Total number of exemplars appended to the WAL.
 * `prometheus_remote_write_wal_out_of_order_samples_total` (counter): Total number of out of order samples ingestion failed attempts.
+* `prometheus_remote_write_wal_metadata_updates_total` (counter): Total number of metadata updates sent through the WAL.
 * `prometheus_remote_write_wal_samples_appended_total` (counter): Total number of samples appended to the WAL.
 * `prometheus_remote_write_wal_storage_active_series` (gauge): Current number of active series being tracked by the WAL.
 * `prometheus_remote_write_wal_storage_created_series_total` (counter): Total number of created series appended to the WAL.

docs/sources/reference/components/prometheus/prometheus.scrape.md

@@ -91,8 +91,14 @@ You can use the following arguments with `prometheus.scrape`:
 > **EXPERIMENTAL**: The `honor_metadata` argument is an [experimental][] feature.
 > Enabling it may increase resource consumption, particularly if a lot of metrics with different names are ingested.
 > Not all downstream components may be compatible with Prometheus metadata yet.
-> For example, `otelcol.receiver.prometheus` may work, but `prometheus.remote_write` may not.
-> Support for more components will be added soon.
+> Currently, the compatible components are:
+>
+> * `otelcol.receiver.prometheus`
+> * `prometheus.remote_write` only when configured for Remote Write v2.
+> * `prometheus.write_queue`
+> 
+> Metadata support for Remote Write v1 in `prometheus.remote_write` will be added soon.
+> 
 > Experimental features are subject to frequent breaking changes, and may be removed with no equivalent replacement.
 > To enable and use an experimental feature, you must set the `stability.level` [flag][] to `experimental`.
 

internal/cmd/integration-tests/common/metadata_asserts.go

@@ -10,8 +10,12 @@ import (
 	"golang.org/x/exp/maps"
 )
 
-// Default metrics metadata
-var PromDefaultMetricMetadata = map[string]Metadata{
+// Default metrics metadata that the prometheus.write_queue component creates in Mimir.
+// For some reason it causes _bucket, _count, and _sum metrics to be visible on Mimir's /metadata endpoint.
+// This is not normal. When Prometheus or Alloy's prometheus.remote_write components send metrics to Mimir,
+// only the metric family is visible on Mimir's /metadata endpoint.
+// TODO: Look into why the write_queue components causes Mimir to behave this way.
+var WriteQueueDefaultMetricMetadata = map[string]Metadata{
 	"golang_counter":                {Type: "counter", Help: "The counter description string"},
 	"golang_gauge":                  {Type: "gauge", Help: "The gauge description string"},
 	"golang_histogram_bucket":       {Type: "histogram", Help: "The histogram description string"},
@@ -23,11 +27,8 @@ var PromDefaultMetricMetadata = map[string]Metadata{
 	"golang_summary":                {Type: "summary", Help: "The summary description string"},
 }
 
-// Default metrics metadata for OTLP generated metrics.
-// When you send metrics to Mimir's OTLP endpoint,
-// Mimir doesn't generate metadata for
-// _bucket, _count, and _sum metrics of classic histograms.
-var OtlpDefaultMetricMetadata = map[string]Metadata{
+// Default metrics metadata that the prometheus.remote_write and otelcol.exporter.otlphttp components create in Mimir.
+var PromDefaultMetricMetadata = map[string]Metadata{
 	"golang_counter":         {Type: "counter", Help: "The counter description string"},
 	"golang_gauge":           {Type: "gauge", Help: "The gauge description string"},
 	"golang_histogram":       {Type: "histogram", Help: "The histogram description string"},

internal/cmd/integration-tests/docker-compose.yaml

@@ -1,7 +1,7 @@
 name: alloy-integration-tests
 services:
   mimir:
-    image: grafana/mimir:2.17.0
+    image: grafana/mimir:3.0.0
     volumes:
       - ./configs/mimir:/etc/mimir-config
     entrypoint:

internal/cmd/integration-tests/tests/otlp-metadata/otlp_metadata_test.go

@@ -26,8 +26,8 @@ func TestOtlpMetadata(t *testing.T) {
 
 	// Make sure we got the expected metrics before checking metadata
 	common.MimirMetricsTest(t, metadataTestMetrics, metadataTestHistogram, testName)
-	expectedMetadata := make(map[string]common.Metadata, len(common.OtlpDefaultMetricMetadata)+len(common.PromDefaultNativeHistogramMetadata))
-	for k, v := range common.OtlpDefaultMetricMetadata {
+	expectedMetadata := make(map[string]common.Metadata, len(common.PromDefaultMetricMetadata)+len(common.PromDefaultNativeHistogramMetadata))
+	for k, v := range common.PromDefaultMetricMetadata {
 		expectedMetadata[toTestMetric(k)] = v
 	}
 	for k, v := range common.PromDefaultNativeHistogramMetadata {

internal/cmd/integration-tests/tests/prom-metadata/config.alloy

@@ -30,3 +30,30 @@ prometheus.write.queue "prom_metadata" {
         }
     }
 }
+
+prometheus.scrape "prom_metadata_remote_write_v2" {
+  targets = [
+    {"__address__" = "prom-gen:9001"},
+  ]
+  params = { "metric-prefix" = ["prom_metadata_remote_write_v2_"] }
+  forward_to = [
+    prometheus.remote_write.prom_metadata_remote_write_v2.receiver,
+  ]
+  scrape_classic_histograms = true
+  scrape_native_histograms = true
+  scrape_interval = "1s"
+  scrape_timeout = "500ms"
+  // Required for metadata support
+  honor_metadata = true
+}
+
+prometheus.remote_write "prom_metadata_remote_write_v2" {
+    external_labels = {
+      test_name = "prom_metadata_remote_write_v2",
+    }
+    endpoint {
+        url = "http://mimir:9009/api/v1/push"
+        protobuf_message = "io.prometheus.write.v2.Request"
+        send_native_histograms = true
+    }
+}

internal/cmd/integration-tests/tests/prom-metadata/prom_metadata_remote_write_test.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/grafana/alloy/internal/cmd/integration-tests/common"
+)
+
+func TestPromMetadataRemoteWriteV2(t *testing.T) {
+	testName := "prom_metadata_remote_write_v2"
+	toTestMetric := func(metricName string) string {
+		return fmt.Sprintf("%s_%s", testName, metricName)
+	}
+
+	// Metadata queries cannot use a series matcher so the metrics all need to be unqiue to the test
+	metadataTestMetrics := make([]string, 0, len(common.PromDefaultMetrics))
+	for _, metricName := range common.PromDefaultMetrics {
+		metadataTestMetrics = append(metadataTestMetrics, toTestMetric(metricName))
+	}
+
+	// Make sure we got the expected metrics before checking metadata
+	common.MimirMetricsTest(t, metadataTestMetrics, nil, testName)
+	expectedMetadata := make(map[string]common.Metadata, len(common.PromDefaultMetricMetadata)+len(common.PromDefaultNativeHistogramMetadata))
+	for k, v := range common.PromDefaultMetricMetadata {
+		expectedMetadata[toTestMetric(k)] = v
+	}
+	common.MimirMetadataTest(t, expectedMetadata)
+}

internal/cmd/integration-tests/tests/prom-metadata/prom_metadata_writequeue_test.go

@@ -26,8 +26,8 @@ func TestPromMetadataWriteQueue(t *testing.T) {
 
 	// Make sure we got the expected metrics before checking metadata
 	common.MimirMetricsTest(t, metadataTestMetrics, metadataTestHistogram, testName)
-	expectedMetadata := make(map[string]common.Metadata, len(common.PromDefaultMetricMetadata)+len(common.PromDefaultNativeHistogramMetadata))
-	for k, v := range common.PromDefaultMetricMetadata {
+	expectedMetadata := make(map[string]common.Metadata, len(common.WriteQueueDefaultMetricMetadata)+len(common.PromDefaultNativeHistogramMetadata))
+	for k, v := range common.WriteQueueDefaultMetricMetadata {
 		expectedMetadata[toTestMetric(k)] = v
 	}
 	for k, v := range common.PromDefaultNativeHistogramMetadata {