From 7a341293f0e8de0cbdeaa4fb12f851153b8af515 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Mon, 7 Apr 2025 14:11:45 +0900 Subject: [PATCH] Implement metadata api query params Signed-off-by: SungJin1212 --- CHANGELOG.md | 1 + integration/e2ecortex/client.go | 4 +- integration/ingester_metadata_test.go | 200 ++++++++++++ integration/otlp_test.go | 2 +- pkg/distributor/distributor.go | 3 +- pkg/distributor/distributor_test.go | 2 +- pkg/ingester/client/ingester.pb.go | 304 +++++++++++++----- pkg/ingester/client/ingester.proto | 3 + pkg/ingester/ingester.go | 4 +- pkg/ingester/ingester_test.go | 6 +- pkg/ingester/user_metrics_metadata.go | 35 +- pkg/ingester/user_metrics_metadata_test.go | 135 ++++++++ pkg/querier/distributor_queryable.go | 2 +- pkg/querier/metadata_handler.go | 65 +++- pkg/querier/metadata_handler_test.go | 195 +++++++++-- pkg/querier/querier_test.go | 4 +- .../metadata_merge_querier.go | 7 +- .../metadata_merge_querier_test.go | 5 +- pkg/querier/testutils.go | 2 +- 19 files changed, 842 insertions(+), 137 deletions(-) create mode 100644 integration/ingester_metadata_test.go create mode 100644 pkg/ingester/user_metrics_metadata_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 736a494f6da..2593b628386 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526 * [FEATURE] Update prometheus alertmanager version to v0.28.0 and add new integration msteamsv2, jira, and rocketchat. #6590 * [FEATURE] Ingester: Support out-of-order native histogram ingestion. It automatically enabled when `-ingester.out-of-order-time-window > 0` and `-blocks-storage.tsdb.enable-native-histograms=true`. #6626 #6663 +* [ENHANCEMENT] Querier: Support query parameters to metadata api (/api/v1/metadata) to allow user to limit metadata to return. #6681 * [ENHANCEMENT] Query Frontend: Add new limit `-frontend.max-query-response-size` for total query response size after decompression in query frontend. #6607 * [ENHANCEMENT] Alertmanager: Add nflog and silences maintenance metrics. #6659 * [ENHANCEMENT] Querier: limit label APIs to query only ingesters if `start` param is not been specified. #6618 diff --git a/integration/e2ecortex/client.go b/integration/e2ecortex/client.go index bd0568eb9f8..9067b60c078 100644 --- a/integration/e2ecortex/client.go +++ b/integration/e2ecortex/client.go @@ -115,9 +115,9 @@ func NewPromQueryClient(address string) (*Client, error) { } // Push the input timeseries to the remote endpoint -func (c *Client) Push(timeseries []prompb.TimeSeries) (*http.Response, error) { +func (c *Client) Push(timeseries []prompb.TimeSeries, metadata ...prompb.MetricMetadata) (*http.Response, error) { // Create write request - data, err := proto.Marshal(&prompb.WriteRequest{Timeseries: timeseries}) + data, err := proto.Marshal(&prompb.WriteRequest{Timeseries: timeseries, Metadata: metadata}) if err != nil { return nil, err } diff --git a/integration/ingester_metadata_test.go b/integration/ingester_metadata_test.go new file mode 100644 index 00000000000..0d0893bf665 --- /dev/null +++ b/integration/ingester_metadata_test.go @@ -0,0 +1,200 @@ +//go:build requires_docker +// +build requires_docker + +package integration + +import ( + "fmt" + "strings" + "testing" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/prompb" + "github.com/stretchr/testify/require" + + "github.com/cortexproject/cortex/integration/e2e" + e2edb "github.com/cortexproject/cortex/integration/e2e/db" + "github.com/cortexproject/cortex/integration/e2ecortex" +) + +func TestIngesterMetadata(t *testing.T) { + s, err := e2e.NewScenario(networkName) + require.NoError(t, err) + defer s.Close() + + // Start dependencies. + consul := e2edb.NewConsul() + require.NoError(t, s.StartAndWaitReady(consul)) + + baseFlags := mergeFlags(AlertmanagerLocalFlags(), BlocksStorageFlags()) + + minio := e2edb.NewMinio(9000, baseFlags["-blocks-storage.s3.bucket-name"]) + require.NoError(t, s.StartAndWaitReady(minio)) + + flags := mergeFlags(baseFlags, map[string]string{ + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + // consul + "-ring.store": "consul", + "-consul.hostname": consul.NetworkHTTPEndpoint(), + }) + + // Start Cortex components + distributor := e2ecortex.NewDistributor("distributor", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") + ingester := e2ecortex.NewIngester("ingester", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") + querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") + require.NoError(t, s.StartAndWaitReady(distributor, ingester, querier)) + + // Wait until distributor has updated the ring. + require.NoError(t, distributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_ring_members"}, e2e.WithLabelMatchers( + labels.MustNewMatcher(labels.MatchEqual, "name", "ingester"), + labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE")))) + + // Wait until querier has updated the ring. + require.NoError(t, querier.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_ring_members"}, e2e.WithLabelMatchers( + labels.MustNewMatcher(labels.MatchEqual, "name", "ingester"), + labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE")))) + + client, err := e2ecortex.NewClient(distributor.HTTPEndpoint(), querier.HTTPEndpoint(), "", "", userID) + require.NoError(t, err) + + metadataMetricNum := 5 + metadataPerMetrics := 2 + metadata := make([]prompb.MetricMetadata, 0, metadataMetricNum) + for i := 0; i < metadataMetricNum; i++ { + for j := 0; j < metadataPerMetrics; j++ { + metadata = append(metadata, prompb.MetricMetadata{ + MetricFamilyName: fmt.Sprintf("metadata_name_%d", i), + Help: fmt.Sprintf("metadata_help_%d_%d", i, j), + Unit: fmt.Sprintf("metadata_unit_%d_%d", i, j), + }) + } + } + res, err := client.Push(nil, metadata...) + require.NoError(t, err) + require.Equal(t, 200, res.StatusCode) + + testMetadataQueryParams(t, client, metadataMetricNum, metadataPerMetrics) +} + +func TestIngesterMetadataWithTenantFederation(t *testing.T) { + s, err := e2e.NewScenario(networkName) + require.NoError(t, err) + defer s.Close() + + // Start dependencies. + consul := e2edb.NewConsul() + require.NoError(t, s.StartAndWaitReady(consul)) + + baseFlags := mergeFlags(AlertmanagerLocalFlags(), BlocksStorageFlags()) + + minio := e2edb.NewMinio(9000, baseFlags["-blocks-storage.s3.bucket-name"]) + require.NoError(t, s.StartAndWaitReady(minio)) + + flags := mergeFlags(baseFlags, map[string]string{ + // tenant federation + "-tenant-federation.enabled": "true", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + // consul + "-ring.store": "consul", + "-consul.hostname": consul.NetworkHTTPEndpoint(), + }) + + // Start Cortex components + distributor := e2ecortex.NewDistributor("distributor", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") + ingester := e2ecortex.NewIngester("ingester", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") + querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") + require.NoError(t, s.StartAndWaitReady(distributor, ingester, querier)) + + // Wait until distributor has updated the ring. + require.NoError(t, distributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_ring_members"}, e2e.WithLabelMatchers( + labels.MustNewMatcher(labels.MatchEqual, "name", "ingester"), + labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE")))) + + // Wait until querier has updated the ring. + require.NoError(t, querier.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_ring_members"}, e2e.WithLabelMatchers( + labels.MustNewMatcher(labels.MatchEqual, "name", "ingester"), + labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE")))) + + metadataMetricNum := 5 + metadataPerMetrics := 2 + metadata := make([]prompb.MetricMetadata, 0, metadataMetricNum) + for i := 0; i < metadataMetricNum; i++ { + for j := 0; j < metadataPerMetrics; j++ { + metadata = append(metadata, prompb.MetricMetadata{ + MetricFamilyName: fmt.Sprintf("metadata_name_%d", i), + Help: fmt.Sprintf("metadata_help_%d_%d", i, j), + Unit: fmt.Sprintf("metadata_unit_%d_%d", i, j), + }) + } + } + + numUsers := 2 + tenantIDs := make([]string, numUsers) + for u := 0; u < numUsers; u++ { + tenantIDs[u] = fmt.Sprintf("user-%d", u) + c, err := e2ecortex.NewClient(distributor.HTTPEndpoint(), querier.HTTPEndpoint(), "", "", tenantIDs[u]) + require.NoError(t, err) + + res, err := c.Push(nil, metadata...) + require.NoError(t, err) + require.Equal(t, 200, res.StatusCode) + } + + client, err := e2ecortex.NewClient(distributor.HTTPEndpoint(), querier.HTTPEndpoint(), "", "", strings.Join(tenantIDs, "|")) + require.NoError(t, err) + + testMetadataQueryParams(t, client, metadataMetricNum, metadataPerMetrics) +} + +func testMetadataQueryParams(t *testing.T, client *e2ecortex.Client, metadataMetricNum, metadataPerMetrics int) { + t.Run("test no parameter", func(t *testing.T) { + result, err := client.Metadata("", "") + require.NoError(t, err) + require.Equal(t, metadataMetricNum, len(result)) + + for _, v := range result { + require.Equal(t, metadataPerMetrics, len(v)) + } + }) + + t.Run("test name parameter", func(t *testing.T) { + t.Run("existing name", func(t *testing.T) { + name := "metadata_name_0" + result, err := client.Metadata(name, "") + require.NoError(t, err) + m, ok := result[name] + require.True(t, ok) + require.Equal(t, metadataPerMetrics, len(m)) + }) + t.Run("existing name with limit 0", func(t *testing.T) { + name := "metadata_name_0" + result, err := client.Metadata(name, "0") + require.NoError(t, err) + require.Equal(t, 0, len(result)) + }) + t.Run("non-existing name", func(t *testing.T) { + result, err := client.Metadata("dummy", "") + require.NoError(t, err) + require.Equal(t, 0, len(result)) + }) + }) + + t.Run("test limit parameter", func(t *testing.T) { + t.Run("less than length of metadata", func(t *testing.T) { + result, err := client.Metadata("", "3") + require.NoError(t, err) + require.Equal(t, 3, len(result)) + }) + t.Run("limit: 0", func(t *testing.T) { + result, err := client.Metadata("", "0") + require.NoError(t, err) + require.Equal(t, 0, len(result)) + }) + t.Run("invalid limit", func(t *testing.T) { + _, err := client.Metadata("", "dummy") + require.Error(t, err) + }) + }) +} diff --git a/integration/otlp_test.go b/integration/otlp_test.go index df5b632b8aa..7eda34e55ec 100644 --- a/integration/otlp_test.go +++ b/integration/otlp_test.go @@ -88,7 +88,7 @@ func TestOTLP(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{"__name__", "foo"}, labelNames) - metadataResult, err := c.Metadata("series_1", "") + metadataResult, err := c.Metadata("series_1_total", "") require.NoError(t, err) require.Equal(t, 1, len(metadataResult)) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index f9b347e56e4..d366767eb79 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -1460,13 +1460,12 @@ func (d *Distributor) metricsForLabelMatchersCommon(ctx context.Context, from, t } // MetricsMetadata returns all metric metadata of a user. -func (d *Distributor) MetricsMetadata(ctx context.Context) ([]scrape.MetricMetadata, error) { +func (d *Distributor) MetricsMetadata(ctx context.Context, req *ingester_client.MetricsMetadataRequest) ([]scrape.MetricMetadata, error) { replicationSet, err := d.GetIngestersForMetadata(ctx) if err != nil { return nil, err } - req := &ingester_client.MetricsMetadataRequest{} // TODO(gotjosh): We only need to look in all the ingesters if shardByAllLabels is enabled. resps, err := d.ForReplicationSet(ctx, replicationSet, d.cfg.ZoneResultsQuorumMetadata, false, func(ctx context.Context, client ingester_client.IngesterClient) (interface{}, error) { return client.MetricsMetadata(ctx, req) diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 16177ad62b1..7113860fa1e 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -2774,7 +2774,7 @@ func TestDistributor_MetricsMetadata(t *testing.T) { require.NoError(t, err) // Assert on metric metadata - metadata, err := ds[0].MetricsMetadata(ctx) + metadata, err := ds[0].MetricsMetadata(ctx, &client.MetricsMetadataRequest{Limit: -1, LimitPerMetric: -1, Metric: ""}) require.NoError(t, err) assert.Equal(t, 10, len(metadata)) diff --git a/pkg/ingester/client/ingester.pb.go b/pkg/ingester/client/ingester.pb.go index 374348afae7..ae8937d9ed5 100644 --- a/pkg/ingester/client/ingester.pb.go +++ b/pkg/ingester/client/ingester.pb.go @@ -1077,6 +1077,9 @@ func (m *MetricsForLabelMatchersStreamResponse) GetMetric() []*cortexpb.Metric { } type MetricsMetadataRequest struct { + Limit int64 `protobuf:"varint,1,opt,name=limit,proto3" json:"limit,omitempty"` + LimitPerMetric int64 `protobuf:"varint,2,opt,name=limit_per_metric,json=limitPerMetric,proto3" json:"limit_per_metric,omitempty"` + Metric string `protobuf:"bytes,3,opt,name=metric,proto3" json:"metric,omitempty"` } func (m *MetricsMetadataRequest) Reset() { *m = MetricsMetadataRequest{} } @@ -1111,6 +1114,27 @@ func (m *MetricsMetadataRequest) XXX_DiscardUnknown() { var xxx_messageInfo_MetricsMetadataRequest proto.InternalMessageInfo +func (m *MetricsMetadataRequest) GetLimit() int64 { + if m != nil { + return m.Limit + } + return 0 +} + +func (m *MetricsMetadataRequest) GetLimitPerMetric() int64 { + if m != nil { + return m.LimitPerMetric + } + return 0 +} + +func (m *MetricsMetadataRequest) GetMetric() string { + if m != nil { + return m.Metric + } + return "" +} + type MetricsMetadataResponse struct { Metadata []*cortexpb.MetricMetadata `protobuf:"bytes,1,rep,name=metadata,proto3" json:"metadata,omitempty"` } @@ -1484,91 +1508,93 @@ func init() { func init() { proto.RegisterFile("ingester.proto", fileDescriptor_60f6df4f3586b478) } var fileDescriptor_60f6df4f3586b478 = []byte{ - // 1339 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0x4b, 0x6f, 0x14, 0xc7, + // 1369 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x58, 0x4b, 0x6f, 0x14, 0xc7, 0x13, 0xdf, 0xf1, 0x3e, 0xec, 0xad, 0x7d, 0xb0, 0x6e, 0x1b, 0xbc, 0x0c, 0x7f, 0xc6, 0x30, 0x88, 0x7f, 0xac, 0x24, 0xd8, 0xe0, 0x24, 0x12, 0xe4, 0x85, 0x6c, 0x30, 0x60, 0xc0, 0x18, 0xc6, 0x86, - 0x44, 0x51, 0xa2, 0xd1, 0x78, 0xb7, 0xb1, 0x27, 0xcc, 0x63, 0x99, 0xee, 0x41, 0x90, 0x53, 0xa2, - 0x7c, 0x80, 0xe4, 0x98, 0x6b, 0x6e, 0xf9, 0x00, 0xf9, 0x10, 0x1c, 0x39, 0xe4, 0x80, 0x72, 0x40, - 0x61, 0x91, 0xa2, 0x1c, 0xc9, 0x37, 0x88, 0xa6, 0x1f, 0xf3, 0xf2, 0xf8, 0x41, 0x04, 0xb9, 0xed, - 0x54, 0xfd, 0xaa, 0xba, 0xea, 0xd7, 0x55, 0x5d, 0xb5, 0xd0, 0xb6, 0xbd, 0x4d, 0x4c, 0x28, 0x0e, - 0x66, 0x07, 0x81, 0x4f, 0x7d, 0x54, 0xeb, 0xf9, 0x01, 0xc5, 0x0f, 0xd5, 0xc9, 0x4d, 0x7f, 0xd3, - 0x67, 0xa2, 0xb9, 0xe8, 0x17, 0xd7, 0xaa, 0xe7, 0x36, 0x6d, 0xba, 0x15, 0x6e, 0xcc, 0xf6, 0x7c, - 0x77, 0x8e, 0x03, 0x07, 0x81, 0xff, 0x35, 0xee, 0x51, 0xf1, 0x35, 0x37, 0xb8, 0xb7, 0x29, 0x15, - 0x1b, 0xe2, 0x07, 0x37, 0xd5, 0x3f, 0x81, 0x86, 0x81, 0xad, 0xbe, 0x81, 0xef, 0x87, 0x98, 0x50, - 0x34, 0x0b, 0xa3, 0xf7, 0x43, 0x1c, 0xd8, 0x98, 0x74, 0x95, 0x63, 0xe5, 0x99, 0xc6, 0xfc, 0xe4, - 0xac, 0x80, 0xdf, 0x0a, 0x71, 0xf0, 0x48, 0xc0, 0x0c, 0x09, 0xd2, 0xcf, 0x43, 0x93, 0x9b, 0x93, - 0x81, 0xef, 0x11, 0x8c, 0xe6, 0x60, 0x34, 0xc0, 0x24, 0x74, 0xa8, 0xb4, 0x3f, 0x98, 0xb3, 0xe7, - 0x38, 0x43, 0xa2, 0xf4, 0x6b, 0xd0, 0xca, 0x68, 0xd0, 0x87, 0x00, 0xd4, 0x76, 0x31, 0x29, 0x0a, - 0x62, 0xb0, 0x31, 0xbb, 0x6e, 0xbb, 0x78, 0x8d, 0xe9, 0x16, 0x2b, 0x8f, 0x9f, 0x4d, 0x97, 0x8c, - 0x14, 0x5a, 0xff, 0x49, 0x81, 0x66, 0x3a, 0x4e, 0xf4, 0x2e, 0x20, 0x42, 0xad, 0x80, 0x9a, 0x0c, - 0x44, 0x2d, 0x77, 0x60, 0xba, 0x91, 0x53, 0x65, 0xa6, 0x6c, 0x74, 0x98, 0x66, 0x5d, 0x2a, 0x56, - 0x08, 0x9a, 0x81, 0x0e, 0xf6, 0xfa, 0x59, 0xec, 0x08, 0xc3, 0xb6, 0xb1, 0xd7, 0x4f, 0x23, 0x4f, - 0xc3, 0x98, 0x6b, 0xd1, 0xde, 0x16, 0x0e, 0x48, 0xb7, 0x9c, 0xe5, 0xe9, 0xba, 0xb5, 0x81, 0x9d, - 0x15, 0xae, 0x34, 0x62, 0x94, 0xfe, 0xb3, 0x02, 0x93, 0x4b, 0x0f, 0xb1, 0x3b, 0x70, 0xac, 0xe0, - 0x3f, 0x09, 0xf1, 0xcc, 0xb6, 0x10, 0x0f, 0x16, 0x85, 0x48, 0x52, 0x31, 0x7e, 0x09, 0x13, 0x2c, - 0xb4, 0x35, 0x1a, 0x60, 0xcb, 0x8d, 0x6f, 0xe4, 0x3c, 0x34, 0x7a, 0x5b, 0xa1, 0x77, 0x2f, 0x73, - 0x25, 0x53, 0xd2, 0x59, 0x72, 0x21, 0x17, 0x22, 0x90, 0xb8, 0x95, 0xb4, 0xc5, 0xd5, 0xca, 0xd8, - 0x48, 0xa7, 0xac, 0xaf, 0xc1, 0xc1, 0x1c, 0x01, 0xaf, 0xe1, 0xc6, 0x7f, 0x53, 0x00, 0xb1, 0x74, - 0xee, 0x58, 0x4e, 0x88, 0x89, 0x24, 0xf5, 0x28, 0x80, 0x13, 0x49, 0x4d, 0xcf, 0x72, 0x31, 0x23, - 0xb3, 0x6e, 0xd4, 0x99, 0xe4, 0x86, 0xe5, 0xe2, 0x1d, 0x38, 0x1f, 0x79, 0x05, 0xce, 0xcb, 0x7b, - 0x72, 0x5e, 0x39, 0xa6, 0xec, 0x83, 0x73, 0x34, 0x09, 0x55, 0xc7, 0x76, 0x6d, 0xda, 0xad, 0x32, - 0x8f, 0xfc, 0x43, 0x3f, 0x0b, 0x13, 0x99, 0xac, 0x04, 0x53, 0xc7, 0xa1, 0xc9, 0xd3, 0x7a, 0xc0, - 0xe4, 0x8c, 0xab, 0xba, 0xd1, 0x70, 0x12, 0xa8, 0xfe, 0x29, 0x1c, 0x4e, 0x59, 0xe6, 0x6e, 0x72, - 0x1f, 0xf6, 0xbf, 0x2a, 0x30, 0x7e, 0x5d, 0x12, 0x45, 0xde, 0x74, 0x91, 0xc6, 0xd9, 0x97, 0x53, - 0xd9, 0xff, 0x0b, 0x1a, 0xf5, 0x0f, 0x44, 0x19, 0x88, 0xa8, 0x45, 0xbe, 0xd3, 0xd0, 0x48, 0xca, - 0x40, 0xa6, 0x0b, 0x71, 0x1d, 0x10, 0xfd, 0x23, 0xe8, 0x26, 0x66, 0x39, 0xb2, 0xf6, 0x34, 0x46, - 0xd0, 0xb9, 0x4d, 0x70, 0xb0, 0x46, 0x2d, 0x2a, 0x89, 0xd2, 0xbf, 0x1b, 0x81, 0xf1, 0x94, 0x50, - 0xb8, 0x3a, 0x29, 0xdf, 0x73, 0xdb, 0xf7, 0xcc, 0xc0, 0xa2, 0xbc, 0x24, 0x15, 0xa3, 0x15, 0x4b, - 0x0d, 0x8b, 0xe2, 0xa8, 0x6a, 0xbd, 0xd0, 0x35, 0x45, 0x23, 0x44, 0x8c, 0x55, 0x8c, 0xba, 0x17, - 0xba, 0xbc, 0xfa, 0xa3, 0x4b, 0xb0, 0x06, 0xb6, 0x99, 0xf3, 0x54, 0x66, 0x9e, 0x3a, 0xd6, 0xc0, - 0x5e, 0xce, 0x38, 0x9b, 0x85, 0x89, 0x20, 0x74, 0x70, 0x1e, 0x5e, 0x61, 0xf0, 0xf1, 0x48, 0x95, - 0xc5, 0x9f, 0x80, 0x96, 0xd5, 0xa3, 0xf6, 0x03, 0x2c, 0xcf, 0xaf, 0xb2, 0xf3, 0x9b, 0x5c, 0x28, - 0x42, 0x38, 0x01, 0x2d, 0xc7, 0xb7, 0xfa, 0xb8, 0x6f, 0x6e, 0x38, 0x7e, 0xef, 0x1e, 0xe9, 0xd6, - 0x38, 0x88, 0x0b, 0x17, 0x99, 0x4c, 0xff, 0x0a, 0x26, 0x22, 0x0a, 0x96, 0x2f, 0x66, 0x49, 0x98, - 0x82, 0xd1, 0x90, 0xe0, 0xc0, 0xb4, 0xfb, 0xa2, 0x21, 0x6b, 0xd1, 0xe7, 0x72, 0x1f, 0x9d, 0x82, - 0x4a, 0xdf, 0xa2, 0x16, 0x4b, 0xb8, 0x31, 0x7f, 0x58, 0x5e, 0xf5, 0x36, 0x1a, 0x0d, 0x06, 0xd3, - 0x2f, 0x03, 0x8a, 0x54, 0x24, 0xeb, 0xfd, 0x0c, 0x54, 0x49, 0x24, 0x10, 0xef, 0xc7, 0x91, 0xb4, - 0x97, 0x5c, 0x24, 0x06, 0x47, 0xea, 0x8f, 0x15, 0xd0, 0x56, 0x30, 0x0d, 0xec, 0x1e, 0xb9, 0xe4, - 0x07, 0xd9, 0xca, 0x7a, 0xc3, 0x75, 0x7f, 0x16, 0x9a, 0xb2, 0x74, 0x4d, 0x82, 0xe9, 0xee, 0x0f, - 0x74, 0x43, 0x42, 0xd7, 0x30, 0x4d, 0x3a, 0xa6, 0x92, 0x7e, 0x2f, 0xae, 0xc1, 0xf4, 0x8e, 0x99, + 0x44, 0x51, 0xa2, 0xd1, 0x78, 0xb7, 0xb1, 0x27, 0xcc, 0x8b, 0xe9, 0x5e, 0x04, 0x39, 0x25, 0xca, + 0x07, 0x48, 0x8e, 0xb9, 0xe6, 0x96, 0x0f, 0x90, 0x0f, 0xc1, 0x91, 0x43, 0x0e, 0x28, 0x07, 0x14, + 0x16, 0x29, 0xca, 0x91, 0x7c, 0x83, 0x68, 0xfa, 0x31, 0x2f, 0x8f, 0x1f, 0x44, 0x90, 0xdb, 0x74, + 0xd5, 0xaf, 0xaa, 0xab, 0x7e, 0x5d, 0xdd, 0x55, 0xbb, 0xd0, 0xb6, 0xbd, 0x4d, 0x4c, 0x28, 0x0e, + 0x67, 0x83, 0xd0, 0xa7, 0x3e, 0xaa, 0xf5, 0xfc, 0x90, 0xe2, 0x87, 0xea, 0xe4, 0xa6, 0xbf, 0xe9, + 0x33, 0xd1, 0x5c, 0xf4, 0xc5, 0xb5, 0xea, 0xb9, 0x4d, 0x9b, 0x6e, 0x0d, 0x36, 0x66, 0x7b, 0xbe, + 0x3b, 0xc7, 0x81, 0x41, 0xe8, 0x7f, 0x8d, 0x7b, 0x54, 0xac, 0xe6, 0x82, 0x7b, 0x9b, 0x52, 0xb1, + 0x21, 0x3e, 0xb8, 0xa9, 0xfe, 0x09, 0x34, 0x0c, 0x6c, 0xf5, 0x0d, 0x7c, 0x7f, 0x80, 0x09, 0x45, + 0xb3, 0x30, 0x7a, 0x7f, 0x80, 0x43, 0x1b, 0x93, 0xae, 0x72, 0xac, 0x3c, 0xd3, 0x98, 0x9f, 0x9c, + 0x15, 0xf0, 0x5b, 0x03, 0x1c, 0x3e, 0x12, 0x30, 0x43, 0x82, 0xf4, 0xf3, 0xd0, 0xe4, 0xe6, 0x24, + 0xf0, 0x3d, 0x82, 0xd1, 0x1c, 0x8c, 0x86, 0x98, 0x0c, 0x1c, 0x2a, 0xed, 0x0f, 0xe6, 0xec, 0x39, + 0xce, 0x90, 0x28, 0xfd, 0x1a, 0xb4, 0x32, 0x1a, 0xf4, 0x21, 0x00, 0xb5, 0x5d, 0x4c, 0x8a, 0x82, + 0x08, 0x36, 0x66, 0xd7, 0x6d, 0x17, 0xaf, 0x31, 0xdd, 0x62, 0xe5, 0xf1, 0xb3, 0xe9, 0x92, 0x91, + 0x42, 0xeb, 0x3f, 0x29, 0xd0, 0x4c, 0xc7, 0x89, 0xde, 0x05, 0x44, 0xa8, 0x15, 0x52, 0x93, 0x81, + 0xa8, 0xe5, 0x06, 0xa6, 0x1b, 0x39, 0x55, 0x66, 0xca, 0x46, 0x87, 0x69, 0xd6, 0xa5, 0x62, 0x85, + 0xa0, 0x19, 0xe8, 0x60, 0xaf, 0x9f, 0xc5, 0x8e, 0x30, 0x6c, 0x1b, 0x7b, 0xfd, 0x34, 0xf2, 0x34, + 0x8c, 0xb9, 0x16, 0xed, 0x6d, 0xe1, 0x90, 0x74, 0xcb, 0x59, 0x9e, 0xae, 0x5b, 0x1b, 0xd8, 0x59, + 0xe1, 0x4a, 0x23, 0x46, 0xe9, 0x3f, 0x2b, 0x30, 0xb9, 0xf4, 0x10, 0xbb, 0x81, 0x63, 0x85, 0xff, + 0x49, 0x88, 0x67, 0xb6, 0x85, 0x78, 0xb0, 0x28, 0x44, 0x92, 0x8a, 0xf1, 0x4b, 0x98, 0x60, 0xa1, + 0xad, 0xd1, 0x10, 0x5b, 0x6e, 0x7c, 0x22, 0xe7, 0xa1, 0xd1, 0xdb, 0x1a, 0x78, 0xf7, 0x32, 0x47, + 0x32, 0x25, 0x9d, 0x25, 0x07, 0x72, 0x21, 0x02, 0x89, 0x53, 0x49, 0x5b, 0x5c, 0xad, 0x8c, 0x8d, + 0x74, 0xca, 0xfa, 0x1a, 0x1c, 0xcc, 0x11, 0xf0, 0x1a, 0x4e, 0xfc, 0x37, 0x05, 0x10, 0x4b, 0xe7, + 0x8e, 0xe5, 0x0c, 0x30, 0x91, 0xa4, 0x1e, 0x05, 0x70, 0x22, 0xa9, 0xe9, 0x59, 0x2e, 0x66, 0x64, + 0xd6, 0x8d, 0x3a, 0x93, 0xdc, 0xb0, 0x5c, 0xbc, 0x03, 0xe7, 0x23, 0xaf, 0xc0, 0x79, 0x79, 0x4f, + 0xce, 0x2b, 0xc7, 0x94, 0x7d, 0x70, 0x8e, 0x26, 0xa1, 0xea, 0xd8, 0xae, 0x4d, 0xbb, 0x55, 0xe6, + 0x91, 0x2f, 0xf4, 0xb3, 0x30, 0x91, 0xc9, 0x4a, 0x30, 0x75, 0x1c, 0x9a, 0x3c, 0xad, 0x07, 0x4c, + 0xce, 0xb8, 0xaa, 0x1b, 0x0d, 0x27, 0x81, 0xea, 0x9f, 0xc2, 0xe1, 0x94, 0x65, 0xee, 0x24, 0xf7, + 0x61, 0xff, 0xab, 0x02, 0xe3, 0xd7, 0x25, 0x51, 0xe4, 0x4d, 0x17, 0x69, 0x9c, 0x7d, 0x39, 0x95, + 0xfd, 0xbf, 0xa0, 0x51, 0xff, 0x40, 0x94, 0x81, 0x88, 0x5a, 0xe4, 0x3b, 0x0d, 0x8d, 0xa4, 0x0c, + 0x64, 0xba, 0x10, 0xd7, 0x01, 0xd1, 0x3f, 0x82, 0x6e, 0x62, 0x96, 0x23, 0x6b, 0x4f, 0x63, 0x04, + 0x9d, 0xdb, 0x04, 0x87, 0x6b, 0xd4, 0xa2, 0x92, 0x28, 0xfd, 0xbb, 0x11, 0x18, 0x4f, 0x09, 0x85, + 0xab, 0x93, 0xf2, 0x3d, 0xb7, 0x7d, 0xcf, 0x0c, 0x2d, 0xca, 0x4b, 0x52, 0x31, 0x5a, 0xb1, 0xd4, + 0xb0, 0x28, 0x8e, 0xaa, 0xd6, 0x1b, 0xb8, 0xa6, 0xb8, 0x08, 0x11, 0x63, 0x15, 0xa3, 0xee, 0x0d, + 0x5c, 0x5e, 0xfd, 0xd1, 0x21, 0x58, 0x81, 0x6d, 0xe6, 0x3c, 0x95, 0x99, 0xa7, 0x8e, 0x15, 0xd8, + 0xcb, 0x19, 0x67, 0xb3, 0x30, 0x11, 0x0e, 0x1c, 0x9c, 0x87, 0x57, 0x18, 0x7c, 0x3c, 0x52, 0x65, + 0xf1, 0x27, 0xa0, 0x65, 0xf5, 0xa8, 0xfd, 0x00, 0xcb, 0xfd, 0xab, 0x6c, 0xff, 0x26, 0x17, 0x8a, + 0x10, 0x4e, 0x40, 0xcb, 0xf1, 0xad, 0x3e, 0xee, 0x9b, 0x1b, 0x8e, 0xdf, 0xbb, 0x47, 0xba, 0x35, + 0x0e, 0xe2, 0xc2, 0x45, 0x26, 0xd3, 0xbf, 0x82, 0x89, 0x88, 0x82, 0xe5, 0x8b, 0x59, 0x12, 0xa6, + 0x60, 0x74, 0x40, 0x70, 0x68, 0xda, 0x7d, 0x71, 0x21, 0x6b, 0xd1, 0x72, 0xb9, 0x8f, 0x4e, 0x41, + 0xa5, 0x6f, 0x51, 0x8b, 0x25, 0xdc, 0x98, 0x3f, 0x2c, 0x8f, 0x7a, 0x1b, 0x8d, 0x06, 0x83, 0xe9, + 0x97, 0x01, 0x45, 0x2a, 0x92, 0xf5, 0x7e, 0x06, 0xaa, 0x24, 0x12, 0x88, 0xf7, 0xe3, 0x48, 0xda, + 0x4b, 0x2e, 0x12, 0x83, 0x23, 0xf5, 0xc7, 0x0a, 0x68, 0x2b, 0x98, 0x86, 0x76, 0x8f, 0x5c, 0xf2, + 0xc3, 0x6c, 0x65, 0xbd, 0xe1, 0xba, 0x3f, 0x0b, 0x4d, 0x59, 0xba, 0x26, 0xc1, 0x74, 0xf7, 0x07, + 0xba, 0x21, 0xa1, 0x6b, 0x98, 0x26, 0x37, 0xa6, 0x92, 0x7e, 0x2f, 0xae, 0xc1, 0xf4, 0x8e, 0x99, 0x08, 0x82, 0x66, 0xa0, 0xe6, 0x32, 0x88, 0x60, 0xa8, 0x93, 0xbc, 0xb0, 0xdc, 0xd4, 0x10, 0x7a, - 0xfd, 0x16, 0x9c, 0xdc, 0xc1, 0x59, 0xae, 0x43, 0xf6, 0xef, 0xb2, 0x0b, 0x87, 0x84, 0xcb, 0x15, - 0x4c, 0xad, 0xe8, 0x1a, 0x65, 0xc3, 0xac, 0xc2, 0xd4, 0x36, 0x8d, 0x70, 0xff, 0x3e, 0x8c, 0xb9, - 0x42, 0x26, 0x0e, 0xe8, 0xe6, 0x0f, 0x88, 0x6d, 0x62, 0xa4, 0xfe, 0xb7, 0x02, 0x07, 0x72, 0x33, - 0x29, 0xba, 0x98, 0xbb, 0x81, 0xef, 0x9a, 0x72, 0xa9, 0x4a, 0x6a, 0xb0, 0x1d, 0xc9, 0x97, 0x85, - 0x78, 0xb9, 0x9f, 0x2e, 0xd2, 0x91, 0x4c, 0x91, 0x7a, 0x50, 0x63, 0xad, 0x2f, 0x87, 0xe9, 0x44, - 0x12, 0x0a, 0xa3, 0xe8, 0xa6, 0x65, 0x07, 0x8b, 0x0b, 0xd1, 0x7c, 0xfa, 0xfd, 0xd9, 0xf4, 0x2b, - 0xed, 0x63, 0xdc, 0x7e, 0xa1, 0x6f, 0x0d, 0x28, 0x0e, 0x0c, 0x71, 0x0a, 0x7a, 0x07, 0x6a, 0x7c, - 0x84, 0x76, 0x2b, 0xec, 0xbc, 0x96, 0xac, 0x8d, 0xf4, 0x94, 0x15, 0x10, 0xfd, 0x07, 0x05, 0xaa, - 0x3c, 0xd3, 0x37, 0x55, 0xb0, 0x2a, 0x8c, 0x61, 0xaf, 0xe7, 0xf7, 0x6d, 0x6f, 0x93, 0xbd, 0x38, - 0x55, 0x23, 0xfe, 0x46, 0x48, 0xf4, 0x6f, 0x54, 0x91, 0x4d, 0xd1, 0xa4, 0x0b, 0xd0, 0xca, 0x54, - 0x4e, 0x66, 0x63, 0x52, 0xf6, 0xb5, 0x31, 0x99, 0xd0, 0x4c, 0x6b, 0xd0, 0x49, 0xa8, 0xd0, 0x47, - 0x03, 0xfe, 0x74, 0xb6, 0xe7, 0xc7, 0xa5, 0x35, 0x53, 0xaf, 0x3f, 0x1a, 0x60, 0x83, 0xa9, 0xa3, - 0x68, 0xd8, 0xd0, 0xe7, 0xd7, 0xc7, 0x7e, 0x47, 0x4d, 0xc3, 0x26, 0x1e, 0x0b, 0xbd, 0x6e, 0xf0, - 0x0f, 0xfd, 0x7b, 0x05, 0xda, 0x49, 0xa5, 0x5c, 0xb2, 0x1d, 0xfc, 0x3a, 0x0a, 0x45, 0x85, 0xb1, - 0xbb, 0xb6, 0x83, 0x59, 0x0c, 0xfc, 0xb8, 0xf8, 0xbb, 0x88, 0xa9, 0xb7, 0xaf, 0x42, 0x3d, 0x4e, - 0x01, 0xd5, 0xa1, 0xba, 0x74, 0xeb, 0xf6, 0xc2, 0xf5, 0x4e, 0x09, 0xb5, 0xa0, 0x7e, 0x63, 0x75, - 0xdd, 0xe4, 0x9f, 0x0a, 0x3a, 0x00, 0x0d, 0x63, 0xe9, 0xf2, 0xd2, 0xe7, 0xe6, 0xca, 0xc2, 0xfa, - 0x85, 0x2b, 0x9d, 0x11, 0x84, 0xa0, 0xcd, 0x05, 0x37, 0x56, 0x85, 0xac, 0x3c, 0xff, 0xe7, 0x28, - 0x8c, 0xc9, 0x18, 0xd1, 0x39, 0xa8, 0xdc, 0x0c, 0xc9, 0x16, 0x3a, 0x94, 0x54, 0xea, 0x67, 0x81, - 0x4d, 0xb1, 0xe8, 0x3c, 0x75, 0x6a, 0x9b, 0x9c, 0xf7, 0x9d, 0x5e, 0x42, 0x17, 0xa1, 0x91, 0x5a, - 0x04, 0x51, 0xe1, 0x7f, 0x00, 0xf5, 0x48, 0x46, 0x9a, 0x7d, 0x1a, 0xf4, 0xd2, 0x69, 0x05, 0xad, - 0x42, 0x9b, 0xa9, 0xe4, 0xd6, 0x47, 0xd0, 0xff, 0xa4, 0x49, 0xd1, 0x26, 0xac, 0x1e, 0xdd, 0x41, - 0x1b, 0x87, 0x75, 0x05, 0x1a, 0xa9, 0xdd, 0x06, 0xa9, 0x99, 0x02, 0xca, 0x2c, 0x80, 0x49, 0x70, - 0x05, 0x6b, 0x94, 0x5e, 0x42, 0x77, 0xc4, 0x92, 0x93, 0xde, 0x92, 0x76, 0xf5, 0x77, 0xbc, 0x40, - 0x57, 0x90, 0xf2, 0x12, 0x40, 0xb2, 0x4f, 0xa0, 0xc3, 0x19, 0xa3, 0xf4, 0x42, 0xa5, 0xaa, 0x45, - 0xaa, 0x38, 0xbc, 0x35, 0xe8, 0xe4, 0xd7, 0x92, 0xdd, 0x9c, 0x1d, 0xdb, 0xae, 0x2a, 0x88, 0x6d, - 0x11, 0xea, 0xf1, 0x48, 0x45, 0xdd, 0x82, 0x29, 0xcb, 0x9d, 0xed, 0x3c, 0x7f, 0xf5, 0x12, 0xba, - 0x04, 0xcd, 0x05, 0xc7, 0xd9, 0x8f, 0x1b, 0x35, 0xad, 0x21, 0x79, 0x3f, 0x4e, 0xfc, 0xea, 0xe7, - 0x47, 0x0c, 0xfa, 0x7f, 0xdc, 0xd8, 0xbb, 0x8e, 0x66, 0xf5, 0xad, 0x3d, 0x71, 0xf1, 0x69, 0xdf, - 0xc0, 0xd1, 0x5d, 0x07, 0xda, 0xbe, 0xcf, 0x3c, 0xb5, 0x07, 0xae, 0x80, 0xf5, 0x75, 0x38, 0x90, - 0x9b, 0x6f, 0x48, 0xcb, 0x79, 0xc9, 0x8d, 0x44, 0x75, 0x7a, 0x47, 0xbd, 0xf4, 0xbb, 0xf8, 0xf1, - 0x93, 0xe7, 0x5a, 0xe9, 0xe9, 0x73, 0xad, 0xf4, 0xf2, 0xb9, 0xa6, 0x7c, 0x3b, 0xd4, 0x94, 0x5f, - 0x86, 0x9a, 0xf2, 0x78, 0xa8, 0x29, 0x4f, 0x86, 0x9a, 0xf2, 0xc7, 0x50, 0x53, 0xfe, 0x1a, 0x6a, - 0xa5, 0x97, 0x43, 0x4d, 0xf9, 0xf1, 0x85, 0x56, 0x7a, 0xf2, 0x42, 0x2b, 0x3d, 0x7d, 0xa1, 0x95, - 0xbe, 0xa8, 0xf5, 0x1c, 0x1b, 0x7b, 0x74, 0xa3, 0xc6, 0xfe, 0xfa, 0xbf, 0xf7, 0x4f, 0x00, 0x00, - 0x00, 0xff, 0xff, 0x84, 0xf7, 0x8d, 0x61, 0x65, 0x10, 0x00, 0x00, + 0xfd, 0x16, 0x9c, 0xdc, 0xc1, 0x59, 0xee, 0x86, 0xec, 0xdf, 0x65, 0x00, 0x87, 0x84, 0xcb, 0x15, + 0x4c, 0xad, 0xe8, 0x18, 0x25, 0xc3, 0x71, 0x3e, 0x4a, 0xfa, 0x05, 0x98, 0x81, 0x0e, 0xfb, 0x30, + 0x03, 0x1c, 0x9a, 0x62, 0x0f, 0xc1, 0x24, 0x93, 0xdf, 0xc4, 0x21, 0xf7, 0x87, 0x0e, 0xc5, 0x31, + 0x94, 0x79, 0x51, 0x89, 0x1d, 0x57, 0x61, 0x6a, 0xdb, 0x8e, 0x22, 0xec, 0xf7, 0x61, 0xcc, 0x15, + 0x32, 0x11, 0x78, 0x37, 0x1f, 0x78, 0x6c, 0x13, 0x23, 0xf5, 0xbf, 0x15, 0x38, 0x90, 0xeb, 0x75, + 0x51, 0x98, 0x77, 0x43, 0xdf, 0x35, 0xe5, 0xb0, 0x96, 0xd4, 0x76, 0x3b, 0x92, 0x2f, 0x0b, 0xf1, + 0x72, 0x3f, 0x5d, 0xfc, 0x23, 0x99, 0xe2, 0xf7, 0xa0, 0xc6, 0x9e, 0x14, 0xd9, 0xa4, 0x27, 0x92, + 0x50, 0x18, 0xf5, 0x37, 0x2d, 0x3b, 0x5c, 0x5c, 0x88, 0xfa, 0xde, 0xef, 0xcf, 0xa6, 0x5f, 0x69, + 0xce, 0xe3, 0xf6, 0x0b, 0x7d, 0x2b, 0xa0, 0x38, 0x34, 0xc4, 0x2e, 0xe8, 0x1d, 0xa8, 0xf1, 0xd6, + 0xdc, 0xad, 0xb0, 0xfd, 0x5a, 0xb2, 0xe6, 0xd2, 0xdd, 0x5b, 0x40, 0xf4, 0x1f, 0x14, 0xa8, 0xf2, + 0x4c, 0xdf, 0xd4, 0x45, 0x50, 0x61, 0x0c, 0x7b, 0x3d, 0xbf, 0x6f, 0x7b, 0x9b, 0xec, 0x00, 0xab, + 0x46, 0xbc, 0x46, 0x48, 0xbc, 0x0b, 0x51, 0xa5, 0x37, 0xc5, 0xe5, 0x5f, 0x80, 0x56, 0xa6, 0x22, + 0x33, 0x93, 0x98, 0xb2, 0xaf, 0x49, 0xcc, 0x84, 0x66, 0x5a, 0x83, 0x4e, 0x42, 0x85, 0x3e, 0x0a, + 0xf8, 0x93, 0xdc, 0x9e, 0x1f, 0x97, 0xd6, 0x4c, 0xbd, 0xfe, 0x28, 0xc0, 0x06, 0x53, 0x47, 0xd1, + 0xb0, 0x61, 0x82, 0x1f, 0x1f, 0xfb, 0x8e, 0x8a, 0x97, 0x75, 0x52, 0x51, 0x7b, 0x7c, 0xa1, 0x7f, + 0xaf, 0x40, 0x3b, 0xa9, 0x94, 0x4b, 0xb6, 0x83, 0x5f, 0x47, 0xa1, 0xa8, 0x30, 0x76, 0xd7, 0x76, + 0x30, 0x8b, 0x81, 0x6f, 0x17, 0xaf, 0x8b, 0x98, 0x7a, 0xfb, 0x2a, 0xd4, 0xe3, 0x14, 0x50, 0x1d, + 0xaa, 0x4b, 0xb7, 0x6e, 0x2f, 0x5c, 0xef, 0x94, 0x50, 0x0b, 0xea, 0x37, 0x56, 0xd7, 0x4d, 0xbe, + 0x54, 0xd0, 0x01, 0x68, 0x18, 0x4b, 0x97, 0x97, 0x3e, 0x37, 0x57, 0x16, 0xd6, 0x2f, 0x5c, 0xe9, + 0x8c, 0x20, 0x04, 0x6d, 0x2e, 0xb8, 0xb1, 0x2a, 0x64, 0xe5, 0xf9, 0x3f, 0x47, 0x61, 0x4c, 0xc6, + 0x88, 0xce, 0x41, 0xe5, 0xe6, 0x80, 0x6c, 0xa1, 0x43, 0x49, 0xa5, 0x7e, 0x16, 0xda, 0x14, 0x8b, + 0x1b, 0xad, 0x4e, 0x6d, 0x93, 0xf3, 0x7b, 0xa7, 0x97, 0xd0, 0x45, 0x68, 0xa4, 0x06, 0x4c, 0x54, + 0xf8, 0xdb, 0x42, 0x3d, 0x92, 0x91, 0x66, 0x9f, 0x1c, 0xbd, 0x74, 0x5a, 0x41, 0xab, 0xd0, 0x66, + 0x2a, 0x39, 0x4d, 0x12, 0xf4, 0x3f, 0x69, 0x52, 0x34, 0x61, 0xab, 0x47, 0x77, 0xd0, 0xc6, 0x61, + 0x5d, 0x81, 0x46, 0x6a, 0x66, 0x42, 0x6a, 0xa6, 0x80, 0x32, 0x83, 0x65, 0x12, 0x5c, 0xc1, 0x78, + 0xa6, 0x97, 0xd0, 0x1d, 0x31, 0x3c, 0xa5, 0xa7, 0xaf, 0x5d, 0xfd, 0x1d, 0x2f, 0xd0, 0x15, 0xa4, + 0xbc, 0x04, 0x90, 0xcc, 0x29, 0xe8, 0x70, 0xc6, 0x28, 0x3d, 0xa8, 0xa9, 0x6a, 0x91, 0x2a, 0x0e, + 0x6f, 0x0d, 0x3a, 0xf9, 0x71, 0x67, 0x37, 0x67, 0xc7, 0xb6, 0xab, 0x0a, 0x62, 0x5b, 0x84, 0x7a, + 0xdc, 0xaa, 0x51, 0xb7, 0xa0, 0x7b, 0x73, 0x67, 0x3b, 0xf7, 0x75, 0xbd, 0x84, 0x2e, 0x41, 0x73, + 0xc1, 0x71, 0xf6, 0xe3, 0x46, 0x4d, 0x6b, 0x48, 0xde, 0x8f, 0x13, 0xbf, 0xfa, 0xf9, 0xd6, 0x85, + 0xfe, 0x1f, 0x5f, 0xec, 0x5d, 0x5b, 0xbe, 0xfa, 0xd6, 0x9e, 0xb8, 0x78, 0xb7, 0x6f, 0xe0, 0xe8, + 0xae, 0x8d, 0x72, 0xdf, 0x7b, 0x9e, 0xda, 0x03, 0x57, 0xc0, 0xfa, 0x3a, 0x1c, 0xc8, 0xf5, 0x37, + 0xa4, 0xe5, 0xbc, 0xe4, 0x5a, 0xad, 0x3a, 0xbd, 0xa3, 0x5e, 0xfa, 0x5d, 0xfc, 0xf8, 0xc9, 0x73, + 0xad, 0xf4, 0xf4, 0xb9, 0x56, 0x7a, 0xf9, 0x5c, 0x53, 0xbe, 0x1d, 0x6a, 0xca, 0x2f, 0x43, 0x4d, + 0x79, 0x3c, 0xd4, 0x94, 0x27, 0x43, 0x4d, 0xf9, 0x63, 0xa8, 0x29, 0x7f, 0x0d, 0xb5, 0xd2, 0xcb, + 0xa1, 0xa6, 0xfc, 0xf8, 0x42, 0x2b, 0x3d, 0x79, 0xa1, 0x95, 0x9e, 0xbe, 0xd0, 0x4a, 0x5f, 0xd4, + 0x7a, 0x8e, 0x8d, 0x3d, 0xba, 0x51, 0x63, 0x7f, 0x29, 0xbc, 0xf7, 0x4f, 0x00, 0x00, 0x00, 0xff, + 0xff, 0x73, 0x37, 0x7c, 0x02, 0xbd, 0x10, 0x00, 0x00, } func (x MatchType) String() string { @@ -2209,6 +2235,15 @@ func (this *MetricsMetadataRequest) Equal(that interface{}) bool { } else if this == nil { return false } + if this.Limit != that1.Limit { + return false + } + if this.LimitPerMetric != that1.LimitPerMetric { + return false + } + if this.Metric != that1.Metric { + return false + } return true } func (this *MetricsMetadataResponse) Equal(that interface{}) bool { @@ -2671,8 +2706,11 @@ func (this *MetricsMetadataRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 4) + s := make([]string, 0, 7) s = append(s, "&client.MetricsMetadataRequest{") + s = append(s, "Limit: "+fmt.Sprintf("%#v", this.Limit)+",\n") + s = append(s, "LimitPerMetric: "+fmt.Sprintf("%#v", this.LimitPerMetric)+",\n") + s = append(s, "Metric: "+fmt.Sprintf("%#v", this.Metric)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -4169,6 +4207,23 @@ func (m *MetricsMetadataRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) _ = i var l int _ = l + if len(m.Metric) > 0 { + i -= len(m.Metric) + copy(dAtA[i:], m.Metric) + i = encodeVarintIngester(dAtA, i, uint64(len(m.Metric))) + i-- + dAtA[i] = 0x1a + } + if m.LimitPerMetric != 0 { + i = encodeVarintIngester(dAtA, i, uint64(m.LimitPerMetric)) + i-- + dAtA[i] = 0x10 + } + if m.Limit != 0 { + i = encodeVarintIngester(dAtA, i, uint64(m.Limit)) + i-- + dAtA[i] = 0x8 + } return len(dAtA) - i, nil } @@ -4813,6 +4868,16 @@ func (m *MetricsMetadataRequest) Size() (n int) { } var l int _ = l + if m.Limit != 0 { + n += 1 + sovIngester(uint64(m.Limit)) + } + if m.LimitPerMetric != 0 { + n += 1 + sovIngester(uint64(m.LimitPerMetric)) + } + l = len(m.Metric) + if l > 0 { + n += 1 + l + sovIngester(uint64(l)) + } return n } @@ -5227,6 +5292,9 @@ func (this *MetricsMetadataRequest) String() string { return "nil" } s := strings.Join([]string{`&MetricsMetadataRequest{`, + `Limit:` + fmt.Sprintf("%v", this.Limit) + `,`, + `LimitPerMetric:` + fmt.Sprintf("%v", this.LimitPerMetric) + `,`, + `Metric:` + fmt.Sprintf("%v", this.Metric) + `,`, `}`, }, "") return s @@ -7425,6 +7493,76 @@ func (m *MetricsMetadataRequest) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: MetricsMetadataRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) + } + m.Limit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIngester + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Limit |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LimitPerMetric", wireType) + } + m.LimitPerMetric = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIngester + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LimitPerMetric |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Metric", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIngester + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthIngester + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthIngester + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Metric = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipIngester(dAtA[iNdEx:]) diff --git a/pkg/ingester/client/ingester.proto b/pkg/ingester/client/ingester.proto index 68f343693e6..0cbfac49c93 100644 --- a/pkg/ingester/client/ingester.proto +++ b/pkg/ingester/client/ingester.proto @@ -129,6 +129,9 @@ message MetricsForLabelMatchersStreamResponse { } message MetricsMetadataRequest { + int64 limit = 1; + int64 limit_per_metric = 2; + string metric = 3; } message MetricsMetadataResponse { diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 86a3a3dfdd6..d34fbb98649 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1950,7 +1950,7 @@ func (i *Ingester) metricsForLabelMatchersCommon(ctx context.Context, req *clien } // MetricsMetadata returns all the metric metadata of a user. -func (i *Ingester) MetricsMetadata(ctx context.Context, _ *client.MetricsMetadataRequest) (*client.MetricsMetadataResponse, error) { +func (i *Ingester) MetricsMetadata(ctx context.Context, req *client.MetricsMetadataRequest) (*client.MetricsMetadataResponse, error) { i.stoppedMtx.RLock() if err := i.checkRunningOrStopping(); err != nil { i.stoppedMtx.RUnlock() @@ -1969,7 +1969,7 @@ func (i *Ingester) MetricsMetadata(ctx context.Context, _ *client.MetricsMetadat return &client.MetricsMetadataResponse{}, nil } - return &client.MetricsMetadataResponse{Metadata: userMetadata.toClientMetadata()}, nil + return &client.MetricsMetadataResponse{Metadata: userMetadata.toClientMetadata(req)}, nil } // CheckReady is the readiness handler used to indicate to k8s when the ingesters diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index c22668b828f..ec5ce7ea788 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -834,7 +834,7 @@ func TestIngesterUserLimitExceeded(t *testing.T) { require.Equal(t, expected, res) // Verify metadata - m, err := ing.MetricsMetadata(ctx, nil) + m, err := ing.MetricsMetadata(ctx, &client.MetricsMetadataRequest{Limit: -1, LimitPerMetric: -1, Metric: ""}) require.NoError(t, err) assert.Equal(t, []*cortexpb.MetricMetadata{metadata1}, m.Metadata) @@ -965,7 +965,7 @@ func TestIngesterMetricLimitExceeded(t *testing.T) { assert.Equal(t, expected, res) // Verify metadata - m, err := ing.MetricsMetadata(ctx, nil) + m, err := ing.MetricsMetadata(ctx, &client.MetricsMetadataRequest{Limit: -1, LimitPerMetric: -1, Metric: ""}) require.NoError(t, err) assert.Equal(t, []*cortexpb.MetricMetadata{metadata1}, m.Metadata) @@ -2008,7 +2008,7 @@ func TestIngester_Push(t *testing.T) { assert.Equal(t, testData.expectedExemplarsIngested, exemplarRes.Timeseries) // Read back metadata to see what has been really ingested. - mres, err := i.MetricsMetadata(ctx, &client.MetricsMetadataRequest{}) + mres, err := i.MetricsMetadata(ctx, &client.MetricsMetadataRequest{Limit: -1, LimitPerMetric: -1, Metric: ""}) require.NoError(t, err) require.NotNil(t, mres) diff --git a/pkg/ingester/user_metrics_metadata.go b/pkg/ingester/user_metrics_metadata.go index dcb5fd6bbf4..8f451c884b4 100644 --- a/pkg/ingester/user_metrics_metadata.go +++ b/pkg/ingester/user_metrics_metadata.go @@ -7,6 +7,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/cortexproject/cortex/pkg/cortexpb" + "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -84,18 +85,46 @@ func (mm *userMetricsMetadata) purge(deadline time.Time) { mm.metrics.memMetadataRemovedTotal.WithLabelValues(mm.userID).Add(float64(deleted)) } -func (mm *userMetricsMetadata) toClientMetadata() []*cortexpb.MetricMetadata { +func (mm *userMetricsMetadata) toClientMetadata(req *client.MetricsMetadataRequest) []*cortexpb.MetricMetadata { mm.mtx.RLock() defer mm.mtx.RUnlock() r := make([]*cortexpb.MetricMetadata, 0, len(mm.metricToMetadata)) + if req.Limit == 0 { + return r + } + + if req.Metric != "" { + metadataSet, ok := mm.metricToMetadata[req.Metric] + if !ok { + return r + } + + metadataSet.add(req.LimitPerMetric, &r) + return r + } + + var metrics int64 for _, set := range mm.metricToMetadata { - for m := range set { - r = append(r, &m) + if req.Limit > 0 && metrics >= req.Limit { + break } + set.add(req.LimitPerMetric, &r) + metrics++ } return r } +func (mns metricMetadataSet) add(limitPerMetric int64, r *[]*cortexpb.MetricMetadata) { + var metrics int64 + for m := range mns { + if limitPerMetric > 0 && metrics >= limitPerMetric { + return + } + *r = append(*r, &m) + metrics++ + } +} + type metricMetadataSet map[cortexpb.MetricMetadata]time.Time // If deadline is zero time, all metrics are purged. diff --git a/pkg/ingester/user_metrics_metadata_test.go b/pkg/ingester/user_metrics_metadata_test.go new file mode 100644 index 00000000000..2a28601ced3 --- /dev/null +++ b/pkg/ingester/user_metrics_metadata_test.go @@ -0,0 +1,135 @@ +package ingester + +import ( + "fmt" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + + "github.com/cortexproject/cortex/pkg/cortexpb" + "github.com/cortexproject/cortex/pkg/ingester/client" + "github.com/cortexproject/cortex/pkg/util" + util_math "github.com/cortexproject/cortex/pkg/util/math" + "github.com/cortexproject/cortex/pkg/util/validation" +) + +const ( + defaultLimit = -1 + defaultLimitPerMetric = -1 +) + +func Test_UserMetricsMetadata(t *testing.T) { + userId := "user-1" + + reg := prometheus.NewPedanticRegistry() + ingestionRate := util_math.NewEWMARate(0.2, instanceIngestionRateTickInterval) + inflightPushRequests := util_math.MaxTracker{} + maxInflightQueryRequests := util_math.MaxTracker{} + + m := newIngesterMetrics(reg, + false, + false, + func() *InstanceLimits { + return &InstanceLimits{} + }, + ingestionRate, + &inflightPushRequests, + &maxInflightQueryRequests, + false) + + limits := validation.Limits{} + overrides, err := validation.NewOverrides(limits, nil) + require.NoError(t, err) + limiter := NewLimiter(overrides, nil, util.ShardingStrategyDefault, true, 1, false, "") + + userMetricsMetadata := newMetadataMap(limiter, m, validation.NewValidateMetrics(reg), userId) + + addMetricMetadata := func(name string, i int) { + metadata := &cortexpb.MetricMetadata{ + MetricFamilyName: fmt.Sprintf("%s_%d", name, i), + Type: cortexpb.GAUGE, + Help: fmt.Sprintf("a help for %s", name), + Unit: fmt.Sprintf("a unit for %s", name), + } + + err := userMetricsMetadata.add(name, metadata) + require.NoError(t, err) + } + + metadataNumPerMetric := 3 + for _, m := range []string{"metric1", "metric2"} { + for i := range metadataNumPerMetric { + addMetricMetadata(m, i) + } + } + + tests := []struct { + description string + limit int64 + limitPerMetric int64 + metric string + expectedLength int + }{ + { + description: "limit: 1", + limit: 1, + limitPerMetric: defaultLimitPerMetric, + expectedLength: 3, + }, + { + description: "limit: 0", + limit: 0, + limitPerMetric: defaultLimitPerMetric, + expectedLength: 0, + }, + { + description: "limit_per_metric: 2", + limit: defaultLimit, + limitPerMetric: 2, + expectedLength: 4, + }, + { + description: "limit: 0, limit_per_metric: 2", + limit: 1, + limitPerMetric: 2, + expectedLength: 2, + }, + { + description: "limit: 1, limit_per_metric: 0 (should be ignored)", + limit: 1, + limitPerMetric: 0, + expectedLength: 3, + }, + { + description: "metric: metric1", + limit: defaultLimit, + limitPerMetric: defaultLimitPerMetric, + metric: "metric1", + expectedLength: 3, + }, + { + description: "metric: metric1, limit_per_metric: 2", + limit: defaultLimit, + limitPerMetric: 2, + metric: "metric1", + expectedLength: 2, + }, + { + description: "not exist metric", + limit: 1, + limitPerMetric: defaultLimitPerMetric, + metric: "dummy", + expectedLength: 0, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + req := &client.MetricsMetadataRequest{Limit: test.limit, LimitPerMetric: test.limitPerMetric, Metric: test.metric} + + r := userMetricsMetadata.toClientMetadata(req) + require.Equal(t, test.expectedLength, len(r)) + }) + } +} diff --git a/pkg/querier/distributor_queryable.go b/pkg/querier/distributor_queryable.go index 46965c8eee3..dffe8ae3002 100644 --- a/pkg/querier/distributor_queryable.go +++ b/pkg/querier/distributor_queryable.go @@ -35,7 +35,7 @@ type Distributor interface { LabelNamesStream(context.Context, model.Time, model.Time, *storage.LabelHints, bool, ...*labels.Matcher) ([]string, error) MetricsForLabelMatchers(ctx context.Context, from, through model.Time, hint *storage.SelectHints, partialDataEnabled bool, matchers ...*labels.Matcher) ([]model.Metric, error) MetricsForLabelMatchersStream(ctx context.Context, from, through model.Time, hint *storage.SelectHints, partialDataEnabled bool, matchers ...*labels.Matcher) ([]model.Metric, error) - MetricsMetadata(ctx context.Context) ([]scrape.MetricMetadata, error) + MetricsMetadata(ctx context.Context, req *client.MetricsMetadataRequest) ([]scrape.MetricMetadata, error) } func newDistributorQueryable(distributor Distributor, streamingMetdata bool, labelNamesWithMatchers bool, iteratorFn chunkIteratorFunc, queryIngestersWithin time.Duration, isPartialDataEnabled partialdata.IsCfgEnabledFunc) QueryableWithFilter { diff --git a/pkg/querier/metadata_handler.go b/pkg/querier/metadata_handler.go index e185cdc6084..9eeeb0b1ad7 100644 --- a/pkg/querier/metadata_handler.go +++ b/pkg/querier/metadata_handler.go @@ -2,15 +2,22 @@ package querier import ( "context" + "fmt" "net/http" + "strconv" "github.com/prometheus/prometheus/scrape" + "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/util" ) +const ( + defaultLimit = -1 +) + type MetadataQuerier interface { - MetricsMetadata(ctx context.Context) ([]scrape.MetricMetadata, error) + MetricsMetadata(ctx context.Context, req *client.MetricsMetadataRequest) ([]scrape.MetricMetadata, error) } type metricMetadata struct { @@ -24,20 +31,41 @@ const ( statusError = "error" ) -type metadataResult struct { +type metadataSuccessResult struct { Status string `json:"status"` - Data map[string][]metricMetadata `json:"data,omitempty"` - Error string `json:"error,omitempty"` + Data map[string][]metricMetadata `json:"data"` +} + +type metadataErrorResult struct { + Status string `json:"status"` + Error string `json:"error"` } // MetadataHandler returns metric metadata held by Cortex for a given tenant. // It is kept and returned as a set. func MetadataHandler(m MetadataQuerier) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - resp, err := m.MetricsMetadata(r.Context()) + limit, err := validateLimits("limit", w, r) + if err != nil { + return + } + + limitPerMetric, err := validateLimits("limit_per_metric", w, r) + if err != nil { + return + } + + metric := r.FormValue("metric") + req := &client.MetricsMetadataRequest{ + Limit: int64(limit), + LimitPerMetric: int64(limitPerMetric), + Metric: metric, + } + + resp, err := m.MetricsMetadata(r.Context(), req) if err != nil { w.WriteHeader(http.StatusBadRequest) - util.WriteJSONResponse(w, metadataResult{Status: statusError, Error: err.Error()}) + util.WriteJSONResponse(w, metadataErrorResult{Status: statusError, Error: err.Error()}) return } @@ -45,7 +73,17 @@ func MetadataHandler(m MetadataQuerier) http.Handler { metrics := map[string][]metricMetadata{} for _, m := range resp { ms, ok := metrics[m.MetricFamily] + // We have to check limit both ingester and here since the ingester only check + // for one user, it cannot handle the case when the mergeMetadataQuerier + // (tenant-federation) is used. + if limitPerMetric > 0 && len(ms) >= limitPerMetric { + continue + } + if !ok { + if limit >= 0 && len(metrics) >= limit { + break + } // Most metrics will only hold 1 copy of the same metadata. ms = make([]metricMetadata, 0, 1) metrics[m.MetricFamily] = ms @@ -53,6 +91,19 @@ func MetadataHandler(m MetadataQuerier) http.Handler { metrics[m.MetricFamily] = append(ms, metricMetadata{Type: string(m.Type), Help: m.Help, Unit: m.Unit}) } - util.WriteJSONResponse(w, metadataResult{Status: statusSuccess, Data: metrics}) + util.WriteJSONResponse(w, metadataSuccessResult{Status: statusSuccess, Data: metrics}) }) } + +func validateLimits(name string, w http.ResponseWriter, r *http.Request) (int, error) { + v := defaultLimit + if s := r.FormValue(name); s != "" { + var err error + if v, err = strconv.Atoi(s); err != nil { + w.WriteHeader(http.StatusBadRequest) + util.WriteJSONResponse(w, metadataErrorResult{Status: statusError, Error: fmt.Sprintf("%s must be a number", name)}) + return 0, err + } + } + return v, nil +} diff --git a/pkg/querier/metadata_handler_test.go b/pkg/querier/metadata_handler_test.go index c47daa1e014..a2c35f2fe98 100644 --- a/pkg/querier/metadata_handler_test.go +++ b/pkg/querier/metadata_handler_test.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "net/http/httptest" + "net/url" "testing" "github.com/prometheus/prometheus/scrape" @@ -16,47 +17,193 @@ func TestMetadataHandler_Success(t *testing.T) { t.Parallel() d := &MockDistributor{} - d.On("MetricsMetadata", mock.Anything).Return( + d.On("MetricsMetadata", mock.Anything, mock.Anything).Return( []scrape.MetricMetadata{ {MetricFamily: "alertmanager_dispatcher_aggregation_groups", Help: "Number of active aggregation groups", Type: "gauge", Unit: ""}, + {MetricFamily: "go_threads", Help: "Number of OS threads created", Type: "gauge", Unit: ""}, + {MetricFamily: "go_threads", Help: "Number of OS threads that were created", Type: "gauge", Unit: ""}, }, nil) - handler := MetadataHandler(d) - - request, err := http.NewRequest("GET", "/metadata", nil) - require.NoError(t, err) - - recorder := httptest.NewRecorder() - handler.ServeHTTP(recorder, request) + fullResponseJson := ` + { + "status": "success", + "data": { + "alertmanager_dispatcher_aggregation_groups": [ + { + "help": "Number of active aggregation groups", + "type": "gauge", + "unit": "" + } + ], + "go_threads": [ + { + "help": "Number of OS threads created", + "type": "gauge", + "unit": "" + }, + { + "help": "Number of OS threads that were created", + "type": "gauge", + "unit": "" + } + ] + } + } + ` - require.Equal(t, http.StatusOK, recorder.Result().StatusCode) - responseBody, err := io.ReadAll(recorder.Result().Body) - require.NoError(t, err) + emptyDataResponseJson := ` + { + "status": "success", + "data": {} + } + ` - expectedJSON := ` - { - "status": "success", - "data": { - "alertmanager_dispatcher_aggregation_groups": [ + tests := []struct { + description string + queryParams url.Values + expectedCode int + expectedJson string + }{ + { + description: "no params", + queryParams: url.Values{}, + expectedCode: http.StatusOK, + expectedJson: fullResponseJson, + }, + { + description: "limit: -1", + queryParams: url.Values{ + "limit": []string{"-1"}, + }, + expectedCode: http.StatusOK, + expectedJson: fullResponseJson, + }, + { + description: "limit: 0", + queryParams: url.Values{ + "limit": []string{"0"}, + }, + expectedCode: http.StatusOK, + expectedJson: emptyDataResponseJson, + }, + { + description: "limit: 1", + queryParams: url.Values{ + "limit": []string{"1"}, + }, + expectedCode: http.StatusOK, + expectedJson: ` { - "help": "Number of active aggregation groups", - "type": "gauge", - "unit": "" + "status": "success", + "data": { + "alertmanager_dispatcher_aggregation_groups": [ + { + "help": "Number of active aggregation groups", + "type": "gauge", + "unit": "" + } + ] + } } - ] - } + `, + }, + { + description: "limit: invalid", + queryParams: url.Values{ + "limit": []string{"aaa"}, + }, + expectedCode: http.StatusBadRequest, + expectedJson: ` + { + "status": "error", + "error": "limit must be a number" + } + `, + }, + { + description: "limit_per_metric: -1", + queryParams: url.Values{ + "limit_per_metric": []string{"-1"}, + }, + expectedCode: http.StatusOK, + expectedJson: fullResponseJson, + }, + { + description: "limit_per_metric: 0, should be ignored", + queryParams: url.Values{ + "limit_per_metric": []string{"0"}, + }, + expectedCode: http.StatusOK, + expectedJson: fullResponseJson, + }, + { + description: "limit_per_metric: 1", + queryParams: url.Values{ + "limit_per_metric": []string{"1"}, + }, + expectedCode: http.StatusOK, + expectedJson: ` + { + "status": "success", + "data": { + "alertmanager_dispatcher_aggregation_groups": [ + { + "help": "Number of active aggregation groups", + "type": "gauge", + "unit": "" + } + ], + "go_threads": [ + { + "help": "Number of OS threads created", + "type": "gauge", + "unit": "" + } + ] + } + } + `, + }, + { + description: "limit_per_metric: invalid", + queryParams: url.Values{ + "limit_per_metric": []string{"aaa"}, + }, + expectedCode: http.StatusBadRequest, + expectedJson: ` + { + "status": "error", + "error": "limit_per_metric must be a number" + } + `, + }, } - ` - require.JSONEq(t, expectedJSON, string(responseBody)) + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + handler := MetadataHandler(d) + + request, err := http.NewRequest("GET", "/metadata", nil) + request.URL.RawQuery = test.queryParams.Encode() + require.NoError(t, err) + + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + + require.Equal(t, test.expectedCode, recorder.Result().StatusCode) + responseBody, err := io.ReadAll(recorder.Result().Body) + require.NoError(t, err) + require.JSONEq(t, test.expectedJson, string(responseBody)) + }) + } } func TestMetadataHandler_Error(t *testing.T) { t.Parallel() d := &MockDistributor{} - d.On("MetricsMetadata", mock.Anything).Return([]scrape.MetricMetadata{}, fmt.Errorf("no user id")) + d.On("MetricsMetadata", mock.Anything, mock.Anything).Return([]scrape.MetricMetadata{}, fmt.Errorf("no user id")) handler := MetadataHandler(d) diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index c87bfce0e68..d2865408abe 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -1392,7 +1392,7 @@ func (m *errDistributor) MetricsForLabelMatchersStream(ctx context.Context, from return nil, errDistributorError } -func (m *errDistributor) MetricsMetadata(ctx context.Context) ([]scrape.MetricMetadata, error) { +func (m *errDistributor) MetricsMetadata(ctx context.Context, request *client.MetricsMetadataRequest) ([]scrape.MetricMetadata, error) { return nil, errDistributorError } @@ -1448,7 +1448,7 @@ func (d *emptyDistributor) MetricsForLabelMatchersStream(ctx context.Context, fr return nil, nil } -func (d *emptyDistributor) MetricsMetadata(ctx context.Context) ([]scrape.MetricMetadata, error) { +func (d *emptyDistributor) MetricsMetadata(ctx context.Context, request *client.MetricsMetadataRequest) ([]scrape.MetricMetadata, error) { return nil, nil } diff --git a/pkg/querier/tenantfederation/metadata_merge_querier.go b/pkg/querier/tenantfederation/metadata_merge_querier.go index 4a51a19d653..611bfbe1f55 100644 --- a/pkg/querier/tenantfederation/metadata_merge_querier.go +++ b/pkg/querier/tenantfederation/metadata_merge_querier.go @@ -10,6 +10,7 @@ import ( "github.com/prometheus/prometheus/scrape" "github.com/weaveworks/common/user" + "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/querier" "github.com/cortexproject/cortex/pkg/tenant" "github.com/cortexproject/cortex/pkg/util/concurrency" @@ -45,7 +46,7 @@ type metadataSelectJob struct { } // MetricsMetadata returns aggregated metadata for multiple tenants -func (m *mergeMetadataQuerier) MetricsMetadata(ctx context.Context) ([]scrape.MetricMetadata, error) { +func (m *mergeMetadataQuerier) MetricsMetadata(ctx context.Context, req *client.MetricsMetadataRequest) ([]scrape.MetricMetadata, error) { log, ctx := spanlogger.New(ctx, "mergeMetadataQuerier.MetricsMetadata") defer log.Span.Finish() @@ -57,7 +58,7 @@ func (m *mergeMetadataQuerier) MetricsMetadata(ctx context.Context) ([]scrape.Me m.tenantsPerMetadataQuery.Observe(float64(len(tenantIds))) if len(tenantIds) == 1 { - return m.upstream.MetricsMetadata(ctx) + return m.upstream.MetricsMetadata(ctx, req) } jobs := make([]interface{}, len(tenantIds)) @@ -79,7 +80,7 @@ func (m *mergeMetadataQuerier) MetricsMetadata(ctx context.Context) ([]scrape.Me return fmt.Errorf("unexpected type %T", jobIntf) } - res, err := job.querier.MetricsMetadata(user.InjectOrgID(ctx, job.id)) + res, err := job.querier.MetricsMetadata(user.InjectOrgID(ctx, job.id), req) if err != nil { return errors.Wrapf(err, "error exemplars querying %s %s", job.id, err) } diff --git a/pkg/querier/tenantfederation/metadata_merge_querier_test.go b/pkg/querier/tenantfederation/metadata_merge_querier_test.go index 58e5f955ba5..a9a93147338 100644 --- a/pkg/querier/tenantfederation/metadata_merge_querier_test.go +++ b/pkg/querier/tenantfederation/metadata_merge_querier_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/weaveworks/common/user" + "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/tenant" ) @@ -51,7 +52,7 @@ type mockMetadataQuerier struct { tenantIdToMetadata map[string][]scrape.MetricMetadata } -func (m *mockMetadataQuerier) MetricsMetadata(ctx context.Context) ([]scrape.MetricMetadata, error) { +func (m *mockMetadataQuerier) MetricsMetadata(ctx context.Context, _ *client.MetricsMetadataRequest) ([]scrape.MetricMetadata, error) { // Due to lint check for `ensure the query path is supporting multiple tenants` ids, err := tenant.TenantIDs(ctx) if err != nil { @@ -137,7 +138,7 @@ func Test_mergeMetadataQuerier_MetricsMetadata(t *testing.T) { } mergeMetadataQuerier := NewMetadataQuerier(&upstream, defaultMaxConcurrency, reg) - metadata, err := mergeMetadataQuerier.MetricsMetadata(user.InjectOrgID(context.Background(), test.orgId)) + metadata, err := mergeMetadataQuerier.MetricsMetadata(user.InjectOrgID(context.Background(), test.orgId), &client.MetricsMetadataRequest{Limit: -1, LimitPerMetric: -1, Metric: ""}) require.NoError(t, err) require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(test.expectedMetrics), "cortex_querier_federated_tenants_per_metadata_query")) require.Equal(t, test.expectedResults, metadata) diff --git a/pkg/querier/testutils.go b/pkg/querier/testutils.go index 0ee4a414640..37b1bd2b179 100644 --- a/pkg/querier/testutils.go +++ b/pkg/querier/testutils.go @@ -58,7 +58,7 @@ func (m *MockDistributor) MetricsForLabelMatchersStream(ctx context.Context, fro return args.Get(0).([]model.Metric), args.Error(1) } -func (m *MockDistributor) MetricsMetadata(ctx context.Context) ([]scrape.MetricMetadata, error) { +func (m *MockDistributor) MetricsMetadata(ctx context.Context, request *client.MetricsMetadataRequest) ([]scrape.MetricMetadata, error) { args := m.Called(ctx) return args.Get(0).([]scrape.MetricMetadata), args.Error(1) }