From 41213c6999128244a7a41da6b504ebc32f28f04d Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Fri, 8 Dec 2023 11:21:48 -0800 Subject: [PATCH 1/2] log split queries at query frontend Signed-off-by: Ben Ye --- pkg/frontend/transport/handler.go | 2 + pkg/frontend/transport/handler_test.go | 13 ++-- pkg/frontend/transport/roundtripper.go | 4 + pkg/querier/stats/stats.go | 16 ++++ pkg/querier/stats/stats.pb.go | 101 ++++++++++++++++++------- pkg/querier/stats/stats.proto | 3 + 6 files changed, 104 insertions(+), 35 deletions(-) diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index 700145312f9..f6554acb252 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -294,6 +294,7 @@ func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString u numSamples := stats.LoadFetchedSamples() numChunkBytes := stats.LoadFetchedChunkBytes() numDataBytes := stats.LoadFetchedDataBytes() + splitQueries := stats.LoadSplitQueries() // Track stats. f.querySeconds.WithLabelValues(userID).Add(wallTime.Seconds()) @@ -324,6 +325,7 @@ func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString u "fetched_samples_count", numSamples, "fetched_chunks_bytes", numChunkBytes, "fetched_data_bytes", numDataBytes, + "split_queries", splitQueries, "status_code", statusCode, "response_size", contentLength, }, stats.LoadExtraFields()...) diff --git a/pkg/frontend/transport/handler_test.go b/pkg/frontend/transport/handler_test.go index 801bda6d81b..dae62012fb8 100644 --- a/pkg/frontend/transport/handler_test.go +++ b/pkg/frontend/transport/handler_test.go @@ -318,11 +318,11 @@ func TestReportQueryStatsFormat(t *testing.T) { tests := map[string]testCase{ "should not include query and header details if empty": { - expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000`, + expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 split_queries=0 status_code=200 response_size=1000`, }, "should include query length and string at the end": { queryString: url.Values(map[string][]string{"query": {"up"}}), - expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000 query_length=2 param_query=up`, + expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 split_queries=0 status_code=200 response_size=1000 query_length=2 param_query=up`, }, "should include query stats": { queryStats: &querier_stats.QueryStats{ @@ -333,22 +333,23 @@ func TestReportQueryStatsFormat(t *testing.T) { FetchedSamplesCount: 300, FetchedChunkBytes: 1024, FetchedDataBytes: 2048, + SplitQueries: 10, }, }, - expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=3 fetched_series_count=100 fetched_chunks_count=200 fetched_samples_count=300 fetched_chunks_bytes=1024 fetched_data_bytes=2048 status_code=200 response_size=1000`, + expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=3 fetched_series_count=100 fetched_chunks_count=200 fetched_samples_count=300 fetched_chunks_bytes=1024 fetched_data_bytes=2048 split_queries=10 status_code=200 response_size=1000`, }, "should include user agent": { header: http.Header{"User-Agent": []string{"Grafana"}}, - expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000 user_agent=Grafana`, + expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 split_queries=0 status_code=200 response_size=1000 user_agent=Grafana`, }, "should include response error": { responseErr: errors.New("foo_err"), - expectedLog: `level=error msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000 error=foo_err`, + expectedLog: `level=error msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 split_queries=0 status_code=200 response_size=1000 error=foo_err`, }, "should include query priority": { queryString: url.Values(map[string][]string{"query": {"up"}}), header: http.Header{util.QueryPriorityHeaderKey: []string{"99"}}, - expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000 query_length=2 priority=99 param_query=up`, + expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 split_queries=0 status_code=200 response_size=1000 query_length=2 priority=99 param_query=up`, }, } diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go index 583fc22d04a..3c0ddc28630 100644 --- a/pkg/frontend/transport/roundtripper.go +++ b/pkg/frontend/transport/roundtripper.go @@ -8,6 +8,8 @@ import ( "github.com/weaveworks/common/httpgrpc" "github.com/weaveworks/common/httpgrpc/server" + + querier_stats "github.com/cortexproject/cortex/pkg/querier/stats" ) // GrpcRoundTripper is similar to http.RoundTripper, but works with HTTP requests converted to protobuf messages. @@ -39,6 +41,8 @@ func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, er return nil, err } + stats := querier_stats.FromContext(r.Context()) + stats.AddSplitQueries(1) resp, err := a.roundTripper.RoundTripGRPC(r.Context(), req) if err != nil { return nil, err diff --git a/pkg/querier/stats/stats.go b/pkg/querier/stats/stats.go index 94a386c9c0b..81b9724abb7 100644 --- a/pkg/querier/stats/stats.go +++ b/pkg/querier/stats/stats.go @@ -180,6 +180,22 @@ func (s *QueryStats) LoadFetchedChunks() uint64 { return atomic.LoadUint64(&s.FetchedChunksCount) } +func (s *QueryStats) AddSplitQueries(count uint64) { + if s == nil { + return + } + + atomic.AddUint64(&s.SplitQueries, count) +} + +func (s *QueryStats) LoadSplitQueries() uint64 { + if s == nil { + return 0 + } + + return atomic.LoadUint64(&s.SplitQueries) +} + // Merge the provided Stats into this one. func (s *QueryStats) Merge(other *QueryStats) { if s == nil || other == nil { diff --git a/pkg/querier/stats/stats.pb.go b/pkg/querier/stats/stats.pb.go index 9ce42b53dec..a50af9a64a6 100644 --- a/pkg/querier/stats/stats.pb.go +++ b/pkg/querier/stats/stats.pb.go @@ -48,6 +48,9 @@ type Stats struct { FetchedSamplesCount uint64 `protobuf:"varint,7,opt,name=fetched_samples_count,json=fetchedSamplesCount,proto3" json:"fetched_samples_count,omitempty"` // The limit hit when executing the query LimitHit string `protobuf:"bytes,8,opt,name=limit_hit,json=limitHit,proto3" json:"limit_hit,omitempty"` + // The total number of split queries sent after going through all the middlewares. + // It includes the number of requests that might be discarded by the queue. + SplitQueries uint64 `protobuf:"varint,9,opt,name=split_queries,json=splitQueries,proto3" json:"split_queries,omitempty"` } func (m *Stats) Reset() { *m = Stats{} } @@ -138,6 +141,13 @@ func (m *Stats) GetLimitHit() string { return "" } +func (m *Stats) GetSplitQueries() uint64 { + if m != nil { + return m.SplitQueries + } + return 0 +} + func init() { proto.RegisterType((*Stats)(nil), "stats.Stats") proto.RegisterMapType((map[string]string)(nil), "stats.Stats.ExtraFieldsEntry") @@ -146,35 +156,36 @@ func init() { func init() { proto.RegisterFile("stats.proto", fileDescriptor_b4756a0aec8b9d44) } var fileDescriptor_b4756a0aec8b9d44 = []byte{ - // 436 bytes of a gzipped FileDescriptorProto + // 458 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x92, 0x31, 0x6f, 0xd3, 0x40, - 0x14, 0xc7, 0x7d, 0x75, 0x5c, 0xe2, 0x33, 0x43, 0x38, 0x82, 0xe4, 0x46, 0xe2, 0x6a, 0x98, 0x3c, - 0x20, 0x07, 0x85, 0x05, 0x81, 0x84, 0x2a, 0xb7, 0x45, 0xcc, 0x0e, 0x13, 0x8b, 0x75, 0x89, 0x2f, - 0xce, 0xa9, 0xb6, 0xaf, 0xb2, 0xcf, 0x80, 0x37, 0x3e, 0x02, 0x23, 0x1f, 0x81, 0x8f, 0x92, 0x31, - 0x63, 0xa7, 0x42, 0x1c, 0x06, 0xc6, 0x7e, 0x04, 0x74, 0x77, 0x76, 0x81, 0x6e, 0x7e, 0xef, 0xf7, - 0xfe, 0x4f, 0xef, 0xff, 0x3f, 0x43, 0xa7, 0x12, 0x44, 0x54, 0xc1, 0x65, 0xc9, 0x05, 0x47, 0x96, - 0x2a, 0x26, 0xe3, 0x94, 0xa7, 0x5c, 0x75, 0xa6, 0xf2, 0x4b, 0xc3, 0x09, 0x4e, 0x39, 0x4f, 0x33, - 0x3a, 0x55, 0xd5, 0xa2, 0x5e, 0x4d, 0x93, 0xba, 0x24, 0x82, 0xf1, 0xa2, 0xe3, 0x47, 0x77, 0x39, - 0x29, 0x1a, 0x8d, 0x9e, 0xfe, 0x32, 0xa1, 0x35, 0x97, 0xab, 0xd1, 0x09, 0xb4, 0x3f, 0x91, 0x2c, - 0x8b, 0x05, 0xcb, 0xa9, 0x0b, 0x3c, 0xe0, 0x3b, 0xb3, 0xa3, 0x40, 0x0b, 0x83, 0x5e, 0x18, 0x9c, - 0x75, 0x8b, 0xc3, 0xe1, 0xe6, 0xfa, 0xd8, 0xf8, 0xf6, 0xe3, 0x18, 0x44, 0x43, 0xa9, 0x7a, 0xcf, - 0x72, 0x8a, 0x9e, 0xc3, 0xf1, 0x8a, 0x8a, 0xe5, 0x9a, 0x26, 0x71, 0x45, 0x4b, 0x46, 0xab, 0x78, - 0xc9, 0xeb, 0x42, 0xb8, 0x07, 0x1e, 0xf0, 0x07, 0x11, 0xea, 0xd8, 0x5c, 0xa1, 0x53, 0x49, 0x50, - 0x00, 0x1f, 0xf6, 0x8a, 0xe5, 0xba, 0x2e, 0x2e, 0xe2, 0x45, 0x23, 0x68, 0xe5, 0x9a, 0x4a, 0xf0, - 0xa0, 0x43, 0xa7, 0x92, 0x84, 0x12, 0xa0, 0x67, 0xb0, 0xdf, 0x12, 0x27, 0x44, 0x90, 0x6e, 0x7c, - 0xa0, 0xc6, 0x47, 0x1d, 0x39, 0x23, 0x82, 0xe8, 0xe9, 0x13, 0x78, 0x9f, 0x7e, 0x16, 0x25, 0x89, - 0x57, 0x8c, 0x66, 0x49, 0xe5, 0x5a, 0x9e, 0xe9, 0x3b, 0xb3, 0xc7, 0x81, 0xce, 0x55, 0xb9, 0x0e, - 0xce, 0xe5, 0xc0, 0x5b, 0xc5, 0xcf, 0x0b, 0x51, 0x36, 0x91, 0x43, 0xff, 0x76, 0xfe, 0x75, 0xa4, - 0xee, 0xeb, 0x1d, 0x1d, 0xfe, 0xe7, 0x48, 0x1d, 0xd8, 0x39, 0x9a, 0xc1, 0x47, 0xb7, 0x19, 0x90, - 0xfc, 0x32, 0xbb, 0x0d, 0xe1, 0x9e, 0x92, 0xf4, 0x76, 0xe7, 0x9a, 0x69, 0xcd, 0x13, 0x68, 0x67, - 0x2c, 0x67, 0x22, 0x5e, 0x33, 0xe1, 0x0e, 0x3d, 0xe0, 0xdb, 0xe1, 0x60, 0x73, 0x2d, 0xa3, 0x55, - 0xed, 0x77, 0x4c, 0x4c, 0xde, 0xc0, 0xd1, 0xdd, 0x4b, 0xd1, 0x08, 0x9a, 0x17, 0xb4, 0x51, 0x4f, - 0x65, 0x47, 0xf2, 0x13, 0x8d, 0xa1, 0xf5, 0x91, 0x64, 0x35, 0x55, 0x89, 0xdb, 0x91, 0x2e, 0x5e, - 0x1d, 0xbc, 0x04, 0xe1, 0xeb, 0xed, 0x0e, 0x1b, 0x57, 0x3b, 0x6c, 0xdc, 0xec, 0x30, 0xf8, 0xd2, - 0x62, 0xf0, 0xbd, 0xc5, 0x60, 0xd3, 0x62, 0xb0, 0x6d, 0x31, 0xf8, 0xd9, 0x62, 0xf0, 0xbb, 0xc5, - 0xc6, 0x4d, 0x8b, 0xc1, 0xd7, 0x3d, 0x36, 0xb6, 0x7b, 0x6c, 0x5c, 0xed, 0xb1, 0xf1, 0x41, 0xff, - 0x74, 0x8b, 0x43, 0xf5, 0xfc, 0x2f, 0xfe, 0x04, 0x00, 0x00, 0xff, 0xff, 0x4d, 0x8f, 0x00, 0x37, - 0x91, 0x02, 0x00, 0x00, + 0x14, 0xc7, 0x7d, 0x4d, 0x52, 0xe2, 0x4b, 0x91, 0xc2, 0x11, 0x24, 0x37, 0x12, 0xd7, 0x00, 0x4b, + 0x06, 0xe4, 0xa0, 0xb0, 0x20, 0x90, 0x50, 0x95, 0xb6, 0x88, 0x15, 0x87, 0x89, 0xc5, 0xba, 0x24, + 0x97, 0xe4, 0x54, 0xc7, 0x17, 0xec, 0x67, 0x20, 0x1b, 0x2b, 0x1b, 0x23, 0x1f, 0x81, 0x8f, 0x92, + 0x31, 0x63, 0xa7, 0x42, 0x9c, 0x85, 0xb1, 0x1f, 0x01, 0xdd, 0x3b, 0xbb, 0xd0, 0x6e, 0xf7, 0xde, + 0xef, 0xfd, 0x9f, 0xee, 0xff, 0xbf, 0xa3, 0x8d, 0x14, 0x04, 0xa4, 0xfe, 0x32, 0xd1, 0xa0, 0x59, + 0x0d, 0x8b, 0x76, 0x6b, 0xa6, 0x67, 0x1a, 0x3b, 0x3d, 0x73, 0xb2, 0xb0, 0xcd, 0x67, 0x5a, 0xcf, + 0x22, 0xd9, 0xc3, 0x6a, 0x94, 0x4d, 0x7b, 0x93, 0x2c, 0x11, 0xa0, 0x74, 0x5c, 0xf0, 0xc3, 0xdb, + 0x5c, 0xc4, 0x2b, 0x8b, 0x1e, 0x7f, 0xab, 0xd2, 0xda, 0xd0, 0xac, 0x66, 0xc7, 0xd4, 0xfd, 0x2c, + 0xa2, 0x28, 0x04, 0xb5, 0x90, 0x1e, 0xe9, 0x90, 0x6e, 0xa3, 0x7f, 0xe8, 0x5b, 0xa1, 0x5f, 0x0a, + 0xfd, 0xd3, 0x62, 0xf1, 0xa0, 0xbe, 0xbe, 0x3c, 0x72, 0x7e, 0xfc, 0x3a, 0x22, 0x41, 0xdd, 0xa8, + 0xde, 0xab, 0x85, 0x64, 0xcf, 0x68, 0x6b, 0x2a, 0x61, 0x3c, 0x97, 0x93, 0x30, 0x95, 0x89, 0x92, + 0x69, 0x38, 0xd6, 0x59, 0x0c, 0xde, 0x5e, 0x87, 0x74, 0xab, 0x01, 0x2b, 0xd8, 0x10, 0xd1, 0x89, + 0x21, 0xcc, 0xa7, 0xf7, 0x4b, 0xc5, 0x78, 0x9e, 0xc5, 0xe7, 0xe1, 0x68, 0x05, 0x32, 0xf5, 0x2a, + 0x28, 0xb8, 0x57, 0xa0, 0x13, 0x43, 0x06, 0x06, 0xb0, 0xa7, 0xb4, 0xdc, 0x12, 0x4e, 0x04, 0x88, + 0x62, 0xbc, 0x8a, 0xe3, 0xcd, 0x82, 0x9c, 0x0a, 0x10, 0x76, 0xfa, 0x98, 0x1e, 0xc8, 0x2f, 0x90, + 0x88, 0x70, 0xaa, 0x64, 0x34, 0x49, 0xbd, 0x5a, 0xa7, 0xd2, 0x6d, 0xf4, 0x1f, 0xfa, 0x36, 0x57, + 0x74, 0xed, 0x9f, 0x99, 0x81, 0x37, 0xc8, 0xcf, 0x62, 0x48, 0x56, 0x41, 0x43, 0xfe, 0xeb, 0xfc, + 0xef, 0x08, 0xef, 0x57, 0x3a, 0xda, 0xbf, 0xe1, 0x08, 0x2f, 0x58, 0x38, 0xea, 0xd3, 0x07, 0xd7, + 0x19, 0x88, 0xc5, 0x32, 0xba, 0x0e, 0xe1, 0x0e, 0x4a, 0x4a, 0xbb, 0x43, 0xcb, 0xac, 0xe6, 0x11, + 0x75, 0x23, 0xb5, 0x50, 0x10, 0xce, 0x15, 0x78, 0xf5, 0x0e, 0xe9, 0xba, 0x83, 0xea, 0xfa, 0xd2, + 0x44, 0x8b, 0xed, 0xb7, 0x0a, 0xd8, 0x13, 0x7a, 0x37, 0x5d, 0x46, 0x0a, 0xc2, 0x8f, 0x19, 0xc6, + 0xe7, 0xb9, 0xb8, 0xee, 0x00, 0x9b, 0xef, 0x6c, 0xaf, 0xfd, 0x9a, 0x36, 0x6f, 0xdb, 0x61, 0x4d, + 0x5a, 0x39, 0x97, 0x2b, 0x7c, 0x4f, 0x37, 0x30, 0x47, 0xd6, 0xa2, 0xb5, 0x4f, 0x22, 0xca, 0x24, + 0x3e, 0x8b, 0x1b, 0xd8, 0xe2, 0xe5, 0xde, 0x0b, 0x32, 0x78, 0xb5, 0xd9, 0x72, 0xe7, 0x62, 0xcb, + 0x9d, 0xab, 0x2d, 0x27, 0x5f, 0x73, 0x4e, 0x7e, 0xe6, 0x9c, 0xac, 0x73, 0x4e, 0x36, 0x39, 0x27, + 0xbf, 0x73, 0x4e, 0xfe, 0xe4, 0xdc, 0xb9, 0xca, 0x39, 0xf9, 0xbe, 0xe3, 0xce, 0x66, 0xc7, 0x9d, + 0x8b, 0x1d, 0x77, 0x3e, 0xd8, 0x9f, 0x39, 0xda, 0xc7, 0x3f, 0xf2, 0xfc, 0x6f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x24, 0xfa, 0x1b, 0xec, 0xb6, 0x02, 0x00, 0x00, } func (this *Stats) Equal(that interface{}) bool { @@ -225,13 +236,16 @@ func (this *Stats) Equal(that interface{}) bool { if this.LimitHit != that1.LimitHit { return false } + if this.SplitQueries != that1.SplitQueries { + return false + } return true } func (this *Stats) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 12) + s := make([]string, 0, 13) s = append(s, "&stats.Stats{") s = append(s, "WallTime: "+fmt.Sprintf("%#v", this.WallTime)+",\n") s = append(s, "FetchedSeriesCount: "+fmt.Sprintf("%#v", this.FetchedSeriesCount)+",\n") @@ -253,6 +267,7 @@ func (this *Stats) GoString() string { s = append(s, "FetchedChunksCount: "+fmt.Sprintf("%#v", this.FetchedChunksCount)+",\n") s = append(s, "FetchedSamplesCount: "+fmt.Sprintf("%#v", this.FetchedSamplesCount)+",\n") s = append(s, "LimitHit: "+fmt.Sprintf("%#v", this.LimitHit)+",\n") + s = append(s, "SplitQueries: "+fmt.Sprintf("%#v", this.SplitQueries)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -284,6 +299,11 @@ func (m *Stats) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.SplitQueries != 0 { + i = encodeVarintStats(dAtA, i, uint64(m.SplitQueries)) + i-- + dAtA[i] = 0x48 + } if len(m.LimitHit) > 0 { i -= len(m.LimitHit) copy(dAtA[i:], m.LimitHit) @@ -392,6 +412,9 @@ func (m *Stats) Size() (n int) { if l > 0 { n += 1 + l + sovStats(uint64(l)) } + if m.SplitQueries != 0 { + n += 1 + sovStats(uint64(m.SplitQueries)) + } return n } @@ -424,6 +447,7 @@ func (this *Stats) String() string { `FetchedChunksCount:` + fmt.Sprintf("%v", this.FetchedChunksCount) + `,`, `FetchedSamplesCount:` + fmt.Sprintf("%v", this.FetchedSamplesCount) + `,`, `LimitHit:` + fmt.Sprintf("%v", this.LimitHit) + `,`, + `SplitQueries:` + fmt.Sprintf("%v", this.SplitQueries) + `,`, `}`, }, "") return s @@ -752,6 +776,25 @@ func (m *Stats) Unmarshal(dAtA []byte) error { } m.LimitHit = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SplitQueries", wireType) + } + m.SplitQueries = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStats + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SplitQueries |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipStats(dAtA[iNdEx:]) diff --git a/pkg/querier/stats/stats.proto b/pkg/querier/stats/stats.proto index ef3de5d1b76..96540201f0a 100644 --- a/pkg/querier/stats/stats.proto +++ b/pkg/querier/stats/stats.proto @@ -28,4 +28,7 @@ message Stats { uint64 fetched_samples_count = 7; // The limit hit when executing the query string limit_hit = 8 [(gogoproto.nullable) = true]; + // The total number of split queries sent after going through all the middlewares. + // It includes the number of requests that might be discarded by the queue. + uint64 split_queries = 9; } From 59fef85abb94e5b39c73b4405d3a543bab10ceeb Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Fri, 8 Dec 2023 11:32:56 -0800 Subject: [PATCH 2/2] update changelog Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a53ec907731..d6bd2071b8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * [ENHANCEMENT] Compactor: Add new compactor metric `cortex_compactor_start_duration_seconds`. #5683 * [ENHANCEMENT] Upgraded Docker base images to `alpine:3.18`. #5684 * [ENHANCEMENT] Index Cache: Multi level cache adds config `max_backfill_items` to cap max items to backfill per async operation. #5686 +* [ENHANCEMENT] Query Frontend: Log number of split queries in `query stats` log. #5703 ## 1.16.0 2023-11-20