From 700b1c26d9f779cd1b0b9a07c4667246840cf6b8 Mon Sep 17 00:00:00 2001 From: Ahmed Hassan Date: Thu, 27 Feb 2025 18:04:51 -0800 Subject: [PATCH 1/8] truncate getLabels and getSeries to max range if no start-end time provided Signed-off-by: Ahmed Hassan --- pkg/querier/distributor_queryable_test.go | 3 +- pkg/querier/querier.go | 75 +++++++++++++++-------- pkg/querier/querier_test.go | 62 ++++++++++++++----- pkg/util/validation/validate.go | 3 +- 4 files changed, 102 insertions(+), 41 deletions(-) diff --git a/pkg/querier/distributor_queryable_test.go b/pkg/querier/distributor_queryable_test.go index 457fba03cbc..a382583f07c 100644 --- a/pkg/querier/distributor_queryable_test.go +++ b/pkg/querier/distributor_queryable_test.go @@ -98,7 +98,8 @@ func TestDistributorQuerier_SelectShouldHonorQueryIngestersWithin(t *testing.T) overrides, err := validation.NewOverrides(limits, nil) require.NoError(t, err) - start, end, err := validateQueryTimeRange(ctx, "test", testData.queryMinT, testData.queryMaxT, overrides, 0) + start, end, warnings, err := validateQueryTimeRange(ctx, "test", testData.queryMinT, testData.queryMaxT, overrides, 0) + require.Nil(t, warnings) require.NoError(t, err) // Select hints are passed by Prometheus when querying /series. var hints *storage.SelectHints diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 2915f01fb1f..fd4661adc0c 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -19,6 +19,7 @@ import ( "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/annotations" + v1 "github.com/prometheus/prometheus/web/api/v1" "github.com/thanos-io/promql-engine/engine" "github.com/thanos-io/promql-engine/logicalplan" "github.com/thanos-io/thanos/pkg/strutil" @@ -27,6 +28,7 @@ import ( "github.com/cortexproject/cortex/pkg/querier/batch" "github.com/cortexproject/cortex/pkg/querier/lazyquery" "github.com/cortexproject/cortex/pkg/querier/partialdata" + seriesset "github.com/cortexproject/cortex/pkg/querier/series" querier_stats "github.com/cortexproject/cortex/pkg/querier/stats" "github.com/cortexproject/cortex/pkg/tenant" "github.com/cortexproject/cortex/pkg/util" @@ -304,11 +306,12 @@ type querier struct { ignoreMaxQueryLength bool } -func (q querier) setupFromCtx(ctx context.Context) (context.Context, *querier_stats.QueryStats, string, int64, int64, storage.Querier, []storage.Querier, error) { +func (q querier) setupFromCtx(ctx context.Context) (context.Context, *querier_stats.QueryStats, string, int64, int64, storage.Querier, []storage.Querier, annotations.Annotations, error) { stats := querier_stats.FromContext(ctx) userID, err := tenant.TenantID(ctx) + warnings := annotations.Annotations(nil) if err != nil { - return ctx, stats, userID, 0, 0, nil, nil, err + return ctx, stats, userID, 0, 0, nil, nil, warnings, err } q.limiterHolder.limiterInitializer.Do(func() { @@ -317,14 +320,14 @@ func (q querier) setupFromCtx(ctx context.Context) (context.Context, *querier_st ctx = limiter.AddQueryLimiterToContext(ctx, q.limiterHolder.limiter) - mint, maxt, err := validateQueryTimeRange(ctx, userID, q.mint, q.maxt, q.limits, q.maxQueryIntoFuture) + mint, maxt, warnings, err := validateQueryTimeRange(ctx, userID, q.mint, q.maxt, q.limits, q.maxQueryIntoFuture) if err != nil { - return ctx, stats, userID, 0, 0, nil, nil, err + return ctx, stats, userID, 0, 0, nil, nil, warnings, err } dqr, err := q.distributor.Querier(mint, maxt) if err != nil { - return ctx, stats, userID, 0, 0, nil, nil, err + return ctx, stats, userID, 0, 0, nil, nil, warnings, err } metadataQuerier := dqr @@ -340,18 +343,18 @@ func (q querier) setupFromCtx(ctx context.Context) (context.Context, *querier_st cqr, err := s.Querier(mint, maxt) if err != nil { - return ctx, stats, userID, 0, 0, nil, nil, err + return ctx, stats, userID, 0, 0, nil, nil, warnings, err } queriers = append(queriers, cqr) } - return ctx, stats, userID, mint, maxt, metadataQuerier, queriers, nil + return ctx, stats, userID, mint, maxt, metadataQuerier, queriers, warnings, nil } // Select implements storage.Querier interface. // The bool passed is ignored because the series is always sorted. func (q querier) Select(ctx context.Context, sortSeries bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - ctx, stats, userID, mint, maxt, metadataQuerier, queriers, err := q.setupFromCtx(ctx) + ctx, stats, userID, mint, maxt, metadataQuerier, queriers, warnings, err := q.setupFromCtx(ctx) if err == errEmptyTimeRange { return storage.EmptySeriesSet() } else if err != nil { @@ -370,7 +373,8 @@ func (q querier) Select(ctx context.Context, sortSeries bool, sp *storage.Select } if sp == nil { - mint, maxt, err = validateQueryTimeRange(ctx, userID, mint, maxt, q.limits, q.maxQueryIntoFuture) + mint, maxt, newWarnings, err := validateQueryTimeRange(ctx, userID, mint, maxt, q.limits, q.maxQueryIntoFuture) + warnings.Merge(newWarnings) if err == errEmptyTimeRange { return storage.EmptySeriesSet() } else if err != nil { @@ -383,7 +387,8 @@ func (q querier) Select(ctx context.Context, sortSeries bool, sp *storage.Select // Validate query time range. Even if the time range has already been validated when we created // the querier, we need to check it again here because the time range specified in hints may be // different. - startMs, endMs, err := validateQueryTimeRange(ctx, userID, sp.Start, sp.End, q.limits, q.maxQueryIntoFuture) + startMs, endMs, newWarnings, err := validateQueryTimeRange(ctx, userID, sp.Start, sp.End, q.limits, q.maxQueryIntoFuture) + warnings.Merge(newWarnings) if err == errEmptyTimeRange { return storage.NoopSeriesSet() } else if err != nil { @@ -437,12 +442,12 @@ func (q querier) Select(ctx context.Context, sortSeries bool, sp *storage.Select } } - return storage.NewMergeSeriesSet(result, 0, storage.ChainedSeriesMerge) + return seriesset.NewSeriesSetWithWarnings(storage.NewMergeSeriesSet(result, 0, storage.ChainedSeriesMerge), warnings) } // LabelValues implements storage.Querier. func (q querier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - ctx, stats, userID, mint, maxt, _, queriers, err := q.setupFromCtx(ctx) + ctx, stats, userID, mint, maxt, _, queriers, warnings, err := q.setupFromCtx(ctx) if err == errEmptyTimeRange { return nil, nil, nil } else if err != nil { @@ -456,7 +461,8 @@ func (q querier) LabelValues(ctx context.Context, name string, hints *storage.La startTime := model.Time(mint) endTime := model.Time(maxt) - if maxQueryLength := q.limits.MaxQueryLength(userID); maxQueryLength > 0 && endTime.Sub(startTime) > maxQueryLength { + maxQueryLength := q.limits.MaxQueryLength(userID) + if maxQueryLength > 0 && endTime.Sub(startTime) > maxQueryLength { limitErr := validation.LimitError(fmt.Sprintf(validation.ErrQueryTooLong, endTime.Sub(startTime), maxQueryLength)) return nil, nil, limitErr } @@ -466,9 +472,8 @@ func (q querier) LabelValues(ctx context.Context, name string, hints *storage.La } var ( - g, _ = errgroup.WithContext(ctx) - sets = [][]string{} - warnings = annotations.Annotations(nil) + g, _ = errgroup.WithContext(ctx) + sets = [][]string{} resMtx sync.Mutex ) @@ -505,7 +510,7 @@ func (q querier) LabelValues(ctx context.Context, name string, hints *storage.La } func (q querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - ctx, stats, userID, mint, maxt, _, queriers, err := q.setupFromCtx(ctx) + ctx, stats, userID, mint, maxt, _, queriers, warnings, err := q.setupFromCtx(ctx) if err == errEmptyTimeRange { return nil, nil, nil } else if err != nil { @@ -519,7 +524,8 @@ func (q querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matc startTime := model.Time(mint) endTime := model.Time(maxt) - if maxQueryLength := q.limits.MaxQueryLength(userID); maxQueryLength > 0 && endTime.Sub(startTime) > maxQueryLength { + maxQueryLength := q.limits.MaxQueryLength(userID) + if maxQueryLength > 0 && endTime.Sub(startTime) > maxQueryLength { limitErr := validation.LimitError(fmt.Sprintf(validation.ErrQueryTooLong, endTime.Sub(startTime), maxQueryLength)) return nil, nil, limitErr } @@ -529,9 +535,8 @@ func (q querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matc } var ( - g, _ = errgroup.WithContext(ctx) - sets = [][]string{} - warnings = annotations.Annotations(nil) + g, _ = errgroup.WithContext(ctx) + sets = [][]string{} resMtx sync.Mutex ) @@ -622,11 +627,31 @@ func UseBeforeTimestampQueryable(queryable storage.Queryable, ts time.Time) Quer } } -func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs int64, limits *validation.Overrides, maxQueryIntoFuture time.Duration) (int64, int64, error) { +func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs int64, limits *validation.Overrides, maxQueryIntoFuture time.Duration) (int64, int64, annotations.Annotations, error) { + warnings := annotations.Annotations(nil) now := model.Now() startTime := model.Time(startMs) endTime := model.Time(endMs) + // Truncate time range to MaxQueryLength if time parameters are unspecified + maxQueryLength := limits.MaxQueryLength(userID) + if maxQueryLength > 0 && (startMs == util.TimeToMillis(v1.MinTime) || endMs == util.TimeToMillis(v1.MaxTime)) { + if util.TimeToMillis(v1.MaxTime) == endMs { + endTime = now + } + if startMs == util.TimeToMillis(v1.MinTime) { + startTime = endTime.Add(-maxQueryLength) + } + + warnings.Add(validation.LimitError(fmt.Sprintf(validation.WarningTruncatedQueryLength, maxQueryLength))) + + // Make sure to log it in traces to ease debugging. + level.Debug(spanlogger.FromContext(ctx)).Log( + "msg", "start or end time parameters not specified, response truncated to maximum query length", + "updatedStart", util.FormatTimeModel(startTime), + "updatedEnd", util.FormatTimeModel(endTime)) + } + // Clamp time range based on max query into future. if maxQueryIntoFuture > 0 && endTime.After(now.Add(maxQueryIntoFuture)) { origEndTime := endTime @@ -639,7 +664,7 @@ func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs i "updated", util.FormatTimeModel(endTime)) if endTime.Before(startTime) { - return 0, 0, errEmptyTimeRange + return 0, 0, warnings, errEmptyTimeRange } } @@ -655,7 +680,7 @@ func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs i "updated", util.FormatTimeModel(startTime)) if endTime.Before(startTime) { - return 0, 0, errEmptyTimeRange + return 0, 0, warnings, errEmptyTimeRange } } @@ -664,7 +689,7 @@ func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs i startTime = 0 } - return int64(startTime), int64(endTime), nil + return int64(startTime), int64(endTime), warnings, nil } func EnableExperimentalPromQLFunctions(enablePromQLExperimentalFunctions, enableHoltWinters bool) { diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index c87bfce0e68..b7adfbc52ed 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -23,6 +23,7 @@ import ( "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/util/annotations" + v1 "github.com/prometheus/prometheus/web/api/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -932,27 +933,34 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLength_Labels(t *testing.T) { tests := map[string]struct { startTime time.Time endTime time.Time - expected error + expectedErr error + expectedWarning error ignoreMaxQueryLength bool }{ "time range shorter than maxQueryLength": { startTime: time.Now().Add(-maxQueryLength).Add(time.Hour), endTime: time.Now(), - expected: nil, + expectedErr: nil, ignoreMaxQueryLength: false, }, "time range longer than maxQueryLength": { startTime: time.Now().Add(-maxQueryLength).Add(-time.Hour), endTime: time.Now(), - expected: validation.LimitError("expanding series: the query time range exceeds the limit (query length: 721h0m0s, limit: 720h0m0s)"), + expectedErr: validation.LimitError("expanding series: the query time range exceeds the limit (query length: 721h0m0s, limit: 720h0m0s)"), ignoreMaxQueryLength: false, }, "time range longer than maxQueryLength and ignoreMaxQueryLength is true": { startTime: time.Now().Add(-maxQueryLength).Add(-time.Hour), endTime: time.Now(), - expected: validation.LimitError("expanding series: the query time range exceeds the limit (query length: 721h0m0s, limit: 720h0m0s)"), + expectedErr: validation.LimitError("expanding series: the query time range exceeds the limit (query length: 721h0m0s, limit: 720h0m0s)"), ignoreMaxQueryLength: true, }, + "time range not specified should be truncated with a warning": { + startTime: v1.MinTime, + endTime: v1.MaxTime, + expectedWarning: validation.LimitError(fmt.Sprintf(validation.WarningTruncatedQueryLength, maxQueryLength)), + ignoreMaxQueryLength: false, + }, } for testName, testData := range tests { @@ -978,23 +986,49 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLength_Labels(t *testing.T) { q, err := queryable.Querier(util.TimeToMillis(testData.startTime), util.TimeToMillis(testData.endTime)) require.NoError(t, err) - _, _, err = q.LabelNames(ctx, &storage.LabelHints{Limit: 0}) + _, warnings, err := q.LabelNames(ctx, &storage.LabelHints{Limit: 0}) - if testData.expected != nil { - require.NotNil(t, err) - assert.True(t, strings.Contains(testData.expected.Error(), err.Error())) + // Verify expected error is returned + if testData.expectedErr != nil { + assert.True(t, strings.Contains(testData.expectedErr.Error(), err.Error())) } else { assert.Nil(t, err) } - _, _, err = q.LabelValues(ctx, labels.MetricName, &storage.LabelHints{Limit: 0}) + // Verify expected warning is returned + if testData.expectedWarning != nil { + foundWarning := false + for _, warning := range warnings { + if strings.Contains(testData.expectedWarning.Error(), warning.Error()) { + foundWarning = true + } + } + assert.True(t, foundWarning) + } else { + assert.Nil(t, warnings) + } - if testData.expected != nil { - require.NotNil(t, err) - assert.True(t, strings.Contains(testData.expected.Error(), err.Error())) + _, warnings, err = q.LabelValues(ctx, labels.MetricName, &storage.LabelHints{Limit: 0}) + + // Verify expected error is returned + if testData.expectedErr != nil { + assert.True(t, strings.Contains(testData.expectedErr.Error(), err.Error())) } else { assert.Nil(t, err) } + + // Verify expected warning is returned + if testData.expectedWarning != nil { + foundWarning := false + for _, warning := range warnings { + if strings.Contains(testData.expectedWarning.Error(), warning.Error()) { + foundWarning = true + } + } + assert.True(t, foundWarning) + } else { + assert.Nil(t, warnings) + } }) } } @@ -1157,7 +1191,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { // We apply the validation here again since when initializing querier we change the start/end time, // but when querying series we don't validate again. So we should pass correct hints here. - start, end, err := validateQueryTimeRange(ctx, "test", util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime), overrides, 0) + start, end, _, err := validateQueryTimeRange(ctx, "test", util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime), overrides, 0) // Skipped query will hit errEmptyTimeRange during validation. if !testData.expectedSkipped { require.NoError(t, err) @@ -1318,7 +1352,7 @@ func TestValidateMaxQueryLength(t *testing.T) { limits := DefaultLimitsConfig() overrides, err := validation.NewOverrides(limits, nil) require.NoError(t, err) - startMs, endMs, err := validateQueryTimeRange(ctx, "test", util.TimeToMillis(tc.start), util.TimeToMillis(tc.end), overrides, 0) + startMs, endMs, _, err := validateQueryTimeRange(ctx, "test", util.TimeToMillis(tc.start), util.TimeToMillis(tc.end), overrides, 0) require.NoError(t, err) startTime := model.Time(startMs) endTime := model.Time(endMs) diff --git a/pkg/util/validation/validate.go b/pkg/util/validation/validate.go index 26d352e29b9..a9826678068 100644 --- a/pkg/util/validation/validate.go +++ b/pkg/util/validation/validate.go @@ -36,7 +36,8 @@ const ( unitTooLong = "unit_too_long" // ErrQueryTooLong is used in chunk store, querier and query frontend. - ErrQueryTooLong = "the query time range exceeds the limit (query length: %s, limit: %s)" + ErrQueryTooLong = "the query time range exceeds the limit (query length: %s, limit: %s)" + WarningTruncatedQueryLength = "start or end time parameters not specified, response truncated to maximum query length (%s)" missingMetricName = "missing_metric_name" invalidMetricName = "metric_name_invalid" From 4301baf54915eaee7404e916f62c2a658297088c Mon Sep 17 00:00:00 2001 From: Ahmed Hassan Date: Fri, 28 Feb 2025 00:47:10 -0800 Subject: [PATCH 2/8] use prometheus api helper function for millisecond conversion Signed-off-by: Ahmed Hassan --- pkg/querier/querier.go | 7 ++++--- pkg/querier/querier_test.go | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index fd4661adc0c..a07798830db 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -15,6 +15,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" @@ -635,11 +636,11 @@ func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs i // Truncate time range to MaxQueryLength if time parameters are unspecified maxQueryLength := limits.MaxQueryLength(userID) - if maxQueryLength > 0 && (startMs == util.TimeToMillis(v1.MinTime) || endMs == util.TimeToMillis(v1.MaxTime)) { - if util.TimeToMillis(v1.MaxTime) == endMs { + if maxQueryLength > 0 && (startMs == timestamp.FromTime(v1.MinTime) || endMs == timestamp.FromTime(v1.MaxTime)) { + if endMs == timestamp.FromTime(v1.MaxTime) { endTime = now } - if startMs == util.TimeToMillis(v1.MinTime) { + if startMs == timestamp.FromTime(v1.MinTime) { startTime = endTime.Add(-maxQueryLength) } diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index b7adfbc52ed..b468707f2ad 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -16,6 +16,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/scrape" @@ -983,7 +984,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLength_Labels(t *testing.T) { ctx := user.InjectOrgID(context.Background(), "test") - q, err := queryable.Querier(util.TimeToMillis(testData.startTime), util.TimeToMillis(testData.endTime)) + q, err := queryable.Querier(timestamp.FromTime(testData.startTime), timestamp.FromTime(testData.endTime)) require.NoError(t, err) _, warnings, err := q.LabelNames(ctx, &storage.LabelHints{Limit: 0}) From 22fddd454054bb72aa2fca828897e96e198a21ff Mon Sep 17 00:00:00 2001 From: Ahmed Hassan Date: Thu, 13 Mar 2025 09:33:32 -0700 Subject: [PATCH 3/8] Revert "use prometheus api helper function for millisecond conversion" This reverts commit 4301baf54915eaee7404e916f62c2a658297088c. Signed-off-by: Ahmed Hassan --- pkg/querier/querier.go | 7 +++---- pkg/querier/querier_test.go | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index a07798830db..fd4661adc0c 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -15,7 +15,6 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" @@ -636,11 +635,11 @@ func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs i // Truncate time range to MaxQueryLength if time parameters are unspecified maxQueryLength := limits.MaxQueryLength(userID) - if maxQueryLength > 0 && (startMs == timestamp.FromTime(v1.MinTime) || endMs == timestamp.FromTime(v1.MaxTime)) { - if endMs == timestamp.FromTime(v1.MaxTime) { + if maxQueryLength > 0 && (startMs == util.TimeToMillis(v1.MinTime) || endMs == util.TimeToMillis(v1.MaxTime)) { + if util.TimeToMillis(v1.MaxTime) == endMs { endTime = now } - if startMs == timestamp.FromTime(v1.MinTime) { + if startMs == util.TimeToMillis(v1.MinTime) { startTime = endTime.Add(-maxQueryLength) } diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index b468707f2ad..b7adfbc52ed 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -16,7 +16,6 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/common/promslog" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/timestamp" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/scrape" @@ -984,7 +983,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLength_Labels(t *testing.T) { ctx := user.InjectOrgID(context.Background(), "test") - q, err := queryable.Querier(timestamp.FromTime(testData.startTime), timestamp.FromTime(testData.endTime)) + q, err := queryable.Querier(util.TimeToMillis(testData.startTime), util.TimeToMillis(testData.endTime)) require.NoError(t, err) _, warnings, err := q.LabelNames(ctx, &storage.LabelHints{Limit: 0}) From 670f306f481a71ffa17a436b683ba00b4a0cc24c Mon Sep 17 00:00:00 2001 From: Ahmed Hassan Date: Thu, 13 Mar 2025 09:33:42 -0700 Subject: [PATCH 4/8] Revert "truncate getLabels and getSeries to max range if no start-end time provided" This reverts commit 700b1c26d9f779cd1b0b9a07c4667246840cf6b8. Signed-off-by: Ahmed Hassan --- pkg/querier/distributor_queryable_test.go | 3 +- pkg/querier/querier.go | 75 ++++++++--------------- pkg/querier/querier_test.go | 62 +++++-------------- pkg/util/validation/validate.go | 3 +- 4 files changed, 41 insertions(+), 102 deletions(-) diff --git a/pkg/querier/distributor_queryable_test.go b/pkg/querier/distributor_queryable_test.go index a382583f07c..457fba03cbc 100644 --- a/pkg/querier/distributor_queryable_test.go +++ b/pkg/querier/distributor_queryable_test.go @@ -98,8 +98,7 @@ func TestDistributorQuerier_SelectShouldHonorQueryIngestersWithin(t *testing.T) overrides, err := validation.NewOverrides(limits, nil) require.NoError(t, err) - start, end, warnings, err := validateQueryTimeRange(ctx, "test", testData.queryMinT, testData.queryMaxT, overrides, 0) - require.Nil(t, warnings) + start, end, err := validateQueryTimeRange(ctx, "test", testData.queryMinT, testData.queryMaxT, overrides, 0) require.NoError(t, err) // Select hints are passed by Prometheus when querying /series. var hints *storage.SelectHints diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index fd4661adc0c..2915f01fb1f 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -19,7 +19,6 @@ import ( "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/annotations" - v1 "github.com/prometheus/prometheus/web/api/v1" "github.com/thanos-io/promql-engine/engine" "github.com/thanos-io/promql-engine/logicalplan" "github.com/thanos-io/thanos/pkg/strutil" @@ -28,7 +27,6 @@ import ( "github.com/cortexproject/cortex/pkg/querier/batch" "github.com/cortexproject/cortex/pkg/querier/lazyquery" "github.com/cortexproject/cortex/pkg/querier/partialdata" - seriesset "github.com/cortexproject/cortex/pkg/querier/series" querier_stats "github.com/cortexproject/cortex/pkg/querier/stats" "github.com/cortexproject/cortex/pkg/tenant" "github.com/cortexproject/cortex/pkg/util" @@ -306,12 +304,11 @@ type querier struct { ignoreMaxQueryLength bool } -func (q querier) setupFromCtx(ctx context.Context) (context.Context, *querier_stats.QueryStats, string, int64, int64, storage.Querier, []storage.Querier, annotations.Annotations, error) { +func (q querier) setupFromCtx(ctx context.Context) (context.Context, *querier_stats.QueryStats, string, int64, int64, storage.Querier, []storage.Querier, error) { stats := querier_stats.FromContext(ctx) userID, err := tenant.TenantID(ctx) - warnings := annotations.Annotations(nil) if err != nil { - return ctx, stats, userID, 0, 0, nil, nil, warnings, err + return ctx, stats, userID, 0, 0, nil, nil, err } q.limiterHolder.limiterInitializer.Do(func() { @@ -320,14 +317,14 @@ func (q querier) setupFromCtx(ctx context.Context) (context.Context, *querier_st ctx = limiter.AddQueryLimiterToContext(ctx, q.limiterHolder.limiter) - mint, maxt, warnings, err := validateQueryTimeRange(ctx, userID, q.mint, q.maxt, q.limits, q.maxQueryIntoFuture) + mint, maxt, err := validateQueryTimeRange(ctx, userID, q.mint, q.maxt, q.limits, q.maxQueryIntoFuture) if err != nil { - return ctx, stats, userID, 0, 0, nil, nil, warnings, err + return ctx, stats, userID, 0, 0, nil, nil, err } dqr, err := q.distributor.Querier(mint, maxt) if err != nil { - return ctx, stats, userID, 0, 0, nil, nil, warnings, err + return ctx, stats, userID, 0, 0, nil, nil, err } metadataQuerier := dqr @@ -343,18 +340,18 @@ func (q querier) setupFromCtx(ctx context.Context) (context.Context, *querier_st cqr, err := s.Querier(mint, maxt) if err != nil { - return ctx, stats, userID, 0, 0, nil, nil, warnings, err + return ctx, stats, userID, 0, 0, nil, nil, err } queriers = append(queriers, cqr) } - return ctx, stats, userID, mint, maxt, metadataQuerier, queriers, warnings, nil + return ctx, stats, userID, mint, maxt, metadataQuerier, queriers, nil } // Select implements storage.Querier interface. // The bool passed is ignored because the series is always sorted. func (q querier) Select(ctx context.Context, sortSeries bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - ctx, stats, userID, mint, maxt, metadataQuerier, queriers, warnings, err := q.setupFromCtx(ctx) + ctx, stats, userID, mint, maxt, metadataQuerier, queriers, err := q.setupFromCtx(ctx) if err == errEmptyTimeRange { return storage.EmptySeriesSet() } else if err != nil { @@ -373,8 +370,7 @@ func (q querier) Select(ctx context.Context, sortSeries bool, sp *storage.Select } if sp == nil { - mint, maxt, newWarnings, err := validateQueryTimeRange(ctx, userID, mint, maxt, q.limits, q.maxQueryIntoFuture) - warnings.Merge(newWarnings) + mint, maxt, err = validateQueryTimeRange(ctx, userID, mint, maxt, q.limits, q.maxQueryIntoFuture) if err == errEmptyTimeRange { return storage.EmptySeriesSet() } else if err != nil { @@ -387,8 +383,7 @@ func (q querier) Select(ctx context.Context, sortSeries bool, sp *storage.Select // Validate query time range. Even if the time range has already been validated when we created // the querier, we need to check it again here because the time range specified in hints may be // different. - startMs, endMs, newWarnings, err := validateQueryTimeRange(ctx, userID, sp.Start, sp.End, q.limits, q.maxQueryIntoFuture) - warnings.Merge(newWarnings) + startMs, endMs, err := validateQueryTimeRange(ctx, userID, sp.Start, sp.End, q.limits, q.maxQueryIntoFuture) if err == errEmptyTimeRange { return storage.NoopSeriesSet() } else if err != nil { @@ -442,12 +437,12 @@ func (q querier) Select(ctx context.Context, sortSeries bool, sp *storage.Select } } - return seriesset.NewSeriesSetWithWarnings(storage.NewMergeSeriesSet(result, 0, storage.ChainedSeriesMerge), warnings) + return storage.NewMergeSeriesSet(result, 0, storage.ChainedSeriesMerge) } // LabelValues implements storage.Querier. func (q querier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - ctx, stats, userID, mint, maxt, _, queriers, warnings, err := q.setupFromCtx(ctx) + ctx, stats, userID, mint, maxt, _, queriers, err := q.setupFromCtx(ctx) if err == errEmptyTimeRange { return nil, nil, nil } else if err != nil { @@ -461,8 +456,7 @@ func (q querier) LabelValues(ctx context.Context, name string, hints *storage.La startTime := model.Time(mint) endTime := model.Time(maxt) - maxQueryLength := q.limits.MaxQueryLength(userID) - if maxQueryLength > 0 && endTime.Sub(startTime) > maxQueryLength { + if maxQueryLength := q.limits.MaxQueryLength(userID); maxQueryLength > 0 && endTime.Sub(startTime) > maxQueryLength { limitErr := validation.LimitError(fmt.Sprintf(validation.ErrQueryTooLong, endTime.Sub(startTime), maxQueryLength)) return nil, nil, limitErr } @@ -472,8 +466,9 @@ func (q querier) LabelValues(ctx context.Context, name string, hints *storage.La } var ( - g, _ = errgroup.WithContext(ctx) - sets = [][]string{} + g, _ = errgroup.WithContext(ctx) + sets = [][]string{} + warnings = annotations.Annotations(nil) resMtx sync.Mutex ) @@ -510,7 +505,7 @@ func (q querier) LabelValues(ctx context.Context, name string, hints *storage.La } func (q querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - ctx, stats, userID, mint, maxt, _, queriers, warnings, err := q.setupFromCtx(ctx) + ctx, stats, userID, mint, maxt, _, queriers, err := q.setupFromCtx(ctx) if err == errEmptyTimeRange { return nil, nil, nil } else if err != nil { @@ -524,8 +519,7 @@ func (q querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matc startTime := model.Time(mint) endTime := model.Time(maxt) - maxQueryLength := q.limits.MaxQueryLength(userID) - if maxQueryLength > 0 && endTime.Sub(startTime) > maxQueryLength { + if maxQueryLength := q.limits.MaxQueryLength(userID); maxQueryLength > 0 && endTime.Sub(startTime) > maxQueryLength { limitErr := validation.LimitError(fmt.Sprintf(validation.ErrQueryTooLong, endTime.Sub(startTime), maxQueryLength)) return nil, nil, limitErr } @@ -535,8 +529,9 @@ func (q querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matc } var ( - g, _ = errgroup.WithContext(ctx) - sets = [][]string{} + g, _ = errgroup.WithContext(ctx) + sets = [][]string{} + warnings = annotations.Annotations(nil) resMtx sync.Mutex ) @@ -627,31 +622,11 @@ func UseBeforeTimestampQueryable(queryable storage.Queryable, ts time.Time) Quer } } -func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs int64, limits *validation.Overrides, maxQueryIntoFuture time.Duration) (int64, int64, annotations.Annotations, error) { - warnings := annotations.Annotations(nil) +func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs int64, limits *validation.Overrides, maxQueryIntoFuture time.Duration) (int64, int64, error) { now := model.Now() startTime := model.Time(startMs) endTime := model.Time(endMs) - // Truncate time range to MaxQueryLength if time parameters are unspecified - maxQueryLength := limits.MaxQueryLength(userID) - if maxQueryLength > 0 && (startMs == util.TimeToMillis(v1.MinTime) || endMs == util.TimeToMillis(v1.MaxTime)) { - if util.TimeToMillis(v1.MaxTime) == endMs { - endTime = now - } - if startMs == util.TimeToMillis(v1.MinTime) { - startTime = endTime.Add(-maxQueryLength) - } - - warnings.Add(validation.LimitError(fmt.Sprintf(validation.WarningTruncatedQueryLength, maxQueryLength))) - - // Make sure to log it in traces to ease debugging. - level.Debug(spanlogger.FromContext(ctx)).Log( - "msg", "start or end time parameters not specified, response truncated to maximum query length", - "updatedStart", util.FormatTimeModel(startTime), - "updatedEnd", util.FormatTimeModel(endTime)) - } - // Clamp time range based on max query into future. if maxQueryIntoFuture > 0 && endTime.After(now.Add(maxQueryIntoFuture)) { origEndTime := endTime @@ -664,7 +639,7 @@ func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs i "updated", util.FormatTimeModel(endTime)) if endTime.Before(startTime) { - return 0, 0, warnings, errEmptyTimeRange + return 0, 0, errEmptyTimeRange } } @@ -680,7 +655,7 @@ func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs i "updated", util.FormatTimeModel(startTime)) if endTime.Before(startTime) { - return 0, 0, warnings, errEmptyTimeRange + return 0, 0, errEmptyTimeRange } } @@ -689,7 +664,7 @@ func validateQueryTimeRange(ctx context.Context, userID string, startMs, endMs i startTime = 0 } - return int64(startTime), int64(endTime), warnings, nil + return int64(startTime), int64(endTime), nil } func EnableExperimentalPromQLFunctions(enablePromQLExperimentalFunctions, enableHoltWinters bool) { diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index b7adfbc52ed..c87bfce0e68 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -23,7 +23,6 @@ import ( "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/tsdbutil" "github.com/prometheus/prometheus/util/annotations" - v1 "github.com/prometheus/prometheus/web/api/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -933,34 +932,27 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLength_Labels(t *testing.T) { tests := map[string]struct { startTime time.Time endTime time.Time - expectedErr error - expectedWarning error + expected error ignoreMaxQueryLength bool }{ "time range shorter than maxQueryLength": { startTime: time.Now().Add(-maxQueryLength).Add(time.Hour), endTime: time.Now(), - expectedErr: nil, + expected: nil, ignoreMaxQueryLength: false, }, "time range longer than maxQueryLength": { startTime: time.Now().Add(-maxQueryLength).Add(-time.Hour), endTime: time.Now(), - expectedErr: validation.LimitError("expanding series: the query time range exceeds the limit (query length: 721h0m0s, limit: 720h0m0s)"), + expected: validation.LimitError("expanding series: the query time range exceeds the limit (query length: 721h0m0s, limit: 720h0m0s)"), ignoreMaxQueryLength: false, }, "time range longer than maxQueryLength and ignoreMaxQueryLength is true": { startTime: time.Now().Add(-maxQueryLength).Add(-time.Hour), endTime: time.Now(), - expectedErr: validation.LimitError("expanding series: the query time range exceeds the limit (query length: 721h0m0s, limit: 720h0m0s)"), + expected: validation.LimitError("expanding series: the query time range exceeds the limit (query length: 721h0m0s, limit: 720h0m0s)"), ignoreMaxQueryLength: true, }, - "time range not specified should be truncated with a warning": { - startTime: v1.MinTime, - endTime: v1.MaxTime, - expectedWarning: validation.LimitError(fmt.Sprintf(validation.WarningTruncatedQueryLength, maxQueryLength)), - ignoreMaxQueryLength: false, - }, } for testName, testData := range tests { @@ -986,49 +978,23 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLength_Labels(t *testing.T) { q, err := queryable.Querier(util.TimeToMillis(testData.startTime), util.TimeToMillis(testData.endTime)) require.NoError(t, err) - _, warnings, err := q.LabelNames(ctx, &storage.LabelHints{Limit: 0}) + _, _, err = q.LabelNames(ctx, &storage.LabelHints{Limit: 0}) - // Verify expected error is returned - if testData.expectedErr != nil { - assert.True(t, strings.Contains(testData.expectedErr.Error(), err.Error())) + if testData.expected != nil { + require.NotNil(t, err) + assert.True(t, strings.Contains(testData.expected.Error(), err.Error())) } else { assert.Nil(t, err) } - // Verify expected warning is returned - if testData.expectedWarning != nil { - foundWarning := false - for _, warning := range warnings { - if strings.Contains(testData.expectedWarning.Error(), warning.Error()) { - foundWarning = true - } - } - assert.True(t, foundWarning) - } else { - assert.Nil(t, warnings) - } + _, _, err = q.LabelValues(ctx, labels.MetricName, &storage.LabelHints{Limit: 0}) - _, warnings, err = q.LabelValues(ctx, labels.MetricName, &storage.LabelHints{Limit: 0}) - - // Verify expected error is returned - if testData.expectedErr != nil { - assert.True(t, strings.Contains(testData.expectedErr.Error(), err.Error())) + if testData.expected != nil { + require.NotNil(t, err) + assert.True(t, strings.Contains(testData.expected.Error(), err.Error())) } else { assert.Nil(t, err) } - - // Verify expected warning is returned - if testData.expectedWarning != nil { - foundWarning := false - for _, warning := range warnings { - if strings.Contains(testData.expectedWarning.Error(), warning.Error()) { - foundWarning = true - } - } - assert.True(t, foundWarning) - } else { - assert.Nil(t, warnings) - } }) } } @@ -1191,7 +1157,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { // We apply the validation here again since when initializing querier we change the start/end time, // but when querying series we don't validate again. So we should pass correct hints here. - start, end, _, err := validateQueryTimeRange(ctx, "test", util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime), overrides, 0) + start, end, err := validateQueryTimeRange(ctx, "test", util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime), overrides, 0) // Skipped query will hit errEmptyTimeRange during validation. if !testData.expectedSkipped { require.NoError(t, err) @@ -1352,7 +1318,7 @@ func TestValidateMaxQueryLength(t *testing.T) { limits := DefaultLimitsConfig() overrides, err := validation.NewOverrides(limits, nil) require.NoError(t, err) - startMs, endMs, _, err := validateQueryTimeRange(ctx, "test", util.TimeToMillis(tc.start), util.TimeToMillis(tc.end), overrides, 0) + startMs, endMs, err := validateQueryTimeRange(ctx, "test", util.TimeToMillis(tc.start), util.TimeToMillis(tc.end), overrides, 0) require.NoError(t, err) startTime := model.Time(startMs) endTime := model.Time(endMs) diff --git a/pkg/util/validation/validate.go b/pkg/util/validation/validate.go index a9826678068..26d352e29b9 100644 --- a/pkg/util/validation/validate.go +++ b/pkg/util/validation/validate.go @@ -36,8 +36,7 @@ const ( unitTooLong = "unit_too_long" // ErrQueryTooLong is used in chunk store, querier and query frontend. - ErrQueryTooLong = "the query time range exceeds the limit (query length: %s, limit: %s)" - WarningTruncatedQueryLength = "start or end time parameters not specified, response truncated to maximum query length (%s)" + ErrQueryTooLong = "the query time range exceeds the limit (query length: %s, limit: %s)" missingMetricName = "missing_metric_name" invalidMetricName = "metric_name_invalid" From 3e5e76ad81b61352b72081549ea8e93175d31e79 Mon Sep 17 00:00:00 2001 From: Ahmed Hassan Date: Thu, 13 Mar 2025 09:38:55 -0700 Subject: [PATCH 5/8] label requests query ingesters only if start time not specified Signed-off-by: Ahmed Hassan --- pkg/querier/querier.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 2915f01fb1f..0710e90f692 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -442,7 +442,7 @@ func (q querier) Select(ctx context.Context, sortSeries bool, sp *storage.Select // LabelValues implements storage.Querier. func (q querier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - ctx, stats, userID, mint, maxt, _, queriers, err := q.setupFromCtx(ctx) + ctx, stats, userID, mint, maxt, metadataQuerier, queriers, err := q.setupFromCtx(ctx) if err == errEmptyTimeRange { return nil, nil, nil } else if err != nil { @@ -453,6 +453,12 @@ func (q querier) LabelValues(ctx context.Context, name string, hints *storage.La stats.AddQueryStorageWallTime(time.Since(startT)) }() + // For label values queries without specifying the start time, we prefer to + // only query ingesters and not to query maxQueryLength to avoid OOM kill. + if mint == 0 { + return metadataQuerier.LabelValues(ctx, name, hints, matchers...) + } + startTime := model.Time(mint) endTime := model.Time(maxt) @@ -505,7 +511,7 @@ func (q querier) LabelValues(ctx context.Context, name string, hints *storage.La } func (q querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - ctx, stats, userID, mint, maxt, _, queriers, err := q.setupFromCtx(ctx) + ctx, stats, userID, mint, maxt, metadataQuerier, queriers, err := q.setupFromCtx(ctx) if err == errEmptyTimeRange { return nil, nil, nil } else if err != nil { @@ -516,6 +522,12 @@ func (q querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matc stats.AddQueryStorageWallTime(time.Since(startT)) }() + // For label names queries without specifying the start time, we prefer to + // only query ingesters and not to query maxQueryLength to avoid OOM kill. + if mint == 0 { + return metadataQuerier.LabelNames(ctx, hints, matchers...) + } + startTime := model.Time(mint) endTime := model.Time(maxt) From 86379b6a948d30072360bb7b92261ed313a580f8 Mon Sep 17 00:00:00 2001 From: Ahmed Hassan Date: Thu, 13 Mar 2025 15:53:19 -0700 Subject: [PATCH 6/8] update api docs Signed-off-by: Ahmed Hassan --- docs/api/_index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/_index.md b/docs/api/_index.md index 1ab231ea83d..64e802575db 100644 --- a/docs/api/_index.md +++ b/docs/api/_index.md @@ -390,7 +390,7 @@ GET,POST /api/v1/labels GET,POST /api/v1/labels ``` -Get label names of ingested series. Starting from release v1.18.0, Cortex by default honors the `start` and `end` request parameters and fetches label names from either ingester, store gateway or both. +Get label names of ingested series. Starting from release v1.18.0, Cortex by default honors the `start` and `end` request parameters and fetches label names from either ingester, store gateway or both. The special case is that if `start` param is not specified, Cortex currently fetches labels from data stored in the ingesters. _For more information, please check out the Prometheus [get label names](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names) documentation._ @@ -405,7 +405,7 @@ GET /api/v1/label/{name}/values GET /api/v1/label/{name}/values ``` -Get label values for a given label name. Starting from release v1.18.0, Cortex by default honors the `start` and `end` request parameters and fetches label values from either ingester, store gateway or both. +Get label values for a given label name. Starting from release v1.18.0, Cortex by default honors the `start` and `end` request parameters and fetches label values from either ingester, store gateway or both. The special case is that if `start` param is not specified, Cortex currently fetches label values from data stored in the ingesters. _For more information, please check out the Prometheus [get label values](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values) documentation._ From 751fe4a6f4fc464de185e10b1e77d7684cefba15 Mon Sep 17 00:00:00 2001 From: Ahmed Hassan Date: Mon, 17 Mar 2025 15:33:59 -0700 Subject: [PATCH 7/8] rerun tests Signed-off-by: Ahmed Hassan From 012942027f2e401c0cdbeced15716814af9c13dd Mon Sep 17 00:00:00 2001 From: Ahmed Hassan Date: Tue, 18 Mar 2025 18:12:19 -0700 Subject: [PATCH 8/8] update changelog Signed-off-by: Ahmed Hassan --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 522365a8669..52d147c8454 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526 * [FEATURE] Update prometheus alertmanager version to v0.28.0 and add new integration msteamsv2, jira, and rocketchat. #6590 +* [ENHANCEMENT] Querier: limit label APIs to query only ingesters if `start` param is not been specified. #6618 * [ENHANCEMENT] Alertmanager: Add new limits `-alertmanager.max-silences-count` and `-alertmanager.max-silences-size-bytes` for limiting silences per tenant. #6605 * [ENHANCEMENT] Update prometheus version to v3.1.0. #6583 * [ENHANCEMENT] Add `compactor.auto-forget-delay` for compactor to auto forget compactors after X minutes without heartbeat. #6533