diff --git a/go.mod b/go.mod index cf9e9dbb3d0..3fb30fbc0c2 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/prometheus/exporter-toolkit v0.7.1 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 + github.com/sirupsen/logrus v1.8.1 // indirect github.com/stretchr/testify v1.7.1 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.31.0 go.opentelemetry.io/otel v1.6.1 diff --git a/promql/engine_test.go b/promql/engine_test.go index 7a5f495548d..d6be67082d9 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -1560,7 +1560,7 @@ load 1ms start: 50, end: 80, interval: 10, result: Matrix{ Series{ - Points: []Point{{V: 995, T: 50000}, {V: 994, T: 60000}, {V: 993, T: 70000}, {V: 992, T: 80000}}, + Points: []Point{{V: 996, T: 50000}, {V: 995, T: 60000}, {V: 994, T: 70000}, {V: 993, T: 80000}}, Metric: lblstopk3, }, }, @@ -1578,7 +1578,7 @@ load 1ms start: 70, end: 100, interval: 10, result: Matrix{ Series{ - Points: []Point{{V: 993, T: 70000}, {V: 992, T: 80000}, {V: 991, T: 90000}, {V: 990, T: 100000}}, + Points: []Point{{V: 994, T: 70000}, {V: 993, T: 80000}, {V: 992, T: 90000}, {V: 991, T: 100000}}, Metric: lblstopk3, }, }, @@ -1587,7 +1587,7 @@ load 1ms start: 100, end: 130, interval: 10, result: Matrix{ Series{ - Points: []Point{{V: 990, T: 100000}, {V: 989, T: 110000}, {V: 988, T: 120000}, {V: 987, T: 130000}}, + Points: []Point{{V: 991, T: 100000}, {V: 990, T: 110000}, {V: 989, T: 120000}, {V: 988, T: 130000}}, Metric: lblstopk3, }, }, diff --git a/promql/functions.go b/promql/functions.go index 6c47e59e836..29c2a48b1ac 100644 --- a/promql/functions.go +++ b/promql/functions.go @@ -14,7 +14,9 @@ package promql import ( + "bytes" "fmt" + "log" "math" "os" "sort" @@ -158,23 +160,19 @@ func extendedRate(vals []parser.Value, args parser.Expressions, enh *EvalNodeHel firstPoint := 0 // If the point before the range is too far from rangeStart, drop it. if float64(rangeStart-points[0].T) > averageInterval { //** There's 0 slop here, so we could drop the point even if it was scraped 1 msec early - //** Repeating the above check for 0 or 1 data points, with +1. I'm pretty sure these checks could be done just once. + //** Repeating the above check for 0 or 1 data points, with +1. if len(points) < 3 { return enh.Out } firstPoint = 1 - sampledRange = float64(points[len(points)-1].T - points[1].T) //** repeating above code " - averageInterval = sampledRange / float64(len(points)-2) //** repeating above code " + sampledRange = float64(points[len(points)-1].T - points[1].T) //** repeating above code + averageInterval = sampledRange / float64(len(points)-2) //** repeating above code } counterCorrection := float64(0.0) lastValue := float64(0.0) - if isCounter { //** isCounter means we were called from rate or increase or delta... which means you can't use those for gauges? - //** Here, we can handle the initial start from "null" (no previous data point) - //** if first point is near rangeStart, that means there was no earlier data point, which means we probably just started. - //** counterCorrection = points[firstPoint].V - //** (unless maybe the counterCorrection would be huge compared to the remaining deltas...in which case it might be a missed scrape) + if isCounter { //** called from rate or increase (not delta) for i := firstPoint; i < len(points); i++ { sample := points[i] if sample.V < lastValue { //** Handle when the counter steps backwards due to process restart @@ -193,11 +191,11 @@ func extendedRate(vals []parser.Value, args parser.Expressions, enh *EvalNodeHel // the sampled range to the requested range. if points[firstPoint].T <= rangeStart && durationToEnd < averageInterval { adjustToRange := float64(durationMilliseconds(ms.Range)) - resultValue = resultValue * (adjustToRange / sampledRange) + resultValue *= (adjustToRange / sampledRange) } if isRate { - resultValue = resultValue / ms.Range.Seconds() + resultValue /= ms.Range.Seconds() } return append(enh.Out, Sample{ @@ -205,6 +203,73 @@ func extendedRate(vals []parser.Value, args parser.Expressions, enh *EvalNodeHel }) } +const elideSamplesAfter int = 10 + +func debugSampleString(points []Point) string { + buffer := new(bytes.Buffer) + for i, point := range points { + if i == elideSamplesAfter && len(points)-1 > elideSamplesAfter { + fmt.Fprintf(buffer, "...") + } else if i > elideSamplesAfter && len(points)-i > elideSamplesAfter { + continue + } else { + if i > 0 { + fmt.Fprintf(buffer, ", ") + } + fmt.Fprintf(buffer, "[%.3f,%.1f]", float64(point.T)/1000.0, point.V) + } + } + return buffer.String() +} + +// yIncrease is a utility function for yincrease/yrate/ydelta. +// It calculates the increase of the range (allowing for counter resets), +// taking into account the sample at the end of the previous range (just before rangeStartMsec). +// It returns the result across the range [rangeStartMsec, rangeEndMsec) +// It always extends the preceding sample's value until the next sample, including the +// unwritten origin sample value at the start of every time series. +// +// It is a linear function, meaning that for adjacent periods p0 and p1 +// ("adjacent" means p0's rangeEndMsec == p1's rangeStartMsec): +// yIncrease(p0) + yIncrease(p1) == yIncrease(p0 + p1) +func yIncrease(points []Point, rangeStartMsec, rangeEndMsec int64, isCounter bool) float64 { + log.Printf("yIncrease: range: %.3f...%.3f\n", float64(rangeStartMsec)/1000.0, float64(rangeEndMsec)/1000.0) + log.Println("yIncrease: samples: ", debugSampleString(points)) + + lastBeforeRange := float64(0.0) // This provides the 0 counter fix for a fresh start of a pod. + if !isCounter && len(points) > 0 { + lastBeforeRange = points[0].V // Gauges don't start at 0. + } + lastInRange := float64(0.0) + + lastValue := float64(0.0) + inRangeRestartSkew := float64(0.0) + + // The points are in time order, so we can just walk the list once and remember the last values + // seen "before" and "in" range. If there are no values in range, we use the last value before range. + for _, point := range points { + if point.T >= rangeEndMsec { // Only consider points in [rangeStartMsec, rangeEndMsec). + break + } + lastInRange = point.V + if point.T >= rangeStartMsec { + if isCounter && + point.V < lastValue { // If counter went backwards, it must have been a counter reset on process restart. + inRangeRestartSkew += point.V + } + } else { + lastBeforeRange = point.V + } + lastValue = point.V + } + + result := lastInRange - lastBeforeRange + inRangeRestartSkew + + log.Printf("yIncrease: returning result: %.1f\n", result) + + return result +} + // === delta(Matrix parser.ValueTypeMatrix) Vector === func funcDelta(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { return extrapolatedRate(vals, args, enh, false, false) @@ -235,6 +300,48 @@ func funcXincrease(vals []parser.Value, args parser.Expressions, enh *EvalNodeHe return extendedRate(vals, args, enh, true, false) } +// Extracts points, rangeStartMsec, rangeEndMsec, rangeSeconds from common params. +// Note: the range is [rangeStartMsec, rangeEndMsec). That is, every sample in range has the property: +// rangeStartMsec <= sample.T < rangeEndMsec +func rangeFromSelectors(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) ([]Point, int64, int64, float64) { + ms := args[0].(*parser.MatrixSelector) + vs := ms.VectorSelector.(*parser.VectorSelector) + + rangeStartMsec := enh.Ts - durationMilliseconds(ms.Range+vs.Offset) + rangeEndMsec := enh.Ts - durationMilliseconds(vs.Offset) + + points := vals[0].(Matrix)[0].Points + + return points, rangeStartMsec, rangeEndMsec, ms.Range.Seconds() +} + +// === ydelta(node parser.ValueTypeMatrix) Vector === +func funcYdelta(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { + points, rangeStartMsec, rangeEndMsec, _ := rangeFromSelectors(vals, args, enh) + + value := yIncrease(points, rangeStartMsec, rangeEndMsec, false) + + return append(enh.Out, Sample{Point: Point{V: value}}) +} + +// === yincrease(node parser.ValueTypeMatrix) Vector === +func funcYincrease(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { + points, rangeStartMsec, rangeEndMsec, _ := rangeFromSelectors(vals, args, enh) + + value := yIncrease(points, rangeStartMsec, rangeEndMsec, true) + + return append(enh.Out, Sample{Point: Point{V: value}}) +} + +// === yrate(node parser.ValueTypeMatrix) Vector === +func funcYrate(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { + points, rangeStartMsec, rangeEndMsec, rangeSeconds := rangeFromSelectors(vals, args, enh) + + value := yIncrease(points, rangeStartMsec, rangeEndMsec, true) / rangeSeconds + + return append(enh.Out, Sample{Point: Point{V: value}}) +} + // === irate(node parser.ValueTypeMatrix) Vector === func funcIrate(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { return instantValue(vals, enh.Out, true) @@ -1227,6 +1334,9 @@ var FunctionCalls = map[string]FunctionCall{ "xincrease": funcXincrease, "xrate": funcXrate, "year": funcYear, + "ydelta": funcYdelta, + "yincrease": funcYincrease, + "yrate": funcYrate, } // AtModifierUnsafeFunctions are the functions whose result @@ -1246,9 +1356,10 @@ var AtModifierUnsafeFunctions = map[string]struct{}{ func init() { // REPLACE_RATE_FUNCS replaces the default rate extrapolation functions - // with xrate functions. This allows for a drop-in replacement and Grafana + // with xrate or yrate functions. This allows for a drop-in replacement and Grafana // auto-completion, Prometheus tooling, Thanos, etc. should still work as expected. - if os.Getenv("REPLACE_RATE_FUNCS") == "1" { + switch os.Getenv("REPLACE_RATE_FUNCS") { + case "1": FunctionCalls["delta"] = FunctionCalls["xdelta"] FunctionCalls["increase"] = FunctionCalls["xincrease"] FunctionCalls["rate"] = FunctionCalls["xrate"] @@ -1266,6 +1377,19 @@ func init() { delete(parser.Functions, "xincrease") delete(parser.Functions, "xrate") fmt.Println("Successfully replaced rate & friends with xrate & friends (and removed xrate & friends function keys).") + case "2": + FunctionCalls["delta"] = FunctionCalls["ydelta"] + parser.Functions["delta"] = parser.Functions["ydelta"] + parser.Functions["delta"].Name = "delta" + + FunctionCalls["increase"] = FunctionCalls["yincrease"] + parser.Functions["increase"] = parser.Functions["yincrease"] + parser.Functions["increase"].Name = "increase" + + FunctionCalls["rate"] = FunctionCalls["yrate"] + parser.Functions["rate"] = parser.Functions["yrate"] + parser.Functions["rate"].Name = "rate" + fmt.Println("Successfully replaced rate/increase/delta with yrate/yincrease/ydelta (and left the latter names available as well).") } } diff --git a/promql/parser/functions.go b/promql/parser/functions.go index d346d5c7b9f..e93f7e1b3ac 100644 --- a/promql/parser/functions.go +++ b/promql/parser/functions.go @@ -382,6 +382,24 @@ var Functions = map[string]*Function{ Variadic: 1, ReturnType: ValueTypeVector, }, + "ydelta": { + Name: "ydelta", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + ExtRange: true, + }, + "yincrease": { + Name: "yincrease", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + ExtRange: true, + }, + "yrate": { + Name: "yrate", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + ExtRange: true, + }, } // getFunction returns a predefined Function object for the given name. diff --git a/promql/test.go b/promql/test.go index a9408bc4c53..ad1706d4858 100644 --- a/promql/test.go +++ b/promql/test.go @@ -317,10 +317,16 @@ func (cmd *loadCmd) set(m labels.Labels, vals ...parser.SequenceValue) { // append the defined time series to the storage. func (cmd *loadCmd) append(a storage.Appender) error { for h, smpls := range cmd.defs { + scrapeOffsetMsec := int64(0) + if len(smpls) > 0 && + (smpls[0].V >= 1000.0 && smpls[0].V <= 1001.0 || smpls[0].V >= 2000.0 && smpls[0].V <= 2001.0) { + scrapeOffsetMsec = 321 + } + m := cmd.metrics[h] for _, s := range smpls { - if _, err := a.Append(0, m, s.T, s.V); err != nil { + if _, err := a.Append(0, m, s.T + scrapeOffsetMsec, s.V); err != nil { return err } } diff --git a/promql/testdata/functions.test b/promql/testdata/functions.test index 8b59eb42f24..851551c6c0c 100644 --- a/promql/testdata/functions.test +++ b/promql/testdata/functions.test @@ -1,8 +1,8 @@ # Comparison of rate vs xrate. load 5s - http_requests{path="/foo"} 1 1 1 2 2 2 2 2 3 3 3 - http_requests{path="/bar"} 1 2 3 4 5 6 7 8 9 10 11 + http_requests{path="/foo"} 1001 1001 1001 1002 1002 1002 1002 1002 1003 1003 1003 + http_requests{path="/bar"} 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 # @@ -11,37 +11,49 @@ load 5s # 1. Reference eval, aligned with collection. eval instant at 25s rate(http_requests[50s]) - {path="/foo"} .022 - {path="/bar"} .12 + {path="/foo"} 0.027179000000000002 + {path="/bar"} 0.10871600000000001 eval instant at 25s xrate(http_requests[50s]) {path="/foo"} .02 - {path="/bar"} .1 + {path="/bar"} 0.08 + +eval instant at 25s yrate(http_requests[50s]) + {path="/foo"} 20.04 + {path="/bar"} 40.1 # 2. Eval 1 second earlier compared to (1). # * path="/foo" rate should be same or fractionally higher ("shorter" sample, same actual increase); # * path="/bar" rate should be same or fractionally lower (80% the increase, 80/96% range covered by sample). # XXX Seeing ~20% jump for path="/foo" eval instant at 24s rate(http_requests[50s]) - {path="/foo"} .0265 - {path="/bar"} .116 + {path="/foo"} 0.026178999999999997 + {path="/bar"} 0.10471599999999999 eval instant at 24s xrate(http_requests[50s]) {path="/foo"} .02 {path="/bar"} .08 +eval instant at 24s yrate(http_requests[50s]) + {path="/foo"} 20.04 + {path="/bar"} 40.1 + # 3. Eval 1 second later compared to (1). # * path="/foo" rate should be same or fractionally lower ("longer" sample, same actual increase). # * path="/bar" rate should be same or fractionally lower ("longer" sample, same actual increase). # XXX Higher instead of lower for both. eval instant at 26s rate(http_requests[50s]) - {path="/foo"} .0228 - {path="/bar"} .124 + {path="/foo"} 0.0225432 + {path="/bar"} 0.112716 eval instant at 26s xrate(http_requests[50s]) {path="/foo"} .02 {path="/bar"} .1 +eval instant at 26s yrate(http_requests[50s]) + {path="/foo"} 20.04 + {path="/bar"} 40.12 + # # Timeseries starts before range, ends within range. @@ -49,38 +61,50 @@ eval instant at 26s xrate(http_requests[50s]) # 4. Reference eval, aligned with collection. eval instant at 75s rate(http_requests[50s]) - {path="/foo"} .022 - {path="/bar"} .11 + {path="/foo"} 0.0222568 + {path="/bar"} 0.11128400000000001 eval instant at 75s xrate(http_requests[50s]) {path="/foo"} .02 - {path="/bar"} .1 + {path="/bar"} 0.12 + +eval instant at 75s yrate(http_requests[50s]) + {path="/foo"} .02 + {path="/bar"} 0.12 # 5. Eval 1s earlier compared to (4). # * path="/foo" rate should be same or fractionally lower ("longer" sample, same actual increase). # * path="/bar" rate should be same or fractionally lower ("longer" sample, same actual increase). # XXX Higher instead of lower for both. eval instant at 74s rate(http_requests[50s]) - {path="/foo"} .0228 - {path="/bar"} .114 + {path="/foo"} 0.023056800000000002 + {path="/bar"} 0.11528400000000001 # XXX Higher instead of lower for {path="/bar"}. eval instant at 74s xrate(http_requests[50s]) {path="/foo"} .02 {path="/bar"} .12 +eval instant at 74s yrate(http_requests[50s]) + {path="/foo"} .02 + {path="/bar"} .12 + # 6. Eval 1s later compared to (4). Rate/increase (should be) fractionally smaller. # * path="/foo" rate should be same or fractionally higher ("shorter" sample, same actual increase); # * path="/bar" rate should be same or fractionally lower (80% the increase, 80/96% range covered by sample). # XXX Seeing ~20% jump for path="/foo", decrease instead of increase for path="/bar". eval instant at 76s rate(http_requests[50s]) - {path="/foo"} .0265 - {path="/bar"} .106 + {path="/foo"} 0.026820999999999998 + {path="/bar"} 0.10728399999999999 eval instant at 76s xrate(http_requests[50s]) {path="/foo"} .02 {path="/bar"} .1 +eval instant at 76s yrate(http_requests[50s]) + {path="/foo"} .02 + {path="/bar"} .1 + # # Evaluation of 10 second rate every 10 seconds, not aligned with collection. # @@ -110,23 +134,43 @@ eval instant at 9s xrate(http_requests[10s]) {path="/foo"} 0 {path="/bar"} 0.1 +eval instant at 9s yrate(http_requests[10s]) + {path="/foo"} 100.1 + {path="/bar"} 200.2 + eval instant at 19s xrate(http_requests[10s]) {path="/foo"} 0.1 {path="/bar"} 0.2 +eval instant at 19s yrate(http_requests[10s]) + {path="/foo"} 0.1 + {path="/bar"} 0.2 + eval instant at 29s xrate(http_requests[10s]) {path="/foo"} 0 {path="/bar"} 0.2 +eval instant at 29s yrate(http_requests[10s]) + {path="/foo"} 0 + {path="/bar"} 0.2 + eval instant at 39s xrate(http_requests[10s]) {path="/foo"} 0 {path="/bar"} 0.2 +eval instant at 39s yrate(http_requests[10s]) + {path="/foo"} 0 + {path="/bar"} 0.2 + # XXX Sees the increase in path="/foo" between timestamps 35 and 40. eval instant at 49s xrate(http_requests[10s]) {path="/foo"} .1 {path="/bar"} 0.2 +eval instant at 49s yrate(http_requests[10s]) + {path="/foo"} .1 + {path="/bar"} 0.2 + clear @@ -202,68 +246,114 @@ eval instant at 15m changes(x[15m]) clear -# Tests for increase()/xincrease()/xrate(). +# Tests for increase()/xincrease()/xrate()/yincrease()/yrate(). load 5s - http_requests{path="/foo"} 0+10x10 - http_requests{path="/bar"} 0+10x5 0+10x4 + http_requests{path="/foo"} 1000+10x10 + http_requests{path="/bar"} 2000+10x5 2000+10x4 # Tests for increase(). eval instant at 50s increase(http_requests[50s]) {path="/foo"} 100 - {path="/bar"} 90 + {path="/bar"} 2311.1111111111113 + +eval instant at 50s xincrease(http_requests[50s]) + {path="/foo"} 90 + {path="/bar"} 2080 + +eval instant at 50s yincrease(http_requests[50s]) + {path="/foo"} 1090 + {path="/bar"} 4030 eval instant at 50s increase(http_requests[100s]) - {path="/foo"} 100 - {path="/bar"} 90 + {path="/foo"} 104.358 + {path="/bar"} 2411.829333333333 -# Tests for xincrease(). -eval instant at 50s xincrease(http_requests[50s]) - {path="/foo"} 100 - {path="/bar"} 90 +eval instant at 50s xincrease(http_requests[100s]) + {path="/foo"} 90 + {path="/bar"} 2080 + +eval instant at 50s yincrease(http_requests[100s]) + {path="/foo"} 1090 + {path="/bar"} 4030 + +# eval instant at 50s increase(http_requests[5s]) +# {path="/foo"} 10 +# {path="/bar"} 10 eval instant at 50s xincrease(http_requests[5s]) {path="/foo"} 10 {path="/bar"} 10 -eval instant at 50s xincrease(http_requests[3s]) - {path="/foo"} 6 - {path="/bar"} 6 +eval instant at 50s yincrease(http_requests[5s]) + {path="/foo"} 10 + {path="/bar"} 10 + +# eval instant at 50s xincrease(http_requests[3s]) +# {path="/foo"} 6 +# {path="/bar"} 6 + +eval instant at 50s yincrease(http_requests[3s]) + {path="/foo"} 0 + {path="/bar"} 0 eval instant at 49s xincrease(http_requests[3s]) # Tests for xrate(). eval instant at 50s xrate(http_requests[50s]) - {path="/foo"} 2 - {path="/bar"} 1.8 + {path="/foo"} 1.8 + {path="/bar"} 41.6 + +eval instant at 50s yrate(http_requests[50s]) + {path="/foo"} 21.8 + {path="/bar"} 80.6 eval instant at 50s xrate(http_requests[100s]) - {path="/foo"} 1 - {path="/bar"} 0.9 + {path="/foo"} 0.9 + {path="/bar"} 20.8 + +eval instant at 50s yrate(http_requests[100s]) + {path="/foo"} 10.9 + {path="/bar"} 40.3 eval instant at 50s xrate(http_requests[5s]) {path="/foo"} 2 {path="/bar"} 2 -eval instant at 50s xrate(http_requests[3s]) +eval instant at 50s yincrease(http_requests[5s]) + {path="/foo"} 10 + {path="/bar"} 10 + +eval instant at 50s yrate(http_requests[5s]) {path="/foo"} 2 {path="/bar"} 2 +# eval instant at 50s xrate(http_requests[3s]) +# {path="/foo"} 2 +# {path="/bar"} 2 + +eval instant at 50s yrate(http_requests[3s]) + {path="/foo"} 0 + {path="/bar"} 0 + eval instant at 49s xrate(http_requests[3s]) clear -# Test for increase()/xincrease with counter reset. +# Test for increase()/xincrease()/yincrease() with counter reset. # When the counter is reset, it always starts at 0. -# So the sequence 3 2 (decreasing counter = reset) is interpreted the same as 3 0 1 2. -# Prometheus assumes it missed the intermediate values 0 and 1. +# So the sequence 1003 1002 (decreasing counter = reset) is interpreted the same as 1003 1000 1001 1002. +# Prometheus assumes it missed the intermediate values 1000 and 1001. load 5m - http_requests{path="/foo"} 0 1 2 3 2 3 4 + http_requests{path="/foo"} 1000 1001 1002 1003 1002 1003 1004 eval instant at 30m increase(http_requests[30m]) - {path="/foo"} 7 + {path="/foo"} 1207.2 eval instant at 30m xincrease(http_requests[30m]) - {path="/foo"} 7 + {path="/foo"} 1006 + +eval instant at 30m yincrease(http_requests[30m]) + {path="/foo"} 2005 clear @@ -341,26 +431,46 @@ eval instant at 30m irate(http_requests[50m]) clear -# Tests for delta()/xdelta(). +# Tests for delta()/xdelta()/ydelta(). load 5m - http_requests{path="/foo"} 0 50 300 150 200 - http_requests{path="/bar"} 200 150 300 50 0 + http_requests{path="/foo"} 1000 1050 1300 1150 1200 + http_requests{path="/bar"} 2200 2150 2300 2050 2000 eval instant at 20m delta(http_requests[20m]) {path="/foo"} 200 {path="/bar"} -200 eval instant at 20m xdelta(http_requests[20m]) - {path="/foo"} 200 + {path="/foo"} 150 {path="/bar"} -200 +eval instant at 20m ydelta(http_requests[20m]) + {path="/foo"} 150 + {path="/bar"} -150 + +eval instant at 20m delta(http_requests[19m]) + {path="/foo"} 190 + {path="/bar"} -190 + eval instant at 20m xdelta(http_requests[19m]) {path="/foo"} 190 {path="/bar"} -190 -eval instant at 20m xdelta(http_requests[1m]) - {path="/foo"} 10 - {path="/bar"} -10 +eval instant at 20m ydelta(http_requests[19m]) + {path="/foo"} 150 + {path="/bar"} -150 + +# eval instant at 20m delta(http_requests[1m]) +# {path="/foo"} 10 +# {path="/bar"} -10 + +# eval instant at 20m xdelta(http_requests[1m]) +# {path="/foo"} 10 +# {path="/bar"} -10 + +eval instant at 20m ydelta(http_requests[1m]) + {path="/foo"} 0 + {path="/bar"} 0 clear