From b58cba3f44c9963189de7977eaf69a4891adc6ad Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Fri, 7 Aug 2020 18:40:42 +0200 Subject: [PATCH 1/2] Revert "Merge pull request #12408: [BEAM-10602] Display Python streaming metrics in Grafana dashboard" This reverts commit cdc24753054d3aa23805511879ec712f952e2a92, reversing changes made to 835805dfc9beaa7b3aecc690dd49c2ae04f14f98. Revert "Merge pull request #12451: [BEAM-10602] Use python_streaming_pardo_5 table for latency results" This reverts commit 2f47b824507b3a1486ed6cdc8fb4ea6639c924ca, reversing changes made to d971ba13b8ed31de3413558f701d752cbf202d47. --- .../perftests_metrics/ParDo_Load_Tests.json | 89 +++++++------------ 1 file changed, 33 insertions(+), 56 deletions(-) diff --git a/.test-infra/metrics/grafana/dashboards/perftests_metrics/ParDo_Load_Tests.json b/.test-infra/metrics/grafana/dashboards/perftests_metrics/ParDo_Load_Tests.json index f39b684b6703..47dc505c0254 100644 --- a/.test-infra/metrics/grafana/dashboards/perftests_metrics/ParDo_Load_Tests.json +++ b/.test-infra/metrics/grafana/dashboards/perftests_metrics/ParDo_Load_Tests.json @@ -16,7 +16,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "iteration": 1596019686624, + "iteration": 1588593729528, "links": [], "panels": [ { @@ -78,7 +78,7 @@ ], "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"value\") FROM \"${sdk}_${processingType}_pardo_5\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"", + "query": "SELECT mean(\"value\") FROM \"${sdk}_${processingType}_pardo_1\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -103,7 +103,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "$sdk | ParDo | 2GB, 100 byte records, 5 iterations, 10 counter operations, 3 counters, parallelism 5", + "title": "$sdk | ParDo | 2GB, 100 byte records, 10 iterations", "tooltip": { "shared": true, "sort": 0, @@ -158,7 +158,7 @@ "y": 0 }, "hiddenSeries": false, - "id": 4, + "id": 3, "interval": "1d", "legend": { "avg": false, @@ -195,14 +195,15 @@ }, { "params": [ - "null" + "metric" ], - "type": "fill" + "type": "field" } ], + "measurement": "python_batch_cogbk_2", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"value\") FROM \"${sdk}_${processingType}_pardo_5\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"", + "query": "SELECT mean(\"value\") FROM \"${sdk}_${processingType}_pardo_2\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -227,7 +228,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "$sdk | ParDo | 2GB, 100 byte records, 5 iterations, 10 counter operations, 3 counters, parallelism 5", + "title": "$sdk | ParDo | 2GB, 100 byte records, 200 iterations", "tooltip": { "shared": true, "sort": 0, @@ -282,7 +283,7 @@ "y": 8 }, "hiddenSeries": false, - "id": 5, + "id": 4, "interval": "1d", "legend": { "avg": false, @@ -309,7 +310,7 @@ "steppedLine": false, "targets": [ { - "alias": "", + "alias": "$tag_metric", "groupBy": [ { "params": [ @@ -319,14 +320,15 @@ }, { "params": [ - "null" + "metric" ], - "type": "fill" + "type": "field" } ], + "measurement": "python_batch_cogbk_3", "orderByTime": "ASC", "policy": "default", - "query": "SELECT min, max, sum / count\nFROM\n(\n SELECT max(value)/1000 as min FROM \"${sdk}_${processingType}_pardo_5\" WHERE \"metric\" =~ /min_latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as max FROM \"${sdk}_${processingType}_pardo_5\" WHERE \"metric\" =~ /max_latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as sum FROM \"${sdk}_${processingType}_pardo_5\" WHERE \"metric\" =~ /sum_latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as count FROM \"${sdk}_${processingType}_pardo_5\" WHERE \"metric\" =~ /count_latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n)\n\n", + "query": "SELECT mean(\"value\") FROM \"${sdk}_${processingType}_pardo_3\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -351,7 +353,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Latency $sdk | ParDo | 2GB, 100 byte records, 5 iterations, 10 counter operations, 3 counters, parallelism 5 (latency)", + "title": "$sdk | ParDo | 2GB, 100 byte records, 10 counter increments", "tooltip": { "shared": true, "sort": 0, @@ -406,7 +408,7 @@ "y": 8 }, "hiddenSeries": false, - "id": 3, + "id": 5, "interval": "1d", "legend": { "avg": false, @@ -433,24 +435,12 @@ "steppedLine": false, "targets": [ { - "alias": "", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], + "alias": "$tag_metric", + "groupBy": [], + "measurement": "python_batch_cogbk_4", "orderByTime": "ASC", "policy": "default", - "query": "SELECT min, max, sum / count\nFROM\n(\n SELECT max(value)/1000 as min FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /loadgenerator\\/impulse.*_min/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as max FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /loadgenerator\\/impulse.*_max/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as sum FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /loadgenerator\\/impulse.*_sum/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as count FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /loadgenerator\\/impulse.*_count/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n)\n\n", + "query": "SELECT mean(\"value\") FROM \"${sdk}_${processingType}_pardo_4\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -461,10 +451,6 @@ "value" ], "type": "field" - }, - { - "params": [], - "type": "mean" } ] ], @@ -475,7 +461,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Checkpoint duration $sdk | ParDo | 2GB, 100 byte records, 5 iterations, 10 counter operations, 3 counters, parallelism 5", + "title": "$sdk | ParDo | 2GB, 100 byte records, 100 counter increments", "tooltip": { "shared": true, "sort": 0, @@ -519,18 +505,15 @@ "refresh": false, "schemaVersion": 22, "style": "dark", - "tags": [ - "performance tests" - ], + "tags": ["performance tests"], "templating": { "list": [ { "allValue": null, "current": { "selected": false, - "tags": [], - "text": "streaming", - "value": "streaming" + "text": "batch", + "value": "batch" }, "hide": 0, "includeAll": false, @@ -539,14 +522,12 @@ "name": "processingType", "options": [ { - "$$hashKey": "object:283", - "selected": false, + "selected": true, "text": "batch", "value": "batch" }, { - "$$hashKey": "object:284", - "selected": true, + "selected": false, "text": "streaming", "value": "streaming" } @@ -559,9 +540,8 @@ "allValue": null, "current": { "selected": false, - "tags": [], - "text": "python", - "value": "python" + "text": "java", + "value": "java" }, "hide": 0, "includeAll": false, @@ -570,19 +550,16 @@ "name": "sdk", "options": [ { - "$$hashKey": "object:272", - "selected": false, + "selected": true, "text": "java", "value": "java" }, { - "$$hashKey": "object:273", - "selected": true, + "selected": false, "text": "python", "value": "python" }, { - "$$hashKey": "object:274", "selected": false, "text": "go", "value": "go" @@ -619,5 +596,5 @@ "variables": { "list": [] }, - "version": 2 -} \ No newline at end of file + "version": 1 +} From 026a972a67188f544b95368547ac01ba8c083326 Mon Sep 17 00:00:00 2001 From: Maximilian Michels Date: Fri, 7 Aug 2020 18:07:12 +0200 Subject: [PATCH 2/2] [BEAM-10602] Add latency/checkpoint duration as separate panels in ParDo Load Test The Flink streaming tests were reported in a separate table and made avaible through this dashboard: https://apache-beam-testing.appspot.com/explore?dashboard=5751884853805056 Turns out, this is not optimal for the new Grafana-based dashboard. We have to change the table name because the query capability of InfluxDb is very limited. This way the results will be shown together with the other Runners' load test results. --- .../job_LoadTests_ParDo_Flink_Python.groovy | 16 +- .../perftests_metrics/ParDo_Load_Tests.json | 285 +++++++++++++++++- 2 files changed, 282 insertions(+), 19 deletions(-) diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy b/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy index 29678339d02e..284ee6ddf42c 100644 --- a/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy +++ b/.test-infra/jenkins/job_LoadTests_ParDo_Flink_Python.groovy @@ -53,12 +53,15 @@ String now = new Date().format("MMddHHmmss", TimeZone.getTimeZone('UTC')) * GROUP BY test_id, timestamp * ORDER BY timestamp * ); + * + * Subsumed by the new Grafana dashboard: + * http://metrics.beam.apache.org/d/MOi-kf3Zk/pardo-load-tests?orgId=1&var-processingType=streaming&var-sdk=python */ def batchScenarios = { datasetName -> [ [ - title : 'ParDo Python Load test: 20M 100 byte records 10 times', + title : 'ParDo Python Load test: 20M 100 byte records 10 iterations', test : 'apache_beam.testing.load_tests.pardo_test', runner : CommonTestProperties.Runner.PORTABLE, pipelineOptions: [ @@ -161,19 +164,20 @@ def streamingScenarios = { datasetName -> test : 'apache_beam.testing.load_tests.pardo_test', runner : CommonTestProperties.Runner.PORTABLE, pipelineOptions: [ - job_name : 'load-tests-python-flink-streaming-pardo-5-' + now, + job_name : 'load-tests-python-flink-streaming-pardo-1-' + now, project : 'apache-beam-testing', publish_to_big_query : true, metrics_dataset : datasetName, + // Keep the old name to not break the legacy dashboard metrics_table : 'python_flink_streaming_pardo_5', - influx_measurement : 'python_streaming_pardo_5', + influx_measurement : 'python_streaming_pardo_1', input_options : '\'{' + '"num_records": 2000000,' + '"key_size": 10,' + '"value_size": 90}\'', - iterations : 5, - number_of_counter_operations: 10, - number_of_counters : 3, + iterations : 10, + number_of_counter_operations: 0, + number_of_counters : 0, parallelism : 5, // Turn on streaming mode (flags are indicated with null values) streaming : null, diff --git a/.test-infra/metrics/grafana/dashboards/perftests_metrics/ParDo_Load_Tests.json b/.test-infra/metrics/grafana/dashboards/perftests_metrics/ParDo_Load_Tests.json index 47dc505c0254..ba6b3d238281 100644 --- a/.test-infra/metrics/grafana/dashboards/perftests_metrics/ParDo_Load_Tests.json +++ b/.test-infra/metrics/grafana/dashboards/perftests_metrics/ParDo_Load_Tests.json @@ -16,7 +16,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "iteration": 1588593729528, + "iteration": 1596815471813, "links": [], "panels": [ { @@ -500,20 +500,273 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "BeamInfluxDB", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 6, + "interval": "1d", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_metric", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "metric" + ], + "type": "field" + } + ], + "measurement": "python_batch_cogbk_3", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT min, max, sum / count\nFROM\n(\n SELECT max(value)/1000 as min FROM \"${sdk}_${processingType}_pardo_1\" WHERE \"metric\" =~ /min_latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as max FROM \"${sdk}_${processingType}_pardo_1\" WHERE \"metric\" =~ /max_latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as sum FROM \"${sdk}_${processingType}_pardo_1\" WHERE \"metric\" =~ /sum_latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as count FROM \"${sdk}_${processingType}_pardo_1\" WHERE \"metric\" =~ /count_latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n)\n,\n-- Can be removed once the data above is available\n(\n SELECT max(value)/1000 as min FROM \"${sdk}_${processingType}_pardo_5\" WHERE \"metric\" =~ /min_latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as max FROM \"${sdk}_${processingType}_pardo_5\" WHERE \"metric\" =~ /max_latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as sum FROM \"${sdk}_${processingType}_pardo_5\" WHERE \"metric\" =~ /sum_latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as count FROM \"${sdk}_${processingType}_pardo_5\" WHERE \"metric\" =~ /count_latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "latency | $sdk | ParDo | 2GB, 100 byte records, 5 iterations, 10 counter operations, 3 counters, parallelism 5", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": true, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:589", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:590", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "BeamInfluxDB", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 7, + "interval": "1d", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_metric", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "metric" + ], + "type": "field" + } + ], + "measurement": "python_batch_cogbk_3", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT min, max, sum / count\nFROM\n(\n SELECT max(value)/1000 as min FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /loadgenerator\\/impulse.*_min/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as max FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /loadgenerator\\/impulse.*_max/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as sum FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /loadgenerator\\/impulse.*_sum/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as count FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /loadgenerator\\/impulse.*_count/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "checkpoint duration | $sdk | ParDo | 2GB, 100 byte records, 5 iterations, 10 counter operations, 3 counters, parallelism 5", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": true, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:589", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:590", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "refresh": false, "schemaVersion": 22, "style": "dark", - "tags": ["performance tests"], + "tags": [ + "performance tests" + ], "templating": { "list": [ { "allValue": null, "current": { - "selected": false, - "text": "batch", - "value": "batch" + "selected": true, + "tags": [], + "text": "streaming", + "value": "streaming" }, "hide": 0, "includeAll": false, @@ -522,12 +775,14 @@ "name": "processingType", "options": [ { - "selected": true, + "$$hashKey": "object:57", + "selected": false, "text": "batch", "value": "batch" }, { - "selected": false, + "$$hashKey": "object:58", + "selected": true, "text": "streaming", "value": "streaming" } @@ -540,8 +795,9 @@ "allValue": null, "current": { "selected": false, - "text": "java", - "value": "java" + "tags": [], + "text": "python", + "value": "python" }, "hide": 0, "includeAll": false, @@ -550,16 +806,19 @@ "name": "sdk", "options": [ { - "selected": true, + "$$hashKey": "object:374", + "selected": false, "text": "java", "value": "java" }, { - "selected": false, + "$$hashKey": "object:375", + "selected": true, "text": "python", "value": "python" }, { + "$$hashKey": "object:376", "selected": false, "text": "go", "value": "go" @@ -596,5 +855,5 @@ "variables": { "list": [] }, - "version": 1 -} + "version": 8 +} \ No newline at end of file