From 843bb63644098acfb2ab63b77ab38838fc6f05eb Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Fri, 8 Dec 2023 13:29:12 -0800 Subject: [PATCH 01/14] Add support for analyzing benchmark results --- packages/web_benchmarks/CHANGELOG.md | 4 + packages/web_benchmarks/README.md | 43 +++ packages/web_benchmarks/lib/analysis.dart | 142 ++++++++ .../lib/src/benchmark_result.dart | 16 +- packages/web_benchmarks/pubspec.yaml | 3 +- .../test/src/analysis_test.dart | 332 ++++++++++++++++++ 6 files changed, 533 insertions(+), 7 deletions(-) create mode 100644 packages/web_benchmarks/lib/analysis.dart create mode 100644 packages/web_benchmarks/test/src/analysis_test.dart diff --git a/packages/web_benchmarks/CHANGELOG.md b/packages/web_benchmarks/CHANGELOG.md index cb26fd04c5f..5641ddfb3ab 100644 --- a/packages/web_benchmarks/CHANGELOG.md +++ b/packages/web_benchmarks/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.1.0 + +* Adds `computeAverage` and `computeDelta` methods to support analysis of benchmark results. + ## 1.0.1 * Adds `parse` constructors for the `BenchmarkResults` and `BenchmarkScore` classes. diff --git a/packages/web_benchmarks/README.md b/packages/web_benchmarks/README.md index cc38fc92cea..4968a5102d8 100644 --- a/packages/web_benchmarks/README.md +++ b/packages/web_benchmarks/README.md @@ -13,3 +13,46 @@ app's code and assets. Additionally, the server communicates with the browser to extract the performance traces. [1]: https://github.com/flutter/packages/blob/master/packages/web_benchmarks/testing/web_benchmarks_test.dart + +# Analyzing benchmark results + +After running web benchmarks, you may want to analyze the results or compare +with the results from other benchmark runs. The `web_benchmarks` package +supports the following analysis operations: + +* compute the delta between two benchmark results +* compute the average of a set of benchmark results + +```dart +import 'dart:convert'; +import 'dart:io'; + +import 'package:web_benchmarks/analysis.dart'; + +void main() { + final baseline = '/path/to/benchmark_baseline.json'; + final test1 = '/path/to/benchmark_test_1.json'; + final test2 = '/path/to/benchmark_test_2.json'; + final baselineFile = File.fromUri(Uri.parse(baseline)); + final testFile1 = File.fromUri(Uri.parse(test1)); + final testFile2 = File.fromUri(Uri.parse(test2)); + + final baselineResults = BenchmarkResults.parse( + jsonDecode(baselineFile.readAsStringSync()), + ); + final testResults1 = BenchmarkResults.parse( + jsonDecode(testFile1.readAsStringSync()), + ); + final testResults2 = BenchmarkResults.parse( + jsonDecode(testFile2.readAsStringSync()), + ); + + // Compute the delta between [baselineResults] and [testResults1]. + final Map>> delta = computeDelta(baselineResults, testResults1); + print(delta); + + // Compute the average of [testResults] and [testResults2]. + final BenchmarkResults average = computeAverage([testResults1, testResults2]); + print(average.toJson()); +} +``` diff --git a/packages/web_benchmarks/lib/analysis.dart b/packages/web_benchmarks/lib/analysis.dart new file mode 100644 index 00000000000..92f2e6ea75f --- /dev/null +++ b/packages/web_benchmarks/lib/analysis.dart @@ -0,0 +1,142 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:collection/collection.dart'; +import 'server.dart'; + +export 'src/benchmark_result.dart'; + +/// Returns the average of the benchmark results in [results]. +/// +/// Each element in [results] is expected to have identical benchmark names and +/// metrics; otherwise, an [Exception] will be thrown. +BenchmarkResults computeAverage(List results) { + if (results.isEmpty) { + throw Exception('Cannot take average of empty list.'); + } + + BenchmarkResults totalSum = results.first; + for (int i = 1; i < results.length; i++) { + final BenchmarkResults current = results[i]; + totalSum = totalSum._sumWith(current); + } + + final Map>> average = totalSum.toJson(); + for (final String benchmark in totalSum.scores.keys) { + final List scoresForBenchmark = totalSum.scores[benchmark]!; + for (int i = 0; i < scoresForBenchmark.length; i++) { + final BenchmarkScore score = scoresForBenchmark[i]; + final double averageValue = score.value / results.length; + average[benchmark]![i][BenchmarkScore.valueKey] = averageValue; + } + } + return BenchmarkResults.parse(average); +} + +/// Computes the delta between [test] and [baseline], and returns the results +/// as a JSON object where each benchmark score entry contains a new field +/// 'delta' with the metric value comparison. +Map>> computeDelta( + BenchmarkResults baseline, + BenchmarkResults test, +) { + for (final String benchmarkName in test.scores.keys) { + // Lookup this benchmark in the baseline. + final List? baselineScores = baseline.scores[benchmarkName]; + if (baselineScores == null) { + continue; + } + + final List testScores = test.scores[benchmarkName]!; + for (final BenchmarkScore score in testScores) { + // Lookup this metric in the baseline. + final BenchmarkScore? baselineScore = baselineScores + .firstWhereOrNull((BenchmarkScore s) => s.metric == score.metric); + if (baselineScore == null) { + continue; + } + + // Add the delta to the [testMetric]. + _benchmarkDeltas[score] = (score.value - baselineScore.value).toDouble(); + } + } + return test._toJsonWithDeltas(); +} + +/// An expando to hold benchmark delta values computed during a [computeDelta] +/// operation. +Expando _benchmarkDeltas = Expando(); + +extension _AnalysisExtension on BenchmarkResults { + /// Returns the JSON representation of this [BenchmarkResults] instance with + /// an added field 'delta' that contains the delta for this metric as computed + /// by the [compareBenchmarks] method. + Map>> _toJsonWithDeltas() { + return scores.map>>( + (String benchmarkName, List scores) { + return MapEntry>>( + benchmarkName, + scores.map>( + (BenchmarkScore score) { + final double? delta = _benchmarkDeltas[score]; + return { + ...score.toJson(), + if (delta != null) 'delta': delta, + }; + }, + ).toList(), + ); + }, + ); + } + + /// Sums this [BenchmarkResults] instance with [other] by adding the values + /// of each matching benchmark score. + /// + /// Returns a [BenchmarkResults] object with the summed values. + /// + /// When [throwExceptionOnMismatch] is true (default), the set of benchmark + /// names and metric names in [other] are expected to be identical to those in + /// [scores], or else an [Exception] will be thrown. + BenchmarkResults _sumWith( + BenchmarkResults other, { + bool throwExceptionOnMismatch = true, + }) { + final Map>> sum = toJson(); + for (final String benchmark in scores.keys) { + // Look up this benchmark in [other]. + final List? matchingBenchmark = other.scores[benchmark]; + if (matchingBenchmark == null) { + if (throwExceptionOnMismatch) { + throw Exception( + 'Cannot sum benchmarks because [other] is missing an entry for ' + 'benchmark "$benchmark".', + ); + } + continue; + } + + final List scoresForBenchmark = scores[benchmark]!; + for (int i = 0; i < scoresForBenchmark.length; i++) { + final BenchmarkScore score = scoresForBenchmark[i]; + // Look up this score in the [matchingBenchmark] from [other]. + final BenchmarkScore? matchingScore = matchingBenchmark + .firstWhereOrNull((BenchmarkScore s) => s.metric == score.metric); + if (matchingScore == null) { + if (throwExceptionOnMismatch) { + throw Exception( + 'Cannot sum benchmarks because benchmark "$benchmark" is missing ' + 'a score for metric ${score.metric}.', + ); + } + continue; + } + + final num sumScore = score.value + matchingScore.value; + sum[benchmark]![i][BenchmarkScore.valueKey] = sumScore; + } + } + return BenchmarkResults.parse(sum); + } +} diff --git a/packages/web_benchmarks/lib/src/benchmark_result.dart b/packages/web_benchmarks/lib/src/benchmark_result.dart index 445dc939489..ab1e828b0bc 100644 --- a/packages/web_benchmarks/lib/src/benchmark_result.dart +++ b/packages/web_benchmarks/lib/src/benchmark_result.dart @@ -14,13 +14,17 @@ class BenchmarkScore { /// Deserializes a JSON object to create a [BenchmarkScore] object. factory BenchmarkScore.parse(Map json) { - final String metric = json[_metricKey]! as String; - final double value = (json[_valueKey]! as num).toDouble(); + final String metric = json[metricKey]! as String; + final double value = (json[valueKey]! as num).toDouble(); return BenchmarkScore(metric: metric, value: value); } - static const String _metricKey = 'metric'; - static const String _valueKey = 'value'; + /// The key for the value [metric] in the [BenchmarkScore] JSON + /// representation. + static const String metricKey = 'metric'; + + /// The key for the value [value] in the [BenchmarkScore] JSON representation. + static const String valueKey = 'value'; /// The name of the metric that this score is categorized under. /// @@ -34,8 +38,8 @@ class BenchmarkScore { /// Serializes the benchmark metric to a JSON object. Map toJson() { return { - _metricKey: metric, - _valueKey: value, + metricKey: metric, + valueKey: value, }; } } diff --git a/packages/web_benchmarks/pubspec.yaml b/packages/web_benchmarks/pubspec.yaml index 7c090ca62ad..117d8dd246e 100644 --- a/packages/web_benchmarks/pubspec.yaml +++ b/packages/web_benchmarks/pubspec.yaml @@ -2,13 +2,14 @@ name: web_benchmarks description: A benchmark harness for performance-testing Flutter apps in Chrome. repository: https://github.com/flutter/packages/tree/main/packages/web_benchmarks issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+web_benchmarks%22 -version: 1.0.1 +version: 1.1.0 environment: sdk: ">=3.2.0 <4.0.0" flutter: ">=3.16.0" dependencies: + collection: ^1.18.0 flutter: sdk: flutter flutter_test: diff --git a/packages/web_benchmarks/test/src/analysis_test.dart b/packages/web_benchmarks/test/src/analysis_test.dart new file mode 100644 index 00000000000..267ddd67672 --- /dev/null +++ b/packages/web_benchmarks/test/src/analysis_test.dart @@ -0,0 +1,332 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:web_benchmarks/analysis.dart'; +import 'package:web_benchmarks/server.dart'; + +void main() { + group('averageBenchmarkResults', () { + test('succeeds for identical benchmark names and metrics', () { + final BenchmarkResults result1 = BenchmarkResults( + >{ + 'foo': [ + BenchmarkScore(metric: 'foo.bar', value: 6), + BenchmarkScore(metric: 'foo.baz', value: 10), + ], + 'bar': [ + BenchmarkScore(metric: 'bar.foo', value: 2.4), + ], + }, + ); + final BenchmarkResults result2 = BenchmarkResults( + >{ + 'foo': [ + BenchmarkScore(metric: 'foo.bar', value: 4), + BenchmarkScore(metric: 'foo.baz', value: 10), + ], + 'bar': [ + BenchmarkScore(metric: 'bar.foo', value: 1.2), + ], + }, + ); + final BenchmarkResults average = + computeAverage([result1, result2]); + expect( + average.toJson(), + >>{ + 'foo': >[ + {'metric': 'foo.bar', 'value': 5}, + {'metric': 'foo.baz', 'value': 10}, + ], + 'bar': >[ + {'metric': 'bar.foo', 'value': 1.7999999999999998}, + ], + }, + ); + }); + + test('fails for mismatched benchmark names', () { + final BenchmarkResults result1 = BenchmarkResults( + >{ + 'foo': [BenchmarkScore(metric: 'foo.bar', value: 6)], + }, + ); + final BenchmarkResults result2 = BenchmarkResults( + >{ + 'foo1': [BenchmarkScore(metric: 'foo.bar', value: 4)], + }, + ); + expect( + () { + computeAverage([result1, result2]); + }, + throwsException, + ); + }); + + test('fails for mismatched benchmark metrics', () { + final BenchmarkResults result1 = BenchmarkResults( + >{ + 'foo': [BenchmarkScore(metric: 'foo.bar', value: 6)], + }, + ); + final BenchmarkResults result2 = BenchmarkResults( + >{ + 'foo': [BenchmarkScore(metric: 'foo.boo', value: 4)], + }, + ); + expect( + () { + computeAverage([result1, result2]); + }, + throwsException, + ); + }); + }); + + test('computeDelta', () { + final BenchmarkResults benchmark1 = + BenchmarkResults.parse(testBenchmarkResults1); + final BenchmarkResults benchmark2 = + BenchmarkResults.parse(testBenchmarkResults2); + final Map>> delta = + computeDelta(benchmark1, benchmark2); + expect(delta, expectedBenchmarkDelta); + }); +} + +final Map>> testBenchmarkResults1 = + >>{ + 'foo': >[ + {'metric': 'preroll_frame.average', 'value': 60.5}, + {'metric': 'preroll_frame.outlierAverage', 'value': 1400}, + {'metric': 'preroll_frame.outlierRatio', 'value': 20.2}, + {'metric': 'preroll_frame.noise', 'value': 0.85}, + {'metric': 'apply_frame.average', 'value': 80.0}, + {'metric': 'apply_frame.outlierAverage', 'value': 200.6}, + {'metric': 'apply_frame.outlierRatio', 'value': 2.5}, + {'metric': 'apply_frame.noise', 'value': 0.4}, + {'metric': 'drawFrameDuration.average', 'value': 2058.9}, + { + 'metric': 'drawFrameDuration.outlierAverage', + 'value': 24000, + }, + { + 'metric': 'drawFrameDuration.outlierRatio', + 'value': 12.05, + }, + {'metric': 'drawFrameDuration.noise', 'value': 0.34}, + {'metric': 'totalUiFrame.average', 'value': 4166}, + ], + 'bar': >[ + {'metric': 'preroll_frame.average', 'value': 60.5}, + {'metric': 'preroll_frame.outlierAverage', 'value': 1400}, + {'metric': 'preroll_frame.outlierRatio', 'value': 20.2}, + {'metric': 'preroll_frame.noise', 'value': 0.85}, + {'metric': 'apply_frame.average', 'value': 80.0}, + {'metric': 'apply_frame.outlierAverage', 'value': 200.6}, + {'metric': 'apply_frame.outlierRatio', 'value': 2.5}, + {'metric': 'apply_frame.noise', 'value': 0.4}, + {'metric': 'drawFrameDuration.average', 'value': 2058.9}, + { + 'metric': 'drawFrameDuration.outlierAverage', + 'value': 24000, + }, + { + 'metric': 'drawFrameDuration.outlierRatio', + 'value': 12.05, + }, + {'metric': 'drawFrameDuration.noise', 'value': 0.34}, + {'metric': 'totalUiFrame.average', 'value': 4166}, + ], +}; + +final Map>> testBenchmarkResults2 = + >>{ + 'foo': >[ + {'metric': 'preroll_frame.average', 'value': 65.5}, + {'metric': 'preroll_frame.outlierAverage', 'value': 1410}, + {'metric': 'preroll_frame.outlierRatio', 'value': 20.0}, + {'metric': 'preroll_frame.noise', 'value': 1.5}, + {'metric': 'apply_frame.average', 'value': 50.0}, + {'metric': 'apply_frame.outlierAverage', 'value': 100.0}, + {'metric': 'apply_frame.outlierRatio', 'value': 2.55}, + {'metric': 'apply_frame.noise', 'value': 0.9}, + {'metric': 'drawFrameDuration.average', 'value': 2000.0}, + { + 'metric': 'drawFrameDuration.outlierAverage', + 'value': 20000 + }, + { + 'metric': 'drawFrameDuration.outlierRatio', + 'value': 11.05 + }, + {'metric': 'drawFrameDuration.noise', 'value': 1.34}, + {'metric': 'totalUiFrame.average', 'value': 4150}, + ], + 'bar': >[ + {'metric': 'preroll_frame.average', 'value': 65.5}, + {'metric': 'preroll_frame.outlierAverage', 'value': 1410}, + {'metric': 'preroll_frame.outlierRatio', 'value': 20.0}, + {'metric': 'preroll_frame.noise', 'value': 1.5}, + {'metric': 'apply_frame.average', 'value': 50.0}, + {'metric': 'apply_frame.outlierAverage', 'value': 100.0}, + {'metric': 'apply_frame.outlierRatio', 'value': 2.55}, + {'metric': 'apply_frame.noise', 'value': 0.9}, + {'metric': 'drawFrameDuration.average', 'value': 2000.0}, + { + 'metric': 'drawFrameDuration.outlierAverage', + 'value': 20000 + }, + { + 'metric': 'drawFrameDuration.outlierRatio', + 'value': 11.05 + }, + {'metric': 'drawFrameDuration.noise', 'value': 1.34}, + {'metric': 'totalUiFrame.average', 'value': 4150}, + ], +}; + +final Map>> expectedBenchmarkDelta = + >>{ + 'foo': >[ + { + 'metric': 'preroll_frame.average', + 'value': 65.5, + 'delta': 5.0 + }, + { + 'metric': 'preroll_frame.outlierAverage', + 'value': 1410.0, + 'delta': 10.0, + }, + { + 'metric': 'preroll_frame.outlierRatio', + 'value': 20.0, + 'delta': -0.1999999999999993, + }, + { + 'metric': 'preroll_frame.noise', + 'value': 1.5, + 'delta': 0.65, + }, + { + 'metric': 'apply_frame.average', + 'value': 50.0, + 'delta': -30.0, + }, + { + 'metric': 'apply_frame.outlierAverage', + 'value': 100.0, + 'delta': -100.6, + }, + { + 'metric': 'apply_frame.outlierRatio', + 'value': 2.55, + 'delta': 0.04999999999999982, + }, + { + 'metric': 'apply_frame.noise', + 'value': 0.9, + 'delta': 0.5, + }, + { + 'metric': 'drawFrameDuration.average', + 'value': 2000.0, + 'delta': -58.90000000000009, + }, + { + 'metric': 'drawFrameDuration.outlierAverage', + 'value': 20000.0, + 'delta': -4000.0, + }, + { + 'metric': 'drawFrameDuration.outlierRatio', + 'value': 11.05, + 'delta': -1.0, + }, + { + 'metric': 'drawFrameDuration.noise', + 'value': 1.34, + 'delta': 1.0, + }, + { + 'metric': 'totalUiFrame.average', + 'value': 4150.0, + 'delta': -16.0, + }, + ], + 'bar': >[ + { + 'metric': 'preroll_frame.average', + 'value': 65.5, + 'delta': 5.0, + }, + { + 'metric': 'preroll_frame.outlierAverage', + 'value': 1410.0, + 'delta': 10.0, + }, + { + 'metric': 'preroll_frame.outlierRatio', + 'value': 20.0, + 'delta': -0.1999999999999993, + }, + { + 'metric': 'preroll_frame.noise', + 'value': 1.5, + 'delta': 0.65, + }, + { + 'metric': 'apply_frame.average', + 'value': 50.0, + 'delta': -30.0, + }, + { + 'metric': 'apply_frame.outlierAverage', + 'value': 100.0, + 'delta': -100.6, + }, + { + 'metric': 'apply_frame.outlierRatio', + 'value': 2.55, + 'delta': 0.04999999999999982, + }, + { + 'metric': 'apply_frame.noise', + 'value': 0.9, + 'delta': 0.5, + }, + { + 'metric': 'drawFrameDuration.average', + 'value': 2000.0, + 'delta': -58.90000000000009, + }, + { + 'metric': 'drawFrameDuration.outlierAverage', + 'value': 20000.0, + 'delta': -4000.0, + }, + { + 'metric': 'drawFrameDuration.outlierRatio', + 'value': 11.05, + 'delta': -1.0, + }, + { + 'metric': 'drawFrameDuration.noise', + 'value': 1.34, + 'delta': 1.0, + }, + { + 'metric': 'totalUiFrame.average', + 'value': 4150.0, + 'delta': -16.0, + }, + ], +}; From d91711a55805293dbcbae9fbb595e0093b3a1ddc Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Fri, 8 Dec 2023 13:42:42 -0800 Subject: [PATCH 02/14] remove import --- packages/web_benchmarks/test/src/analysis_test.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/web_benchmarks/test/src/analysis_test.dart b/packages/web_benchmarks/test/src/analysis_test.dart index 267ddd67672..d21e4f4679a 100644 --- a/packages/web_benchmarks/test/src/analysis_test.dart +++ b/packages/web_benchmarks/test/src/analysis_test.dart @@ -8,7 +8,6 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:web_benchmarks/analysis.dart'; -import 'package:web_benchmarks/server.dart'; void main() { group('averageBenchmarkResults', () { From 4d8073e7c93b34cbc894ebf132bcdc857ddba800 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Fri, 8 Dec 2023 14:24:09 -0800 Subject: [PATCH 03/14] use code-excerpt --- packages/web_benchmarks/README.md | 40 +++++++++---------- .../example/analyze_example.dart | 36 +++++++++++++++++ 2 files changed, 56 insertions(+), 20 deletions(-) create mode 100644 packages/web_benchmarks/example/analyze_example.dart diff --git a/packages/web_benchmarks/README.md b/packages/web_benchmarks/README.md index 4968a5102d8..73f57f1458b 100644 --- a/packages/web_benchmarks/README.md +++ b/packages/web_benchmarks/README.md @@ -23,6 +23,7 @@ supports the following analysis operations: * compute the delta between two benchmark results * compute the average of a set of benchmark results + ```dart import 'dart:convert'; import 'dart:io'; @@ -30,29 +31,28 @@ import 'dart:io'; import 'package:web_benchmarks/analysis.dart'; void main() { - final baseline = '/path/to/benchmark_baseline.json'; - final test1 = '/path/to/benchmark_test_1.json'; - final test2 = '/path/to/benchmark_test_2.json'; - final baselineFile = File.fromUri(Uri.parse(baseline)); - final testFile1 = File.fromUri(Uri.parse(test1)); - final testFile2 = File.fromUri(Uri.parse(test2)); - - final baselineResults = BenchmarkResults.parse( - jsonDecode(baselineFile.readAsStringSync()), - ); - final testResults1 = BenchmarkResults.parse( - jsonDecode(testFile1.readAsStringSync()), - ); - final testResults2 = BenchmarkResults.parse( - jsonDecode(testFile2.readAsStringSync()), - ); + final BenchmarkResults baselineResults = + _benchmarkResultsFromFile('/path/to/benchmark_baseline.json'); + final BenchmarkResults testResults1 = + _benchmarkResultsFromFile('/path/to/benchmark_test_1.json'); + final BenchmarkResults testResults2 = + _benchmarkResultsFromFile('/path/to/benchmark_test_2.json'); // Compute the delta between [baselineResults] and [testResults1]. - final Map>> delta = computeDelta(baselineResults, testResults1); - print(delta); + final Map>> delta = + computeDelta(baselineResults, testResults1); + stdout.writeln(delta); // Compute the average of [testResults] and [testResults2]. - final BenchmarkResults average = computeAverage([testResults1, testResults2]); - print(average.toJson()); + final BenchmarkResults average = + computeAverage([testResults1, testResults2]); + stdout.writeln(average.toJson()); +} + +BenchmarkResults _benchmarkResultsFromFile(String path) { + final File file = File.fromUri(Uri.parse(path)); + final Map fileContentAsJson = + jsonDecode(file.readAsStringSync()) as Map; + return BenchmarkResults.parse(fileContentAsJson); } ``` diff --git a/packages/web_benchmarks/example/analyze_example.dart b/packages/web_benchmarks/example/analyze_example.dart new file mode 100644 index 00000000000..49c98da2e43 --- /dev/null +++ b/packages/web_benchmarks/example/analyze_example.dart @@ -0,0 +1,36 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// #docregion analyze +import 'dart:convert'; +import 'dart:io'; + +import 'package:web_benchmarks/analysis.dart'; + +void main() { + final BenchmarkResults baselineResults = + _benchmarkResultsFromFile('/path/to/benchmark_baseline.json'); + final BenchmarkResults testResults1 = + _benchmarkResultsFromFile('/path/to/benchmark_test_1.json'); + final BenchmarkResults testResults2 = + _benchmarkResultsFromFile('/path/to/benchmark_test_2.json'); + + // Compute the delta between [baselineResults] and [testResults1]. + final Map>> delta = + computeDelta(baselineResults, testResults1); + stdout.writeln(delta); + + // Compute the average of [testResults] and [testResults2]. + final BenchmarkResults average = + computeAverage([testResults1, testResults2]); + stdout.writeln(average.toJson()); +} + +BenchmarkResults _benchmarkResultsFromFile(String path) { + final File file = File.fromUri(Uri.parse(path)); + final Map fileContentAsJson = + jsonDecode(file.readAsStringSync()) as Map; + return BenchmarkResults.parse(fileContentAsJson); +} +// #enddocregion analyze \ No newline at end of file From b614de0e6cac70dc252bb8de54e3ac63dff1b8ad Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Fri, 8 Dec 2023 14:51:43 -0800 Subject: [PATCH 04/14] add new line --- packages/web_benchmarks/example/analyze_example.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web_benchmarks/example/analyze_example.dart b/packages/web_benchmarks/example/analyze_example.dart index 49c98da2e43..c95e4f8b443 100644 --- a/packages/web_benchmarks/example/analyze_example.dart +++ b/packages/web_benchmarks/example/analyze_example.dart @@ -33,4 +33,4 @@ BenchmarkResults _benchmarkResultsFromFile(String path) { jsonDecode(file.readAsStringSync()) as Map; return BenchmarkResults.parse(fileContentAsJson); } -// #enddocregion analyze \ No newline at end of file +// #enddocregion analyze From 0c6f947e284925994d02abdd54469dac6335a7b9 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Mon, 11 Dec 2023 09:50:29 -0800 Subject: [PATCH 05/14] review comments --- packages/web_benchmarks/lib/analysis.dart | 55 +++++-------------- .../lib/src/benchmark_result.dart | 7 +++ .../test/src/analysis_test.dart | 4 +- 3 files changed, 23 insertions(+), 43 deletions(-) diff --git a/packages/web_benchmarks/lib/analysis.dart b/packages/web_benchmarks/lib/analysis.dart index 92f2e6ea75f..e6d3ddb0c1f 100644 --- a/packages/web_benchmarks/lib/analysis.dart +++ b/packages/web_benchmarks/lib/analysis.dart @@ -13,31 +13,30 @@ export 'src/benchmark_result.dart'; /// metrics; otherwise, an [Exception] will be thrown. BenchmarkResults computeAverage(List results) { if (results.isEmpty) { - throw Exception('Cannot take average of empty list.'); + throw ArgumentError('Cannot take average of empty list.'); } - BenchmarkResults totalSum = results.first; - for (int i = 1; i < results.length; i++) { - final BenchmarkResults current = results[i]; - totalSum = totalSum._sumWith(current); - } + final BenchmarkResults totalSum = results.reduce( + (BenchmarkResults sum, BenchmarkResults next) => sum._sumWith(next), + ); - final Map>> average = totalSum.toJson(); + final BenchmarkResults average = totalSum; for (final String benchmark in totalSum.scores.keys) { final List scoresForBenchmark = totalSum.scores[benchmark]!; for (int i = 0; i < scoresForBenchmark.length; i++) { final BenchmarkScore score = scoresForBenchmark[i]; final double averageValue = score.value / results.length; - average[benchmark]![i][BenchmarkScore.valueKey] = averageValue; + average.scores[benchmark]![i] = + BenchmarkScore(metric: score.metric, value: averageValue); } } - return BenchmarkResults.parse(average); + return average; } -/// Computes the delta between [test] and [baseline], and returns the results -/// as a JSON object where each benchmark score entry contains a new field -/// 'delta' with the metric value comparison. -Map>> computeDelta( +/// Computes the delta for each matching metric in [test] and [baseline], +/// assigns the delta values to each [BenchmarkScore] in [test], and then +/// returns the modified [test] object. +BenchmarkResults computeDelta( BenchmarkResults baseline, BenchmarkResults test, ) { @@ -58,39 +57,13 @@ Map>> computeDelta( } // Add the delta to the [testMetric]. - _benchmarkDeltas[score] = (score.value - baselineScore.value).toDouble(); + score.delta = (score.value - baselineScore.value).toDouble(); } } - return test._toJsonWithDeltas(); + return test; } -/// An expando to hold benchmark delta values computed during a [computeDelta] -/// operation. -Expando _benchmarkDeltas = Expando(); - extension _AnalysisExtension on BenchmarkResults { - /// Returns the JSON representation of this [BenchmarkResults] instance with - /// an added field 'delta' that contains the delta for this metric as computed - /// by the [compareBenchmarks] method. - Map>> _toJsonWithDeltas() { - return scores.map>>( - (String benchmarkName, List scores) { - return MapEntry>>( - benchmarkName, - scores.map>( - (BenchmarkScore score) { - final double? delta = _benchmarkDeltas[score]; - return { - ...score.toJson(), - if (delta != null) 'delta': delta, - }; - }, - ).toList(), - ); - }, - ); - } - /// Sums this [BenchmarkResults] instance with [other] by adding the values /// of each matching benchmark score. /// diff --git a/packages/web_benchmarks/lib/src/benchmark_result.dart b/packages/web_benchmarks/lib/src/benchmark_result.dart index ab1e828b0bc..f61a44c3c5b 100644 --- a/packages/web_benchmarks/lib/src/benchmark_result.dart +++ b/packages/web_benchmarks/lib/src/benchmark_result.dart @@ -35,11 +35,18 @@ class BenchmarkScore { /// The result of measuring a particular metric in this benchmark run. final num value; + /// Optional delta value describing the difference between this metric's score + /// and the score of a matching metric from another [BenchmarkResults]. + /// + /// This value may be assigned by the [computeDelta] analysis method. + num? delta; + /// Serializes the benchmark metric to a JSON object. Map toJson() { return { metricKey: metric, valueKey: value, + if (delta != null) 'delta': delta, }; } } diff --git a/packages/web_benchmarks/test/src/analysis_test.dart b/packages/web_benchmarks/test/src/analysis_test.dart index d21e4f4679a..e2e223b8149 100644 --- a/packages/web_benchmarks/test/src/analysis_test.dart +++ b/packages/web_benchmarks/test/src/analysis_test.dart @@ -94,9 +94,9 @@ void main() { BenchmarkResults.parse(testBenchmarkResults1); final BenchmarkResults benchmark2 = BenchmarkResults.parse(testBenchmarkResults2); - final Map>> delta = + final BenchmarkResults delta = computeDelta(benchmark1, benchmark2); - expect(delta, expectedBenchmarkDelta); + expect(delta.toJson(), expectedBenchmarkDelta); }); } From 81108341cd9cf9caba95262d9f50db7ca898fd32 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Mon, 11 Dec 2023 09:56:55 -0800 Subject: [PATCH 06/14] formatting --- packages/web_benchmarks/lib/src/benchmark_result.dart | 2 +- packages/web_benchmarks/test/src/analysis_test.dart | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/web_benchmarks/lib/src/benchmark_result.dart b/packages/web_benchmarks/lib/src/benchmark_result.dart index f61a44c3c5b..d89453302f9 100644 --- a/packages/web_benchmarks/lib/src/benchmark_result.dart +++ b/packages/web_benchmarks/lib/src/benchmark_result.dart @@ -37,7 +37,7 @@ class BenchmarkScore { /// Optional delta value describing the difference between this metric's score /// and the score of a matching metric from another [BenchmarkResults]. - /// + /// /// This value may be assigned by the [computeDelta] analysis method. num? delta; diff --git a/packages/web_benchmarks/test/src/analysis_test.dart b/packages/web_benchmarks/test/src/analysis_test.dart index e2e223b8149..852cb4c6408 100644 --- a/packages/web_benchmarks/test/src/analysis_test.dart +++ b/packages/web_benchmarks/test/src/analysis_test.dart @@ -94,8 +94,7 @@ void main() { BenchmarkResults.parse(testBenchmarkResults1); final BenchmarkResults benchmark2 = BenchmarkResults.parse(testBenchmarkResults2); - final BenchmarkResults delta = - computeDelta(benchmark1, benchmark2); + final BenchmarkResults delta = computeDelta(benchmark1, benchmark2); expect(delta.toJson(), expectedBenchmarkDelta); }); } From 26ce9113118c3040c7dc09fc5a4f5cc8eb4c6269 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Mon, 11 Dec 2023 10:02:29 -0800 Subject: [PATCH 07/14] fix readme --- packages/web_benchmarks/README.md | 5 ++--- packages/web_benchmarks/example/analyze_example.dart | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/web_benchmarks/README.md b/packages/web_benchmarks/README.md index 73f57f1458b..ce98a22b194 100644 --- a/packages/web_benchmarks/README.md +++ b/packages/web_benchmarks/README.md @@ -39,9 +39,8 @@ void main() { _benchmarkResultsFromFile('/path/to/benchmark_test_2.json'); // Compute the delta between [baselineResults] and [testResults1]. - final Map>> delta = - computeDelta(baselineResults, testResults1); - stdout.writeln(delta); + final BenchmarkResults delta = computeDelta(baselineResults, testResults1); + stdout.writeln(delta.toJson()); // Compute the average of [testResults] and [testResults2]. final BenchmarkResults average = diff --git a/packages/web_benchmarks/example/analyze_example.dart b/packages/web_benchmarks/example/analyze_example.dart index c95e4f8b443..bf2721ac594 100644 --- a/packages/web_benchmarks/example/analyze_example.dart +++ b/packages/web_benchmarks/example/analyze_example.dart @@ -17,9 +17,8 @@ void main() { _benchmarkResultsFromFile('/path/to/benchmark_test_2.json'); // Compute the delta between [baselineResults] and [testResults1]. - final Map>> delta = - computeDelta(baselineResults, testResults1); - stdout.writeln(delta); + final BenchmarkResults delta = computeDelta(baselineResults, testResults1); + stdout.writeln(delta.toJson()); // Compute the average of [testResults] and [testResults2]. final BenchmarkResults average = From bbaba5baf56f5c1b39ead5e578ac53360341a5a8 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Mon, 11 Dec 2023 13:18:33 -0800 Subject: [PATCH 08/14] review comments --- packages/web_benchmarks/lib/analysis.dart | 31 ++++++++++++++----- .../lib/src/benchmark_result.dart | 11 +++++-- .../test/src/benchmark_result_test.dart | 11 +++++-- 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/packages/web_benchmarks/lib/analysis.dart b/packages/web_benchmarks/lib/analysis.dart index e6d3ddb0c1f..e177e1e0189 100644 --- a/packages/web_benchmarks/lib/analysis.dart +++ b/packages/web_benchmarks/lib/analysis.dart @@ -33,13 +33,14 @@ BenchmarkResults computeAverage(List results) { return average; } -/// Computes the delta for each matching metric in [test] and [baseline], -/// assigns the delta values to each [BenchmarkScore] in [test], and then -/// returns the modified [test] object. +/// Computes the delta for each matching metric in [test] and [baseline], and +/// returns a new [BenchmarkResults] object where each [BenchmarkScore] contains +/// a [delta] value. BenchmarkResults computeDelta( BenchmarkResults baseline, BenchmarkResults test, ) { + final BenchmarkResults delta = test.copy(); for (final String benchmarkName in test.scores.keys) { // Lookup this benchmark in the baseline. final List? baselineScores = baseline.scores[benchmarkName]; @@ -48,22 +49,36 @@ BenchmarkResults computeDelta( } final List testScores = test.scores[benchmarkName]!; - for (final BenchmarkScore score in testScores) { + final List deltaScores = delta.scores[benchmarkName]!; + + for (int i = 0; i < testScores.length; i++) { + final BenchmarkScore testScore = testScores[i]; // Lookup this metric in the baseline. final BenchmarkScore? baselineScore = baselineScores - .firstWhereOrNull((BenchmarkScore s) => s.metric == score.metric); + .firstWhereOrNull((BenchmarkScore s) => s.metric == testScore.metric); if (baselineScore == null) { continue; } - // Add the delta to the [testMetric]. - score.delta = (score.value - baselineScore.value).toDouble(); + final BenchmarkScore scoreWithDelta = BenchmarkScore( + metric: testScore.metric, + value: testScore.value, + delta: (testScore.value - baselineScore.value).toDouble(), + ); + deltaScores + ..removeAt(i) + ..insert(i, scoreWithDelta); } } - return test; + return delta; } extension _AnalysisExtension on BenchmarkResults { + /// Returns a new [BenchmarkResults] object that is a copy of [this]. + BenchmarkResults copy() { + return BenchmarkResults.parse(toJson()); + } + /// Sums this [BenchmarkResults] instance with [other] by adding the values /// of each matching benchmark score. /// diff --git a/packages/web_benchmarks/lib/src/benchmark_result.dart b/packages/web_benchmarks/lib/src/benchmark_result.dart index d89453302f9..9e9dfa32fb0 100644 --- a/packages/web_benchmarks/lib/src/benchmark_result.dart +++ b/packages/web_benchmarks/lib/src/benchmark_result.dart @@ -10,13 +10,15 @@ class BenchmarkScore { BenchmarkScore({ required this.metric, required this.value, + this.delta, }); /// Deserializes a JSON object to create a [BenchmarkScore] object. factory BenchmarkScore.parse(Map json) { final String metric = json[metricKey]! as String; final double value = (json[valueKey]! as num).toDouble(); - return BenchmarkScore(metric: metric, value: value); + final num? delta = json[deltaKey] as num?; + return BenchmarkScore(metric: metric, value: value, delta: delta); } /// The key for the value [metric] in the [BenchmarkScore] JSON @@ -26,6 +28,9 @@ class BenchmarkScore { /// The key for the value [value] in the [BenchmarkScore] JSON representation. static const String valueKey = 'value'; + /// The key for the value [delta] in the [BenchmarkScore] JSON representation. + static const String deltaKey = 'delta'; + /// The name of the metric that this score is categorized under. /// /// Scores collected over time under the same name can be visualized as a @@ -39,14 +44,14 @@ class BenchmarkScore { /// and the score of a matching metric from another [BenchmarkResults]. /// /// This value may be assigned by the [computeDelta] analysis method. - num? delta; + final num? delta; /// Serializes the benchmark metric to a JSON object. Map toJson() { return { metricKey: metric, valueKey: value, - if (delta != null) 'delta': delta, + if (delta != null) deltaKey: delta, }; } } diff --git a/packages/web_benchmarks/test/src/benchmark_result_test.dart b/packages/web_benchmarks/test/src/benchmark_result_test.dart index 18765d00646..65adee8556a 100644 --- a/packages/web_benchmarks/test/src/benchmark_result_test.dart +++ b/packages/web_benchmarks/test/src/benchmark_result_test.dart @@ -10,8 +10,8 @@ void main() { test('$BenchmarkResults', () { final Map data = { 'foo': >[ - {'metric': 'foo.bar', 'value': 12.34}, - {'metric': 'foo.baz', 'value': 10}, + {'metric': 'foo.bar', 'value': 12.34, 'delta': -0.2}, + {'metric': 'foo.baz', 'value': 10, 'delta': 3.3}, ], 'bar': >[ {'metric': 'bar.foo', 'value': 1.23}, @@ -27,11 +27,14 @@ void main() { expect(fooBenchmarks.length, 2); expect(fooBenchmarks[0].metric, 'foo.bar'); expect(fooBenchmarks[0].value, 12.34); + expect(fooBenchmarks[0].delta, -0.2); expect(fooBenchmarks[1].metric, 'foo.baz'); expect(fooBenchmarks[1].value, 10); + expect(fooBenchmarks[1].delta, 3.3); expect(barBenchmarks.length, 1); expect(barBenchmarks[0].metric, 'bar.foo'); expect(barBenchmarks[0].value, 1.23); + expect(barBenchmarks[0].delta, isNull); expect(benchmarkResults.toJson(), data); }); @@ -39,12 +42,14 @@ void main() { test('$BenchmarkScore', () { final Map data = { 'metric': 'foo', - 'value': 1.234 + 'value': 1.234, + 'delta': -0.4, }; final BenchmarkScore score = BenchmarkScore.parse(data); expect(score.metric, 'foo'); expect(score.value, 1.234); + expect(score.delta, -0.4); expect(score.toJson(), data); }); From e6e5f062a744f13a80adfc4f793427b67ed25955 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Mon, 11 Dec 2023 15:05:38 -0800 Subject: [PATCH 09/14] fixes --- packages/web_benchmarks/lib/analysis.dart | 21 ++++--------------- .../lib/src/benchmark_result.dart | 2 +- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/packages/web_benchmarks/lib/analysis.dart b/packages/web_benchmarks/lib/analysis.dart index e177e1e0189..dc61846dd77 100644 --- a/packages/web_benchmarks/lib/analysis.dart +++ b/packages/web_benchmarks/lib/analysis.dart @@ -40,7 +40,7 @@ BenchmarkResults computeDelta( BenchmarkResults baseline, BenchmarkResults test, ) { - final BenchmarkResults delta = test.copy(); + final Map>> delta = test.toJson(); for (final String benchmarkName in test.scores.keys) { // Lookup this benchmark in the baseline. final List? baselineScores = baseline.scores[benchmarkName]; @@ -49,8 +49,6 @@ BenchmarkResults computeDelta( } final List testScores = test.scores[benchmarkName]!; - final List deltaScores = delta.scores[benchmarkName]!; - for (int i = 0; i < testScores.length; i++) { final BenchmarkScore testScore = testScores[i]; // Lookup this metric in the baseline. @@ -60,25 +58,14 @@ BenchmarkResults computeDelta( continue; } - final BenchmarkScore scoreWithDelta = BenchmarkScore( - metric: testScore.metric, - value: testScore.value, - delta: (testScore.value - baselineScore.value).toDouble(), - ); - deltaScores - ..removeAt(i) - ..insert(i, scoreWithDelta); + delta[benchmarkName]![i][BenchmarkScore.deltaKey] = + (testScore.value - baselineScore.value).toDouble(); } } - return delta; + return BenchmarkResults.parse(delta); } extension _AnalysisExtension on BenchmarkResults { - /// Returns a new [BenchmarkResults] object that is a copy of [this]. - BenchmarkResults copy() { - return BenchmarkResults.parse(toJson()); - } - /// Sums this [BenchmarkResults] instance with [other] by adding the values /// of each matching benchmark score. /// diff --git a/packages/web_benchmarks/lib/src/benchmark_result.dart b/packages/web_benchmarks/lib/src/benchmark_result.dart index 9e9dfa32fb0..ac80c747345 100644 --- a/packages/web_benchmarks/lib/src/benchmark_result.dart +++ b/packages/web_benchmarks/lib/src/benchmark_result.dart @@ -69,7 +69,7 @@ class BenchmarkResults { final List scores = (json[key]! as List) .cast>() .map(BenchmarkScore.parse) - .toList(); + .toList(growable: false); results[key] = scores; } return BenchmarkResults(results); From 5bbfaa8fa69f8e4ce4182c667c564f8a3c9e8d65 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Mon, 11 Dec 2023 15:37:24 -0800 Subject: [PATCH 10/14] don't serialize --- packages/web_benchmarks/lib/analysis.dart | 32 ++++++++++++++++++----- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/packages/web_benchmarks/lib/analysis.dart b/packages/web_benchmarks/lib/analysis.dart index dc61846dd77..30121168209 100644 --- a/packages/web_benchmarks/lib/analysis.dart +++ b/packages/web_benchmarks/lib/analysis.dart @@ -40,29 +40,38 @@ BenchmarkResults computeDelta( BenchmarkResults baseline, BenchmarkResults test, ) { - final Map>> delta = test.toJson(); + final Map> delta = + >{}; for (final String benchmarkName in test.scores.keys) { - // Lookup this benchmark in the baseline. + final List testScores = test.scores[benchmarkName]!; final List? baselineScores = baseline.scores[benchmarkName]; if (baselineScores == null) { + delta[benchmarkName] = List.generate( + testScores.length, + (int i) => testScores[i]._copyWith(), + ); continue; } - final List testScores = test.scores[benchmarkName]!; + final List scoresWithDelta = []; for (int i = 0; i < testScores.length; i++) { final BenchmarkScore testScore = testScores[i]; - // Lookup this metric in the baseline. final BenchmarkScore? baselineScore = baselineScores .firstWhereOrNull((BenchmarkScore s) => s.metric == testScore.metric); if (baselineScore == null) { + scoresWithDelta.add(testScore._copyWith()); continue; } - delta[benchmarkName]![i][BenchmarkScore.deltaKey] = - (testScore.value - baselineScore.value).toDouble(); + scoresWithDelta.add( + testScore._copyWith( + delta: (testScore.value - baselineScore.value).toDouble(), + ), + ); } + delta[benchmarkName] = scoresWithDelta; } - return BenchmarkResults.parse(delta); + return BenchmarkResults(delta); } extension _AnalysisExtension on BenchmarkResults { @@ -115,3 +124,12 @@ extension _AnalysisExtension on BenchmarkResults { return BenchmarkResults.parse(sum); } } + +extension _CopyExtension on BenchmarkScore { + BenchmarkScore _copyWith({String? metric, num? value, num? delta}) => + BenchmarkScore( + metric: metric ?? this.metric, + value: value ?? this.value, + delta: delta ?? this.delta, + ); +} From 511b1e7dc12a9e1748120a2c91bd39ddf783b382 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Mon, 11 Dec 2023 15:40:15 -0800 Subject: [PATCH 11/14] shorten --- packages/web_benchmarks/lib/analysis.dart | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/web_benchmarks/lib/analysis.dart b/packages/web_benchmarks/lib/analysis.dart index 30121168209..d7da9e33dd9 100644 --- a/packages/web_benchmarks/lib/analysis.dart +++ b/packages/web_benchmarks/lib/analysis.dart @@ -58,14 +58,11 @@ BenchmarkResults computeDelta( final BenchmarkScore testScore = testScores[i]; final BenchmarkScore? baselineScore = baselineScores .firstWhereOrNull((BenchmarkScore s) => s.metric == testScore.metric); - if (baselineScore == null) { - scoresWithDelta.add(testScore._copyWith()); - continue; - } - scoresWithDelta.add( testScore._copyWith( - delta: (testScore.value - baselineScore.value).toDouble(), + delta: baselineScore == null + ? null + : (testScore.value - baselineScore.value).toDouble(), ), ); } From 3a8f2ea855b5b023c602612d69d89ec44d77a9e1 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Mon, 11 Dec 2023 15:43:04 -0800 Subject: [PATCH 12/14] shorten further --- packages/web_benchmarks/lib/analysis.dart | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/packages/web_benchmarks/lib/analysis.dart b/packages/web_benchmarks/lib/analysis.dart index d7da9e33dd9..9575e96ac09 100644 --- a/packages/web_benchmarks/lib/analysis.dart +++ b/packages/web_benchmarks/lib/analysis.dart @@ -53,20 +53,17 @@ BenchmarkResults computeDelta( continue; } - final List scoresWithDelta = []; - for (int i = 0; i < testScores.length; i++) { - final BenchmarkScore testScore = testScores[i]; - final BenchmarkScore? baselineScore = baselineScores - .firstWhereOrNull((BenchmarkScore s) => s.metric == testScore.metric); - scoresWithDelta.add( - testScore._copyWith( + delta[benchmarkName] = testScores.map( + (BenchmarkScore testScore) { + final BenchmarkScore? baselineScore = baselineScores.firstWhereOrNull( + (BenchmarkScore s) => s.metric == testScore.metric); + return testScore._copyWith( delta: baselineScore == null ? null : (testScore.value - baselineScore.value).toDouble(), - ), - ); - } - delta[benchmarkName] = scoresWithDelta; + ); + }, + ).toList(); } return BenchmarkResults(delta); } From bb858421eb779b07d7b0e8bb79087abf28f86b9f Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Mon, 11 Dec 2023 15:43:56 -0800 Subject: [PATCH 13/14] shorten --- packages/web_benchmarks/lib/analysis.dart | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/web_benchmarks/lib/analysis.dart b/packages/web_benchmarks/lib/analysis.dart index 9575e96ac09..94b3912f884 100644 --- a/packages/web_benchmarks/lib/analysis.dart +++ b/packages/web_benchmarks/lib/analysis.dart @@ -45,17 +45,9 @@ BenchmarkResults computeDelta( for (final String benchmarkName in test.scores.keys) { final List testScores = test.scores[benchmarkName]!; final List? baselineScores = baseline.scores[benchmarkName]; - if (baselineScores == null) { - delta[benchmarkName] = List.generate( - testScores.length, - (int i) => testScores[i]._copyWith(), - ); - continue; - } - delta[benchmarkName] = testScores.map( (BenchmarkScore testScore) { - final BenchmarkScore? baselineScore = baselineScores.firstWhereOrNull( + final BenchmarkScore? baselineScore = baselineScores?.firstWhereOrNull( (BenchmarkScore s) => s.metric == testScore.metric); return testScore._copyWith( delta: baselineScore == null From 43fd82451d7c42a71ac0e5d09a743bffa8f49382 Mon Sep 17 00:00:00 2001 From: Kenzie Schmoll Date: Mon, 11 Dec 2023 15:54:32 -0800 Subject: [PATCH 14/14] stop serializing in sumwith --- packages/web_benchmarks/lib/analysis.dart | 32 +++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/web_benchmarks/lib/analysis.dart b/packages/web_benchmarks/lib/analysis.dart index 94b3912f884..73b5d6e78c5 100644 --- a/packages/web_benchmarks/lib/analysis.dart +++ b/packages/web_benchmarks/lib/analysis.dart @@ -73,7 +73,8 @@ extension _AnalysisExtension on BenchmarkResults { BenchmarkResults other, { bool throwExceptionOnMismatch = true, }) { - final Map>> sum = toJson(); + final Map> sum = + >{}; for (final String benchmark in scores.keys) { // Look up this benchmark in [other]. final List? matchingBenchmark = other.scores[benchmark]; @@ -88,26 +89,25 @@ extension _AnalysisExtension on BenchmarkResults { } final List scoresForBenchmark = scores[benchmark]!; - for (int i = 0; i < scoresForBenchmark.length; i++) { - final BenchmarkScore score = scoresForBenchmark[i]; + sum[benchmark] = + scoresForBenchmark.map((BenchmarkScore score) { // Look up this score in the [matchingBenchmark] from [other]. final BenchmarkScore? matchingScore = matchingBenchmark .firstWhereOrNull((BenchmarkScore s) => s.metric == score.metric); - if (matchingScore == null) { - if (throwExceptionOnMismatch) { - throw Exception( - 'Cannot sum benchmarks because benchmark "$benchmark" is missing ' - 'a score for metric ${score.metric}.', - ); - } - continue; + if (matchingScore == null && throwExceptionOnMismatch) { + throw Exception( + 'Cannot sum benchmarks because benchmark "$benchmark" is missing ' + 'a score for metric ${score.metric}.', + ); } - - final num sumScore = score.value + matchingScore.value; - sum[benchmark]![i][BenchmarkScore.valueKey] = sumScore; - } + return score._copyWith( + value: matchingScore == null + ? score.value + : score.value + matchingScore.value, + ); + }).toList(); } - return BenchmarkResults.parse(sum); + return BenchmarkResults(sum); } }