From 218ace725c34f0e7f46b4db0fdc84ddc1fe35003 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Tue, 1 Oct 2024 12:52:42 -0700 Subject: [PATCH 1/9] Add flutter frame timings to benchmark metrics --- packages/web_benchmarks/CHANGELOG.md | 4 ++ packages/web_benchmarks/lib/src/recorder.dart | 40 +++++++++++++++++++ packages/web_benchmarks/pubspec.yaml | 2 +- .../test/src/analysis_test.dart | 4 -- .../benchmark/web_benchmarks_test.dart | 11 ++++- 5 files changed, 54 insertions(+), 7 deletions(-) diff --git a/packages/web_benchmarks/CHANGELOG.md b/packages/web_benchmarks/CHANGELOG.md index e399fd367ee0..689e885de892 100644 --- a/packages/web_benchmarks/CHANGELOG.md +++ b/packages/web_benchmarks/CHANGELOG.md @@ -1,3 +1,7 @@ +## 3.1.0-wip + +* Add `flutter_frame.total_time`, `flutter_frame.build_time`, and `flutter_frame.raster_time` +metrics to benchmark results. These values are derived from the Flutter `FrameTiming` API. ## 3.0.0 * **Breaking change:** removed the `initialPage` parameter from the `serveWebBenchmark` diff --git a/packages/web_benchmarks/lib/src/recorder.dart b/packages/web_benchmarks/lib/src/recorder.dart index 85d841f1f8cd..7ce1fd5fc8b0 100644 --- a/packages/web_benchmarks/lib/src/recorder.dart +++ b/packages/web_benchmarks/lib/src/recorder.dart @@ -37,6 +37,26 @@ const String kProfilePrerollFrame = 'preroll_frame'; /// to the renderer. const String kProfileApplyFrame = 'apply_frame'; +/// A benchmark metric that tracks the timespan between vsync start and raster +/// finish for a Flutter frame. +/// +/// This value corresponds to [FrameTiming.totalSpan] from the Flutter Engine. +const String kFlutterFrameTotalTime = 'flutter_frame.total_time'; + +/// A benchmark metric that tracks the duration to build the Flutter frame on +/// the Dart UI thread. +/// +/// This value corresponds to [FrameTiming.buildDuration] from the Flutter +/// Engine. +const String kFlutterFrameBuildTime = 'flutter_frame.build_time'; + +/// A benchmark metric that tracks the duration to rasterize the Flutter frame +/// on the Dart raster thread. +/// +/// This value corresponds to [FrameTiming.rasterDuration] from the Flutter +/// Engine. +const String kFlutterFrameRasterTime = 'flutter_frame.raster_time'; + /// Measures the amount of time [action] takes. Duration timeAction(VoidCallback action) { final Stopwatch stopwatch = Stopwatch()..start(); @@ -432,6 +452,26 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder { ); }); + binding.addTimingsCallback((List frameTimings) { + for (final FrameTiming frameTiming in frameTimings) { + localProfile.addDataPoint( + kFlutterFrameTotalTime, + frameTiming.totalSpan, + reported: false, + ); + localProfile.addDataPoint( + kFlutterFrameBuildTime, + frameTiming.buildDuration, + reported: false, + ); + localProfile.addDataPoint( + kFlutterFrameRasterTime, + frameTiming.rasterDuration, + reported: false, + ); + } + }); + binding._beginRecording(this, widget); try { diff --git a/packages/web_benchmarks/pubspec.yaml b/packages/web_benchmarks/pubspec.yaml index 94964f30230c..54ca82807e41 100644 --- a/packages/web_benchmarks/pubspec.yaml +++ b/packages/web_benchmarks/pubspec.yaml @@ -2,7 +2,7 @@ name: web_benchmarks description: A benchmark harness for performance-testing Flutter apps in Chrome. repository: https://github.com/flutter/packages/tree/main/packages/web_benchmarks issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+web_benchmarks%22 -version: 3.0.0 +version: 3.1.0-wip environment: sdk: ^3.3.0 diff --git a/packages/web_benchmarks/test/src/analysis_test.dart b/packages/web_benchmarks/test/src/analysis_test.dart index 852cb4c64083..c584f58085c8 100644 --- a/packages/web_benchmarks/test/src/analysis_test.dart +++ b/packages/web_benchmarks/test/src/analysis_test.dart @@ -2,10 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - import 'package:flutter_test/flutter_test.dart'; import 'package:web_benchmarks/analysis.dart'; diff --git a/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart b/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart index 63b33dc8a14e..73479b0b9cd5 100644 --- a/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart +++ b/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart @@ -12,6 +12,13 @@ import 'package:web_benchmarks/src/common.dart'; import 'test_infra/common.dart'; +const List _sharedExpectedMetrics = [ + 'drawFrameDuration', + 'flutter_frame.total_time', + 'flutter_frame.build_time', + 'flutter_frame.raster_time', +]; + Future main() async { test( 'Can run a web benchmark', @@ -92,11 +99,11 @@ Future _runBenchmarks({ // The skwasm renderer doesn't have preroll or apply frame steps in its rendering. final List expectedMetrics = compilationOptions.useWasm - ? ['drawFrameDuration'] + ? _sharedExpectedMetrics : [ 'preroll_frame', 'apply_frame', - 'drawFrameDuration', + ..._sharedExpectedMetrics, ]; for (final String benchmarkName in benchmarkNames) { From d2c62d858c6750f1b33984d5a29d2df10b2d4970 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Tue, 1 Oct 2024 13:01:45 -0700 Subject: [PATCH 2/9] remove callback in finally --- packages/web_benchmarks/lib/src/recorder.dart | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/web_benchmarks/lib/src/recorder.dart b/packages/web_benchmarks/lib/src/recorder.dart index 7ce1fd5fc8b0..c631adde00c7 100644 --- a/packages/web_benchmarks/lib/src/recorder.dart +++ b/packages/web_benchmarks/lib/src/recorder.dart @@ -452,7 +452,9 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder { ); }); - binding.addTimingsCallback((List frameTimings) { + late void Function(List frameTimings) frameTimingsCallback; + binding.addTimingsCallback( + frameTimingsCallback = (List frameTimings) { for (final FrameTiming frameTiming in frameTimings) { localProfile.addDataPoint( kFlutterFrameTotalTime, @@ -480,6 +482,7 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder { } finally { stopListeningToEngineBenchmarkValues(kProfilePrerollFrame); stopListeningToEngineBenchmarkValues(kProfileApplyFrame); + binding.removeTimingsCallback(frameTimingsCallback); } } } From 0e368fb63ac178a1dec8d1d38448b6f603e0c0f0 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Tue, 1 Oct 2024 14:45:51 -0700 Subject: [PATCH 3/9] Stop passing strings everywhere --- packages/web_benchmarks/lib/src/common.dart | 42 ++++++++++++ packages/web_benchmarks/lib/src/recorder.dart | 66 +++++++------------ .../benchmark/web_benchmarks_test.dart | 20 +++--- 3 files changed, 73 insertions(+), 55 deletions(-) diff --git a/packages/web_benchmarks/lib/src/common.dart b/packages/web_benchmarks/lib/src/common.dart index a32f65c77da7..4d85b5270f36 100644 --- a/packages/web_benchmarks/lib/src/common.dart +++ b/packages/web_benchmarks/lib/src/common.dart @@ -18,3 +18,45 @@ const String kEndOfBenchmarks = '__end_of_benchmarks__'; /// The default initial path for the URL that will be loaded upon opening the /// benchmark app or reloading it in Chrome. const String defaultInitialPath = 'index.html'; + +/// The computed values for each benchmark metric. +enum BenchmarkMetric { + /// A benchmark metric that includes frame-related computations prior to + /// submitting layer and picture operations to the underlying renderer, such as + /// HTML and CanvasKit. During this phase we compute transforms, clips, and + /// other information needed for rendering. + prerollFrame('preroll_frame'), + + /// A benchmark metric that includes submitting layer and picture information + /// to the renderer. + applyFrame('apply_frame'), + + /// A benchmark metric that measures the time spent in [PlatformDispatcher]'s + /// onDrawFrame callback. + drawFrame('draw_frame'), + + /// A benchmark metric that tracks the timespan between vsync start and raster + /// finish for a Flutter frame. + /// + /// This value corresponds to [FrameTiming.totalSpan] from the Flutter Engine. + flutterFrameTotalTime('flutter_frame.total_time'), + + /// A benchmark metric that tracks the duration to build the Flutter frame on + /// the Dart UI thread. + /// + /// This value corresponds to [FrameTiming.buildDuration] from the Flutter + /// Engine. + flutterFrameBuildTime('flutter_frame.build_time'), + + /// A benchmark metric that tracks the duration to rasterize the Flutter frame + /// on the Dart raster thread. + /// + /// This value corresponds to [FrameTiming.rasterDuration] from the Flutter + /// Engine. + flutterFrameRasterTime('flutter_frame.raster_time'); + + const BenchmarkMetric(this.label); + + /// The metric name used in the recorded benchmark data. + final String label; +} diff --git a/packages/web_benchmarks/lib/src/recorder.dart b/packages/web_benchmarks/lib/src/recorder.dart index c631adde00c7..073781ab4dc4 100644 --- a/packages/web_benchmarks/lib/src/recorder.dart +++ b/packages/web_benchmarks/lib/src/recorder.dart @@ -27,36 +27,6 @@ const int _kWarmUpSampleCount = 200; /// The total number of samples collected by a benchmark. const int kTotalSampleCount = _kWarmUpSampleCount + kMeasuredSampleCount; -/// A benchmark metric that includes frame-related computations prior to -/// submitting layer and picture operations to the underlying renderer, such as -/// HTML and CanvasKit. During this phase we compute transforms, clips, and -/// other information needed for rendering. -const String kProfilePrerollFrame = 'preroll_frame'; - -/// A benchmark metric that includes submitting layer and picture information -/// to the renderer. -const String kProfileApplyFrame = 'apply_frame'; - -/// A benchmark metric that tracks the timespan between vsync start and raster -/// finish for a Flutter frame. -/// -/// This value corresponds to [FrameTiming.totalSpan] from the Flutter Engine. -const String kFlutterFrameTotalTime = 'flutter_frame.total_time'; - -/// A benchmark metric that tracks the duration to build the Flutter frame on -/// the Dart UI thread. -/// -/// This value corresponds to [FrameTiming.buildDuration] from the Flutter -/// Engine. -const String kFlutterFrameBuildTime = 'flutter_frame.build_time'; - -/// A benchmark metric that tracks the duration to rasterize the Flutter frame -/// on the Dart raster thread. -/// -/// This value corresponds to [FrameTiming.rasterDuration] from the Flutter -/// Engine. -const String kFlutterFrameRasterTime = 'flutter_frame.raster_time'; - /// Measures the amount of time [action] takes. Duration timeAction(VoidCallback action) { final Stopwatch stopwatch = Stopwatch()..start(); @@ -270,7 +240,7 @@ abstract class SceneBuilderRecorder extends Recorder { PlatformDispatcher.instance.onDrawFrame = () { final FlutterView? view = PlatformDispatcher.instance.implicitView; try { - _profile.record('drawFrameDuration', () { + _profile.record(BenchmarkMetric.drawFrame.label, () { final SceneBuilder sceneBuilder = SceneBuilder(); onDrawFrame(sceneBuilder); _profile.record('sceneBuildDuration', () { @@ -410,8 +380,11 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder { @mustCallSuper void frameDidDraw() { endMeasureFrame(); - profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed, - reported: true); + profile.addDataPoint( + BenchmarkMetric.drawFrame.label, + _drawFrameStopwatch.elapsed, + reported: true, + ); if (shouldContinue()) { PlatformDispatcher.instance.scheduleFrame(); @@ -437,16 +410,18 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder { _RecordingWidgetsBinding.ensureInitialized(); final Widget widget = createWidget(); - registerEngineBenchmarkValueListener(kProfilePrerollFrame, (num value) { + registerEngineBenchmarkValueListener(BenchmarkMetric.prerollFrame.label, + (num value) { localProfile.addDataPoint( - kProfilePrerollFrame, + BenchmarkMetric.prerollFrame.label, Duration(microseconds: value.toInt()), reported: false, ); }); - registerEngineBenchmarkValueListener(kProfileApplyFrame, (num value) { + registerEngineBenchmarkValueListener(BenchmarkMetric.applyFrame.label, + (num value) { localProfile.addDataPoint( - kProfileApplyFrame, + BenchmarkMetric.applyFrame.label, Duration(microseconds: value.toInt()), reported: false, ); @@ -457,17 +432,17 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder { frameTimingsCallback = (List frameTimings) { for (final FrameTiming frameTiming in frameTimings) { localProfile.addDataPoint( - kFlutterFrameTotalTime, + BenchmarkMetric.flutterFrameTotalTime.label, frameTiming.totalSpan, reported: false, ); localProfile.addDataPoint( - kFlutterFrameBuildTime, + BenchmarkMetric.flutterFrameBuildTime.label, frameTiming.buildDuration, reported: false, ); localProfile.addDataPoint( - kFlutterFrameRasterTime, + BenchmarkMetric.flutterFrameRasterTime.label, frameTiming.rasterDuration, reported: false, ); @@ -480,8 +455,8 @@ abstract class WidgetRecorder extends Recorder implements FrameRecorder { await _runCompleter.future; return localProfile; } finally { - stopListeningToEngineBenchmarkValues(kProfilePrerollFrame); - stopListeningToEngineBenchmarkValues(kProfileApplyFrame); + stopListeningToEngineBenchmarkValues(BenchmarkMetric.prerollFrame.label); + stopListeningToEngineBenchmarkValues(BenchmarkMetric.applyFrame.label); binding.removeTimingsCallback(frameTimingsCallback); } } @@ -551,8 +526,11 @@ abstract class WidgetBuildRecorder extends Recorder implements FrameRecorder { // Only record frames that show the widget. if (showWidget) { endMeasureFrame(); - profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed, - reported: true); + profile.addDataPoint( + BenchmarkMetric.drawFrame.label, + _drawFrameStopwatch.elapsed, + reported: true, + ); } if (shouldContinue()) { diff --git a/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart b/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart index 73479b0b9cd5..8a9308a39fa1 100644 --- a/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart +++ b/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart @@ -12,11 +12,11 @@ import 'package:web_benchmarks/src/common.dart'; import 'test_infra/common.dart'; -const List _sharedExpectedMetrics = [ - 'drawFrameDuration', - 'flutter_frame.total_time', - 'flutter_frame.build_time', - 'flutter_frame.raster_time', +final List _wasmExpectedMetrics = [ + BenchmarkMetric.drawFrame.label, + BenchmarkMetric.flutterFrameTotalTime.label, + BenchmarkMetric.flutterFrameBuildTime.label, + BenchmarkMetric.flutterFrameRasterTime.label, ]; Future main() async { @@ -99,12 +99,10 @@ Future _runBenchmarks({ // The skwasm renderer doesn't have preroll or apply frame steps in its rendering. final List expectedMetrics = compilationOptions.useWasm - ? _sharedExpectedMetrics - : [ - 'preroll_frame', - 'apply_frame', - ..._sharedExpectedMetrics, - ]; + ? _wasmExpectedMetrics + : BenchmarkMetric.values + .map((BenchmarkMetric metric) => metric.label) + .toList(); for (final String benchmarkName in benchmarkNames) { for (final String metricName in expectedMetrics) { From 79a9d5b26d88fd486e3494f5fbfd890c319d6255 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Tue, 1 Oct 2024 14:53:37 -0700 Subject: [PATCH 4/9] renames --- packages/web_benchmarks/CHANGELOG.md | 1 + packages/web_benchmarks/lib/metrics.dart | 5 +++ packages/web_benchmarks/lib/src/common.dart | 42 ----------------- packages/web_benchmarks/lib/src/metrics.dart | 45 +++++++++++++++++++ packages/web_benchmarks/lib/src/recorder.dart | 1 + 5 files changed, 52 insertions(+), 42 deletions(-) create mode 100644 packages/web_benchmarks/lib/metrics.dart create mode 100644 packages/web_benchmarks/lib/src/metrics.dart diff --git a/packages/web_benchmarks/CHANGELOG.md b/packages/web_benchmarks/CHANGELOG.md index 689e885de892..ca0882052c43 100644 --- a/packages/web_benchmarks/CHANGELOG.md +++ b/packages/web_benchmarks/CHANGELOG.md @@ -2,6 +2,7 @@ * Add `flutter_frame.total_time`, `flutter_frame.build_time`, and `flutter_frame.raster_time` metrics to benchmark results. These values are derived from the Flutter `FrameTiming` API. +* Expose a new library `metrics.dart` that contains definitions for the benchmark metrics. ## 3.0.0 * **Breaking change:** removed the `initialPage` parameter from the `serveWebBenchmark` diff --git a/packages/web_benchmarks/lib/metrics.dart b/packages/web_benchmarks/lib/metrics.dart new file mode 100644 index 000000000000..cfc5d9fcc294 --- /dev/null +++ b/packages/web_benchmarks/lib/metrics.dart @@ -0,0 +1,5 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +export 'src/metrics.dart'; diff --git a/packages/web_benchmarks/lib/src/common.dart b/packages/web_benchmarks/lib/src/common.dart index 4d85b5270f36..a32f65c77da7 100644 --- a/packages/web_benchmarks/lib/src/common.dart +++ b/packages/web_benchmarks/lib/src/common.dart @@ -18,45 +18,3 @@ const String kEndOfBenchmarks = '__end_of_benchmarks__'; /// The default initial path for the URL that will be loaded upon opening the /// benchmark app or reloading it in Chrome. const String defaultInitialPath = 'index.html'; - -/// The computed values for each benchmark metric. -enum BenchmarkMetric { - /// A benchmark metric that includes frame-related computations prior to - /// submitting layer and picture operations to the underlying renderer, such as - /// HTML and CanvasKit. During this phase we compute transforms, clips, and - /// other information needed for rendering. - prerollFrame('preroll_frame'), - - /// A benchmark metric that includes submitting layer and picture information - /// to the renderer. - applyFrame('apply_frame'), - - /// A benchmark metric that measures the time spent in [PlatformDispatcher]'s - /// onDrawFrame callback. - drawFrame('draw_frame'), - - /// A benchmark metric that tracks the timespan between vsync start and raster - /// finish for a Flutter frame. - /// - /// This value corresponds to [FrameTiming.totalSpan] from the Flutter Engine. - flutterFrameTotalTime('flutter_frame.total_time'), - - /// A benchmark metric that tracks the duration to build the Flutter frame on - /// the Dart UI thread. - /// - /// This value corresponds to [FrameTiming.buildDuration] from the Flutter - /// Engine. - flutterFrameBuildTime('flutter_frame.build_time'), - - /// A benchmark metric that tracks the duration to rasterize the Flutter frame - /// on the Dart raster thread. - /// - /// This value corresponds to [FrameTiming.rasterDuration] from the Flutter - /// Engine. - flutterFrameRasterTime('flutter_frame.raster_time'); - - const BenchmarkMetric(this.label); - - /// The metric name used in the recorded benchmark data. - final String label; -} diff --git a/packages/web_benchmarks/lib/src/metrics.dart b/packages/web_benchmarks/lib/src/metrics.dart new file mode 100644 index 000000000000..003dbf02ec81 --- /dev/null +++ b/packages/web_benchmarks/lib/src/metrics.dart @@ -0,0 +1,45 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// The metrics measured for each benchmark. +enum BenchmarkMetric { + /// A benchmark metric that includes frame-related computations prior to + /// submitting layer and picture operations to the underlying renderer, such as + /// HTML and CanvasKit. During this phase we compute transforms, clips, and + /// other information needed for rendering. + prerollFrame('preroll_frame'), + + /// A benchmark metric that includes submitting layer and picture information + /// to the renderer. + applyFrame('apply_frame'), + + /// A benchmark metric that measures the time spent in [PlatformDispatcher]'s + /// onDrawFrame callback. + drawFrame('draw_frame'), + + /// A benchmark metric that tracks the timespan between vsync start and raster + /// finish for a Flutter frame. + /// + /// This value corresponds to [FrameTiming.totalSpan] from the Flutter Engine. + flutterFrameTotalTime('flutter_frame.total_time'), + + /// A benchmark metric that tracks the duration to build the Flutter frame on + /// the Dart UI thread. + /// + /// This value corresponds to [FrameTiming.buildDuration] from the Flutter + /// Engine. + flutterFrameBuildTime('flutter_frame.build_time'), + + /// A benchmark metric that tracks the duration to rasterize the Flutter frame + /// on the Dart raster thread. + /// + /// This value corresponds to [FrameTiming.rasterDuration] from the Flutter + /// Engine. + flutterFrameRasterTime('flutter_frame.raster_time'); + + const BenchmarkMetric(this.label); + + /// The metric name used in the recorded benchmark data. + final String label; +} diff --git a/packages/web_benchmarks/lib/src/recorder.dart b/packages/web_benchmarks/lib/src/recorder.dart index 073781ab4dc4..82a490c942a2 100644 --- a/packages/web_benchmarks/lib/src/recorder.dart +++ b/packages/web_benchmarks/lib/src/recorder.dart @@ -18,6 +18,7 @@ import 'package:meta/meta.dart'; import 'package:web/web.dart' as html; import 'common.dart'; +import 'metrics.dart'; /// The number of samples from warm-up iterations. /// From 11d1e20356dcd7ddd06e2dea70c50c8d21217e69 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Tue, 1 Oct 2024 14:54:04 -0700 Subject: [PATCH 5/9] new line --- packages/web_benchmarks/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/web_benchmarks/CHANGELOG.md b/packages/web_benchmarks/CHANGELOG.md index ca0882052c43..7a5b8d5366da 100644 --- a/packages/web_benchmarks/CHANGELOG.md +++ b/packages/web_benchmarks/CHANGELOG.md @@ -3,6 +3,7 @@ * Add `flutter_frame.total_time`, `flutter_frame.build_time`, and `flutter_frame.raster_time` metrics to benchmark results. These values are derived from the Flutter `FrameTiming` API. * Expose a new library `metrics.dart` that contains definitions for the benchmark metrics. + ## 3.0.0 * **Breaking change:** removed the `initialPage` parameter from the `serveWebBenchmark` From 3d6bed3b31dee87358df7be643b0eb75996cf577 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Tue, 1 Oct 2024 14:59:16 -0700 Subject: [PATCH 6/9] fix imports --- packages/web_benchmarks/lib/src/metrics.dart | 17 +++++++++++++++++ .../test_app/benchmark/web_benchmarks_test.dart | 13 +++---------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/web_benchmarks/lib/src/metrics.dart b/packages/web_benchmarks/lib/src/metrics.dart index 003dbf02ec81..940fc5a32ddd 100644 --- a/packages/web_benchmarks/lib/src/metrics.dart +++ b/packages/web_benchmarks/lib/src/metrics.dart @@ -43,3 +43,20 @@ enum BenchmarkMetric { /// The metric name used in the recorded benchmark data. final String label; } + +/// The list of expected benchmark metrics for the current compilation mode, as +/// determined by the value of [useWasm]. +List expectedBenchmarkMetrics({required bool useWasm}) { + return [ + // The skwasm renderer doesn't have preroll or apply frame steps in its + // rendering. + if (!useWasm) ...[ + BenchmarkMetric.prerollFrame, + BenchmarkMetric.applyFrame, + ], + BenchmarkMetric.drawFrame, + BenchmarkMetric.flutterFrameTotalTime, + BenchmarkMetric.flutterFrameBuildTime, + BenchmarkMetric.flutterFrameRasterTime, + ]; +} diff --git a/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart b/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart index 8a9308a39fa1..1bb715e423f6 100644 --- a/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart +++ b/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart @@ -7,18 +7,12 @@ import 'dart:io'; import 'package:test/test.dart'; +import 'package:web_benchmarks/metrics.dart'; import 'package:web_benchmarks/server.dart'; import 'package:web_benchmarks/src/common.dart'; import 'test_infra/common.dart'; -final List _wasmExpectedMetrics = [ - BenchmarkMetric.drawFrame.label, - BenchmarkMetric.flutterFrameTotalTime.label, - BenchmarkMetric.flutterFrameBuildTime.label, - BenchmarkMetric.flutterFrameRasterTime.label, -]; - Future main() async { test( 'Can run a web benchmark', @@ -98,9 +92,8 @@ Future _runBenchmarks({ ); // The skwasm renderer doesn't have preroll or apply frame steps in its rendering. - final List expectedMetrics = compilationOptions.useWasm - ? _wasmExpectedMetrics - : BenchmarkMetric.values + final List expectedMetrics = + expectedBenchmarkMetrics(useWasm: compilationOptions.useWasm) .map((BenchmarkMetric metric) => metric.label) .toList(); From 2ce45368a30ec85ef4fe5b2f715b5ee71083dac7 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Tue, 1 Oct 2024 14:59:52 -0700 Subject: [PATCH 7/9] remove comment --- .../testing/test_app/benchmark/web_benchmarks_test.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart b/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart index 1bb715e423f6..a6900a126d31 100644 --- a/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart +++ b/packages/web_benchmarks/testing/test_app/benchmark/web_benchmarks_test.dart @@ -91,7 +91,6 @@ Future _runBenchmarks({ compilationOptions: compilationOptions, ); - // The skwasm renderer doesn't have preroll or apply frame steps in its rendering. final List expectedMetrics = expectedBenchmarkMetrics(useWasm: compilationOptions.useWasm) .map((BenchmarkMetric metric) => metric.label) From cdd1a14a4513bd707041c5b48235a2d02f886795 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Tue, 1 Oct 2024 15:27:03 -0700 Subject: [PATCH 8/9] dart doc --- packages/web_benchmarks/lib/src/metrics.dart | 32 +++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/web_benchmarks/lib/src/metrics.dart b/packages/web_benchmarks/lib/src/metrics.dart index 940fc5a32ddd..7e1ca03db531 100644 --- a/packages/web_benchmarks/lib/src/metrics.dart +++ b/packages/web_benchmarks/lib/src/metrics.dart @@ -2,37 +2,39 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -/// The metrics measured for each benchmark. +/// The names for the metrics collected by the benchmark recorder. enum BenchmarkMetric { - /// A benchmark metric that includes frame-related computations prior to - /// submitting layer and picture operations to the underlying renderer, such as - /// HTML and CanvasKit. During this phase we compute transforms, clips, and - /// other information needed for rendering. + /// The name for the benchmark metric that includes frame-related computations + /// prior to submitting layer and picture operations to the underlying + /// renderer, such as HTML and CanvasKit. + /// + /// During this phase we compute transforms, clips, and other information + /// needed for rendering. prerollFrame('preroll_frame'), - /// A benchmark metric that includes submitting layer and picture information - /// to the renderer. + /// The name for the benchmark metric that includes submitting layer and + /// picture information to the renderer. applyFrame('apply_frame'), - /// A benchmark metric that measures the time spent in [PlatformDispatcher]'s - /// onDrawFrame callback. + /// The name for the benchmark metric that measures the time spent in + /// [PlatformDispatcher]'s onDrawFrame callback. drawFrame('draw_frame'), - /// A benchmark metric that tracks the timespan between vsync start and raster - /// finish for a Flutter frame. + /// The name for the benchmark metric that tracks the timespan between vsync + /// start and raster finish for a Flutter frame. /// /// This value corresponds to [FrameTiming.totalSpan] from the Flutter Engine. flutterFrameTotalTime('flutter_frame.total_time'), - /// A benchmark metric that tracks the duration to build the Flutter frame on - /// the Dart UI thread. + /// The name for the benchmark metric that tracks the duration to build the + /// Flutter frame on the Dart UI thread. /// /// This value corresponds to [FrameTiming.buildDuration] from the Flutter /// Engine. flutterFrameBuildTime('flutter_frame.build_time'), - /// A benchmark metric that tracks the duration to rasterize the Flutter frame - /// on the Dart raster thread. + /// The name for the benchmark metric that tracks the duration to rasterize + /// the Flutter frame on the Dart raster thread. /// /// This value corresponds to [FrameTiming.rasterDuration] from the Flutter /// Engine. From d1811d43158675d6d7b0c740d41f7266886cbdd6 Mon Sep 17 00:00:00 2001 From: Kenzie Davisson Date: Tue, 1 Oct 2024 15:36:14 -0700 Subject: [PATCH 9/9] fixes --- packages/web_benchmarks/lib/src/metrics.dart | 6 +++++- packages/web_benchmarks/lib/src/runner.dart | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/web_benchmarks/lib/src/metrics.dart b/packages/web_benchmarks/lib/src/metrics.dart index 7e1ca03db531..e80692ee58b2 100644 --- a/packages/web_benchmarks/lib/src/metrics.dart +++ b/packages/web_benchmarks/lib/src/metrics.dart @@ -46,13 +46,17 @@ enum BenchmarkMetric { final String label; } +/// The name for the benchmark metric that records the 'averageTotalUIFrameTime' +/// from the Blink trace summary. +const String totalUiFrameAverage = 'totalUiFrame.average'; + /// The list of expected benchmark metrics for the current compilation mode, as /// determined by the value of [useWasm]. List expectedBenchmarkMetrics({required bool useWasm}) { return [ // The skwasm renderer doesn't have preroll or apply frame steps in its // rendering. - if (!useWasm) ...[ + if (!useWasm) ...[ BenchmarkMetric.prerollFrame, BenchmarkMetric.applyFrame, ], diff --git a/packages/web_benchmarks/lib/src/runner.dart b/packages/web_benchmarks/lib/src/runner.dart index a78ef20bbc49..4e52a88f61ab 100644 --- a/packages/web_benchmarks/lib/src/runner.dart +++ b/packages/web_benchmarks/lib/src/runner.dart @@ -19,6 +19,7 @@ import 'benchmark_result.dart'; import 'browser.dart'; import 'common.dart'; import 'compilation_options.dart'; +import 'metrics.dart'; /// The default port number used by the local benchmark server. const int defaultBenchmarkServerPort = 9999; @@ -224,11 +225,11 @@ class BenchmarkServer { if (latestPerformanceTrace != null) { final BlinkTraceSummary? traceSummary = BlinkTraceSummary.fromJson(latestPerformanceTrace!); - profile['totalUiFrame.average'] = + profile[totalUiFrameAverage] = traceSummary?.averageTotalUIFrameTime.inMicroseconds; profile['scoreKeys'] ??= []; // using dynamic for consistency with JSON - (profile['scoreKeys'] as List).add('totalUiFrame.average'); + (profile['scoreKeys'] as List).add(totalUiFrameAverage); latestPerformanceTrace = null; } collectedProfiles.add(profile);