diff --git a/.gitignore b/.gitignore index 0e88f1b4da..380abaa646 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ packages .pub app/build pubspec.lock +config.yaml diff --git a/agent/bin/agent.dart b/agent/bin/agent.dart new file mode 100644 index 0000000000..d15ce1462e --- /dev/null +++ b/agent/bin/agent.dart @@ -0,0 +1,259 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:http/http.dart'; +import 'package:meta/meta.dart'; +import 'package:stack_trace/stack_trace.dart'; + +import 'package:cocoon_agent/src/adb.dart'; +import 'package:cocoon_agent/src/analysis.dart'; +import 'package:cocoon_agent/src/firebase.dart'; +import 'package:cocoon_agent/src/framework.dart'; +import 'package:cocoon_agent/src/gallery.dart'; +import 'package:cocoon_agent/src/golem.dart'; +import 'package:cocoon_agent/src/perf_tests.dart'; +import 'package:cocoon_agent/src/refresh.dart'; +import 'package:cocoon_agent/src/size_tests.dart'; +import 'package:cocoon_agent/src/utils.dart'; + +/// Agents periodically poll the server for more tasks. This sleep period is +/// used to prevent us from DDoS-ing the server. +const Duration _sleepBetweenBuilds = const Duration(seconds: 10); + +final List _streamSubscriptions = []; + +bool _exiting = false; + +Future main(List args) async { + Config.initialize(args); + + print('Agent configuration:'); + print(config); + + Agent agent = new Agent( + baseCocoonUrl: config.baseCocoonUrl, + agentId: config.agentId, + httpClient: new AuthenticatedClient(config.agentId, config.authToken) + ); + + _listenToShutdownSignals(); + while(!_exiting) { + try { + await _captureAsyncStacks(agent.performNextTaskIfAny); + } catch(error, chain) { + print('Caught: $error\n${(chain as Chain).terse}'); + } + + // TODO(yjbanov): report health status after running the task + await new Future.delayed(_sleepBetweenBuilds); + } +} + +void _listenToShutdownSignals() { + _streamSubscriptions.addAll([ + ProcessSignal.SIGINT.watch().listen((_) { + print('\nReceived SIGINT. Shutting down.'); + _stop(ProcessSignal.SIGINT); + }), + ProcessSignal.SIGTERM.watch().listen((_) { + print('\nReceived SIGTERM. Shutting down.'); + _stop(ProcessSignal.SIGTERM); + }), + ]); +} + +Future _stop(ProcessSignal signal) async { + _exiting = true; + for (StreamSubscription sub in _streamSubscriptions) { + await sub.cancel(); + } + _streamSubscriptions.clear(); + // TODO(yjbanov): stop processes launched by tasks, if any + await new Future.delayed(const Duration(seconds: 1)); + exit(0); +} + +class Agent { + Agent({@required this.baseCocoonUrl, @required this.agentId, @required this.httpClient}); + + final String baseCocoonUrl; + final String agentId; + final Client httpClient; + + /// Makes a REST API request to Cocoon. + Future _cocoon(String apiPath, dynamic json) async { + String url = '$baseCocoonUrl/api/$apiPath'; + Response resp = await httpClient.post(url, body: JSON.encode(json)); + return JSON.decode(resp.body); + } + + Future performNextTaskIfAny() async { + Map reservation = await reserveTask(); + if (reservation['TaskEntity'] != null) { + String taskName = reservation['TaskEntity']['Task']['Name']; + String taskKey = reservation['TaskEntity']['Key']; + String revision = reservation['ChecklistEntity']['Checklist']['Commit']['Sha']; + section('Task info'); + print('name : $taskName'); + print('key : $taskKey'); + print('revision : $revision'); + + try { + await _captureAsyncStacks(() async { + await getFlutterAt(revision); + int golemRevision = await computeGolemRevision(); + DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision); + String dartSdkVersion = await getDartVersion(); + Task task = getTask(taskName, revision, revisionTimestamp, dartSdkVersion); + TaskRunner runner = new TaskRunner(revision, golemRevision, [task]); + BuildResult result = await _runTask(runner); + // TODO(yjbanov): upload logs + if (result.succeeded) { + await updateTaskStatus(taskKey, 'Succeeded'); + await _uploadDataToFirebase(result); + } else { + await updateTaskStatus(taskKey, 'Failed'); + } + }); + } catch(error, chain) { + // TODO(yjbanov): upload logs + print('Caught: $error\n${(chain as Chain).terse}'); + await updateTaskStatus(taskKey, 'Failed'); + } + } + } + + Future _screenOff() async { + try { + await (await adb()).sendToSleep(); + } catch(error, stackTrace) { + print('Failed to turn off screen: $error\n$stackTrace'); + } + } + + Future _runTask(TaskRunner runner) async { + // Load-balance tests across attached devices + await pickNextDevice(); + try { + return await runner.run(); + } finally { + await _screenOff(); + } + } + + Task getTask(String taskName, String revision, DateTime revisionTimestamp, String dartSdkVersion) { + List allTasks = [ + createComplexLayoutScrollPerfTest(), + createFlutterGalleryStartupTest(), + createComplexLayoutStartupTest(), + createFlutterGalleryBuildTest(), + createComplexLayoutBuildTest(), + createGalleryTransitionTest(), + createBasicMaterialAppSizeTest(), + createAnalyzerCliTest(sdk: dartSdkVersion, commit: revision, timestamp: revisionTimestamp), + createAnalyzerServerTest(sdk: dartSdkVersion, commit: revision, timestamp: revisionTimestamp), + createRefreshTest(commit: revision, timestamp: revisionTimestamp), + ]; + + return allTasks.firstWhere( + (Task t) => t.name == taskName, + orElse: () { + throw 'Task $taskName not found'; + } + ); + } + + Future> reserveTask() => _cocoon('reserve-task', { + 'AgentID': agentId + }); + + Future getFlutterAt(String revision) async { + String currentRevision = await getCurrentFlutterRepoCommit(); + + // This agent will likely run multiple tasks in the same checklist and + // therefore the same revision. It would be too costly to have to reinstall + // Flutter every time. + if (currentRevision == revision) { + return; + } + + await getFlutter(revision); + } + + Future updateTaskStatus(String taskKey, String newStatus) async { + await _cocoon('update-task-status', { + 'TaskKey': taskKey, + 'NewStatus': newStatus, + }); + } +} + +class AuthenticatedClient extends BaseClient { + AuthenticatedClient(this._agentId, this._authToken); + + final String _agentId; + final String _authToken; + final Client _delegate = new Client(); + + Future send(Request request) async { + request.headers['Agent-ID'] = _agentId; + request.headers['Agent-Auth-Token'] = _authToken; + StreamedResponse resp = await _delegate.send(request); + + if (resp.statusCode != 200) + throw 'HTTP error ${resp.statusCode}:\n${(await Response.fromStream(resp)).body}'; + + return resp; + } +} + +Future _uploadDataToFirebase(BuildResult result) async { + List> golemData = >[]; + + for (TaskResult taskResult in result.results) { + // TODO(devoncarew): We should also upload the fact that these tasks failed. + if (taskResult.data == null) + continue; + + Map data = new Map.from(taskResult.data.json); + + if (taskResult.data.benchmarkScoreKeys != null) { + for (String scoreKey in taskResult.data.benchmarkScoreKeys) { + String benchmarkName = '${taskResult.task.name}.$scoreKey'; + if (registeredBenchmarkNames.contains(benchmarkName)) { + golemData.add({ + 'benchmark_name': benchmarkName, + 'golem_revision': result.golemRevision, + 'score': taskResult.data.json[scoreKey], + }); + } + } + } + + data['__metadata__'] = { + 'success': taskResult.succeeded, + 'revision': taskResult.revision, + 'message': taskResult.message, + }; + + data['__golem__'] = golemData; + + uploadToFirebase(taskResult.task.name, data); + } +} + +Future _captureAsyncStacks(Future callback()) { + Completer completer = new Completer(); + Chain.capture(() async { + await callback(); + completer.complete(); + }, onError: (error, Chain chain) async { + completer.completeError(error, chain); + }); + return completer.future; +} diff --git a/agent/config.sample.yaml b/agent/config.sample.yaml new file mode 100644 index 0000000000..9abcc04a34 --- /dev/null +++ b/agent/config.sample.yaml @@ -0,0 +1,34 @@ +# Sample agent configuration, with explanations for every parameter. + +# The URL of the Cocoon server. +# +# To make test an agent with your dev server, set this to http://localhost:8080. +# +# For production server _always_ use HTTPS. +base_cocoon_url: https://flutter-dashboard.appspot.com + +# The ID of this agent. +agent_id: agent-with-ios-devices-#5 + +# Agent attached this token to its HTTP requests to authenticate itself. +# +# To acquire a token, open `base_cocoon_url` in Chrome, open Chrome Dev Tools, +# select Console tab, then: +# +# If `agent_id` is not yet registered with Cocoon, enter: +# +# cocoon.createAgent(['-a', 'AGENT_ID', '-c', 'CAPABILITY#1', ...]) +# +# Replace AGENT_ID with the value of `agent_id`, and enter the capabilities +# offered by the agent. The `auth_token` value will be printed to the console. +# +# If `agent_id` is already registered with Cocoon, enter: +# +# cocoon.authAgent(['-a', 'AGENT_ID']) +# +# Replace AGENT_ID with the value of `agent_id`. +auth_token: blahblahblah + +# Contact sethladd@google.com, yjbanov@google.com or devoncarew@google.com to +# get a Firebase token. +firebase_flutter_dashboard_token: blahblahblah diff --git a/agent/lib/src/adb.dart b/agent/lib/src/adb.dart new file mode 100644 index 0000000000..7fa3794314 --- /dev/null +++ b/agent/lib/src/adb.dart @@ -0,0 +1,130 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:math' as math; + +import 'utils.dart'; + +typedef Future AdbGetter(); + +/// Get an instance of [Adb]. +/// +/// See [realAdbGetter] for signature. This can be overwritten for testing. +AdbGetter adb = realAdbGetter; + +Adb _currentDevice; + +/// Picks a random Android device out of connected devices and sets it as +/// [_currentDevice]. +Future pickNextDevice() async { + List allDevices = (await Adb.deviceIds) + .map((String id) => new Adb(deviceId: id)) + .toList(); + // TODO(yjbanov): filter out and warn about those with low battery level + _currentDevice = allDevices[new math.Random().nextInt(allDevices.length)]; +} + +Future realAdbGetter() async { + if (_currentDevice == null) + await pickNextDevice(); + return _currentDevice; +} + +class Adb { + Adb({String this.deviceId}); + + final String deviceId; + + // Parses information about a device. Example: + // + // 015d172c98400a03 device usb:340787200X product:nakasi model:Nexus_7 device:grouper + static final RegExp _kDeviceRegex = new RegExp(r'^(\S+)\s+(\S+)(.*)'); + + static Future> get deviceIds async { + List output = (await eval(config.adbPath, ['devices', '-l'], canFail: false, onKill: _adbTimeout())) + .trim().split('\n'); + List results = []; + for (String line in output) { + // Skip lines like: * daemon started successfully * + if (line.startsWith('* daemon ')) + continue; + + if (line.startsWith('List of devices')) + continue; + + if (_kDeviceRegex.hasMatch(line)) { + Match match = _kDeviceRegex.firstMatch(line); + + String deviceID = match[1]; + String deviceState = match[2]; + + if (!const ['unauthorized', 'offline'].contains(deviceState)) { + results.add(deviceID); + } + } else { + throw 'Failed to parse device from adb output: $line'; + } + } + + return results; + } + + /// Whether the device is awake. + Future isAwake() async { + return await _getWakefulness() == 'Awake'; + } + + /// Whether the device is asleep. + Future isAsleep() async { + return await _getWakefulness() == 'Asleep'; + } + + /// Wake up the device if it is not awake using [togglePower]. + Future wakeUp() async { + if (!(await isAwake())) + await togglePower(); + } + + /// Send the device to sleep mode if it is not asleep using [togglePower]. + Future sendToSleep() async { + if (!(await isAsleep())) + await togglePower(); + } + + /// Sends `KEYCODE_POWER` (26), which causes the device to toggle its mode + /// between awake and asleep. + Future togglePower() async { + await shellExec('input', const ['keyevent', '26']); + } + + /// Unlocks the device by sending `KEYCODE_MENU` (82). + /// + /// This only works when the device doesn't have a secure unlock pattern. + Future unlock() async { + await wakeUp(); + await shellExec('input', const ['keyevent', '82']); + } + + /// Retrieves device's wakefulness state. + /// + /// See: https://android.googlesource.com/platform/frameworks/base/+/master/core/java/android/os/PowerManagerInternal.java + Future _getWakefulness() async { + String powerInfo = await shellEval('dumpsys', ['power']); + String wakefulness = grep('mWakefulness=', from: powerInfo).single.split('=')[1].trim(); + return wakefulness; + } + + /// Executes [command] on `adb shell` and returns its exit code. + Future shellExec(String command, List arguments, {Map env}) async { + await exec(config.adbPath, ['shell', command]..addAll(arguments), env: env, canFail: false, onKill: _adbTimeout()); + } + + /// Executes [command] on `adb shell` and returns its standard output as a [String]. + Future shellEval(String command, List arguments, {Map env}) { + return eval(config.adbPath, ['shell', command]..addAll(arguments), env: env, canFail: false, onKill: _adbTimeout()); + } + + static Future _adbTimeout() => new Future.delayed(const Duration(seconds: 5)); +} diff --git a/agent/lib/src/analysis.dart b/agent/lib/src/analysis.dart new file mode 100644 index 0000000000..6b7431403c --- /dev/null +++ b/agent/lib/src/analysis.dart @@ -0,0 +1,107 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as path; + +import 'benchmarks.dart'; +import 'framework.dart'; +import 'utils.dart'; + +Task createAnalyzerCliTest({ + @required String sdk, + @required String commit, + @required DateTime timestamp +}) { + return new AnalyzerCliTask(sdk, commit, timestamp); +} + +Task createAnalyzerServerTest({ + @required String sdk, + @required String commit, + @required DateTime timestamp +}) { + return new AnalyzerServerTask(sdk, commit, timestamp); +} + +abstract class AnalyzerTask extends Task { + AnalyzerTask(String name) : super(name); + + Benchmark benchmark; + + @override + Future run() async { + section(benchmark.name); + await runBenchmark(benchmark, iterations: 3, warmUpBenchmark: true); + return benchmark.bestResult; + } +} + +class AnalyzerCliTask extends AnalyzerTask { + AnalyzerCliTask(String sdk, String commit, DateTime timestamp) : super('analyzer_cli__analysis_time') { + this.benchmark = new FlutterAnalyzeBenchmark(onCancel, sdk, commit, timestamp); + } +} + +class AnalyzerServerTask extends AnalyzerTask { + AnalyzerServerTask(String sdk, String commit, DateTime timestamp) : super('analyzer_server__analysis_time') { + this.benchmark = new FlutterAnalyzeAppBenchmark(onCancel, sdk, commit, timestamp); + } +} + +class FlutterAnalyzeBenchmark extends Benchmark { + FlutterAnalyzeBenchmark(Future onCancel, this.sdk, this.commit, this.timestamp) + : super('flutter analyze --flutter-repo', onCancel); + + final String sdk; + final String commit; + final DateTime timestamp; + + File get benchmarkFile => file(path.join(config.flutterDirectory.path, 'analysis_benchmark.json')); + + @override + TaskResultData get lastResult => new TaskResultData.fromFile(benchmarkFile); + + @override + Future run() async { + rm(benchmarkFile); + await inDirectory(config.flutterDirectory, () async { + await flutter('analyze', onCancel, options: ['--flutter-repo', '--benchmark']); + }); + return addBuildInfo(benchmarkFile, timestamp: timestamp, expected: 25.0, sdk: sdk, commit: commit); + } +} + +class FlutterAnalyzeAppBenchmark extends Benchmark { + FlutterAnalyzeAppBenchmark(Future onCancel, this.sdk, this.commit, this.timestamp) + : super('analysis server mega_gallery', onCancel); + + final String sdk; + final String commit; + final DateTime timestamp; + + @override + TaskResultData get lastResult => new TaskResultData.fromFile(benchmarkFile); + + Directory get megaDir => dir(path.join(config.flutterDirectory.path, 'dev/benchmarks/mega_gallery')); + File get benchmarkFile => file(path.join(megaDir.path, 'analysis_benchmark.json')); + + Future init() { + return inDirectory(config.flutterDirectory, () async { + await dart(['dev/tools/mega_gallery.dart'], onCancel); + }); + } + + @override + Future run() async { + rm(benchmarkFile); + await inDirectory(megaDir, () async { + await flutter('analyze', onCancel, options: ['--watch', '--benchmark']); + }); + return addBuildInfo(benchmarkFile, timestamp: timestamp, expected: 10.0, sdk: sdk, commit: commit); + } +} diff --git a/agent/lib/src/benchmarks.dart b/agent/lib/src/benchmarks.dart new file mode 100644 index 0000000000..90900a32df --- /dev/null +++ b/agent/lib/src/benchmarks.dart @@ -0,0 +1,57 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'framework.dart'; + +abstract class Benchmark { + Benchmark(this.name, this.onCancel); + + final String name; + final Future onCancel; + + TaskResultData bestResult; + + Future init() => new Future.value(); + + Future run(); + TaskResultData get lastResult; + + String toString() => name; +} + +Future runBenchmark(Benchmark benchmark, { + int iterations: 1, + bool warmUpBenchmark: false +}) async { + await benchmark.init(); + + List allRuns = []; + + num minValue; + + if (warmUpBenchmark) + await benchmark.run(); + + while (iterations > 0) { + iterations--; + + print(''); + + try { + num result = await benchmark.run(); + allRuns.add(result); + + if (minValue == null || result < minValue) { + benchmark.bestResult = benchmark.lastResult; + minValue = result; + } + } catch (error) { + print('benchmark failed with error: $error'); + } + } + + return minValue; +} diff --git a/agent/lib/src/buildbot.dart b/agent/lib/src/buildbot.dart new file mode 100644 index 0000000000..44566fc131 --- /dev/null +++ b/agent/lib/src/buildbot.dart @@ -0,0 +1,141 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:http/http.dart' as http; + +const String _jsonEndpoint = 'https://build.chromium.org/p/client.flutter/json'; + +/// Looks up the latest `git` commit SHA that succeeded both on Linux and Mac +/// buildbots. +Future getLatestGreenRevision() async { + BuilderClient linuxBuilder = new BuilderClient('Linux'); + BuilderClient macBuilder = new BuilderClient('Mac'); + List linuxBuildNumbers = await linuxBuilder.listRecentBuilds(); + List macBuildNumbers = await macBuilder.listRecentBuilds(); + Set greenLinuxRevisions = new Set(); + Set greenMacRevisions = new Set(); + + // Keep fetching builds until we find a build that was green both on Mac + // and Linux buildbots. + while (greenLinuxRevisions.intersection(greenMacRevisions).isEmpty && + linuxBuildNumbers.isNotEmpty && + macBuildNumbers.isNotEmpty) { + BuildInfo linuxBuild = await linuxBuilder.getBuild(linuxBuildNumbers.removeLast()); + if (linuxBuild.isGreen) + greenLinuxRevisions.add(linuxBuild.revision); + + BuildInfo macBuild = await macBuilder.getBuild(macBuildNumbers.removeLast()); + if (macBuild.isGreen) + greenMacRevisions.add(macBuild.revision); + } + + Set intersection = greenLinuxRevisions.intersection(greenMacRevisions); + if (intersection.isEmpty) { + // No builds that are green on both Mac and Linux + return null; + } + + return intersection.single; +} + +class BuildInfo { + BuildInfo(this.builderName, this.number, this.isGreen, this.revision); + + final String builderName; + final int number; + final bool isGreen; + final String revision; +} + +class BuilderClient { + BuilderClient(this.builderName); + + final String builderName; + + String get builderUrl => '${_jsonEndpoint}/builders/$builderName'; + + Future getBuild(int buildNumber) async { + Map buildJson = await _getJson('$builderUrl/builds/$buildNumber'); + + return new BuildInfo( + builderName, + buildNumber, + _isGreen(buildJson), + _getBuildProperty(buildJson, 'git_revision') + ); + } + + Future> listRecentBuilds() async { + Map resp = await _getJson('$builderUrl/builds'); + return resp.keys.map(int.parse).toList(); + } +} + +Future _getJson(String url) async { + return JSON.decode((await http.get(url)).body); +} + +/// Properties are encoded as: +/// +/// { +/// "properties": [ +/// [ +/// "name1", +/// value1, +/// ... things we don't care about ... +/// ], +/// [ +/// "name2", +/// value2, +/// ... things we don't care about ... +/// ] +/// ] +/// } +dynamic _getBuildProperty(Map buildJson, String propertyName) { + List> properties = buildJson['properties']; + for (List property in properties) { + if (property[0] == propertyName) + return property[1]; + } + return null; +} + +/// Parses out whether the build was successful. +/// +/// Successes are encoded like this: +/// +/// "text": [ +/// "build", +/// "successful" +/// ] +/// +/// Exceptions are encoded like this: +/// +/// "text": [ +/// "exception", +/// "steps", +/// "exception", +/// "flutter build apk material_gallery" +/// ] +/// +/// Errors are encoded like this: +/// +/// "text": [ +/// "failed", +/// "steps", +/// "failed", +/// "flutter build ios simulator stocks" +/// ] +bool _isGreen(Map buildJson) { + if (buildJson['text'] == null || buildJson['text'].length < 2) { + stderr.writeln('WARNING: failed to parse "text" property out of build JSON'); + return false; + } + + return buildJson['text'][1] == 'successful'; +} diff --git a/agent/lib/src/firebase.dart b/agent/lib/src/firebase.dart new file mode 100644 index 0000000000..4633a656e3 --- /dev/null +++ b/agent/lib/src/firebase.dart @@ -0,0 +1,35 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:firebase_rest/firebase_rest.dart'; + +import 'utils.dart'; + +const firebaseBaseUrl = 'https://purple-butterfly-3000.firebaseio.com'; + +Firebase _measurements() { + String firebaseToken = config.firebaseFlutterDashboardToken; + return new Firebase(Uri.parse("$firebaseBaseUrl/measurements"), + auth: firebaseToken); +} + +Future uploadToFirebase(String measurementKey, dynamic jsonData) async { + Firebase ref = _measurements().child(measurementKey); + await ref.child('current').set(jsonData); + await ref.child('history').push(jsonData); +} + +Future firebaseDownloadCurrent(String measurementKey) async { + DataSnapshot snapshot = await _measurements() + .child(measurementKey) + .child('current') + .get(); + + if (!snapshot.exists) + return null; + + return snapshot.val; +} diff --git a/agent/lib/src/framework.dart b/agent/lib/src/framework.dart new file mode 100644 index 0000000000..497823a3ef --- /dev/null +++ b/agent/lib/src/framework.dart @@ -0,0 +1,183 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; +import 'dart:convert'; + +import 'utils.dart'; + +/// Maximum amount of time a single task is allowed to take to run. +/// +/// If exceeded the task is considered to have failed. +const Duration taskTimeout = const Duration(minutes: 7); + +/// Maximum amount of time cancelling a task is allowed to take. +const Duration cancelTimeout = const Duration(minutes: 1); + +/// Represents a unit of work performed on the dashboard build box that can +/// succeed, fail and be retried independently of others. +abstract class Task { + Task(this.name); + + /// The name of the task that shows up in log messages. + /// + /// This should be as unique as possible to avoid confusion. + final String name; + + final Completer _cancelCompleter = new Completer(); + + /// Signals that the task must be cancelled immediately. + /// + /// Task implementations must obey this signal by killing pending processes + /// and reclaiming subscriptions to streams, such as network requests. + /// + /// The signal may be sent any time whether the task is running or + /// not and must be robust enough to handle it without throwing. + Future get onCancel => _cancelCompleter.future; + + /// Performs actual work. + Future run(); + + void cancel() { + _cancelCompleter.complete(); + } +} + +/// Runs a queue of tasks; collects results. +class TaskRunner { + TaskRunner(this.revision, this.golemRevision, this.tasks); + + /// Flutter repository revision at which this runner ran. + final String revision; + + /// Revision _number_ as understood by Golem. + /// + /// See [computeGolemRevision]. + final int golemRevision; + final List tasks; + + Future run() async { + List results = []; + + for (Task task in tasks) { + section('Running task ${task.name}'); + TaskResult result; + try { + TaskResultData data = await task.run().timeout(taskTimeout); + if (data != null) + result = new TaskResult.success(task, revision, data); + else + result = new TaskResult.failure(task, revision, 'Task data missing'); + } catch (taskError, taskErrorStack) { + String message = '${task.name} failed: $taskError'; + if (taskErrorStack != null) { + message += '\n\n$taskErrorStack'; + } + try { + task.cancel(); + } catch (cancelError, cancelErrorStack) { + message += '\n\nAttempted to cancel tasks, but failed due to: $cancelError'; + if (cancelErrorStack != null) { + message += '\n\n$cancelErrorStack'; + } + } + print(''); + print(message); + result = new TaskResult.failure(task, revision, message); + } + results.add(result); + section('Task ${task.name} ${result.succeeded ? "succeeded" : "failed"}.'); + } + + return new BuildResult(golemRevision, results); + } +} + +/// All results accumulated from a build session. +class BuildResult { + BuildResult(this.golemRevision, this.results); + + final int golemRevision; + + /// Individual task results. + final List results; + + /// Whether the overall build failed. + /// + /// We consider the build as failed if at least one task fails. + bool get failed => results.any((TaskResult res) => res.failed); + + /// The opposite of [failed], i.e. all tasks succeeded. + bool get succeeded => !failed; + + /// The number of failed tasks. + int get failedTaskCount => results.fold(0, (int previous, TaskResult res) => previous + (res.failed ? 1 : 0)); +} + +/// A result of running a single task. +class TaskResult { + + /// Constructs a successful result. + TaskResult.success(this.task, this.revision, this.data) + : this.succeeded = true, + this.message = 'success'; + + /// Constructs an unsuccessful result. + TaskResult.failure(this.task, this.revision, this.message) + : this.succeeded = false, + this.data = null; + + /// The task that was run. + final Task task; + + /// Whether the task succeeded. + final bool succeeded; + + /// The revision of Flutter repo at which this result was generated. + final String revision; + + /// Data generated by the task that will be uploaded to Firebase. + final TaskResultData data; + + /// Whether the task failed. + bool get failed => !succeeded; + + /// Explains the result in a human-readable format. + final String message; +} + +/// Data generated by the task that will be uploaded to Firebase. +class TaskResultData { + TaskResultData(this.json, {this.benchmarkScoreKeys}) { + const JsonEncoder prettyJson = const JsonEncoder.withIndent(' '); + if (benchmarkScoreKeys != null) { + for (String key in benchmarkScoreKeys) { + if (!json.containsKey(key)) { + throw 'Invalid Golem score key "$key". It does not exist in task result data ${prettyJson.convert(json)}'; + } + } + } + } + + factory TaskResultData.fromFile(File file, {List benchmarkScoreKeys}) { + return new TaskResultData(JSON.decode(file.readAsStringSync()), benchmarkScoreKeys: benchmarkScoreKeys); + } + + /// Task-specific JSON data + final Map json; + + /// Keys in [json] that store scores that will be submitted to Golem. + /// + /// Each key is also part of a benchmark's name tracked by Golem. + /// A benchmark name is computed by combining [Task.name] with a key + /// separated by a dot. For example, if a task's name is + /// `"complex_layout__start_up"` and score key is + /// `"engineEnterTimestampMicros"`, the score will be submitted to Golem under + /// `"complex_layout__start_up.engineEnterTimestampMicros"`. + /// + /// This convention reduces the amount of configuration that needs to be done + /// to submit benchmark scores to Golem. + final List benchmarkScoreKeys; +} diff --git a/agent/lib/src/gallery.dart b/agent/lib/src/gallery.dart new file mode 100644 index 0000000000..447aded40f --- /dev/null +++ b/agent/lib/src/gallery.dart @@ -0,0 +1,46 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'adb.dart'; +import 'framework.dart'; +import 'utils.dart'; + +Task createGalleryTransitionTest() => new GalleryTransitionTest(); + +class GalleryTransitionTest extends Task { + GalleryTransitionTest() : super('flutter_gallery__transition_perf'); + + @override + Future run() async { + Adb device = await adb(); + device.unlock(); + Directory galleryDirectory = dir('${config.flutterDirectory.path}/examples/flutter_gallery'); + await inDirectory(galleryDirectory, () async { + await pub('get', onCancel); + await flutter('drive', onCancel, options: [ + '--profile', + '--trace-startup', + '-t', + 'test_driver/transitions_perf.dart', + '-d', + device.deviceId, + ]); + }); + + // Route paths contains slashes, which Firebase doesn't accept in keys, so we + // remove them. + Map original = JSON.decode(file('${galleryDirectory.path}/build/transition_durations.timeline.json').readAsStringSync()); + Map clean = new Map.fromIterable( + original.keys, + key: (String key) => key.replaceAll('/', ''), + value: (String key) => original[key] + ); + + return new TaskResultData(clean); + } +} diff --git a/agent/lib/src/golem.dart b/agent/lib/src/golem.dart new file mode 100644 index 0000000000..e86b706f6b --- /dev/null +++ b/agent/lib/src/golem.dart @@ -0,0 +1,47 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'utils.dart'; + +/// Benchmark names registered in Golem. +/// +/// This list must be in-sync with benchmarks listed in `flutterBenchmarks` +/// variable defined in +/// https://chrome-internal.googlesource.com/golem/+/master/config/benchmarks.dart +const registeredBenchmarkNames = const [ + 'complex_layout__start_up.engineEnterTimestampMicros', + 'complex_layout__start_up.timeToFirstFrameMicros', + 'complex_layout__build.aot_snapshot_build_millis', + 'complex_layout__build.aot_snapshot_size_vmisolate', + 'complex_layout__build.aot_snapshot_size_isolate', + 'complex_layout__build.aot_snapshot_size_instructions', + 'complex_layout__build.aot_snapshot_size_rodata', + 'complex_layout__build.aot_snapshot_size_total', + + 'complex_layout_scroll_perf__timeline_summary.average_frame_build_time_millis', + 'complex_layout_scroll_perf__timeline_summary.missed_frame_build_budget_count', + 'complex_layout_scroll_perf__timeline_summary.worst_frame_build_time_millis', + + 'flutter_gallery__start_up.engineEnterTimestampMicros', + 'flutter_gallery__start_up.timeToFirstFrameMicros', + 'flutter_gallery__build.aot_snapshot_build_millis', + 'flutter_gallery__build.aot_snapshot_size_vmisolate', + 'flutter_gallery__build.aot_snapshot_size_isolate', + 'flutter_gallery__build.aot_snapshot_size_instructions', + 'flutter_gallery__build.aot_snapshot_size_rodata', + 'flutter_gallery__build.aot_snapshot_size_total', +]; + +/// Computes a golem-compliant revision number. +/// +/// Golem does not understand git commit hashes. It needs a monotonically +/// increasing integer number as the revision (Subversion legacy). +Future computeGolemRevision() async { + String gitRevs = await inDirectory(config.flutterDirectory, () async { + return eval('git', ['rev-list', 'origin/master', '--topo-order', '--first-parent', 'origin/master']); + }); + return gitRevs.split('\n').length; +} diff --git a/agent/lib/src/perf_tests.dart b/agent/lib/src/perf_tests.dart new file mode 100644 index 0000000000..b00aa0e0eb --- /dev/null +++ b/agent/lib/src/perf_tests.dart @@ -0,0 +1,145 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert' show JSON; + +import 'adb.dart'; +import 'framework.dart'; +import 'utils.dart'; + +Task createComplexLayoutScrollPerfTest() { + return new PerfTest( + 'complex_layout_scroll_perf__timeline_summary', + '${config.flutterDirectory.path}/dev/benchmarks/complex_layout', + 'test_driver/scroll_perf.dart', + 'complex_layout_scroll_perf' + ); +} + +Task createFlutterGalleryStartupTest() { + return new StartupTest('flutter_gallery__start_up', '${config.flutterDirectory.path}/examples/flutter_gallery'); +} + +Task createComplexLayoutStartupTest() { + return new StartupTest('complex_layout__start_up', '${config.flutterDirectory.path}/dev/benchmarks/complex_layout'); +} + +Task createFlutterGalleryBuildTest() { + return new BuildTest('flutter_gallery__build', '${config.flutterDirectory.path}/examples/flutter_gallery'); +} + +Task createComplexLayoutBuildTest() { + return new BuildTest('complex_layout__build', '${config.flutterDirectory.path}/dev/benchmarks/complex_layout'); +} + +/// Measure application startup performance. +class StartupTest extends Task { + + StartupTest(String name, this.testDirectory) : super(name); + + final String testDirectory; + + Future run() async { + return await inDirectory(testDirectory, () async { + Adb device = await adb(); + device.unlock(); + await pub('get', onCancel); + await flutter('run', onCancel, options: [ + '--profile', + '--trace-startup', + '-d', + device.deviceId + ]); + Map data = JSON.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync()); + return new TaskResultData(data, benchmarkScoreKeys: [ + 'engineEnterTimestampMicros', + 'timeToFirstFrameMicros', + ]); + }); + } +} + +/// Measures application runtime performance, specifically per-frame +/// performance. +class PerfTest extends Task { + + PerfTest(String name, this.testDirectory, this.testTarget, this.timelineFileName) + : super(name); + + final String testDirectory; + final String testTarget; + final String timelineFileName; + + @override + Future run() { + return inDirectory(testDirectory, () async { + Adb device = await adb(); + device.unlock(); + await pub('get', onCancel); + await flutter('drive', onCancel, options: [ + '--profile', + '--trace-startup', // Enables "endless" timeline event buffering. + '-t', + testTarget, + '-d', + device.deviceId + ]); + Map data = JSON.decode(file('$testDirectory/build/${timelineFileName}.timeline_summary.json').readAsStringSync()); + return new TaskResultData(data, benchmarkScoreKeys: [ + 'average_frame_build_time_millis', + 'worst_frame_build_time_millis', + 'missed_frame_build_budget_count', + ]); + }); + } +} + + +class BuildTest extends Task { + + BuildTest(String name, this.testDirectory) : super(name); + + final String testDirectory; + + Future run() async { + return await inDirectory(testDirectory, () async { + Adb device = await adb(); + device.unlock(); + await pub('get', onCancel); + + var watch = new Stopwatch()..start(); + await flutter('build', onCancel, options: [ + 'aot', + '--profile', + '--no-pub', + '--target-platform', 'android-arm' // Generate blobs instead of assembly. + ]); + watch.stop(); + + var vmisolateSize = file("$testDirectory/build/aot/snapshot_aot_vmisolate").lengthSync(); + var isolateSize = file("$testDirectory/build/aot/snapshot_aot_isolate").lengthSync(); + var instructionsSize = file("$testDirectory/build/aot/snapshot_aot_instr").lengthSync(); + var rodataSize = file("$testDirectory/build/aot/snapshot_aot_rodata").lengthSync(); + var totalSize = vmisolateSize + isolateSize + instructionsSize + rodataSize; + + Map data = { + 'aot_snapshot_build_millis': watch.elapsedMilliseconds, + 'aot_snapshot_size_vmisolate': vmisolateSize, + 'aot_snapshot_size_isolate': isolateSize, + 'aot_snapshot_size_instructions': instructionsSize, + 'aot_snapshot_size_rodata': rodataSize, + 'aot_snapshot_size_total': totalSize, + }; + return new TaskResultData(data, benchmarkScoreKeys: [ + 'aot_snapshot_build_millis', + 'aot_snapshot_size_vmisolate', + 'aot_snapshot_size_isolate', + 'aot_snapshot_size_instructions', + 'aot_snapshot_size_rodata', + 'aot_snapshot_size_total', + ]); + }); + } +} diff --git a/agent/lib/src/refresh.dart b/agent/lib/src/refresh.dart new file mode 100644 index 0000000000..056945b913 --- /dev/null +++ b/agent/lib/src/refresh.dart @@ -0,0 +1,73 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; + +import 'package:path/path.dart' as path; + +import 'adb.dart'; +import 'benchmarks.dart'; +import 'framework.dart'; +import 'utils.dart'; + +Task createRefreshTest({ + String commit, + DateTime timestamp +}) => new EditRefreshTask(commit, timestamp); + +class EditRefreshTask extends Task { + EditRefreshTask(this.commit, this.timestamp) + : super('mega_gallery__refresh_time') { + assert(commit != null); + assert(timestamp != null); + } + + final String commit; + final DateTime timestamp; + + @override + Future run() async { + Adb device = await adb(); + device.unlock(); + Benchmark benchmark = new EditRefreshBenchmark(commit, timestamp, onCancel); + section(benchmark.name); + await runBenchmark(benchmark, iterations: 3, warmUpBenchmark: true); + return benchmark.bestResult; + } +} + +class EditRefreshBenchmark extends Benchmark { + EditRefreshBenchmark(this.commit, this.timestamp, Future onCancel) + : super('edit refresh', onCancel); + + final String commit; + final DateTime timestamp; + + Directory get megaDir => dir(path.join(config.flutterDirectory.path, 'dev/benchmarks/mega_gallery')); + File get benchmarkFile => file(path.join(megaDir.path, 'refresh_benchmark.json')); + + @override + TaskResultData get lastResult => new TaskResultData.fromFile(benchmarkFile); + + Future init() { + return inDirectory(config.flutterDirectory, () async { + await dart(['dev/tools/mega_gallery.dart'], onCancel); + }); + } + + @override + Future run() async { + Adb device = await adb(); + rm(benchmarkFile); + int exitCode = await inDirectory(megaDir, () async { + return await flutter( + 'run', onCancel, options: ['-d', device.deviceId, '--resident', '--benchmark'], canFail: true + ); + }); + if (exitCode != 0) + return new Future.error(exitCode); + return addBuildInfo(benchmarkFile, timestamp: timestamp, expected: 200, commit: commit); + } +} diff --git a/agent/lib/src/size_tests.dart b/agent/lib/src/size_tests.dart new file mode 100644 index 0000000000..e398a30c67 --- /dev/null +++ b/agent/lib/src/size_tests.dart @@ -0,0 +1,46 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; + +import 'framework.dart'; +import 'utils.dart'; + +Task createBasicMaterialAppSizeTest() => new BasicMaterialAppSizeTest(); + +class BasicMaterialAppSizeTest extends Task { + BasicMaterialAppSizeTest() : super('basic_material_app__size'); + + @override + Future run() async { + const sampleAppName = 'sample_flutter_app'; + Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName'); + + if (await sampleDir.exists()) + rrm(sampleDir); + + int apkSizeInBytes; + + await inDirectory(Directory.systemTemp, () async { + await flutter('create', onCancel, options: [sampleAppName]); + + if (!(await sampleDir.exists())) + throw 'Failed to create sample Flutter app in ${sampleDir.path}'; + + await inDirectory(sampleDir, () async { + await pub('get', onCancel); + await flutter('build', onCancel, options: ['clean']); + await flutter('build', onCancel, options: ['apk', '--release']); + apkSizeInBytes = await file('${sampleDir.path}/build/app.apk').length(); + }); + }); + + return new TaskResultData({ + 'release_size_in_bytes': apkSizeInBytes + }, benchmarkScoreKeys: [ + 'release_size_in_bytes' + ]); + } +} diff --git a/agent/lib/src/utils.dart b/agent/lib/src/utils.dart new file mode 100644 index 0000000000..4a5c2adc39 --- /dev/null +++ b/agent/lib/src/utils.dart @@ -0,0 +1,425 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as path; +import 'package:yaml/yaml.dart'; + +/// Virtual current working directory, which affect functions, such as [exec]. +String cwd = Directory.current.path; + +Config _config; +Config get config => _config; + +class BuildFailedError extends Error { + BuildFailedError(this.message); + + final String message; + + @override + String toString() => message; +} + +void fail(String message) { + throw new BuildFailedError(message); +} + +void rm(FileSystemEntity entity) { + if (entity.existsSync()) + entity.deleteSync(); +} + +/// Remove recursively. +void rrm(FileSystemEntity entity) { + if (entity.existsSync()) + entity.deleteSync(recursive: true); +} + +List ls(Directory directory) => directory.listSync(); + +Directory dir(String path) => new Directory(path); + +File file(String path) => new File(path); + +void copy(File sourceFile, Directory targetDirectory, { String name }) { + File target = file(path.join(targetDirectory.path, name ?? path.basename(sourceFile.path))); + target.writeAsBytesSync(sourceFile.readAsBytesSync()); +} + +FileSystemEntity move(FileSystemEntity whatToMove, { Directory to, String name }) { + return whatToMove.renameSync(path.join(to.path, name ?? path.basename(whatToMove.path))); +} + +/// Equivalent of `mkdir directory`. +void mkdir(Directory directory) { + directory.createSync(); +} + +/// Equivalent of `mkdir -p directory`. +void mkdirs(Directory directory) { + directory.createSync(recursive: true); +} + +bool exists(FileSystemEntity entity) => entity.existsSync(); + +void section(String title) { + print(''); + print('••• $title •••'); +} + +Future getDartVersion() async { + // The Dart VM returns the version text to stderr. + ProcessResult result = Process.runSync(dartBin, ['--version']); + String version = result.stderr.trim(); + + // Convert: + // Dart VM version: 1.17.0-dev.2.0 (Tue May 3 12:14:52 2016) on "macos_x64" + // to: + // 1.17.0-dev.2.0 + if (version.indexOf('(') != -1) + version = version.substring(0, version.indexOf('(')).trim(); + if (version.indexOf(':') != -1) + version = version.substring(version.indexOf(':') + 1).trim(); + + return version.replaceAll('"', "'"); +} + +Future getCurrentFlutterRepoCommit() { + if (!dir('${config.flutterDirectory.path}/.git').existsSync()) { + return null; + } + + return inDirectory(config.flutterDirectory, () { + return eval('git', ['rev-parse', 'HEAD']); + }); +} + +Future getFlutterRepoCommitTimestamp(String commit) { + // git show -s --format=%at 4b546df7f0b3858aaaa56c4079e5be1ba91fbb65 + return inDirectory(config.flutterDirectory, () async { + String unixTimestamp = await eval('git', ['show', '-s', '--format=%at', commit]); + int secondsSinceEpoch = int.parse(unixTimestamp); + return new DateTime.fromMillisecondsSinceEpoch(secondsSinceEpoch * 1000); + }); +} + +Future startProcess(String executable, List arguments, + {Map env, Future onKill}) async { + Process proc = await Process.start(executable, arguments, environment: env, workingDirectory: cwd); + + if (onKill != null) { + bool processExited = false; + + proc.exitCode.then((_) { + processExited = true; + }); + + onKill.then((_) { + if (!processExited) { + print('Caught signal to kill process (PID: ${proc.pid}): $executable ${arguments.join(' ')}'); + bool killed = proc.kill(ProcessSignal.SIGKILL); + print('Process ${killed ? "was killed successfully" : "could not be killed"}.'); + } + }); + } + + return proc; +} + +/// Executes a command and returns its exit code. +Future exec(String executable, List arguments, + {Map env, bool canFail: false, Future onKill}) async { + print('Executing: $executable ${arguments.join(' ')}'); + Process proc = await startProcess(executable, arguments, env: env, onKill: onKill); + + proc.stdout + .transform(UTF8.decoder) + .transform(const LineSplitter()) + .listen(print); + proc.stderr + .transform(UTF8.decoder) + .transform(const LineSplitter()) + .listen(stderr.writeln); + + int exitCode = await proc.exitCode; + + if (exitCode != 0 && !canFail) + fail('Executable failed with exit code ${exitCode}.'); + + return exitCode; +} + +/// Executes a command and returns its standard output as a String. +Future eval(String executable, List arguments, + {Map env, bool canFail: false, Future onKill}) async { + print('Executing: $executable ${arguments.join(' ')}'); + Process proc = await startProcess(executable, arguments, env: env, onKill: onKill); + stderr.addStream(proc.stderr); + String output = await UTF8.decodeStream(proc.stdout); + int exitCode = await proc.exitCode; + + if (exitCode != 0 && !canFail) + fail('Executable failed with exit code ${exitCode}.'); + + return output.trimRight(); +} + +Future flutter(String command, Future onKill, {List options: const[], bool canFail: false}) { + if (onKill == null) { + throw 'flutter command must obey onKill signal'; + } + + List args = [command] + ..addAll(options); + return exec(path.join(config.flutterDirectory.path, 'bin', 'flutter'), args, canFail: canFail, onKill: onKill); +} + +String get dartBin => path.join(config.flutterDirectory.path, 'bin/cache/dart-sdk/bin/dart'); + +Future dart(List args, Future onKill) => exec(dartBin, args, onKill: onKill); + +Future pub(String command, Future onKill) { + return exec( + path.join(config.flutterDirectory.path, 'bin/cache/dart-sdk/bin/pub'), + [command], + onKill: onKill + ); +} + +Future inDirectory(dynamic directory, Future action()) async { + String previousCwd = cwd; + try { + cd(directory); + return await action(); + } finally { + cd(previousCwd); + } +} + +void cd(dynamic directory) { + Directory d; + if (directory is String) { + cwd = directory; + d = dir(directory); + } else if (directory is Directory) { + cwd = directory.path; + d = directory; + } else { + throw 'Unsupported type ${directory.runtimeType} of $directory'; + } + + if (!d.existsSync()) + throw 'Cannot cd into directory that does not exist: $directory'; +} + +class Config { + Config({ + this.baseCocoonUrl, + this.agentId, + this.firebaseFlutterDashboardToken, + this.authToken, + this.flutterDirectory + }); + + static final ArgParser _argParser = new ArgParser() + ..addOption( + 'config-file', + abbr: 'c', + defaultsTo: 'config.yaml' + ); + + static void initialize(List rawArgs) { + ArgResults args = _argParser.parse(rawArgs); + File agentConfigFile = file(args['config-file']); + + if (!agentConfigFile.existsSync()) { + throw ('Agent config file not found: ${agentConfigFile.path}.'); + } + + Map agentConfig = loadYaml(agentConfigFile.readAsStringSync()); + String baseCocoonUrl = agentConfig['base_cocoon_url'] ?? 'https://flutter-dashboard.appspot.com'; + String agentId = requireConfigProperty(agentConfig, 'agent_id'); + String firebaseFlutterDashboardToken = requireConfigProperty(agentConfig, 'firebase_flutter_dashboard_token'); + String authToken = requireConfigProperty(agentConfig, 'auth_token'); + Directory flutterDirectory = dir('${Platform.environment['HOME']}/.cocoon/flutter'); + mkdirs(flutterDirectory); + + _config = new Config( + baseCocoonUrl: baseCocoonUrl, + agentId: agentId, + firebaseFlutterDashboardToken: firebaseFlutterDashboardToken, + authToken: authToken, + flutterDirectory: flutterDirectory + ); + } + + final String baseCocoonUrl; + final String agentId; + final String firebaseFlutterDashboardToken; + final String authToken; + final Directory flutterDirectory; + + String get adbPath { + String androidHome = Platform.environment['ANDROID_HOME']; + + if (androidHome == null) + throw 'ANDROID_HOME environment variable missing. This variable must ' + 'point to the Android SDK directory containing platform-tools.'; + + File adbPath = file(path.join(androidHome, 'platform-tools/adb')); + + if (!adbPath.existsSync()) + throw 'adb not found at: $adbPath'; + + return adbPath.absolute.path; + } + + @override + String toString() => +''' +baseCocoonUrl: $baseCocoonUrl +agentId: $agentId +flutterDirectory: $flutterDirectory +adbPath: $adbPath +'''.trim(); +} + +String requireEnvVar(String name) { + String value = Platform.environment[name]; + + if (value == null) + fail('${name} environment variable is missing. Quitting.'); + + return value; +} + +dynamic/*=T*/ requireConfigProperty(Map*/> map, String propertyName) { + if (!map.containsKey(propertyName)) + fail('Configuration property not found: $propertyName'); + + return map[propertyName]; +} + +String jsonEncode(dynamic data) { + return new JsonEncoder.withIndent(' ').convert(data) + '\n'; +} + +Future getFlutter(String revision) async { + section('Get Flutter!'); + + if (exists(dir('${config.flutterDirectory.path}/.git'))) { + bool hasLocalChanges = await inDirectory(config.flutterDirectory, () async { + String unstagedChanges = await eval('git', ['diff', '--numstat']); + String stagedChanges = await eval('git', ['diff', '--numstat', '--cached']); + return unstagedChanges.trim().isNotEmpty || stagedChanges.trim().isNotEmpty; + }); + + if (hasLocalChanges) { + section('WARNING'); + print( + 'Pending changes detected in the local Flutter repo. Will skip syncing ' + 'Flutter repo. The build will continue but it will marked as failed.' + ); + return false; + } + rrm(config.flutterDirectory); + } + + Future timeout = new Future.delayed(const Duration(minutes: 10)); + + await inDirectory(config.flutterDirectory.parent, () async { + await exec('git', ['clone', 'https://github.com/flutter/flutter.git'], onKill: timeout); + }); + + await inDirectory(config.flutterDirectory, () async { + await exec('git', ['checkout', revision], onKill: timeout); + }); + + await flutter('config', timeout, options: ['--no-analytics']); + + section('flutter doctor'); + await flutter('doctor', timeout); + + section('flutter update-packages'); + await flutter('update-packages', timeout); + return true; +} + +void checkNotNull(Object o1, [Object o2 = 1, Object o3 = 1, Object o4 = 1, + Object o5 = 1, Object o6 = 1, Object o7 = 1, Object o8 = 1, Object o9 = 1, Object o10 = 1]) { + if (o1 == null) + throw 'o1 is null'; + + if (o2 == null) + throw 'o2 is null'; + + if (o3 == null) + throw 'o3 is null'; + + if (o4 == null) + throw 'o4 is null'; + + if (o5 == null) + throw 'o5 is null'; + + if (o6 == null) + throw 'o6 is null'; + + if (o7 == null) + throw 'o7 is null'; + + if (o8 == null) + throw 'o8 is null'; + + if (o9 == null) + throw 'o9 is null'; + + if (o10 == null) + throw 'o10 is null'; +} + +/// Add benchmark values to a JSON results file. +/// +/// If the file contains information about how long the benchmark took to run +/// (a `time` field), then return that info. +// TODO(yjbanov): move this data to __metadata__ +num addBuildInfo(File jsonFile, { + num expected, + String sdk, + String commit, + DateTime timestamp +}) { + Map json; + + if (jsonFile.existsSync()) + json = JSON.decode(jsonFile.readAsStringSync()); + else + json = {}; + + if (expected != null) + json['expected'] = expected; + if (sdk != null) + json['sdk'] = sdk; + if (commit != null) + json['commit'] = commit; + if (timestamp != null) + json['timestamp'] = timestamp.millisecondsSinceEpoch; + + jsonFile.writeAsStringSync(jsonEncode(json)); + + // Return the elapsed time of the benchmark (if any). + return json['time']; +} + +/// Splits [from] into lines and selects those that contain [pattern]. +Iterable grep(Pattern pattern, {@required String from}) { + return from.split('\n').where((String line) { + return line.contains(pattern); + }); +} diff --git a/agent/pubspec.yaml b/agent/pubspec.yaml new file mode 100644 index 0000000000..e07d96e722 --- /dev/null +++ b/agent/pubspec.yaml @@ -0,0 +1,29 @@ +name: cocoon_agent +version: 0.0.1 +author: Flutter Authors +description: Scripts that run Cocoon checklist tasks +homepage: https://github.com/flutter/cocoon + +environment: + sdk: '>=1.12.0 <2.0.0' + +dependencies: + args: ^0.13.0 + browser: ^0.10.0 + charted: ^0.4.0 + dart_to_js_script_rewriter: ^1.0.0 + firebase: ^0.6.6 + firebase_rest: ^0.1.3 + http: ^0.11.3 + intl: ^0.12.7 + meta: ^0.12.1 + path: ^1.3.0 + stack_trace: ^1.6.5 + yaml: ^2.1.10 + +dev_dependencies: + test: ^0.12.13 + collection: ^1.8.0 + +transformers: + - dart_to_js_script_rewriter diff --git a/agent/test/src/adb_test.dart b/agent/test/src/adb_test.dart new file mode 100644 index 0000000000..65118210d5 --- /dev/null +++ b/agent/test/src/adb_test.dart @@ -0,0 +1,190 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:test/test.dart'; +import 'package:collection/collection.dart'; + +import 'package:cocoon_agent/src/adb.dart'; + +main() { + group('adb', () { + Adb device; + + setUp(() { + FakeAdb.resetLog(); + adb = null; + device = new FakeAdb(); + }); + + tearDown(() { + adb = realAdbGetter; + }); + + group('isAwake/isAsleep', () { + test('reads Awake', () async { + FakeAdb.pretendAwake(); + expect(await device.isAwake(), isTrue); + expect(await device.isAsleep(), isFalse); + }); + + test('reads Asleep', () async { + FakeAdb.pretendAsleep(); + expect(await device.isAwake(), isFalse); + expect(await device.isAsleep(), isTrue); + }); + }); + + group('togglePower', () { + test('sends power event', () async { + await device.togglePower(); + expectLog([ + cmd(command: 'input', arguments: ['keyevent', '26']), + ]); + }); + }); + + group('wakeUp', () { + test('when awake', () async { + FakeAdb.pretendAwake(); + await device.wakeUp(); + expectLog([ + cmd(command: 'dumpsys', arguments: ['power']), + ]); + }); + + test('when asleep', () async { + FakeAdb.pretendAsleep(); + await device.wakeUp(); + expectLog([ + cmd(command: 'dumpsys', arguments: ['power']), + cmd(command: 'input', arguments: ['keyevent', '26']), + ]); + }); + }); + + group('sendToSleep', () { + test('when asleep', () async { + FakeAdb.pretendAsleep(); + await device.sendToSleep(); + expectLog([ + cmd(command: 'dumpsys', arguments: ['power']), + ]); + }); + + test('when awake', () async { + FakeAdb.pretendAwake(); + await device.sendToSleep(); + expectLog([ + cmd(command: 'dumpsys', arguments: ['power']), + cmd(command: 'input', arguments: ['keyevent', '26']), + ]); + }); + }); + + group('unlock', () { + test('sends unlock event', () async { + FakeAdb.pretendAwake(); + await device.unlock(); + expectLog([ + cmd(command: 'dumpsys', arguments: ['power']), + cmd(command: 'input', arguments: ['keyevent', '82']), + ]); + }); + }); + }); +} + +void expectLog(List log) { + expect(FakeAdb.commandLog, log); +} + +CommandArgs cmd({String command, List arguments, Map env}) => new CommandArgs( + command: command, + arguments: arguments, + env: env +); + +typedef dynamic ExitErrorFactory(); + +class CommandArgs { + CommandArgs({this.command, this.arguments, this.env}); + + final String command; + final List arguments; + final Map env; + + @override + String toString() => 'CommandArgs(command: $command, arguments: $arguments, env: $env)'; + + @override + operator==(Object other) { + if (other.runtimeType != CommandArgs) + return false; + + CommandArgs otherCmd = other; + return otherCmd.command == this.command && + const ListEquality().equals(otherCmd.arguments, this.arguments) && + const MapEquality().equals(otherCmd.env, this.env); + } + + @override + int get hashCode => 17 * (17 * command.hashCode + _hashArguments) + _hashEnv; + + int get _hashArguments => arguments != null + ? const ListEquality().hash(arguments) + : null.hashCode; + + int get _hashEnv => env != null + ? const MapEquality().hash(env) + : null.hashCode; +} + +class FakeAdb extends Adb { + FakeAdb({String deviceId: null}) : super(deviceId: deviceId); + + static String output = ''; + static ExitErrorFactory exitErrorFactory = () => null; + + static List commandLog = []; + + static void resetLog() { + commandLog.clear(); + } + + static void pretendAwake() { + output = ''' + mWakefulness=Awake + '''; + } + + static void pretendAsleep() { + output = ''' + mWakefulness=Asleep + '''; + } + + @override + Future shellEval(String command, List arguments, {Map env}) async { + commandLog.add(new CommandArgs( + command: command, + arguments: arguments, + env: env + )); + return output; + } + + @override + Future shellExec(String command, List arguments, {Map env}) async { + commandLog.add(new CommandArgs( + command: command, + arguments: arguments, + env: env + )); + dynamic exitError = exitErrorFactory(); + if (exitError != null) + throw exitError; + } +} diff --git a/agent/test/src/utils_test.dart b/agent/test/src/utils_test.dart new file mode 100644 index 0000000000..18b1916623 --- /dev/null +++ b/agent/test/src/utils_test.dart @@ -0,0 +1,18 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:test/test.dart'; +import 'package:cocoon_agent/src/utils.dart'; + +main() { + group('grep', () { + test('greps lines', () { + expect(grep('b', from: 'ab\ncd\nba'), ['ab', 'ba']); + }); + + test('understands RegExp', () { + expect(grep(new RegExp('^b'), from: 'ab\nba'), ['ba']); + }); + }); +} diff --git a/app/web/build.html b/app/web/build.html new file mode 100644 index 0000000000..d560df72f1 --- /dev/null +++ b/app/web/build.html @@ -0,0 +1,35 @@ + + + + + + + + + Flutter Build History + + + + + + + +
+ +
+ + + + + diff --git a/app/web/styles.css b/app/web/buildStyles.css similarity index 94% rename from app/web/styles.css rename to app/web/buildStyles.css index c035e035d1..96d0b47491 100644 --- a/app/web/styles.css +++ b/app/web/buildStyles.css @@ -290,7 +290,8 @@ td.stats-value { background-color: #CCCCFF; } .task-in-progress { - background-color: blue; + background: linear-gradient(0deg, blue 0%, blue 40%, white 41%, white 59%, blue 60%, blue 100%);; + animation: inProgressAnimation 2s infinite linear; } .task-succeeded { background-color: green; @@ -307,3 +308,12 @@ td.stats-value { .task-unknown { background-color: #6600cc; } + +@keyframes inProgressAnimation { + 0% { + transform: rotateZ(0deg); + } + 100% { + transform: rotateZ(180deg); + } +} diff --git a/app/web/buildbot.js b/app/web/buildbot.js new file mode 100644 index 0000000000..f2d41bf01d --- /dev/null +++ b/app/web/buildbot.js @@ -0,0 +1,106 @@ +// Globally visible list of current build statuses encoded as: +// +// { +// "Mac": true, +// "Linux": false +// } +var buildStatuses = {}; + +function allBuildsGreen() { + var allGreen = true; + for (var builderName in buildStatuses) { + if (buildStatuses.hasOwnProperty(builderName)) { + allGreen = allGreen && buildStatuses[builderName] === true; + } + } + return allGreen; +} + +(function() { + const url = 'https://build.chromium.org/p/client.flutter/json/builders/'; + + function getBuildStatus(builderName) { + var urlWithBuilder = url + builderName + '/'; + + return fetch(urlWithBuilder + 'builds').then(function(response){ + if (response.status !== 200) { + console.error('Error status listing builds: ' + response.status); + return Promise.reject(new Error(response.statusText)); + } + + return response.json(); + }).then(function(data) { + var keys = Object.keys(data); + var latest = keys[keys.length-1]; + return Promise.resolve(latest); + }).then(function(latestBuildNum) { + return Promise.resolve(fetch(urlWithBuilder + 'builds/' + latestBuildNum)); + }).then(function(response) { + if (response.status !== 200) { + console.error('Error status retrieving build info: ' + response.status); + return Promise.reject(new Error(response.statusText)); + } + + return response.json(); + }).then(function(data) { + var isSuccessful = data['text'] && data['text'][1] === 'successful'; + var elem = document.querySelector('#buildbot-' + builderName.toLowerCase().replace(' ', '-') + '-status'); + buildStatuses[builderName] = isSuccessful; + if (isSuccessful) { + elem.classList.remove('buildbot-sad'); + elem.classList.add('buildbot-happy'); + } else { + elem.classList.remove('buildbot-happy'); + elem.classList.add('buildbot-sad'); + } + }).catch(function(err) { + console.error(err); + }); + } + + function subscribeToDashboardStatus() { + buildStatuses['dashboard'] = null; + whenFirebaseReady.then(function(ref) { + ref.child('measurements').on("value", function(snapshot) { + var status = snapshot.child('dashboard_bot_status').child('current').val(); + buildStatuses['dashboard'] = (status.success === true); + }, function (errorObject) { + console.log("The read failed: " + errorObject.code); + }); + }); + } + + function refreshAllBuildStatuses() { + Promise.all([ + getBuildStatus('Linux'), + getBuildStatus('Linux Engine'), + getBuildStatus('Mac'), + getBuildStatus('Mac Engine') + ]).then(function() { + var dashboardStatusSpan = document.querySelector('#dashboard-status'); + var dashboardLogLink = document.querySelector('#dashboard-log-link'); + dashboardStatusSpan.classList.remove('buildbot-sad'); + if (allBuildsGreen()) { + // Show dashboard status green iff it and all other builds are green. + dashboardLogLink.style.color = '#4CAF50'; + } else if (buildStatuses['dashboard'] === false) { + // The dashboard is explicitly broken. Go into the broken build mode. + dashboardLogLink.style.color = 'red'; + dashboardStatusSpan.classList.add('buildbot-sad'); + } else { + // If one of the buildbots is red the dashboard status is irrelevant. + // Showing it red won't add any useful information, and showing it green + // is misleading. + dashboardLogLink.style.color = '#DDD'; + } + + setTimeout(refreshAllBuildStatuses, 5 * 1000); + }, function() { + // Schedule a fetch even if the previous one fails, but wait a little longer + setTimeout(refreshAllBuildStatuses, 60 * 1000); + }); + } + + subscribeToDashboardStatus(); + refreshAllBuildStatuses(); +})(); diff --git a/app/web/charts.dart b/app/web/charts.dart new file mode 100644 index 0000000000..e8232dd5d8 --- /dev/null +++ b/app/web/charts.dart @@ -0,0 +1,286 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:html' hide Event; + +import 'package:charted/charts/charts.dart'; +import 'package:firebase/firebase.dart'; + +Firebase firebase; +CartesianArea analysisChartArea; +CartesianArea dartdocChartArea; +CartesianArea refreshChartArea; + +Map repoMeasurements = {}; +Map galleryMeasurements = {}; +Map refreshMeasurements = {}; + +void main() { + _updateAnalysisChart(); + _updateDartdocChart(); + _updateRefreshChart(); + + firebase = new Firebase("https://purple-butterfly-3000.firebaseio.com/"); + firebase.onAuth().listen((context) { + _listenForChartChanges(); + }); +} + +void _listenForChartChanges() { + Firebase repoAnalysis = firebase.child("measurements/analyzer_cli__analysis_time/history"); + Firebase galleryAnalysis = firebase.child("measurements/analyzer_server__analysis_time/history"); + Firebase refreshTimes = firebase.child("measurements/mega_gallery__refresh_time/history"); + + DateTime startDate = new DateTime.now().subtract(new Duration(days: 90)); + + Query repoQuery = repoAnalysis + .orderByChild('timestamp') + .startAt(key: 'timestamp', value: startDate.millisecondsSinceEpoch) + .limitToLast(2000); + Query galleryQuery = galleryAnalysis + .orderByChild('timestamp') + .startAt(key: 'timestamp', value: startDate.millisecondsSinceEpoch) + .limitToLast(2000); + Query refreshQuery = refreshTimes + .orderByChild('timestamp') + .startAt(key: 'timestamp', value: startDate.millisecondsSinceEpoch) + .limitToLast(2000); + + repoQuery.onValue.listen((Event event) { + repoMeasurements = {}; + event.snapshot.forEach((DataSnapshot snapshot) { + Measurement measurement = new Measurement(snapshot.val()); + if (measurement.timestampMillis != null) { + repoMeasurements[measurement.timestampMillis] = measurement; + } + }); + _updateCharts(); + }); + galleryQuery.onValue.listen((Event event) { + galleryMeasurements = {}; + event.snapshot.forEach((DataSnapshot snapshot) { + Measurement measurement = new Measurement(snapshot.val()); + if (measurement.timestampMillis != null) { + galleryMeasurements[measurement.timestampMillis] = measurement; + } + }); + _updateCharts(); + }); + refreshQuery.onValue.listen((Event event) { + refreshMeasurements = {}; + event.snapshot.forEach((DataSnapshot snapshot) { + Measurement measurement = new Measurement(snapshot.val()); + if (measurement.timestampMillis != null) { + refreshMeasurements[measurement.timestampMillis] = measurement; + } + }); + _updateCharts(); + }); +} + +void _updateCharts() { + List times = new List.from(new Set() + ..addAll(repoMeasurements.keys) + ..addAll(galleryMeasurements.keys) + )..sort(); + List analysisData = times.map((int time) { + return [time, repoMeasurements[time]?.time, galleryMeasurements[time]?.time]; + }).toList(); + _updateAnalysisChart(analysisData); + + times = repoMeasurements.keys.toList()..sort(); + List dartdocData = times + .map((int time) => [time, repoMeasurements[time].missingDartDocs]) + .where((List tuple) => tuple[1] != null) + .toList(); + _updateDartdocChart(dartdocData); + + times = refreshMeasurements.keys.toList()..sort(); + List refreshData = times + .map((int time) => [time, refreshMeasurements[time].time]) + .toList(); + _updateRefreshChart(refreshData); +} + +class Measurement { + Measurement(this.map); + + final Map map; + + num get expected => map['expected']; + num get issues => map['issues']; + String get sdk => map['sdk']; + num get time => map['time']; + num get missingDartDocs => map['missingDartDocs']; + num get timestampMillis => map['timestamp']; + + String get commit => map['commit'] is String ? map['commit'] : null; + + DateTime get date => new DateTime.fromMillisecondsSinceEpoch(timestampMillis); + + String get dateString { + DateTime d = date; + return '${d.year.toString().padLeft(4, '0')}-' + '${d.month.toString().padLeft(2, '0')}-' + '${d.day.toString().padLeft(2, '0')}'; + } + + String toString() => '${time}s (${dateString})'; +} + +final List _analysisColumnSpecs = [ + new ChartColumnSpec(label: 'Time', type: ChartColumnSpec.TYPE_TIMESTAMP), + new ChartColumnSpec(label: 'flutter_repo', type: ChartColumnSpec.TYPE_NUMBER, formatter: _printDurationValSeconds), + new ChartColumnSpec(label: 'mega_gallery', type: ChartColumnSpec.TYPE_NUMBER, formatter: _printDurationValSeconds) +]; + +void _updateAnalysisChart([List data = const []]) { + if (data == null || data.length < 2) + data = _createPlaceholderData([0.0, 0.0]); + + if (analysisChartArea == null) { + DivElement chartElement = document.querySelector('#analysis-perf-chart'); + DivElement legendHost = chartElement.querySelector('.chart-legend-host'); + ChartSeries series = new ChartSeries("Flutter Analysis Times", [1, 2], new LineChartRenderer()); + + analysisChartArea = new CartesianArea( + chartElement.querySelector('.chart-host'), + null, + _createChartConfig(legendHost, series) + ); + analysisChartArea.addChartBehavior(new Hovercard(builder: (int columnIndex, int rowIndex) { + List data = analysisChartArea.data.rows.elementAt(rowIndex); + ChartColumnSpec spec = analysisChartArea.data.columns.elementAt(columnIndex); + Measurement measurement = (columnIndex == 1 ? repoMeasurements : galleryMeasurements)[data[0]]; + return _createTooltip(spec, measurement, unitsLabel: 's'); + })); + analysisChartArea.addChartBehavior(new AxisLabelTooltip()); + } + + analysisChartArea.data = new ChartData(_analysisColumnSpecs, data); + analysisChartArea.draw(); +} + +final List _dartdocColumnSpecs = [ + new ChartColumnSpec(label: 'Time', type: ChartColumnSpec.TYPE_TIMESTAMP), + new ChartColumnSpec(label: 'Burndown', type: ChartColumnSpec.TYPE_NUMBER) +]; + +void _updateDartdocChart([List data]) { + if (data == null || data.length < 2) + data = _createPlaceholderData([0]); + + if (dartdocChartArea == null) { + DivElement chartElement = document.querySelector('#documentation-chart'); + DivElement legendHost = chartElement.querySelector('.chart-legend-host'); + ChartSeries series = new ChartSeries('Dartdoc Burndown', [1], new LineChartRenderer()); + + dartdocChartArea = new CartesianArea( + chartElement.querySelector('.chart-host'), + null, + _createChartConfig(legendHost, series) + ); + dartdocChartArea.addChartBehavior(new Hovercard(builder: (int columnIndex, int rowIndex) { + List data = dartdocChartArea.data.rows.elementAt(rowIndex); + ChartColumnSpec spec = dartdocChartArea.data.columns.elementAt(columnIndex); + Measurement measurement = repoMeasurements[data[0]]; + return _createTooltip(spec, measurement, value: measurement.missingDartDocs); + })); + dartdocChartArea.addChartBehavior(new AxisLabelTooltip()); + } + + dartdocChartArea.data = new ChartData(_dartdocColumnSpecs, data); + dartdocChartArea.draw(); +} + +final List _refreshColumnSpecs = [ + new ChartColumnSpec(label: 'Time', type: ChartColumnSpec.TYPE_TIMESTAMP), + new ChartColumnSpec(label: 'Refresh', type: ChartColumnSpec.TYPE_NUMBER, formatter: _printDurationValMillis) +]; + +void _updateRefreshChart([List data = const []]) { + if (data == null || data.length < 2) + data = _createPlaceholderData([0]); + + if (refreshChartArea == null) { + DivElement chartElement = document.querySelector('#refresh-perf-chart'); + DivElement legendHost = chartElement.querySelector('.chart-legend-host'); + ChartSeries series = new ChartSeries("Edit Refresh Times", [1], new LineChartRenderer()); + + refreshChartArea = new CartesianArea( + chartElement.querySelector('.chart-host'), + null, + _createChartConfig(legendHost, series) + ); + refreshChartArea.addChartBehavior(new Hovercard(builder: (int columnIndex, int rowIndex) { + List data = refreshChartArea.data.rows.elementAt(rowIndex); + ChartColumnSpec spec = refreshChartArea.data.columns.elementAt(columnIndex); + Measurement measurement = refreshMeasurements[data[0]]; + return _createTooltip(spec, measurement, unitsLabel: 'ms'); + })); + refreshChartArea.addChartBehavior(new AxisLabelTooltip()); + } + + refreshChartArea.data = new ChartData(_refreshColumnSpecs, data); + refreshChartArea.draw(); +} + +List _createPlaceholderData(List templateItems) { + DateTime now = new DateTime.now(); + return [ + [now.subtract(new Duration(days: 30)).millisecondsSinceEpoch]..addAll(templateItems), + [now.millisecondsSinceEpoch]..addAll(templateItems), + ]; +} + +ChartConfig _createChartConfig(DivElement legendHost, ChartSeries series) { + ChartConfig config = new ChartConfig([series], [0]); + config.legend = new ChartLegend(legendHost); + return config; +} + +String _printDurationValSeconds(num val) { + if (val == null) return ''; + return val.toStringAsFixed(1) + 's'; +} + +String _printDurationValMillis(num val) { + if (val == null) return ''; + return _formatWithThousandsSeparator(val.toInt()) + 'ms'; +} + +Element _createTooltip(ChartColumnSpec spec, Measurement measurement, { + dynamic value, + String unitsLabel: '' +}) { + Element element = div('', className: 'hovercard-single'); + + if (measurement == null) { + element.text = 'No data'; + } else { + if (value == null) + value = measurement.time; + element.children.add(div(spec.label, className: 'hovercard-title')); + element.children.add(div('$value$unitsLabel', className: 'hovercard-value')); + element.children.add(div('${measurement.date}', className: 'hovercard-value')); + if (measurement.commit != null) + element.children.add(div('(commit ${measurement.commit.substring(0, 10)})', className: 'hovercard-value')); + } + + return element; +} + +DivElement div(String text, { String className }) { + DivElement element = new DivElement()..text = text; + if (className != null) + element.className = className; + return element; +} + +String _formatWithThousandsSeparator(int value) { + String str = value.toString(); + if (str.length > 3) + str = str.substring(0, str.length - 3) + ',' + str.substring(str.length - 3); + return str; +} diff --git a/app/web/charts.html b/app/web/charts.html new file mode 100644 index 0000000000..c711d82222 --- /dev/null +++ b/app/web/charts.html @@ -0,0 +1,120 @@ + + + + + + + + + + + + + Flutter Dashboard + + + + + + + + + + + +
+
+
+
Analysis Times
+
+
+
+
+
+
+
+
+
Dartdoc Burndown
+
+
+
+
+
+
+
+ +
+
+
+
App Size
+
+
+
+
+
+
+
+
+
Edit/Refresh Times
+
+
+
+
+
+
+
+ + + diff --git a/app/web/firebase.js b/app/web/firebase.js new file mode 100644 index 0000000000..57afad938a --- /dev/null +++ b/app/web/firebase.js @@ -0,0 +1,255 @@ +var ref = new Firebase("https://purple-butterfly-3000.firebaseio.com/"); + +var whenFirebaseReady = new Promise(function(resolve, reject) { + ref.onAuth(function(authData) { + if (authData) { + console.log("User " + authData.uid + " is logged in with " + + authData.provider + " and has displayName '" + + authData.google.displayName + "' and email " + authData.google.email); + // Save the user's profile into the database so we can + // use them in Security and Firebase Rules. + // We don't trust this data, so don't rely on the email. + ref.child("users").child(authData.uid).set({ + provider: authData.provider, + name: authData.google.displayName, + email: authData.google.email || 'undefined' + }).then(function(snapshot) { + resolve(ref); + }, function(error) { + console.error(error); + }); + } else { + console.log("User is logged out"); + } + }); +}); + +(function() { + whenFirebaseReady.then(function() { + console.log('Connected to Firebase'); + getData(); + }); + + function authHandler(error, authData) { + if (error) { + console.log("Login Failed!", error); + } else { + console.log("Authenticated successfully with payload:", authData); + } + } + + function getData() { + ref.child('measurements').on("value", function(snapshot) { + removeExistingBoxes(); + generateBoxes(snapshot.val()); + updateLastJobRanTime(snapshot.child('dashboard_bot_status').child('current').val()); + }, function (errorObject) { + console.log("The read failed: " + errorObject.code); + }); + } + + function removeExistingBoxes() { + var boxesFromFirebase = document.querySelectorAll('.from-firebase'); + Array.prototype.forEach.call(boxesFromFirebase, function (box) { + box.parentNode.removeChild(box); + }); + } + + function _cloneTemplate(measurementType) { + var tmplId = '#' + measurementType + '_tmpl'; + var tmpl = document.querySelector(tmplId); + if (tmpl == undefined) { + console.error('Template for ' + tmplId + ' not found'); + return null; + } + var clone = document.importNode(tmpl.content, true); + return clone; + } + + function _getTitleForTemplate(measurementType, measurementName) { + return measurementName.substring(0, measurementName.length-measurementType.length); + } + + function _parseHtml(string) { + var div = document.createElement('div'); + div.innerHTML = string; + return div.firstChild; + } + + function _comma(str) { + if (str.length > 3) + str = str.substring(0, str.length - 3) + ',' + str.substring(str.length - 3); + return str; + } + + var generators = { + '__analysis_time': function(measurementType, measurementName, data) { + var clone = _cloneTemplate(measurementType); + if (clone == null) return; + var title = _getTitleForTemplate(measurementType, measurementName); + var targetPercent = Math.round((data.expected / data.time) * 100); + + clone.querySelector('.metric-number').textContent = parseFloat(data.time).toFixed(1); + clone.querySelector('.metric-name').textContent = title; + clone.querySelector('.metric-target').textContent = data.expected.toFixed(1); + document.querySelector('#container').appendChild(clone); + }, + + '__refresh_time': function(measurementType, measurementName, data) { + var clone = _cloneTemplate(measurementType); + if (clone == null) return; + var title = _getTitleForTemplate(measurementType, measurementName); + var targetPercent = Math.round((data.expected / data.time) * 100); + + clone.querySelector('.metric-number').textContent = _comma(data.time.toString()); + clone.querySelector('.metric-name').textContent = title; + clone.querySelector('.metric-target').textContent = data.expected; + document.querySelector('#container').appendChild(clone); + }, + + '__start_up': function(measurementType, measurementName, data) { + var clone = _cloneTemplate(measurementType); + if (clone == null) return; + var title = _getTitleForTemplate(measurementType, measurementName); + clone.querySelector('.metric-name').textContent = title; + var timeToFirstFrame = _comma((data.timeToFirstFrameMicros / 1000).toFixed(0)); + clone.querySelector('.metric-number').textContent = timeToFirstFrame; + clone.querySelector('.time-to-framework-init').textContent = _comma((data.timeToFrameworkInitMicros / 1000).toFixed(0)); + clone.querySelector('.time-after-init-to-first-frame').textContent = _comma((data.timeAfterFrameworkInitMicros / 1000).toFixed(0)); + + document.querySelector('#container').appendChild(clone); + }, + + '__timeline_summary': function(measurementType, measurementName, data) { + var clone = _cloneTemplate(measurementType); + if (clone == null) return; + var title = _getTitleForTemplate(measurementType, measurementName); + if (title.endsWith('_scroll_perf')) + title = title.substring(0, title.length - '_scroll_perf'.length) + clone.querySelector('.metric-name').textContent = title; + clone.querySelector('.average_frame_build_time_millis').textContent = data.average_frame_build_time_millis.toFixed(2); + clone.querySelector('.frame_count').textContent = data.frame_count; + clone.querySelector('.metric-number').textContent = data.missed_frame_build_budget_count; + + if (data.missed_frame_build_budget_count > 0) { + clone.querySelector('.metric-section').appendChild( + _parseHtml('' + data.missed_frame_build_budget_count + ' missed') + ); + } + + var framesContainer = clone.querySelector('.frames'); + data.frame_build_times.forEach(function (frameBuildTime) { + var div; + if (frameBuildTime > 24000) { + div = _parseHtml('
'); + } else if (frameBuildTime > 8000) { + div = _parseHtml('
'); + } else { + div = _parseHtml('
'); + } + framesContainer.appendChild(div); + }); + + document.querySelector('#container').appendChild(clone); + }, + + '__transition_perf': function(measurementType, measurementName, data) { + let clone = _cloneTemplate(measurementType); + if (clone == null) return; + + let title = _getTitleForTemplate(measurementType, measurementName); + clone.querySelector('.metric-name').textContent = title; + let chart = clone.querySelector('.frames'); + + for (let screenName in data) { + if (data.hasOwnProperty(screenName)) { + if (screenName == '__metadata__') { + continue; + } + let durations = data[screenName]; + durations.forEach(function(duration) { + let div; + if (duration > 300000) { + div = _parseHtml(`
`); + } else if (duration > 100000) { + div = _parseHtml(`
`); + } else { + div = _parseHtml(`
`); + } + chart.appendChild(div); + }); + // Add padding between screens + chart.appendChild(_parseHtml('
')); + } + } + + document.querySelector('#container').appendChild(clone); + }, + + '__size': function(measurementType, measurementName, data) { + var clone = _cloneTemplate(measurementType); + if (clone == null) return; + var title = _getTitleForTemplate(measurementType, measurementName); + clone.querySelector('.metric-name').textContent = title; + // Formats with thousands separator + clone.querySelector('.metric-number').textContent = + Math.round(data.release_size_in_bytes / 1024).toLocaleString('en', { useGrouping: true }); + document.querySelector('#container').appendChild(clone); + } + }; + + var ignoredMeasurements = [ + 'dashboard_bot_status', + 'golem_data' + ]; + + function updateLastJobRanTime(buildData) { + var lastJobRanTime = document.querySelector('#last-job-ran-time'); + var eightHoursInMillis = 8 * 3600 * 1000; + var buildAge = Date.now() - Date.parse(buildData.build_timestamp); + var buildOutOfDateLabel = document.querySelector('#build-out-of-date'); + if (allBuildsGreen() && buildAge > eightHoursInMillis) { + buildOutOfDateLabel.style.display = 'block'; + } else { + buildOutOfDateLabel.style.display = 'none'; + } + + if (lastJobRanTime) { + lastJobRanTime.textContent = buildData.build_timestamp; + } + + var dashboardLogLink = document.querySelector('#dashboard-log-link'); + dashboardLogLink.href = `https://pantheon.corp.google.com/m/cloudstorage/b/flutter-dashboard/o/${ buildData.revision }/output.txt`; + } + + function generateBoxes(measurements) { + for (var measurementName in measurements) { + if (measurements.hasOwnProperty(measurementName) && ignoredMeasurements.indexOf(measurementName) != 0) { + var measurementType = measurementName + .substring(measurementName.indexOf('__'), measurementName.length); + var generator = generators[measurementType]; + if (generator === undefined) { + console.error('WARNING: Did not find generator for ' + measurementName + + ' of type ' + measurementType); + continue; + } + generator(measurementType, measurementName, measurements[measurementName]['current']); + } + } + + updateLastUpdatedTime(); + } + + function updateLastUpdatedTime() { + document.querySelector('#firebase-last-updated-time').textContent = new Date(); + } + + document.getElementById('firebase-login').addEventListener('click', function() { + ref.authWithOAuthPopup("google", authHandler, {'scope': "email"}); + }); + + document.getElementById('firebase-logout').addEventListener('click', function() { + console.log('Logging out'); + ref.unauth(); + }); +})(); diff --git a/app/web/github.js b/app/web/github.js new file mode 100644 index 0000000000..b2855839a6 --- /dev/null +++ b/app/web/github.js @@ -0,0 +1,39 @@ +(function() { + + var ACCESS_TOKEN = '6a93c73f10808b9a92d4ec8a91b9e823c192d204'; + + function getMilestone(milestoneNumber) { + var url = 'https://api.github.com/repos/flutter/flutter/milestones/' + + milestoneNumber + '?access_token=' + ACCESS_TOKEN; + + return fetch(url).then(function(response) { + return response.json(); + }); + } + + function updateLastUpdatedTime() { + document.querySelector('#github-last-updated-time').textContent = new Date(); + } + + function displayMilestone() { + getMilestone(7).then(function(data) { + function daysBetween(d1, d2) { + var start = Math.floor( d1.getTime() / (3600*24*1000)); + var end = Math.floor( d2.getTime() / (3600*24*1000)); + return start - end; + } + + var card = document.querySelector('#github_issues'); + card.querySelector('.metric-number').textContent = data.open_issues; + var targetDue = new Date(data.due_on); + var daysAway = daysBetween(targetDue, new Date()); + card.querySelector('.days-away-number').textContent = daysAway; + + updateLastUpdatedTime(); + + setTimeout(displayMilestone, 60*60*1000); // one hour refresh + }); + } + + displayMilestone(); +})(); \ No newline at end of file diff --git a/app/web/index.html b/app/web/index.html index 746b8ece5b..4eca504113 100644 --- a/app/web/index.html +++ b/app/web/index.html @@ -6,30 +6,71 @@ found in the LICENSE file. --> - + Flutter Dashboard - - - + + - -
- + +
+
- - +
+ +
diff --git a/app/web/status.html b/app/web/status.html new file mode 100644 index 0000000000..1cff898f1c --- /dev/null +++ b/app/web/status.html @@ -0,0 +1,208 @@ + + + + + + + + + Flutter Dashboard + + + + + + + +
+
+
+
Open issues (Gallery)
+
+
days away
+
+
+ +
+
+
Buildbot (status)
+
+ + + + + + + + + + + + + + + + + + + + + + ⬆ + +
+
+
+ +
+ Build out of date! +
+
+ +
+
+
+ + +
+ +
+ Firebase updated + • + Github updated + • + Job ran at +
+ + +
+ + + + + + + + + + + + + + + + + + diff --git a/app/web/statusStyles.css b/app/web/statusStyles.css new file mode 100644 index 0000000000..c88c236778 --- /dev/null +++ b/app/web/statusStyles.css @@ -0,0 +1,243 @@ +/* Copyright (c) 2016 The Chromium Authors. All rights reserved. */ +/* Use of this source code is governed by a BSD-style license that can be */ +/* found in the LICENSE file. */ + +@import url(https://fonts.googleapis.com/css?family=Roboto:300); + +body { + background: #eee; + font: 18px Roboto, sans-serif; + font-weight: 300; + color: #333; + margin: 0; + padding: 10px 10px 38px 10px; +} + +@keyframes flash-broken-build { + 0% { + background-color: #eee; + } + 50% { + background-color: #ff8566; + } + 100% { + background-color: #eee; + } +} + +#container { + display: flex; + flex-flow: row wrap; +} + +span.last-updated { + font-size: 14px; +} + +.card { + margin: 10px; + width: 400px; + background: #f7f7f7; + padding: 20px; + border-radius: 4px; + box-shadow: 0 1px 3px rgba(0,0,0,0.12), 0 1px 2px rgba(0,0,0,0.24); + text-align: center; + + /* center the inside contents */ + display: flex; + justify-content: center; + overflow: hidden; +} + +.card .metric-data { + font-size: 90px; + margin-bottom: 15px; +} + +.card .metric-type { + font-size: 23px; +} + +.card .metric-name-wrapper { + color: rgba(0, 0, 0, 0.54); +} + +.card .metric-unit { + color: rgba(0, 0, 0, 0.54); + font-size: 72px; +} + +.card .build-status-card { + font-size: 60px; + display: flex; + align-items: center; + justify-content: space-between; + margin-top: 25px; +} + +.card-inside { + width: 100%; +} + +footer { + position: fixed; + bottom: 0; + left: 0; + right: 0; + padding: 2px 19px 10px 19px; + z-index: 2; + background: inherit; + display: flex; +} + +footer span { + vertical-align: bottom; +} + +footer i.material-icons { + vertical-align: bottom; +} + +h3 { + text-align: center; + margin: 0; + margin-bottom: 1em; + font-size: 1.5em; +} + +.metric-section { + margin-top: 10px; + margin-bottom: 10px; +} + +span.metric-section + span.metric-section { + padding-left: 30px; +} + +.metric { + font-weight: 300; +} + +.warning { + font-weight: 400; + color: #FF2222; +} + +.congrats { + color: #468263; +} + +.row { + display: flex; + flex-direction: row; +} + +.frames { + display: flex; + position: relative; + flex-direction: row; + align-items: flex-end; + width: 100%; + border: 1px solid #81C784; } + +.frame-budget-indicator { + position: absolute; + left: 0; + right: 0; + bottom: 0; + border-top: 1px dashed #81C784; + text-align: right; + font-size: 70%; + padding-right: 5px; + padding-top: 2px; + background-color: #E8F5E9; + z-index: 1; } + +.frame-60FPS-indicator { + position: absolute; + left: 0; + right: 0; + bottom: 0; + border-top: 1px dashed #81C784; + text-align: right; + font-size: 70%; + padding-right: 5px; + padding-top: 2px; + background-color: #F9FBE7; + z-index: 0; +} + +.frames .frame { + width: 2px; + z-index: 2; +} + +.good-frame { + background-color: #468263; +} + +.bad-frame { + background-color: #FF2222; +} + +.terrible-frame { + background-color: #910000; +} + +table.stats-table { + width: 100%; +} + +table.stats-table td { + text-align: center; +} + +td.stats-label { + width: 33%; + text-align: end; + color: gray; + font-size: small; +} + +td.stats-value { + font-weight: bold; + width: 33%; + text-align: start; +} + +.buildbot-happy { + fill: #4CAF50; +} + +.buildbot-sad { + fill: #F44336; + animation-name: buildstatus-crazy; + animation-duration: 4s; + animation-iteration-count: infinite; +} + +@keyframes buildstatus-crazy { + 0% { + transform: rotateZ(-30deg) scale(1); + } + 50% { + transform: rotateZ(30deg) scale(1.5); + } + 100% { + transform: rotateZ(-30deg) scale(1); + } +} + +#build-out-of-date { + display: none; + position: fixed; + background-color: rgba(255, 0, 0, 0.7); + left: -40px; + top: 50%; + transform: rotateZ(-14deg); + width: 110%; + height: 21px; + text-align: center; + padding: 15px; + z-index: 3; +}