refresh chromebot status

app/main.go

@@ -30,6 +30,7 @@ func init() {
 	registerRPC("/api/get-status", commands.GetStatus)
 	registerRPC("/api/refresh-github-commits", commands.RefreshGithubCommits)
 	registerRPC("/api/refresh-travis-status", commands.RefreshTravisStatus)
+	registerRPC("/api/refresh-chromebot-status", commands.RefreshChromebotStatus)
 	registerRPC("/api/reserve-task", commands.ReserveTask)
 	registerRPC("/api/update-task-status", commands.UpdateTaskStatus)
 

commands/refresh_chromebot_status.go

@@ -0,0 +1,203 @@
+// Copyright (c) 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package commands
+
+import (
+	"cocoon/db"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+
+	"google.golang.org/appengine/urlfetch"
+)
+
+// RefreshChromebotStatusResult contains chromebot status results.
+type RefreshChromebotStatusResult struct {
+	Results []*ChromebotResult
+}
+
+// ChromebotResult describes a chromebot build result.
+type ChromebotResult struct {
+	Commit string
+	State  db.TaskStatus
+}
+
+// RefreshChromebotStatus pulls down the latest chromebot builds and updates the
+// corresponding task statuses.
+func RefreshChromebotStatus(cocoon *db.Cocoon, inputJSON []byte) (interface{}, error) {
+	linuxResults, err := refreshChromebot(cocoon, "linux_bot", "Linux")
+
+	if err != nil {
+		return nil, err
+	}
+
+	macResults, err := refreshChromebot(cocoon, "mac_bot", "Mac")
+
+	if err != nil {
+		return nil, err
+	}
+
+	var allResults []*ChromebotResult
+	allResults = append(allResults, linuxResults...)
+	allResults = append(allResults, macResults...)
+	return RefreshChromebotStatusResult{allResults}, nil
+}
+
+func refreshChromebot(cocoon *db.Cocoon, taskName string, builderName string) ([]*ChromebotResult, error) {
+	tasks, err := cocoon.QueryPendingTasks(taskName)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if len(tasks) == 0 {
+		// Short-circuit. Don't bother fetching Chromebot data if there are no tasks to
+		// to update.
+		return make([]*ChromebotResult, 0), nil
+	}
+
+	buildStatuses, err := fetchChromebotBuildStatuses(cocoon, builderName)
+
+	if err != nil {
+		return nil, err
+	}
+
+	for _, fullTask := range tasks {
+		for _, status := range buildStatuses {
+			if status.Commit == fullTask.ChecklistEntity.Checklist.Commit.Sha {
+				task := fullTask.TaskEntity.Task
+				task.Status = status.State
+				cocoon.PutTask(fullTask.TaskEntity.Key, task)
+			}
+		}
+	}
+
+	return buildStatuses, nil
+}
+
+func fetchChromebotBuildStatuses(cocoon *db.Cocoon, builderName string) ([]*ChromebotResult, error) {
+	builderURL := fmt.Sprintf("https://build.chromium.org/p/client.flutter/json/builders/%v", builderName)
+
+	jsonResponse, err := fetchJSON(cocoon, builderURL)
+
+	if err != nil {
+		return nil, err
+	}
+
+	buildIds := jsonResponse.(map[string]interface{})["cachedBuilds"].([]interface{})
+
+	if len(buildIds) > 10 {
+		// Build IDs are sorted in ascending order, to get the latest 10 builds we
+		// grab the tail.
+		buildIds = buildIds[len(buildIds)-10:]
+	}
+
+	var results []*ChromebotResult
+	for i := len(buildIds) - 1; i >= 0; i-- {
+		buildID := buildIds[i]
+		buildJSON, err := fetchJSON(cocoon, fmt.Sprintf("%v/builds/%v", builderURL, buildID))
+
+		if err != nil {
+			return nil, err
+		}
+
+		results = append(results, &ChromebotResult{
+			Commit: getBuildProperty(buildJSON.(map[string]interface{}), "git_revision"),
+			State:  getStatus(buildJSON.(map[string]interface{})),
+		})
+	}
+
+	return results, nil
+}
+
+func fetchJSON(cocoon *db.Cocoon, url string) (interface{}, error) {
+	httpClient := urlfetch.Client(cocoon.Ctx)
+	response, err := httpClient.Get(url)
+	if err != nil {
+		return nil, err
+	}
+
+	defer response.Body.Close()
+
+	body, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var js interface{}
+	if json.Unmarshal(body, &js) != nil {
+		return nil, err
+	}
+
+	return js, nil
+}
+
+// Properties are encoded as:
+//
+//     {
+//       "properties": [
+//         [
+//           "name1",
+//           value1,
+//           ... things we don't care about ...
+//         ],
+//         [
+//           "name2",
+//           value2,
+//           ... things we don't care about ...
+//         ]
+//       ]
+//     }
+func getBuildProperty(buildJSON map[string]interface{}, propertyName string) string {
+	properties := buildJSON["properties"].([]interface{})
+	for _, property := range properties {
+		if property.([]interface{})[0] == propertyName {
+			return property.([]interface{})[1].(string)
+		}
+	}
+	return ""
+}
+
+// Parses out whether the build was successful.
+//
+// Successes are encoded like this:
+//
+//     "text": [
+//       "build",
+//       "successful"
+//     ]
+//
+// Exceptions are encoded like this:
+//
+//     "text": [
+//       "exception",
+//       "steps",
+//       "exception",
+//       "flutter build apk material_gallery"
+//     ]
+//
+// Errors are encoded like this:
+//
+//     "text": [
+//       "failed",
+//       "steps",
+//       "failed",
+//       "flutter build ios simulator stocks"
+//     ]
+//
+// In-progress builds are encoded like this:
+//
+//    "text": []
+//
+func getStatus(buildJSON map[string]interface{}) db.TaskStatus {
+	text := buildJSON["text"].([]interface{})
+	if text == nil || len(text) < 2 {
+		return db.TaskInProgress
+	} else if text[1].(string) == "successful" {
+		return db.TaskSucceeded
+	} else {
+		return db.TaskFailed
+	}
+}

commands/refresh_travis_status.go

@@ -39,22 +39,8 @@ func RefreshTravisStatus(cocoon *db.Cocoon, inputJSON []byte) (interface{}, erro
 		return RefreshTravisStatusResult{}, nil
 	}
 
-	// Maps from checklist key to checklist
-	checklists := make(map[string]*db.ChecklistEntity)
-	for _, taskEntity := range travisTasks {
-		key := taskEntity.Task.ChecklistKey
-		keyString := taskEntity.Task.ChecklistKey.Encode()
-		if checklists[keyString] == nil {
-			checklists[keyString], err = cocoon.GetChecklist(key)
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-
 	// Fetch data from Travis
 	httpClient := urlfetch.Client(cocoon.Ctx)
-
 	travisResp, err := httpClient.Get("https://api.travis-ci.org/repos/flutter/flutter/builds")
 	if err != nil {
 		return nil, err
@@ -73,9 +59,9 @@ func RefreshTravisStatus(cocoon *db.Cocoon, inputJSON []byte) (interface{}, erro
 		return nil, err
 	}
 
-	for _, taskEntity := range travisTasks {
-		task := taskEntity.Task
-		checklistEntity := checklists[task.ChecklistKey.Encode()]
+	for _, fullTask := range travisTasks {
+		task := fullTask.TaskEntity.Task
+		checklistEntity := fullTask.ChecklistEntity
 		for _, travisResult := range travisResults {
 			if travisResult.Commit == checklistEntity.Checklist.Commit.Sha {
 				if travisResult.State == "finished" {
@@ -85,7 +71,7 @@ func RefreshTravisStatus(cocoon *db.Cocoon, inputJSON []byte) (interface{}, erro
 				} else {
 					task.Status = db.TaskFailed
 				}
-				cocoon.PutTask(taskEntity.Key, task)
+				cocoon.PutTask(fullTask.TaskEntity.Key, task)
 			}
 		}
 	}

db/db.go

@@ -170,22 +170,30 @@ func (c *Cocoon) QueryTasks(checklistKey *datastore.Key) ([]*TaskEntity, error)
 	return c.runTaskQuery(query)
 }
 
+// FullTask contains information about a Task as well as surrounding metadata.
+// It is generally more expensive to query this data than to query just the task
+// records.
+type FullTask struct {
+	TaskEntity      *TaskEntity
+	ChecklistEntity *ChecklistEntity
+}
+
 // QueryPendingTasks lists the latest tasks with the given name that are not yet
 // in a final status.
 //
 // See also IsFinal.
-func (c *Cocoon) QueryPendingTasks(name string) ([]*TaskEntity, error) {
+func (c *Cocoon) QueryPendingTasks(taskName string) ([]*FullTask, error) {
 	checklists, err := c.QueryLatestChecklists()
 
 	if err != nil {
 		return nil, err
 	}
 
-	tasks := make([]*TaskEntity, 0, 20)
+	tasks := make([]*FullTask, 0, 20)
 	for i := len(checklists) - 1; i >= 0; i-- {
 		query := datastore.NewQuery("Task").
 			Ancestor(checklists[i].Key).
-			Filter("Name =", name).
+			Filter("Name =", taskName).
 			Order("-CreateTimestamp").
 			Limit(20)
 		candidates, err := c.runTaskQuery(query)
@@ -196,7 +204,10 @@ func (c *Cocoon) QueryPendingTasks(name string) ([]*TaskEntity, error) {
 
 		for _, candidate := range candidates {
 			if !candidate.Task.Status.IsFinal() {
-				tasks = append(tasks, candidate)
+				tasks = append(tasks, &FullTask{
+					TaskEntity:      candidate,
+					ChecklistEntity: checklists[i],
+				})
 			}
 		}
 	}