cli: add more execution debugging tools

Author: bduffany

cli/cli_command/register/BUILD

@@ -14,6 +14,7 @@ go_library(
         "//cli/cli_command",
         "//cli/download",
         "//cli/execute",
+        "//cli/executions",
         "//cli/explain",
         "//cli/fix",
         "//cli/index",

cli/cli_command/register/register.go

@@ -9,6 +9,7 @@ import (
 	"github.com/buildbuddy-io/buildbuddy/cli/cli_command"
 	"github.com/buildbuddy-io/buildbuddy/cli/download"
 	"github.com/buildbuddy-io/buildbuddy/cli/execute"
+	"github.com/buildbuddy-io/buildbuddy/cli/executions"
 	"github.com/buildbuddy-io/buildbuddy/cli/explain"
 	"github.com/buildbuddy-io/buildbuddy/cli/fix"
 	"github.com/buildbuddy-io/buildbuddy/cli/index"
@@ -63,6 +64,11 @@ func register() {
 			Help:    "Executes arbitrary commands using remote execution.",
 			Handler: execute.HandleExecute,
 		},
+		{
+			Name:    "executions",
+			Help:    "Remote execution tools (see 'bb executions --help')",
+			Handler: executions.HandleExecutions,
+		},
 		{
 			Name:    "fix",
 			Help:    "Applies fixes to WORKSPACE and BUILD files.",

cli/execute/BUILD

@@ -8,6 +8,7 @@ go_library(
     importpath = "github.com/buildbuddy-io/buildbuddy/cli/execute",
     deps = [
         "//cli/arg",
+        "//cli/executions",
         "//cli/log",
         "//cli/login",
         "//proto:remote_execution_go_proto",

cli/execute/execute.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/buildbuddy-io/buildbuddy/cli/arg"
+	"github.com/buildbuddy-io/buildbuddy/cli/executions"
 	"github.com/buildbuddy-io/buildbuddy/cli/log"
 	"github.com/buildbuddy-io/buildbuddy/cli/login"
 	"github.com/buildbuddy-io/buildbuddy/server/real_environment"
@@ -45,6 +46,7 @@ var (
 	inputRoot       = flags.String("input_root", "", "Input root directory. By default, the action will have no inputs. Incompatible with --input_root_digest.")
 	inputRootDigest = flags.String("input_root_digest", "", "Digest of the input root directory. This is useful to re-run an existing action. Users can also use `bb download` to fetch the input tree locally. Incompatible with --input_root.")
 	outputPaths     = flag.New(flags, "output_path", []string{}, "Path to an expected output file or directory. The path should be relative to the workspace root. This flag can be specified more than once.")
+	output          = flags.String("output", "stdio", "Output format: `stdio` to print command stdout/stderr, `id` to print only execution ID, `json` to print execution response and logs as JSON, or `markdown` (or `md`) to print a markdown summary.")
 	// Note: bazel has remote_default_exec_properties but it has somewhat
 	// confusing semantics, so we call this "exec_properties" to avoid
 	// confusion.
@@ -92,6 +94,13 @@ func HandleExecute(args []string) (int, error) {
 		log.Print(usage)
 		return 1, nil
 	}
+	*output = strings.ToLower(*output)
+	if *output == "md" {
+		*output = "markdown"
+	}
+	if *output != "stdio" && *output != "id" && *output != "json" && *output != "markdown" {
+		return -1, fmt.Errorf("invalid --output %q (allowed values: stdio, id, json, markdown, md)", *output)
+	}
 	if err := execute(cmdArgs); err != nil {
 		return -1, err
 	}
@@ -187,14 +196,16 @@ func execute(cmdArgs []string) error {
 	}
 	log.Debugf("Waiting for execution to complete")
 	var rsp *rexec.Response
+	var executionErr error
 	for {
 		msg, err := stream.Recv()
 		if err != nil {
 			return err
 		}
 		if msg.Err != nil {
-			// We failed to execute.
-			return msg.Err
+			// Keep track of execution errors so we can still render available
+			// response logs for structured output modes.
+			executionErr = msg.Err
 		}
 		// Log execution state
 		progress := &repb.ExecutionProgress{}
@@ -213,18 +224,51 @@ func execute(cmdArgs []string) error {
 			break
 		}
 	}
+	// Defensive guard: we should only exit the loop on a Done message, which
+	// always sets rsp. Keep this check to avoid nil dereference if that
+	// invariant is ever broken by future changes.
+	if rsp == nil {
+		return fmt.Errorf("execute stream ended before completion")
+	}
 	log.Debugf("Execution completed in %s", time.Since(stageStart))
-	stageStart = time.Now()
-	log.Debugf("Downloading result")
-	res, err := rexec.GetResult(ctx, env, *instanceName, df, rsp.ExecuteResponse.GetResult())
-	if err != nil {
-		return status.WrapError(err, "execution failed")
+	if executionErr != nil && *output == "stdio" {
+		return executionErr
 	}
-	log.Debugf("Downloaded results in %s", time.Since(stageStart))
-	log.Debugf("End-to-end execution time: %s", time.Since(start))
+	var logs *rexec.ExecutionLogs
+	if *output == "json" || *output == "markdown" {
+		logs, err = rexec.GetExecutionLogs(ctx, env.GetByteStreamClient(), *instanceName, df, rsp.ExecuteResponse)
+		if err != nil {
+			log.Warnf("Could not fetch all execution logs: %s", err)
+		}
+	}
+	switch *output {
+	case "id":
+		if _, err := fmt.Fprintln(os.Stdout, rsp.GetName()); err != nil {
+			return err
+		}
+	case "json":
+		if err := executions.WriteJSONOutput(os.Stdout, rsp.GetName(), rsp.ExecuteResponse, logs); err != nil {
+			return fmt.Errorf("write json output: %w", err)
+		}
+	case "markdown":
+		if _, err := fmt.Fprint(os.Stdout, executions.RenderMarkdownWithDetails(rsp.GetName(), rsp.ExecuteResponse, logs)); err != nil {
+			return err
+		}
+	case "stdio":
+		stageStart = time.Now()
+		log.Debugf("Downloading result")
+		res, err := rexec.GetResult(ctx, env, *instanceName, df, rsp.ExecuteResponse.GetResult())
+		if err != nil {
+			return status.WrapError(err, "execution failed")
+		}
+		log.Debugf("Downloaded results in %s", time.Since(stageStart))
 
-	os.Stdout.Write(res.Stdout)
-	os.Stderr.Write(res.Stderr)
+		os.Stdout.Write(res.Stdout)
+		os.Stderr.Write(res.Stderr)
+	default:
+		return fmt.Errorf("invalid --output %q", *output)
+	}
+	log.Debugf("End-to-end execution time: %s", time.Since(start))
 
 	if *responseJSONFile != "" {
 		b, err := protojson.Marshal(rsp.ExecuteResponse)
@@ -235,6 +279,9 @@ func execute(cmdArgs []string) error {
 			return fmt.Errorf("write response JSON file: %w", err)
 		}
 	}
+	if executionErr != nil {
+		return executionErr
+	}
 
 	return nil
 }

cli/executions/BUILD

@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+package(default_visibility = ["//cli:__subpackages__"])
+
+go_library(
+    name = "executions",
+    srcs = ["executions.go"],
+    importpath = "github.com/buildbuddy-io/buildbuddy/cli/executions",
+    deps = [
+        "//cli/arg",
+        "//cli/log",
+        "//cli/login",
+        "//cli/terminal",
+        "//proto:execution_stats_go_proto",
+        "//proto:remote_execution_go_proto",
+        "//server/util/grpc_client",
+        "//server/util/rexec",
+        "@org_golang_google_grpc//codes",
+        "@org_golang_google_grpc//metadata",
+        "@org_golang_google_protobuf//encoding/protojson",
+        "@org_golang_google_protobuf//types/known/timestamppb",
+    ],
+)