redact user-defined secrets in remote & workflow logs (#11446)

## Summary
- redact user-defined secret values in CI runner logs for `bb remote`
and workflows
- pass secret env var names from execution server to CI runner via
`BUILDBUDDY_SECRET_ENV_VAR_NAMES`
- extend redaction logic to apply dynamic value redaction
(`RedactTextWithValues`)
- add failing-then-passing integration coverage in
`TestAccessingSecrets` by echoing a user-defined secret and asserting
`<REDACTED>`
- add unit tests for `RedactTextWithValues`, including overlap handling
(e.g. `abc` vs `abcdef`) to verify longest-first replacement behavior
- centralize CI runner env-var contract constants in
`enterprise/server/util/ci_runner_env`

## Validation
- `bb remote test
//enterprise/server/test/integration/remote_bazel:remote_bazel_test
--config=remote --test_filter=TestAccessingSecrets --test_output=all
--nocache_test_results`
- `bb remote test //server/util/redact:redact_test --config=remote
--test_filter=TestRedactTextWithValues --test_output=all
--nocache_test_results`

Test invocations:
-
https://app.buildbuddy.io/invocation/d8af9aca-d126-408d-bda4-a62f26cd2522
-
https://app.buildbuddy.io/invocation/1b09c525-6b2e-4b4d-8cd7-20c477ce687d

---------

Co-authored-by: Shelley <shelley@exe.dev>

Author: dan-stowell

enterprise/server/cmd/ci_runner/BUILD

@@ -15,6 +15,7 @@ go_library(
     importpath = "github.com/buildbuddy-io/buildbuddy/enterprise/server/cmd/ci_runner",
     deps = [
         "//enterprise/server/bes_artifacts",
+        "//enterprise/server/util/ci_runner_env",
         "//enterprise/server/workflow/config",
         "//proto:build_event_stream_go_proto",
         "//proto:command_line_go_proto",

enterprise/server/cmd/ci_runner/main.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/bes_artifacts"
+	"github.com/buildbuddy-io/buildbuddy/enterprise/server/util/ci_runner_env"
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/workflow/config"
 	"github.com/buildbuddy-io/buildbuddy/server/build_event_publisher"
 	"github.com/buildbuddy-io/buildbuddy/server/real_environment"
@@ -107,9 +108,8 @@ const (
 	// Env vars set by workflow runner
 	// NOTE: These env vars are not populated for non-private repos.
 
-	buildbuddyAPIKeyEnvVarName = "BUILDBUDDY_API_KEY"
-	repoUserEnvVarName         = "REPO_USER"
-	repoTokenEnvVarName        = "REPO_TOKEN"
+	repoUserEnvVarName  = "REPO_USER"
+	repoTokenEnvVarName = "REPO_TOKEN"
 
 	// Exit code placeholder used when a command doesn't return an exit code on its own.
 	noExitCode         = -1
@@ -142,7 +142,7 @@ const (
 	ansiGray  = "\033[90m"
 	ansiReset = "\033[0m"
 
-	clientIdentityEnvVar = "BB_GRPC_CLIENT_IDENTITY"
+	clientIdentityEnvVar = ci_runner_env.BBGrpcClientIdentityEnvVarName
 
 	// We save the startup options used for the last executed bazel command so we can apply
 	// them on future bazel commands without restarting the Bazel server.
@@ -313,7 +313,7 @@ type buildEventReporter struct {
 	progressCount int32
 }
 
-func newBuildEventReporter(ctx context.Context, besBackend string, apiKey string, forcedInvocationID string, isWorkflow bool) (*buildEventReporter, error) {
+func newBuildEventReporter(ctx context.Context, besBackend string, apiKey string, forcedInvocationID string, isWorkflow bool, redactionValues []string) (*buildEventReporter, error) {
 	iid := forcedInvocationID
 	if iid == "" {
 		var err error
@@ -338,7 +338,7 @@ func newBuildEventReporter(ctx context.Context, besBackend string, apiKey string
 		uploader = ul
 	}
 
-	return &buildEventReporter{apiKey: apiKey, bep: bep, uploader: uploader, log: newInvocationLog(), invocationID: iid, isWorkflow: isWorkflow, childInvocations: []string{}}, nil
+	return &buildEventReporter{apiKey: apiKey, bep: bep, uploader: uploader, log: newInvocationLog(redactionValues), invocationID: iid, isWorkflow: isWorkflow, childInvocations: []string{}}, nil
 }
 
 func (r *buildEventReporter) InvocationID() string {
@@ -664,7 +664,7 @@ func run() error {
 
 	ws := &workspace{
 		startTime:          time.Now(),
-		buildbuddyAPIKey:   os.Getenv(buildbuddyAPIKeyEnvVarName),
+		buildbuddyAPIKey:   os.Getenv(ci_runner_env.BuildBuddyAPIKeyEnvVarName),
 		forcedInvocationID: *invocationID,
 		runID:              runID,
 	}
@@ -685,7 +685,8 @@ func run() error {
 
 	// Use a context without a timeout for the build event reporter, so that even
 	// if the `timeout` is reached, any events will finish getting published
-	buildEventReporter, err := newBuildEventReporter(contextWithoutTimeout, *besBackend, ws.buildbuddyAPIKey, *invocationID, *workflowID != "" /*=isWorkflow*/)
+	redactionValues := parseSecretRedactionValues(os.Getenv(ci_runner_env.BuildBuddySecretEnvVarNamesForRedaction))
+	buildEventReporter, err := newBuildEventReporter(contextWithoutTimeout, *besBackend, ws.buildbuddyAPIKey, *invocationID, *workflowID != "" /*=isWorkflow*/, redactionValues)
 	if err != nil {
 		return err
 	}
@@ -950,20 +951,24 @@ func (r *buildEventReporter) Printf(format string, vals ...interface{}) {
 
 type invocationLog struct {
 	lockingbuffer.LockingBuffer
-	writer        io.Writer
-	writeListener func(s string)
+	writer          io.Writer
+	writeListener   func(s string)
+	redactionValues []string
 }
 
-func newInvocationLog() *invocationLog {
-	invLog := &invocationLog{writeListener: func(s string) {}}
+func newInvocationLog(redactionValues []string) *invocationLog {
+	invLog := &invocationLog{writeListener: func(s string) {}, redactionValues: redactionValues}
 	invLog.writer = io.MultiWriter(&invLog.LockingBuffer, os.Stderr)
 	return invLog
 }
 
 func (invLog *invocationLog) Write(b []byte) (int, error) {
 	output := string(b)
 
-	redacted := redact.RedactText(output)
+	// Use value-aware redaction so user-defined secret values injected into the
+	// runner environment are masked in invocation logs (including overlapping
+	// values handled safely by longest-first replacement in redact package).
+	redacted := redact.RedactTextWithValues(output, invLog.redactionValues)
 
 	invLog.writeListener(redacted)
 	_, err := invLog.writer.Write([]byte(redacted))
@@ -2285,11 +2290,11 @@ func writeBazelrc(path, invocationID, runID, rootDir string) error {
 	if isPushedRefInFork() {
 		lines = append(lines, "common --build_metadata=FORK_REPO_URL="+*pushedRepoURL)
 	}
-	if apiKey := os.Getenv(buildbuddyAPIKeyEnvVarName); apiKey != "" {
+	if apiKey := os.Getenv(ci_runner_env.BuildBuddyAPIKeyEnvVarName); apiKey != "" {
 		lines = append(lines, "common --remote_header=x-buildbuddy-api-key="+apiKey)
 		lines = append(lines, "build:buildbuddy_api_key --remote_header=x-buildbuddy-api-key="+apiKey)
 	}
-	if origin := os.Getenv("BB_GRPC_CLIENT_ORIGIN"); origin != "" {
+	if origin := os.Getenv(ci_runner_env.BBGrpcClientOriginEnvVarName); origin != "" {
 		lines = append(lines, fmt.Sprintf("common --remote_header=%s=%s", usageutil.OriginHeaderName, origin))
 		lines = append(lines, fmt.Sprintf("common --bes_header=%s=%s", usageutil.OriginHeaderName, origin))
 	}
@@ -2782,3 +2787,24 @@ func diskUsage() (*diskUsageStats, error) {
 		usedBytes:     int64(usedBytes),
 	}, nil
 }
+
+func parseSecretRedactionValues(serializedSecretNames string) []string {
+	if serializedSecretNames == "" {
+		return nil
+	}
+	var names []string
+	if err := json.Unmarshal([]byte(serializedSecretNames), &names); err != nil {
+		backendLog.Warningf("Failed to parse %s env var for secret redaction: %s", ci_runner_env.BuildBuddySecretEnvVarNamesForRedaction, err)
+		return nil
+	}
+	values := make([]string, 0, len(names))
+	for _, name := range names {
+		if name == "" {
+			continue
+		}
+		if val, ok := os.LookupEnv(name); ok && val != "" {
+			values = append(values, val)
+		}
+	}
+	return values
+}

enterprise/server/hostedrunner/BUILD

@@ -10,6 +10,7 @@ go_library(
         "//enterprise/server/experiments",
         "//enterprise/server/remote_execution/operation",
         "//enterprise/server/remote_execution/snaputil",
+        "//enterprise/server/util/ci_runner_env",
         "//enterprise/server/util/ci_runner_util",
         "//enterprise/server/workflow/config",
         "//proto:invocation_status_go_proto",

enterprise/server/hostedrunner/hostedrunner.go

@@ -13,6 +13,7 @@ import (
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/experiments"
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/operation"
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/snaputil"
+	"github.com/buildbuddy-io/buildbuddy/enterprise/server/util/ci_runner_env"
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/util/ci_runner_util"
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/workflow/config"
 	"github.com/buildbuddy-io/buildbuddy/server/endpoint_urls/build_buddy_url"
@@ -368,7 +369,7 @@ func (r *runnerService) credentialEnvOverrides(ctx context.Context, req *rnpb.Ru
 
 	// Use env override headers for credentials.
 	envOverrides := []string{
-		"BUILDBUDDY_API_KEY=" + apiKey.Value,
+		ci_runner_env.BuildBuddyAPIKeyEnvVarName + "=" + apiKey.Value,
 		"REPO_USER=" + req.GetGitRepo().GetUsername(),
 		"REPO_TOKEN=" + accessToken,
 	}

enterprise/server/remote_execution/execution_server/BUILD

@@ -13,6 +13,7 @@ go_library(
         "//enterprise/server/remote_execution/action_merger",
         "//enterprise/server/remote_execution/operation",
         "//enterprise/server/tasksize",
+        "//enterprise/server/util/ci_runner_env",
         "//enterprise/server/util/execution",
         "//proto:execution_stats_go_proto",
         "//proto:invocation_status_go_proto",

enterprise/server/remote_execution/execution_server/execution_server.go

@@ -3,6 +3,7 @@ package execution_server
 import (
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"io"
 	"path/filepath"
@@ -18,6 +19,7 @@ import (
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/action_merger"
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/operation"
 	"github.com/buildbuddy-io/buildbuddy/enterprise/server/tasksize"
+	"github.com/buildbuddy-io/buildbuddy/enterprise/server/util/ci_runner_env"
 	"github.com/buildbuddy-io/buildbuddy/proto/invocation_status"
 	"github.com/buildbuddy-io/buildbuddy/server/environment"
 	"github.com/buildbuddy-io/buildbuddy/server/interfaces"
@@ -861,6 +863,20 @@ func (s *ExecutionServer) dispatch(ctx context.Context, req *repb.ExecuteRequest
 			return nil, err
 		}
 		executionTask.Command.EnvironmentVariables = append(executionTask.Command.EnvironmentVariables, envVars...)
+		secretEnvVarNames := make([]string, 0, len(envVars))
+		for _, envVar := range envVars {
+			secretEnvVarNames = append(secretEnvVarNames, envVar.GetName())
+		}
+		if len(secretEnvVarNames) > 0 {
+			serializedNames, err := json.Marshal(secretEnvVarNames)
+			if err != nil {
+				return nil, status.WrapError(err, "marshal secret env var names")
+			}
+			executionTask.Command.EnvironmentVariables = append(executionTask.Command.EnvironmentVariables, &repb.Command_EnvironmentVariable{
+				Name:  ci_runner_env.BuildBuddySecretEnvVarNamesForRedaction,
+				Value: string(serializedNames),
+			})
+		}
 	}
 
 	executionTask.QueuedTimestamp = timestamppb.Now()

enterprise/server/remote_execution/executorplatform/BUILD

@@ -7,6 +7,7 @@ go_library(
     srcs = ["executorplatform.go"],
     importpath = "github.com/buildbuddy-io/buildbuddy/enterprise/server/remote_execution/executorplatform",
     deps = [
+        "//enterprise/server/util/ci_runner_env",
         "//proto:remote_execution_go_proto",
         "//server/environment",
         "//server/interfaces",

enterprise/server/remote_execution/executorplatform/executorplatform.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/buildbuddy-io/buildbuddy/enterprise/server/util/ci_runner_env"
 	"github.com/buildbuddy-io/buildbuddy/server/environment"
 	"github.com/buildbuddy-io/buildbuddy/server/interfaces"
 	"github.com/buildbuddy-io/buildbuddy/server/util/flag"
@@ -257,7 +258,7 @@ func ApplyOverrides(env environment.Env, executorProps *ExecutorProperties, plat
 	// forwarding an env var to the runner.
 	if platformProps.WorkflowID != "" {
 		command.EnvironmentVariables = append(command.EnvironmentVariables, &repb.Command_EnvironmentVariable{
-			Name:  "BB_GRPC_CLIENT_ORIGIN",
+			Name:  ci_runner_env.BBGrpcClientOriginEnvVarName,
 			Value: usageutil.ClientOrigin(),
 		})
 
@@ -270,7 +271,7 @@ func ApplyOverrides(env environment.Env, executorProps *ExecutorProperties, plat
 				return err
 			}
 			command.EnvironmentVariables = append(command.EnvironmentVariables, &repb.Command_EnvironmentVariable{
-				Name:  "BB_GRPC_CLIENT_IDENTITY",
+				Name:  ci_runner_env.BBGrpcClientIdentityEnvVarName,
 				Value: h,
 			})
 		}

enterprise/server/test/integration/remote_bazel/remote_bazel_test.go

@@ -531,16 +531,17 @@ sh_binary(
 		GroupId: env.GroupID1,
 	}
 
-	// Save a secret
+	// Save secrets
 	saveSecret(t, bbClient, ctx, reqCtx, *pubKey, "SECRET_TARGET", ":hello_world")
+	saveSecret(t, bbClient, ctx, reqCtx, *pubKey, "SECRET_MESSAGE", "super_secret_message_for_redaction_test")
 
 	// Run remote bazel
 	runRemoteBazelInSeparateProcess(t, repoDir, bbServer.GRPCAddress(),
 		// Initialize secrets as env vars on the runner
 		"--runner_exec_properties=include-secrets=true",
 		// Use --script here, because otherwise $SECRET_TARGET will be parsed
 		// as a string literal and will not be expanded as an env var
-		"--script=bazel run $SECRET_TARGET --noenable_bzlmod --enable_workspace",
+		"--script=echo secret=$SECRET_MESSAGE && bazel run $SECRET_TARGET --noenable_bzlmod --enable_workspace",
 		fmt.Sprintf("--remote_header=x-buildbuddy-api-key=%s", env.APIKey1))
 
 	// Check the invocation logs to ensure the bazel command successfully ran
@@ -567,6 +568,8 @@ sh_binary(
 	require.NoError(t, err)
 	require.Contains(t, string(logResp.GetBuffer()), "Build completed successfully")
 	require.Contains(t, string(logResp.GetBuffer()), "FUTURE OF BUILDS!")
+	require.NotContains(t, string(logResp.GetBuffer()), "super_secret_message_for_redaction_test")
+	require.Contains(t, string(logResp.GetBuffer()), "secret=<REDACTED>")
 }
 
 func setupSecrets(t *testing.T) (func(*rbetest.Env, *testenv.TestEnv), *string) {

enterprise/server/util/ci_runner_env/BUILD

@@ -0,0 +1,9 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+package(default_visibility = ["//enterprise:__subpackages__"])
+
+go_library(
+    name = "ci_runner_env",
+    srcs = ["ci_runner_env.go"],
+    importpath = "github.com/buildbuddy-io/buildbuddy/enterprise/server/util/ci_runner_env",
+)