diff --git a/cmd/agent/container/deferred_hooks.go b/cmd/agent/container/deferred_hooks.go index 26b89d2a0..1393140c3 100644 --- a/cmd/agent/container/deferred_hooks.go +++ b/cmd/agent/container/deferred_hooks.go @@ -21,6 +21,7 @@ type DeferredHooksCmd struct { Prebuild bool DotfilesRepo string DotfilesScript string + SecretsEnv []string } // NewDeferredHooksCmd creates a new command. @@ -44,6 +45,8 @@ func NewDeferredHooksCmd(flags *flags.GlobalFlags) *cobra.Command { StringVar(&cmd.DotfilesRepo, "dotfiles-repo", "", "Dotfiles repository URL") deferredCmd.Flags(). StringVar(&cmd.DotfilesScript, "dotfiles-script", "", "Dotfiles install script path") + deferredCmd.Flags(). + StringSliceVar(&cmd.SecretsEnv, "secrets-env", []string{}, "Secrets to inject into lifecycle commands (KEY=VALUE)") _ = deferredCmd.MarkFlagRequired("setup-info") return deferredCmd } @@ -65,7 +68,7 @@ func (cmd *DeferredHooksCmd) Run(ctx context.Context) error { Repository: cmd.DotfilesRepo, InstallScript: cmd.DotfilesScript, RemoteUser: config.GetRemoteUser(setupInfo), - }) + }, cmd.SecretsEnv) if err != nil { log.Errorf("deferred hooks setup failed: %v", err) return nil diff --git a/cmd/agent/container/post_attach.go b/cmd/agent/container/post_attach.go index 505940067..5ad8533c9 100644 --- a/cmd/agent/container/post_attach.go +++ b/cmd/agent/container/post_attach.go @@ -17,7 +17,8 @@ import ( // PostAttachCmd runs postAttachCommand hooks as a detached background process. type PostAttachCmd struct { *flags.GlobalFlags - SetupInfo string + SetupInfo string + SecretsEnv []string } // NewPostAttachCmd creates a new command. @@ -34,6 +35,8 @@ func NewPostAttachCmd(flags *flags.GlobalFlags) *cobra.Command { }, } postAttachCmd.Flags().StringVar(&cmd.SetupInfo, "setup-info", "", "The container setup info") + postAttachCmd.Flags(). + StringSliceVar(&cmd.SecretsEnv, "secrets-env", []string{}, "Secrets to inject into lifecycle commands (KEY=VALUE)") _ = postAttachCmd.MarkFlagRequired("setup-info") return postAttachCmd } @@ -51,7 +54,7 @@ func (cmd *PostAttachCmd) Run(ctx context.Context) error { } log.Debugf("running postAttachCommand hooks") - if err := setup.RunPostAttachHooks(ctx, setupInfo); err != nil { + if err := setup.RunPostAttachHooks(ctx, setupInfo, cmd.SecretsEnv); err != nil { log.Errorf("postAttachCommand failed: %v", err) } diff --git a/cmd/agent/container/setup.go b/cmd/agent/container/setup.go index 210225e7c..31240384c 100644 --- a/cmd/agent/container/setup.go +++ b/cmd/agent/container/setup.go @@ -188,6 +188,7 @@ func (cmd *SetupContainerCmd) finalizeSetup(sctx *setupContext) error { cfg := &setup.ContainerSetupConfig{ SetupInfo: sctx.setupInfo, ExtraWorkspaceEnv: sctx.workspaceInfo.CLIOptions.WorkspaceEnv, + SecretsEnv: sctx.workspaceInfo.CLIOptions.SecretsEnv, ChownProjects: cmd.ChownWorkspace, PlatformOptions: &sctx.workspaceInfo.CLIOptions.Platform, TunnelClient: sctx.tunnelClient, @@ -238,6 +239,7 @@ func (cmd *SetupContainerCmd) setupPostAttach( if !deferred.Empty() { err = cmd.startDeferredHooks( resolvedSetupInfo, cmd.DotfilesRepo, cmd.DotfilesScript, + sctx.workspaceInfo.CLIOptions.SecretsEnv, ) if err != nil { log.Errorf("failed to start deferred lifecycle hooks: %v", err) @@ -262,16 +264,22 @@ func compressSetupInfo(setupInfo *config.Result) (string, error) { } func (cmd *SetupContainerCmd) startDeferredHooks( - setupInfo, dotfilesRepo, dotfilesScript string, + setupInfo, dotfilesRepo, dotfilesScript string, secretsEnv []string, ) error { return command.StartBackgroundOnce("devsy.deferred-hooks", func() (*exec.Cmd, error) { log.Debugf("starting deferred lifecycle hooks as background process") - return buildDeferredHooksCmd(setupInfo, cmd.Prebuild, dotfilesRepo, dotfilesScript) + return buildDeferredHooksCmd( + setupInfo, + cmd.Prebuild, + dotfilesRepo, + dotfilesScript, + secretsEnv, + ) }) } func buildDeferredHooksCmd( - setupInfo string, prebuild bool, dotfilesRepo, dotfilesScript string, + setupInfo string, prebuild bool, dotfilesRepo, dotfilesScript string, secretsEnv []string, ) (*exec.Cmd, error) { binaryPath, err := os.Executable() if err != nil { @@ -291,6 +299,9 @@ func buildDeferredHooksCmd( if dotfilesScript != "" { args = append(args, "--dotfiles-script", dotfilesScript) } + if len(secretsEnv) > 0 { + args = append(args, "--secrets-env", strings.Join(secretsEnv, ",")) + } return &exec.Cmd{ Path: binaryPath, @@ -472,15 +483,22 @@ func (cmd *SetupContainerCmd) startPostAttachHooks(sctx *setupContext) error { return nil, err } - //nolint:gosec // binaryPath is from os.Executable(), not user input - return exec.Command( - binaryPath, - "agent", - "container", - "post-attach", - "--setup-info", - cmd.SetupInfo, - ), nil + args := []string{ + "agent", "container", "post-attach", + "--setup-info", cmd.SetupInfo, + } + if len(sctx.workspaceInfo.CLIOptions.SecretsEnv) > 0 { + args = append( + args, + "--secrets-env", + strings.Join(sctx.workspaceInfo.CLIOptions.SecretsEnv, ","), + ) + } + + return &exec.Cmd{ + Path: binaryPath, + Args: append([]string{binaryPath}, args...), + }, nil }) } diff --git a/cmd/up.go b/cmd/up.go index 187b69202..30136cc93 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -25,6 +25,7 @@ import ( "github.com/devsy-org/devsy/pkg/log" options2 "github.com/devsy-org/devsy/pkg/options" provider2 "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/secrets" devssh "github.com/devsy-org/devsy/pkg/ssh" "github.com/devsy-org/devsy/pkg/telemetry" "github.com/devsy-org/devsy/pkg/util" @@ -47,6 +48,7 @@ type UpCmd struct { Reconfigure bool SSHConfigPath string + SecretsFile string DotfilesSource string DotfilesScript string @@ -274,6 +276,9 @@ func (cmd *UpCmd) registerWorkspaceFlags(upCmd *cobra.Command) { StringSliceVar(&cmd.WorkspaceEnvFile, "workspace-env-file", []string{}, "The path to files containing a list of extra env variables to put into the workspace, "+ "e.g. MY_ENV_VAR=MY_VALUE") + upCmd.Flags(). + StringVar(&cmd.SecretsFile, "secrets-file", "", + "Path to a dotenv-style file containing KEY=VALUE secrets injected into lifecycle commands") upCmd.Flags(). StringArrayVar(&cmd.InitEnv, "init-env", []string{}, "Extra env variables to inject during the initialization of the workspace, e.g. MY_ENV_VAR=MY_VALUE") @@ -802,6 +807,16 @@ func (cmd *UpCmd) prepareClient( return nil, err } + if cmd.SecretsFile != "" { + parsed, err := secrets.ParseSecretsFile(cmd.SecretsFile) + if err != nil { + return nil, err + } + for k, v := range parsed { + cmd.SecretsEnv = append(cmd.SecretsEnv, k+"="+v) + } + } + cmd.WorkspaceEnv = options2.InheritFromEnvironment( cmd.WorkspaceEnv, inheritedEnvironmentVariables, diff --git a/e2e/tests/up/provider_docker.go b/e2e/tests/up/provider_docker.go index 361be75b1..7b1b93773 100644 --- a/e2e/tests/up/provider_docker.go +++ b/e2e/tests/up/provider_docker.go @@ -553,6 +553,38 @@ var _ = ginkgo.Describe( gomega.Expect(hasCustomMount).To(gomega.BeTrue()) }, ginkgo.SpecTimeout(framework.TimeoutShort())) + ginkgo.It("secrets-file injects env into lifecycle commands", func(ctx context.Context) { + tempDir, err := setupWorkspace( + "tests/up/testdata/docker-secrets-file", + dtc.initialDir, + dtc.f, + ) + framework.ExpectNoError(err) + + secretsDir, err := framework.CreateTempDir() + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func() { _ = os.RemoveAll(secretsDir) }) + + secretsFile := filepath.Join(secretsDir, "secrets.env") + err = os.WriteFile( + secretsFile, + []byte("MY_SECRET=test-value-12345\nANOTHER_SECRET=second-secret-42\n"), + 0o600, + ) + framework.ExpectNoError(err) + + err = dtc.f.DevsyUp(ctx, tempDir, "--secrets-file", secretsFile) + framework.ExpectNoError(err) + + out, err := dtc.execSSH(ctx, tempDir, "cat /tmp/secret-check.out") + framework.ExpectNoError(err) + gomega.Expect(strings.TrimSpace(out)).To(gomega.Equal("test-value-12345")) + + out, err = dtc.execSSH(ctx, tempDir, "cat /tmp/another-secret-check.out") + framework.ExpectNoError(err) + gomega.Expect(strings.TrimSpace(out)).To(gomega.Equal("second-secret-42")) + }, ginkgo.SpecTimeout(framework.TimeoutShort())) + ginkgo.It("multi devcontainer selection", func(ctx context.Context) { tempDir, err := setupWorkspace( "tests/up/testdata/docker-multi-devcontainer", diff --git a/e2e/tests/up/testdata/docker-secrets-file/.devcontainer.json b/e2e/tests/up/testdata/docker-secrets-file/.devcontainer.json new file mode 100644 index 000000000..6fd903a51 --- /dev/null +++ b/e2e/tests/up/testdata/docker-secrets-file/.devcontainer.json @@ -0,0 +1,4 @@ +{ + "image": "ghcr.io/devsy-org/test-images/go:1", + "postCreateCommand": "echo -n $MY_SECRET > /tmp/secret-check.out && echo -n $ANOTHER_SECRET > /tmp/another-secret-check.out" +} diff --git a/pkg/devcontainer/setup/lifecyclehooks.go b/pkg/devcontainer/setup/lifecyclehooks.go index 7338cb98c..bf86a2723 100644 --- a/pkg/devcontainer/setup/lifecyclehooks.go +++ b/pkg/devcontainer/setup/lifecyclehooks.go @@ -133,6 +133,20 @@ type lifecycleEnv struct { remoteEnv map[string]string } +// mergeSecretsEnv merges KEY=VALUE pairs from the --secrets-file flag into the +// lifecycle env map. Existing keys are NOT overridden (config takes precedence). +func mergeSecretsEnv(env map[string]string, secretsEnv []string) { + for _, entry := range secretsEnv { + key, value, ok := strings.Cut(entry, "=") + if !ok { + continue + } + if _, exists := env[key]; !exists { + env[key] = value + } + } +} + func resolveLifecycleEnv( ctx context.Context, setupInfo *config.Result, @@ -222,8 +236,10 @@ func RunPreAttachHooks( setupInfo *config.Result, prebuild bool, dotfiles DotfilesConfig, + secretsEnv []string, ) (DeferredHooks, error) { env := resolveLifecycleEnv(ctx, setupInfo) + mergeSecretsEnv(env.remoteEnv, secretsEnv) all := preAttachPhaseParams(setupInfo, env, prebuild) // Insert the dotfiles phase between postCreate and postStart. @@ -372,8 +388,9 @@ func (d DeferredHooks) Run() error { // RunPostAttachHooks runs postAttachCommand only. // These run after the IDE has been opened and can be long-running. -func RunPostAttachHooks(ctx context.Context, setupInfo *config.Result) error { +func RunPostAttachHooks(ctx context.Context, setupInfo *config.Result, secretsEnv []string) error { env := resolveLifecycleEnv(ctx, setupInfo) + mergeSecretsEnv(env.remoteEnv, secretsEnv) return runHook(hookRunParams{ commands: setupInfo.MergedConfig.PostAttachCommands, diff --git a/pkg/devcontainer/setup/lifecyclehooks_test.go b/pkg/devcontainer/setup/lifecyclehooks_test.go index 014ba0d5c..8a9bc7f84 100644 --- a/pkg/devcontainer/setup/lifecyclehooks_test.go +++ b/pkg/devcontainer/setup/lifecyclehooks_test.go @@ -100,11 +100,11 @@ func (s *LifecycleHookTestSuite) TestLifecycleHooksNoOpWithEmptyConfig() { } // Both functions should return nil with empty config (no commands to run) - deferred, err := RunPreAttachHooks(ctx, result, false, DotfilesConfig{}) + deferred, err := RunPreAttachHooks(ctx, result, false, DotfilesConfig{}, nil) assert.NoError(s.T(), err) assert.True(s.T(), deferred.Empty()) - err = RunPostAttachHooks(ctx, result) + err = RunPostAttachHooks(ctx, result, nil) assert.NoError(s.T(), err) } @@ -252,7 +252,7 @@ func (s *LifecycleHookTestSuite) TestPrebuildIgnoresWaitFor() { } // In prebuild mode, no deferred hooks are returned regardless of waitFor. - deferred, err := RunPreAttachHooks(ctx, result, true, DotfilesConfig{}) + deferred, err := RunPreAttachHooks(ctx, result, true, DotfilesConfig{}, nil) assert.NoError(s.T(), err) assert.True(s.T(), deferred.Empty()) } @@ -643,6 +643,41 @@ func (s *LifecycleHookTestSuite) TestWaitForEmptyPhaseLogsWarning() { `waitFor phase "updateContentCommand" has no commands configured`) } +func (s *LifecycleHookTestSuite) TestMergeSecretsEnv() { + env := map[string]string{"EXISTING": "keep"} + + mergeSecretsEnv(env, []string{"SECRET_KEY=secret_val", "OTHER=data"}) + + assert.Equal(s.T(), "keep", env["EXISTING"]) + assert.Equal(s.T(), "secret_val", env["SECRET_KEY"]) + assert.Equal(s.T(), "data", env["OTHER"]) +} + +func (s *LifecycleHookTestSuite) TestMergeSecretsEnvDoesNotOverride() { + env := map[string]string{"MY_VAR": "original"} + + mergeSecretsEnv(env, []string{"MY_VAR=overridden"}) + + assert.Equal(s.T(), "original", env["MY_VAR"]) +} + +func (s *LifecycleHookTestSuite) TestMergeSecretsEnvNil() { + env := map[string]string{"KEY": "val"} + + mergeSecretsEnv(env, nil) + + assert.Equal(s.T(), "val", env["KEY"]) + assert.Len(s.T(), env, 1) +} + +func (s *LifecycleHookTestSuite) TestMergeSecretsEnvValueWithEquals() { + env := map[string]string{} + + mergeSecretsEnv(env, []string{"CONN=host=db port=5432"}) + + assert.Equal(s.T(), "host=db port=5432", env["CONN"]) +} + func TestLifecycleHookTestSuite(t *testing.T) { suite.Run(t, new(LifecycleHookTestSuite)) } diff --git a/pkg/devcontainer/setup/setup.go b/pkg/devcontainer/setup/setup.go index 73a44b775..2a50ef621 100644 --- a/pkg/devcontainer/setup/setup.go +++ b/pkg/devcontainer/setup/setup.go @@ -39,6 +39,7 @@ type DotfilesConfig struct { type ContainerSetupConfig struct { SetupInfo *config.Result ExtraWorkspaceEnv []string + SecretsEnv []string ChownProjects bool Prebuild bool PlatformOptions *devsy.PlatformOptions @@ -70,7 +71,13 @@ func SetupContainerPreAttach( setupOptionalFeatures(ctx, cfg) log.Debugf("running pre-attach lifecycle hooks") - deferred, err := RunPreAttachHooks(ctx, cfg.SetupInfo, cfg.Prebuild, cfg.Dotfiles) + deferred, err := RunPreAttachHooks( + ctx, + cfg.SetupInfo, + cfg.Prebuild, + cfg.Dotfiles, + cfg.SecretsEnv, + ) if err != nil { return DeferredHooks{}, fmt.Errorf("lifecycle hooks pre-attach: %w", err) } @@ -83,7 +90,7 @@ func SetupContainerPreAttach( // Called after the IDE has been opened. func SetupContainerPostAttach(ctx context.Context, cfg *ContainerSetupConfig) error { log.Debugf("running post-attach lifecycle hooks") - if err := RunPostAttachHooks(ctx, cfg.SetupInfo); err != nil { + if err := RunPostAttachHooks(ctx, cfg.SetupInfo, cfg.SecretsEnv); err != nil { return fmt.Errorf("lifecycle hooks post-attach: %w", err) } diff --git a/pkg/provider/workspace.go b/pkg/provider/workspace.go index 78ae2d33f..aa05fd74b 100644 --- a/pkg/provider/workspace.go +++ b/pkg/provider/workspace.go @@ -216,6 +216,7 @@ type CLIOptions struct { DevContainerID string `json:"devContainerID,omitempty"` WorkspaceEnv []string `json:"workspaceEnv,omitempty"` WorkspaceEnvFile []string `json:"workspaceEnvFile,omitempty"` + SecretsEnv []string `json:"secretsEnv,omitempty"` InitEnv []string `json:"initEnv,omitempty"` Recreate bool `json:"recreate,omitempty"` Prebuild bool `json:"prebuild,omitempty"` diff --git a/pkg/secrets/secrets.go b/pkg/secrets/secrets.go new file mode 100644 index 000000000..9acc43a07 --- /dev/null +++ b/pkg/secrets/secrets.go @@ -0,0 +1,51 @@ +package secrets + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +// ParseSecretsFile reads a dotenv-style secrets file and returns the key-value +// pairs as a map. Lines starting with # are comments, blank lines are ignored, +// and values are split on the first = only (values may contain =). Whitespace +// around keys is trimmed. +func ParseSecretsFile(path string) (map[string]string, error) { + f, err := os.Open(path) // #nosec G304 -- User-specified secrets file path is intentional. + if err != nil { + return nil, fmt.Errorf("open secrets file: %w", err) + } + defer func() { _ = f.Close() }() + + secrets := map[string]string{} + scanner := bufio.NewScanner(f) + lineNum := 0 + for scanner.Scan() { + lineNum++ + line := scanner.Text() + + // Skip blank lines and comments. + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + + key, value, found := strings.Cut(trimmed, "=") + if !found { + return nil, fmt.Errorf("secrets file %s: line %d: missing '=' separator", path, lineNum) + } + + key = strings.TrimSpace(key) + if key == "" { + return nil, fmt.Errorf("secrets file %s: line %d: empty key", path, lineNum) + } + + secrets[key] = value + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read secrets file %s: %w", path, err) + } + + return secrets, nil +} diff --git a/pkg/secrets/secrets_test.go b/pkg/secrets/secrets_test.go new file mode 100644 index 000000000..bde054450 --- /dev/null +++ b/pkg/secrets/secrets_test.go @@ -0,0 +1,145 @@ +package secrets_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/devsy-org/devsy/pkg/secrets" +) + +func TestParseSecretsFile(t *testing.T) { + content := `# Database credentials +DB_HOST=localhost +DB_PASSWORD=s3cr3t + +# API key +API_KEY=abc123 +` + path := writeTemp(t, content) + + got, err := secrets.ParseSecretsFile(path) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 { + t.Fatalf("expected 3 entries, got %d", len(got)) + } + if got["DB_HOST"] != "localhost" { + t.Errorf("DB_HOST = %q, want %q", got["DB_HOST"], "localhost") + } + if got["DB_PASSWORD"] != "s3cr3t" { + t.Errorf("DB_PASSWORD = %q, want %q", got["DB_PASSWORD"], "s3cr3t") + } + if got["API_KEY"] != "abc123" { + t.Errorf("API_KEY = %q, want %q", got["API_KEY"], "abc123") + } +} + +func TestParseSecretsFile_CommentsAndBlankLines(t *testing.T) { + content := ` +# This is a comment + +KEY=value + # indented comment + +OTHER=val +` + path := writeTemp(t, content) + + got, err := secrets.ParseSecretsFile(path) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("expected 2 entries, got %d", len(got)) + } + if got["KEY"] != "value" { + t.Errorf("KEY = %q, want %q", got["KEY"], "value") + } + if got["OTHER"] != "val" { + t.Errorf("OTHER = %q, want %q", got["OTHER"], "val") + } +} + +func TestParseSecretsFile_ValueWithEquals(t *testing.T) { + content := `CONNECTION_STRING=host=db port=5432 user=admin password=p=ss +TOKEN=abc==def +` + path := writeTemp(t, content) + + got, err := secrets.ParseSecretsFile(path) + if err != nil { + t.Fatal(err) + } + if got["CONNECTION_STRING"] != "host=db port=5432 user=admin password=p=ss" { + t.Errorf("CONNECTION_STRING = %q", got["CONNECTION_STRING"]) + } + if got["TOKEN"] != "abc==def" { + t.Errorf("TOKEN = %q, want %q", got["TOKEN"], "abc==def") + } +} + +func TestParseSecretsFile_MissingFile(t *testing.T) { + _, err := secrets.ParseSecretsFile("/nonexistent/path/secrets.env") + if err == nil { + t.Fatal("expected error for missing file") + } +} + +func TestParseSecretsFile_EmptyFile(t *testing.T) { + path := writeTemp(t, "") + + got, err := secrets.ParseSecretsFile(path) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("expected 0 entries, got %d", len(got)) + } +} + +func TestParseSecretsFile_WhitespaceAroundKey(t *testing.T) { + content := " MY_KEY =some_value\n" + path := writeTemp(t, content) + + got, err := secrets.ParseSecretsFile(path) + if err != nil { + t.Fatal(err) + } + if got["MY_KEY"] != "some_value" { + t.Errorf("MY_KEY = %q, want %q", got["MY_KEY"], "some_value") + } +} + +func TestParseSecretsFile_MissingSeparator(t *testing.T) { + content := "INVALID_LINE\n" + path := writeTemp(t, content) + + _, err := secrets.ParseSecretsFile(path) + if err == nil { + t.Fatal("expected error for line without = separator") + } +} + +func TestParseSecretsFile_EmptyValue(t *testing.T) { + content := "EMPTY_VAL=\n" + path := writeTemp(t, content) + + got, err := secrets.ParseSecretsFile(path) + if err != nil { + t.Fatal(err) + } + if got["EMPTY_VAL"] != "" { + t.Errorf("EMPTY_VAL = %q, want empty string", got["EMPTY_VAL"]) + } +} + +func writeTemp(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "secrets.env") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +}