From c347190407fa49445c10aa9c0b25f9dad5c83331 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Thu, 7 May 2026 00:52:06 -0500 Subject: [PATCH] feat(features): add features test command Implement `devsy features test` which builds a temporary container with a dev container feature installed and runs test scripts from the feature's test/ directory to verify correct behavior. This follows the devcontainer spec's feature testing workflow for feature authors. --- cmd/features/root.go | 1 + cmd/features/test.go | 370 +++++++++++++++++++++++++++++++++ cmd/features/test_test.go | 131 ++++++++++++ e2e/tests/features/features.go | 137 ++++++++++++ 4 files changed, 639 insertions(+) create mode 100644 cmd/features/test.go create mode 100644 cmd/features/test_test.go diff --git a/cmd/features/root.go b/cmd/features/root.go index 54b5eb7b2..3d0ffa271 100644 --- a/cmd/features/root.go +++ b/cmd/features/root.go @@ -17,6 +17,7 @@ func NewFeaturesCmd(globalFlags *flags.GlobalFlags) *cobra.Command { featuresCmd.AddCommand(NewInfoCmd(globalFlags)) featuresCmd.AddCommand(NewResolveDepsCmd(globalFlags)) featuresCmd.AddCommand(NewGenerateDocsCmd(globalFlags)) + featuresCmd.AddCommand(NewTestCmd(globalFlags)) return featuresCmd } diff --git a/cmd/features/test.go b/cmd/features/test.go new file mode 100644 index 000000000..739126bcc --- /dev/null +++ b/cmd/features/test.go @@ -0,0 +1,370 @@ +package features + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/devsy-org/devsy/cmd/flags" + devsycopy "github.com/devsy-org/devsy/pkg/copy" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/log" + "github.com/spf13/cobra" +) + +// TestCmd holds the flags for the features test command. +type TestCmd struct { + *flags.GlobalFlags + + BaseImage string + RemoteUser string + Output string + SkipScenarios bool + ProjectFolder string +} + +type testResult struct { + FeatureID string `json:"featureId"` + Passed bool `json:"passed"` + Scenarios []scenarioResult `json:"scenarios"` +} + +type scenarioResult struct { + Name string `json:"name"` + Passed bool `json:"passed"` + Output string `json:"output,omitempty"` +} + +// NewTestCmd creates the features test cobra command. +func NewTestCmd(globalFlags *flags.GlobalFlags) *cobra.Command { + cmd := &TestCmd{GlobalFlags: globalFlags} + testCmd := &cobra.Command{ + Use: "test", + Short: "Test a dev container feature", + Long: `Build a temporary container with a dev container feature installed and +run its test scripts to verify correct behavior. + +The command looks for test scenarios in the feature's test/ directory. +Each .sh file in test/ is executed inside the container. If all scripts +exit 0, the feature test passes.`, + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cobraCmd *cobra.Command, _ []string) error { + return cmd.Run(cobraCmd.Context()) + }, + } + + testCmd.Flags().StringVar( + &cmd.ProjectFolder, "project-folder", "", + "Path to the feature project directory (containing devcontainer-feature.json)", + ) + testCmd.Flags().StringVar( + &cmd.BaseImage, "base-image", "mcr.microsoft.com/devcontainers/base:ubuntu", + "Base image to install the feature into for testing", + ) + testCmd.Flags().StringVar( + &cmd.RemoteUser, "remote-user", "root", + "User to run tests as inside the container", + ) + testCmd.Flags().StringVar( + &cmd.Output, "output", "text", "Output format (text or json)", + ) + testCmd.Flags().BoolVar( + &cmd.SkipScenarios, "skip-scenarios", false, + "Only verify the feature installs successfully without running test scenarios", + ) + _ = testCmd.MarkFlagRequired("project-folder") + + return testCmd +} + +// Run executes the features test workflow. +func (cmd *TestCmd) Run(ctx context.Context) error { + if err := validateOutputFormat(cmd.Output); err != nil { + return err + } + + projectFolder, err := filepath.Abs(cmd.ProjectFolder) + if err != nil { + return fmt.Errorf("resolve project folder: %w", err) + } + + featureCfg, err := config.ParseDevContainerFeature(projectFolder) + if err != nil { + return fmt.Errorf("parse feature metadata: %w", err) + } + + log.Infof("Testing feature %q (v%s)", featureCfg.ID, featureCfg.Version) + + result := cmd.executeTest(ctx, projectFolder, featureCfg) + return cmd.reportResult(result) +} + +func (cmd *TestCmd) executeTest( + ctx context.Context, + projectFolder string, + featureCfg *config.FeatureConfig, +) *testResult { + result := &testResult{ + FeatureID: featureCfg.ID, + Passed: true, + } + + containerID, err := cmd.buildTestContainer(ctx, projectFolder, featureCfg) + if err != nil { + result.Passed = false + result.Scenarios = append(result.Scenarios, scenarioResult{ + Name: "install", + Passed: false, + Output: err.Error(), + }) + return result + } + defer cmd.cleanupContainer(ctx, containerID) + + result.Scenarios = append(result.Scenarios, scenarioResult{ + Name: "install", + Passed: true, + }) + + if cmd.SkipScenarios { + return result + } + + cmd.collectScenarios(ctx, containerID, projectFolder, result) + return result +} + +func (cmd *TestCmd) collectScenarios( + ctx context.Context, + containerID, projectFolder string, + result *testResult, +) { + scenarios, err := cmd.runTestScenarios(ctx, containerID, projectFolder) + if err != nil { + result.Passed = false + result.Scenarios = append(result.Scenarios, scenarioResult{ + Name: "test-execution", + Passed: false, + Output: err.Error(), + }) + return + } + + result.Scenarios = append(result.Scenarios, scenarios...) + for _, s := range scenarios { + if !s.Passed { + result.Passed = false + } + } +} + +func (cmd *TestCmd) buildTestContainer( + ctx context.Context, + featureDir string, + featureCfg *config.FeatureConfig, +) (string, error) { + dockerfile := generateTestDockerfile(cmd.BaseImage, featureCfg, cmd.RemoteUser) + + tmpDir, err := os.MkdirTemp("", "devsy-feature-test-*") + if err != nil { + return "", fmt.Errorf("create temp dir: %w", err) + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + if err := os.WriteFile( + filepath.Join(tmpDir, "Dockerfile"), []byte(dockerfile), 0o600, + ); err != nil { + return "", fmt.Errorf("write Dockerfile: %w", err) + } + + featureDestDir := filepath.Join(tmpDir, "feature") + if err := devsycopy.Directory(featureDir, featureDestDir); err != nil { + return "", fmt.Errorf("copy feature to build context: %w", err) + } + + containerID, err := cmd.dockerBuildAndRun(ctx, tmpDir, featureCfg) + if err != nil { + return "", err + } + + return containerID, nil +} + +func (cmd *TestCmd) dockerBuildAndRun( + ctx context.Context, + buildContext string, + featureCfg *config.FeatureConfig, +) (string, error) { + imageName := fmt.Sprintf("devsy-feature-test-%s:%s", featureCfg.ID, featureCfg.Version) + + log.Infof("Building test container with base image %q", cmd.BaseImage) + buildArgs := []string{ + "build", "-t", imageName, + "-f", filepath.Join(buildContext, "Dockerfile"), + buildContext, + } + + // #nosec G204 -- args are constructed internally from trusted inputs + buildCmd := exec.CommandContext(ctx, "docker", buildArgs...) + var buildOut bytes.Buffer + buildCmd.Stdout = &buildOut + buildCmd.Stderr = &buildOut + if err := buildCmd.Run(); err != nil { + return "", fmt.Errorf("build test image: %s\n%s", err, buildOut.String()) + } + + log.Infof("Starting test container") + runArgs := []string{ + "run", "-d", + "--label", "devsy.feature.test=" + featureCfg.ID, + imageName, + "sleep", "infinity", + } + + // #nosec G204 -- args are constructed internally from trusted inputs + runCmd := exec.CommandContext(ctx, "docker", runArgs...) + var runOut bytes.Buffer + runCmd.Stdout = &runOut + runCmd.Stderr = &runOut + if err := runCmd.Run(); err != nil { + return "", fmt.Errorf("start test container: %s\n%s", err, runOut.String()) + } + + return strings.TrimSpace(runOut.String()), nil +} + +func (cmd *TestCmd) runTestScenarios( + ctx context.Context, + containerID, featureDir string, +) ([]scenarioResult, error) { + testDir := filepath.Join(featureDir, "test") + entries, err := os.ReadDir(testDir) + if err != nil { + if os.IsNotExist(err) { + log.Infof("No test/ directory found, skipping test scenarios") + return nil, nil + } + return nil, fmt.Errorf("read test directory: %w", err) + } + + var results []scenarioResult + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".sh") { + continue + } + + scriptPath := filepath.Join(testDir, entry.Name()) + scriptContent, err := os.ReadFile(scriptPath) // #nosec G304 -- user-specified test path + if err != nil { + results = append(results, scenarioResult{ + Name: entry.Name(), + Passed: false, + Output: fmt.Sprintf("read script: %v", err), + }) + continue + } + + log.Infof("Running test scenario: %s", entry.Name()) + result := cmd.execTestScript(ctx, containerID, entry.Name(), string(scriptContent)) + results = append(results, result) + } + + return results, nil +} + +func (cmd *TestCmd) execTestScript( + ctx context.Context, + containerID, name, script string, +) scenarioResult { + execArgs := []string{ + "exec", + "-u", cmd.RemoteUser, + containerID, + "bash", "-c", script, + } + + // #nosec G204 -- args are constructed internally from trusted inputs + execCmd := exec.CommandContext(ctx, "docker", execArgs...) + var out bytes.Buffer + execCmd.Stdout = &out + execCmd.Stderr = &out + + err := execCmd.Run() + return scenarioResult{ + Name: name, + Passed: err == nil, + Output: out.String(), + } +} + +func (cmd *TestCmd) cleanupContainer(ctx context.Context, containerID string) { + log.Infof("Cleaning up test container") + + // #nosec G204 -- fixed command with internally-resolved container ID + rmCmd := exec.CommandContext(ctx, "docker", "rm", "-f", containerID) + _ = rmCmd.Run() +} + +func (cmd *TestCmd) reportResult(result *testResult) error { + if cmd.Output == outputJSON { + return writeJSON(os.Stdout, result) + } + + return cmd.printTextResult(result) +} + +func (cmd *TestCmd) printTextResult(result *testResult) error { + w := os.Stdout + + if result.Passed { + _, _ = fmt.Fprintf(w, "PASS: Feature %q\n", result.FeatureID) + } else { + _, _ = fmt.Fprintf(w, "FAIL: Feature %q\n", result.FeatureID) + } + + for _, s := range result.Scenarios { + status := "PASS" + if !s.Passed { + status = "FAIL" + } + _, _ = fmt.Fprintf(w, " [%s] %s\n", status, s.Name) + if !s.Passed && s.Output != "" { + for line := range strings.SplitSeq(strings.TrimSpace(s.Output), "\n") { + _, _ = fmt.Fprintf(w, " %s\n", line) + } + } + } + + if !result.Passed { + return fmt.Errorf("feature test failed") + } + return nil +} + +func generateTestDockerfile( + baseImage string, + featureCfg *config.FeatureConfig, + remoteUser string, +) string { + var b strings.Builder + fmt.Fprintf(&b, "FROM %s\n", baseImage) + fmt.Fprintf(&b, "USER %s\n", remoteUser) + b.WriteString("COPY feature/ /tmp/_dev_container_feature/\n") + b.WriteString("WORKDIR /tmp/_dev_container_feature\n") + + for optName, opt := range featureCfg.Options { + envKey := strings.ToUpper(optName) + defaultVal := string(opt.Default) + fmt.Fprintf(&b, "ENV %s=%s\n", envKey, defaultVal) + } + + b.WriteString("RUN chmod +x install.sh && ./install.sh\n") + b.WriteString("WORKDIR /\n") + b.WriteString("RUN rm -rf /tmp/_dev_container_feature\n") + return b.String() +} diff --git a/cmd/features/test_test.go b/cmd/features/test_test.go new file mode 100644 index 000000000..29e7c094b --- /dev/null +++ b/cmd/features/test_test.go @@ -0,0 +1,131 @@ +package features + +import ( + "os" + "path/filepath" + "testing" + + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTestCmd_FlagDefaults(t *testing.T) { + cmd := NewTestCmd(nil) + + baseImageFlag := cmd.Flags().Lookup("base-image") + require.NotNil(t, baseImageFlag) + assert.Equal(t, "mcr.microsoft.com/devcontainers/base:ubuntu", baseImageFlag.DefValue) + + remoteUserFlag := cmd.Flags().Lookup("remote-user") + require.NotNil(t, remoteUserFlag) + assert.Equal(t, "root", remoteUserFlag.DefValue) + + outputFlag := cmd.Flags().Lookup("output") + require.NotNil(t, outputFlag) + assert.Equal(t, "text", outputFlag.DefValue) + + skipScenariosFlag := cmd.Flags().Lookup("skip-scenarios") + require.NotNil(t, skipScenariosFlag) + assert.Equal(t, "false", skipScenariosFlag.DefValue) +} + +func TestTestCmd_AllFlagsRegistered(t *testing.T) { + cmd := NewTestCmd(nil) + expected := []string{"project-folder", "base-image", "remote-user", "output", "skip-scenarios"} + for _, name := range expected { + assert.NotNil(t, cmd.Flags().Lookup(name), "flag %q should be registered", name) + } +} + +func TestTestCmd_InvalidOutputFormat(t *testing.T) { + testCmd := &TestCmd{ + Output: "yaml", + ProjectFolder: "/nonexistent", + } + err := testCmd.Run(t.Context()) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid output format") +} + +func TestTestCmd_MissingFeatureJSON(t *testing.T) { + tmpDir := t.TempDir() + + testCmd := &TestCmd{ + Output: "text", + ProjectFolder: tmpDir, + BaseImage: "ubuntu:22.04", + RemoteUser: "root", + } + err := testCmd.Run(t.Context()) + require.Error(t, err) + assert.Contains(t, err.Error(), "parse feature metadata") +} + +func TestGenerateTestDockerfile_Basic(t *testing.T) { + featureCfg := &config.FeatureConfig{ + ID: "go", + Version: "1.0.0", + Name: "Go", + } + + got := generateTestDockerfile("ubuntu:22.04", featureCfg, "root") + + assert.Contains(t, got, "FROM ubuntu:22.04") + assert.Contains(t, got, "USER root") + assert.Contains(t, got, "COPY feature/ /tmp/_dev_container_feature/") + assert.Contains(t, got, "RUN chmod +x install.sh && ./install.sh") + assert.Contains(t, got, "RUN rm -rf /tmp/_dev_container_feature") +} + +func TestGenerateTestDockerfile_WithOptions(t *testing.T) { + featureCfg := &config.FeatureConfig{ + ID: "node", + Version: "1.0.0", + Options: map[string]config.FeatureConfigOption{ + "version": { + Type: "string", + Default: types.StrBool("18"), + }, + "installYarn": { + Type: "boolean", + Default: types.StrBool("true"), + }, + }, + } + + got := generateTestDockerfile("ubuntu:22.04", featureCfg, "vscode") + + assert.Contains(t, got, "USER vscode") + assert.Contains(t, got, "ENV VERSION=18") + assert.Contains(t, got, "ENV INSTALLYARN=true") +} + +func TestTestCmd_RunTestScenarios_NoTestDir(t *testing.T) { + tmpDir := t.TempDir() + + testCmd := &TestCmd{ + RemoteUser: "root", + } + results, err := testCmd.runTestScenarios(t.Context(), "fake-container-id", tmpDir) + require.NoError(t, err) + assert.Nil(t, results) +} + +func TestTestCmd_RunTestScenarios_SkipsNonSh(t *testing.T) { + tmpDir := t.TempDir() + testDir := filepath.Join(tmpDir, "test") + require.NoError(t, os.MkdirAll(testDir, 0o750)) + require.NoError(t, os.WriteFile( + filepath.Join(testDir, "readme.md"), []byte("# Tests"), 0o600, + )) + require.NoError(t, os.MkdirAll(filepath.Join(testDir, "subdir"), 0o750)) + + testCmd := &TestCmd{ + RemoteUser: "root", + } + results, err := testCmd.runTestScenarios(t.Context(), "fake-container-id", tmpDir) + require.NoError(t, err) + assert.Empty(t, results) +} diff --git a/e2e/tests/features/features.go b/e2e/tests/features/features.go index 16680ba59..9e11f688e 100644 --- a/e2e/tests/features/features.go +++ b/e2e/tests/features/features.go @@ -301,6 +301,143 @@ var _ = ginkgo.Describe("features commands", ginkgo.Label("features"), func() { gomega.Expect(stdout).To(gomega.ContainSubstring("2.0.0")) }, ginkgo.SpecTimeout(framework.TimeoutShort())) }) + + ginkgo.Describe("features test", func() { + ginkgo.It("tests a feature with a passing test script", func(ctx context.Context) { + f := framework.NewDefaultFramework(initialDir + "/bin") + + featureDir, err := os.MkdirTemp("", "e2e-features-test-*") + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func() { _ = os.RemoveAll(featureDir) }) + + // Create feature metadata + featureJSON := `{ + "id": "hello", + "version": "1.0.0", + "name": "Hello Feature", + "description": "A simple feature that creates a hello script" + }` + framework.ExpectNoError(os.WriteFile( + filepath.Join(featureDir, "devcontainer-feature.json"), + []byte(featureJSON), + 0o600, + )) + + // Create install script + installScript := "#!/bin/sh\nset -e\n" + + "echo 'hello-feature' > /usr/local/bin/hello-feature\n" + + "chmod +x /usr/local/bin/hello-feature\n" + framework.ExpectNoError(os.WriteFile( //nolint:gosec // test needs exec permission + filepath.Join(featureDir, "install.sh"), + []byte(installScript), + 0o755, + )) + + // Create test directory with a passing test + testDir := filepath.Join(featureDir, "test") + framework.ExpectNoError(os.MkdirAll(testDir, 0o750)) + testScript := "#!/bin/bash\nset -e\n" + + "test -f /usr/local/bin/hello-feature\n" + framework.ExpectNoError(os.WriteFile( //nolint:gosec // test needs exec permission + filepath.Join(testDir, "test_hello.sh"), + []byte(testScript), + 0o755, + )) + + stdout, _, err := f.ExecCommandCapture(ctx, []string{ + "features", "test", + "--project-folder", featureDir, + "--base-image", "ubuntu:22.04", + }) + framework.ExpectNoError(err) + + gomega.Expect(stdout).To(gomega.ContainSubstring("PASS")) + gomega.Expect(stdout).To(gomega.ContainSubstring("hello")) + }, ginkgo.SpecTimeout(framework.TimeoutModerate())) + + ginkgo.It("reports failure for a failing test script", func(ctx context.Context) { + f := framework.NewDefaultFramework(initialDir + "/bin") + + featureDir, err := os.MkdirTemp("", "e2e-features-test-fail-*") + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func() { _ = os.RemoveAll(featureDir) }) + + featureJSON := `{ + "id": "failing", + "version": "1.0.0", + "name": "Failing Feature" + }` + framework.ExpectNoError(os.WriteFile( + filepath.Join(featureDir, "devcontainer-feature.json"), + []byte(featureJSON), + 0o600, + )) + + installScript := "#!/bin/sh\nset -e\necho installed\n" + framework.ExpectNoError(os.WriteFile( //nolint:gosec // test needs exec permission + filepath.Join(featureDir, "install.sh"), + []byte(installScript), + 0o755, + )) + + testDir := filepath.Join(featureDir, "test") + framework.ExpectNoError(os.MkdirAll(testDir, 0o750)) + testScript := "#!/bin/bash\nexit 1\n" + framework.ExpectNoError(os.WriteFile( //nolint:gosec // test needs exec permission + filepath.Join(testDir, "test_should_fail.sh"), + []byte(testScript), + 0o755, + )) + + stdout, _, err := f.ExecCommandCapture(ctx, []string{ + "features", "test", + "--project-folder", featureDir, + "--base-image", "ubuntu:22.04", + }) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(stdout).To(gomega.ContainSubstring("FAIL")) + }, ginkgo.SpecTimeout(framework.TimeoutModerate())) + + ginkgo.It("outputs JSON when --output=json is specified", func(ctx context.Context) { + f := framework.NewDefaultFramework(initialDir + "/bin") + + featureDir, err := os.MkdirTemp("", "e2e-features-test-json-*") + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func() { _ = os.RemoveAll(featureDir) }) + + featureJSON := `{ + "id": "json-test", + "version": "1.0.0", + "name": "JSON Test Feature" + }` + framework.ExpectNoError(os.WriteFile( + filepath.Join(featureDir, "devcontainer-feature.json"), + []byte(featureJSON), + 0o600, + )) + + installScript := "#!/bin/sh\necho ok\n" + framework.ExpectNoError(os.WriteFile( //nolint:gosec // test needs exec permission + filepath.Join(featureDir, "install.sh"), + []byte(installScript), + 0o755, + )) + + stdout, _, err := f.ExecCommandCapture(ctx, []string{ + "features", "test", + "--project-folder", featureDir, + "--base-image", "ubuntu:22.04", + "--skip-scenarios", + "--output", "json", + }) + framework.ExpectNoError(err) + + var result map[string]any + gomega.Expect(json.Unmarshal([]byte(stdout), &result)).To(gomega.Succeed()) + gomega.Expect(result["featureId"]).To(gomega.Equal("json-test")) + gomega.Expect(result["passed"]).To(gomega.BeTrue()) + }, ginkgo.SpecTimeout(framework.TimeoutModerate())) + }) }) func pushFeatureWithAnnotations(refStr string, annotations map[string]string) {