diff --git a/e2e/framework/command.go b/e2e/framework/command.go index 0d1955ed1..069c8eb33 100644 --- a/e2e/framework/command.go +++ b/e2e/framework/command.go @@ -145,7 +145,9 @@ func (f *Framework) DevsySSH( workspace string, command string, ) (string, error) { - out, err := f.ExecCommandOutput(ctx, []string{"ssh", workspace, "--command", command}) + out, err := execWithSSHRetry(ctx, workspace, func(ctx context.Context) (string, error) { + return f.ExecCommandOutput(ctx, []string{"ssh", workspace, "--command", command}) + }) if err != nil { return "", fmt.Errorf("devsy ssh failed: %s", err.Error()) } diff --git a/e2e/framework/retry.go b/e2e/framework/retry.go index fa8141e1d..3b1d04fc5 100644 --- a/e2e/framework/retry.go +++ b/e2e/framework/retry.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os/exec" "strings" "time" @@ -20,6 +21,16 @@ var dockerPullBackoff = wait.Backoff{ Jitter: 0.1, } +// sshBackoff defines retry timing for transient SSH failures on Windows+WSL +// runners where the devsy agent binary injection can intermittently fail. +// 3 total attempts (1 initial + 2 retries) with waits of ~5s, ~10s. +var sshBackoff = wait.Backoff{ + Steps: 3, + Duration: 5 * time.Second, + Factor: 2.0, + Jitter: 0.1, +} + // retryableDockerPatterns are stderr substrings indicating a transient Docker // registry error that is worth retrying. var retryableDockerPatterns = []string{ @@ -31,6 +42,17 @@ var retryableDockerPatterns = []string{ "503 Service Unavailable", } +// isRetryableSSHError returns true when the error indicates a transient SSH +// failure (exit code 1) typically caused by the devsy agent not yet being +// ready inside the container. +func isRetryableSSHError(err error) bool { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode() == 1 + } + return false +} + // isRetryableDockerError returns true if stderr contains a transient Docker // registry error (rate limits, timeouts, connection resets). func isRetryableDockerError(stderr string) bool { @@ -78,3 +100,41 @@ func execWithDockerRetry( } return lastStdout, lastStderr, err } + +// execWithSSHRetry runs fn and retries if the error indicates a transient SSH +// failure (exit status 1). This handles the case where the devsy agent binary +// injection into a WSL container has not completed yet. +func execWithSSHRetry( + ctx context.Context, + workspace string, + fn func(ctx context.Context) (string, error), +) (string, error) { + var lastOut string + var lastErr error + attempt := 0 + + err := wait.ExponentialBackoffWithContext(ctx, sshBackoff, + func(ctx context.Context) (bool, error) { + attempt++ + lastOut, lastErr = fn(ctx) + if lastErr == nil { + return true, nil // success + } + if isRetryableSSHError(lastErr) { + ginkgo.GinkgoWriter.Printf( + "[retry] ssh %s: attempt %d failed with transient error, retrying: %s\n", + workspace, attempt, lastErr, + ) + return false, nil // retry + } + return false, lastErr // non-retryable, stop immediately + }, + ) + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return lastOut, err + } + if err != nil && lastErr != nil { + return lastOut, fmt.Errorf("after %d attempts: %w", attempt, lastErr) + } + return lastOut, err +} diff --git a/e2e/framework/retry_test.go b/e2e/framework/retry_test.go index 50fb59f46..3271a2b35 100644 --- a/e2e/framework/retry_test.go +++ b/e2e/framework/retry_test.go @@ -1,11 +1,46 @@ package framework import ( + "fmt" + "os/exec" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// exitError runs a shell command that exits with the given code and returns +// the resulting *exec.ExitError. +func exitError(t *testing.T, code string) *exec.ExitError { + t.Helper() + // #nosec G204 -- test helper with controlled exit code argument + err := exec.Command("sh", "-c", "exit "+code).Run() + require.Error(t, err) + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + return exitErr +} + +func TestIsRetryableSSHError_ExitStatus1(t *testing.T) { + assert.True(t, isRetryableSSHError(exitError(t, "1"))) +} + +func TestIsRetryableSSHError_ExitStatus10(t *testing.T) { + assert.False(t, isRetryableSSHError(exitError(t, "10"))) +} + +func TestIsRetryableSSHError_ExitStatus127(t *testing.T) { + assert.False(t, isRetryableSSHError(exitError(t, "127"))) +} + +func TestIsRetryableSSHError_Nil(t *testing.T) { + assert.False(t, isRetryableSSHError(nil)) +} + +func TestIsRetryableSSHError_NonExitError(t *testing.T) { + assert.False(t, isRetryableSSHError(fmt.Errorf("some other error"))) +} + func TestIsRetryableDockerError_RateLimit(t *testing.T) { stderr := `GET https://index.docker.io/v2/library/ubuntu/manifests/latest: ` + `TOOMANYREQUESTS: You have reached your unauthenticated pull rate limit.`