From cba38cf3570a89f7dc48627ea0bb3400c13389f8 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 24 Apr 2026 12:08:07 -0500 Subject: [PATCH 1/2] fix(e2e): add SSH retry logic for flaky Windows WSL tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DevsySSH had no retry logic — it called ExecCommandOutput once and failed immediately. On Windows+WSL runners the devsy agent binary injection intermittently fails, causing SSH to return exit status 1. Add execWithSSHRetry with exponential backoff (3 attempts, 5s initial, factor 2.0, 0.1 jitter) to make DevsySSH resilient to transient agent readiness delays, matching the existing execWithDockerRetry pattern. --- e2e/framework/command.go | 4 ++- e2e/framework/retry.go | 55 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/e2e/framework/command.go b/e2e/framework/command.go index 0d1955ed1..069c8eb33 100644 --- a/e2e/framework/command.go +++ b/e2e/framework/command.go @@ -145,7 +145,9 @@ func (f *Framework) DevsySSH( workspace string, command string, ) (string, error) { - out, err := f.ExecCommandOutput(ctx, []string{"ssh", workspace, "--command", command}) + out, err := execWithSSHRetry(ctx, workspace, func(ctx context.Context) (string, error) { + return f.ExecCommandOutput(ctx, []string{"ssh", workspace, "--command", command}) + }) if err != nil { return "", fmt.Errorf("devsy ssh failed: %s", err.Error()) } diff --git a/e2e/framework/retry.go b/e2e/framework/retry.go index fa8141e1d..c7dd3a1fe 100644 --- a/e2e/framework/retry.go +++ b/e2e/framework/retry.go @@ -20,6 +20,16 @@ var dockerPullBackoff = wait.Backoff{ Jitter: 0.1, } +// sshBackoff defines retry timing for transient SSH failures on Windows+WSL +// runners where the devsy agent binary injection can intermittently fail. +// 3 total attempts (1 initial + 2 retries) with waits of ~5s, ~10s. +var sshBackoff = wait.Backoff{ + Steps: 3, + Duration: 5 * time.Second, + Factor: 2.0, + Jitter: 0.1, +} + // retryableDockerPatterns are stderr substrings indicating a transient Docker // registry error that is worth retrying. var retryableDockerPatterns = []string{ @@ -31,6 +41,13 @@ var retryableDockerPatterns = []string{ "503 Service Unavailable", } +// isRetryableSSHError returns true when the error indicates a transient SSH +// failure (exit status 1) typically caused by the devsy agent not yet being +// ready inside the container. +func isRetryableSSHError(err error) bool { + return err != nil && strings.Contains(err.Error(), "exit status 1") +} + // isRetryableDockerError returns true if stderr contains a transient Docker // registry error (rate limits, timeouts, connection resets). func isRetryableDockerError(stderr string) bool { @@ -78,3 +95,41 @@ func execWithDockerRetry( } return lastStdout, lastStderr, err } + +// execWithSSHRetry runs fn and retries if the error indicates a transient SSH +// failure (exit status 1). This handles the case where the devsy agent binary +// injection into a WSL container has not completed yet. +func execWithSSHRetry( + ctx context.Context, + workspace string, + fn func(ctx context.Context) (string, error), +) (string, error) { + var lastOut string + var lastErr error + attempt := 0 + + err := wait.ExponentialBackoffWithContext(ctx, sshBackoff, + func(ctx context.Context) (bool, error) { + attempt++ + lastOut, lastErr = fn(ctx) + if lastErr == nil { + return true, nil // success + } + if isRetryableSSHError(lastErr) { + ginkgo.GinkgoWriter.Printf( + "[retry] ssh %s: attempt %d failed with transient error, retrying: %s\n", + workspace, attempt, lastErr, + ) + return false, nil // retry + } + return false, lastErr // non-retryable, stop immediately + }, + ) + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return lastOut, err + } + if err != nil && lastErr != nil { + return lastOut, fmt.Errorf("after %d attempts: %w", attempt, lastErr) + } + return lastOut, err +} From a9a97d70ea0fcfa74269ac43d035c932b0a93d93 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Fri, 24 Apr 2026 12:14:47 -0500 Subject: [PATCH 2/2] fix(e2e): use errors.As for SSH exit code check, add unit tests Replace substring match on "exit status 1" with errors.As type assertion on *exec.ExitError and ExitCode() == 1 to avoid false positives on exit codes 10, 127, etc. Add 5 unit tests for isRetryableSSHError covering exit code 1 (true), exit code 10 (false), exit code 127 (false), nil (false), and non-ExitError (false). --- e2e/framework/retry.go | 9 +++++++-- e2e/framework/retry_test.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/e2e/framework/retry.go b/e2e/framework/retry.go index c7dd3a1fe..3b1d04fc5 100644 --- a/e2e/framework/retry.go +++ b/e2e/framework/retry.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os/exec" "strings" "time" @@ -42,10 +43,14 @@ var retryableDockerPatterns = []string{ } // isRetryableSSHError returns true when the error indicates a transient SSH -// failure (exit status 1) typically caused by the devsy agent not yet being +// failure (exit code 1) typically caused by the devsy agent not yet being // ready inside the container. func isRetryableSSHError(err error) bool { - return err != nil && strings.Contains(err.Error(), "exit status 1") + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode() == 1 + } + return false } // isRetryableDockerError returns true if stderr contains a transient Docker diff --git a/e2e/framework/retry_test.go b/e2e/framework/retry_test.go index 50fb59f46..3271a2b35 100644 --- a/e2e/framework/retry_test.go +++ b/e2e/framework/retry_test.go @@ -1,11 +1,46 @@ package framework import ( + "fmt" + "os/exec" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// exitError runs a shell command that exits with the given code and returns +// the resulting *exec.ExitError. +func exitError(t *testing.T, code string) *exec.ExitError { + t.Helper() + // #nosec G204 -- test helper with controlled exit code argument + err := exec.Command("sh", "-c", "exit "+code).Run() + require.Error(t, err) + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + return exitErr +} + +func TestIsRetryableSSHError_ExitStatus1(t *testing.T) { + assert.True(t, isRetryableSSHError(exitError(t, "1"))) +} + +func TestIsRetryableSSHError_ExitStatus10(t *testing.T) { + assert.False(t, isRetryableSSHError(exitError(t, "10"))) +} + +func TestIsRetryableSSHError_ExitStatus127(t *testing.T) { + assert.False(t, isRetryableSSHError(exitError(t, "127"))) +} + +func TestIsRetryableSSHError_Nil(t *testing.T) { + assert.False(t, isRetryableSSHError(nil)) +} + +func TestIsRetryableSSHError_NonExitError(t *testing.T) { + assert.False(t, isRetryableSSHError(fmt.Errorf("some other error"))) +} + func TestIsRetryableDockerError_RateLimit(t *testing.T) { stderr := `GET https://index.docker.io/v2/library/ubuntu/manifests/latest: ` + `TOOMANYREQUESTS: You have reached your unauthenticated pull rate limit.`