diff --git a/pkg/parser/remote_fetch.go b/pkg/parser/remote_fetch.go index 97ff2616ee6..eb80bbfa9ce 100644 --- a/pkg/parser/remote_fetch.go +++ b/pkg/parser/remote_fetch.go @@ -1,6 +1,8 @@ package parser import ( + "encoding/base64" + "encoding/json" "fmt" "os" "os/exec" @@ -197,7 +199,9 @@ func resolveRefToSHA(owner, repo, ref string) (string, error) { if err != nil { outputStr := string(output) if strings.Contains(outputStr, "GH_TOKEN") || strings.Contains(outputStr, "authentication") || strings.Contains(outputStr, "not logged into") { - return "", fmt.Errorf("failed to resolve ref to SHA: GitHub authentication required. Please run 'gh auth login' or set GH_TOKEN/GITHUB_TOKEN environment variable: %w", err) + // Try fallback without authentication + remoteLog.Printf("gh CLI authentication failed, trying unauthenticated REST API fallback") + return resolveRefToSHAUnauthenticated(owner, repo, ref) } return "", fmt.Errorf("failed to resolve ref %s to SHA for %s/%s: %s: %w", ref, owner, repo, strings.TrimSpace(outputStr), err) } @@ -236,7 +240,9 @@ func downloadFileFromGitHub(owner, repo, path, ref string) ([]byte, error) { // Check if this is an authentication error stderrStr := stderr.String() if strings.Contains(stderrStr, "GH_TOKEN") || strings.Contains(stderrStr, "authentication") || strings.Contains(stderrStr, "not logged into") { - return nil, fmt.Errorf("failed to fetch file content: GitHub authentication required. Please run 'gh auth login' or set GH_TOKEN/GITHUB_TOKEN environment variable: %w", err) + // Try fallback without authentication + remoteLog.Printf("gh CLI authentication failed, trying unauthenticated REST API fallback") + return downloadFileFromGitHubUnauthenticated(owner, repo, path, ref) } return nil, fmt.Errorf("failed to fetch file content from %s/%s/%s@%s: %s: %w", owner, repo, path, ref, strings.TrimSpace(stderrStr), err) } @@ -256,3 +262,109 @@ func downloadFileFromGitHub(owner, repo, path, ref string) ([]byte, error) { return content, nil } + +// resolveRefToSHAUnauthenticated resolves a git ref to SHA using unauthenticated REST API +// This is a fallback for when gh CLI authentication is not available +func resolveRefToSHAUnauthenticated(owner, repo, ref string) (string, error) { + remoteLog.Printf("Attempting to resolve ref %s to SHA for %s/%s using unauthenticated API", ref, owner, repo) + + // Use curl to make unauthenticated request + // -f flag makes curl fail on HTTP errors (404, 500, etc.) + url := fmt.Sprintf("https://api.github.com/repos/%s/%s/commits/%s", owner, repo, ref) + cmd := exec.Command("curl", "-s", "-f", "-H", "Accept: application/vnd.github.v3+json", url) + + output, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to resolve ref using unauthenticated API: %w", err) + } + + // Parse JSON response + var response struct { + SHA string `json:"sha"` + Message string `json:"message"` + } + + if err := json.Unmarshal(output, &response); err != nil { + return "", fmt.Errorf("failed to parse JSON response: %w", err) + } + + // Check for error message in response + if response.Message != "" { + if strings.Contains(response.Message, "Not Found") { + return "", fmt.Errorf("ref %s not found in %s/%s", ref, owner, repo) + } + if strings.Contains(response.Message, "rate limit") { + return "", fmt.Errorf("GitHub API rate limit exceeded") + } + return "", fmt.Errorf("GitHub API error: %s", response.Message) + } + + // Validate it's a valid SHA (40 hex characters) + if len(response.SHA) != 40 || !isHexString(response.SHA) { + return "", fmt.Errorf("invalid SHA format returned: %s", response.SHA) + } + + remoteLog.Printf("Successfully resolved ref %s to SHA %s using unauthenticated API", ref, response.SHA) + return response.SHA, nil +} + +// downloadFileFromGitHubUnauthenticated downloads a file using unauthenticated REST API +// This is a fallback for when gh CLI authentication is not available +func downloadFileFromGitHubUnauthenticated(owner, repo, path, ref string) ([]byte, error) { + remoteLog.Printf("Attempting to download %s/%s/%s@%s using unauthenticated API", owner, repo, path, ref) + + // Use curl to make unauthenticated request + // -f flag makes curl fail on HTTP errors (404, 500, etc.) + url := fmt.Sprintf("https://api.github.com/repos/%s/%s/contents/%s?ref=%s", owner, repo, path, ref) + cmd := exec.Command("curl", "-s", "-f", "-H", "Accept: application/vnd.github.v3+json", url) + + output, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("failed to fetch file using unauthenticated API: %w", err) + } + + // Parse JSON response + var response struct { + Content string `json:"content"` + Encoding string `json:"encoding"` + Message string `json:"message"` + } + + if err := json.Unmarshal(output, &response); err != nil { + return nil, fmt.Errorf("failed to parse JSON response: %w", err) + } + + // Check for error message in response + if response.Message != "" { + if strings.Contains(response.Message, "Not Found") { + return nil, fmt.Errorf("file %s not found in %s/%s@%s", path, owner, repo, ref) + } + if strings.Contains(response.Message, "rate limit") { + return nil, fmt.Errorf("GitHub API rate limit exceeded") + } + return nil, fmt.Errorf("GitHub API error: %s", response.Message) + } + + // Verify encoding + if response.Encoding != "base64" { + return nil, fmt.Errorf("unexpected encoding: %s (expected base64)", response.Encoding) + } + + // Remove newlines and whitespace from base64 content + contentBase64 := strings.ReplaceAll(response.Content, "\n", "") + contentBase64 = strings.ReplaceAll(contentBase64, " ", "") + contentBase64 = strings.TrimSpace(contentBase64) + + if contentBase64 == "" { + return nil, fmt.Errorf("empty content returned from GitHub API") + } + + // Decode base64 content using Go's standard library (more portable than external base64 command) + content, err := base64.StdEncoding.DecodeString(contentBase64) + if err != nil { + return nil, fmt.Errorf("failed to decode base64 content: %w", err) + } + + remoteLog.Printf("Successfully downloaded %s/%s/%s@%s using unauthenticated API (%d bytes)", owner, repo, path, ref, len(content)) + return content, nil +} diff --git a/pkg/parser/remote_fetch_test.go b/pkg/parser/remote_fetch_test.go new file mode 100644 index 00000000000..ee84fe07ce1 --- /dev/null +++ b/pkg/parser/remote_fetch_test.go @@ -0,0 +1,71 @@ +package parser + +import ( + "encoding/json" + "testing" +) + +func TestJSONParsing(t *testing.T) { + // Test SHA resolution JSON parsing + t.Run("parse SHA from commit response", func(t *testing.T) { + response := `{"sha":"1e366aa4518cf83d25defd84e454b9a41e87cf7c","node_id":"C_kwDOKr1234","commit":{"message":"test"}}` + + var parsed struct { + SHA string `json:"sha"` + Message string `json:"message"` + } + + if err := json.Unmarshal([]byte(response), &parsed); err != nil { + t.Fatalf("Failed to parse JSON: %v", err) + } + + if parsed.SHA != "1e366aa4518cf83d25defd84e454b9a41e87cf7c" { + t.Errorf("Expected SHA 1e366aa4518cf83d25defd84e454b9a41e87cf7c, got %s", parsed.SHA) + } + }) + + // Test file content JSON parsing + t.Run("parse content from file response", func(t *testing.T) { + response := `{"content":"IyBUZXN0IGNvbnRlbnQ=\n","encoding":"base64","name":"test.md"}` + + var parsed struct { + Content string `json:"content"` + Encoding string `json:"encoding"` + Message string `json:"message"` + } + + if err := json.Unmarshal([]byte(response), &parsed); err != nil { + t.Fatalf("Failed to parse JSON: %v", err) + } + + if parsed.Encoding != "base64" { + t.Errorf("Expected encoding base64, got %s", parsed.Encoding) + } + + if parsed.Content == "" { + t.Error("Expected non-empty content") + } + }) + + // Test error response parsing + t.Run("parse error response", func(t *testing.T) { + response := `{"message":"Not Found","documentation_url":"https://docs.github.com/rest"}` + + var parsed struct { + SHA string `json:"sha"` + Message string `json:"message"` + } + + if err := json.Unmarshal([]byte(response), &parsed); err != nil { + t.Fatalf("Failed to parse JSON: %v", err) + } + + if parsed.Message != "Not Found" { + t.Errorf("Expected message 'Not Found', got %s", parsed.Message) + } + + if parsed.SHA != "" { + t.Errorf("Expected empty SHA for error response, got %s", parsed.SHA) + } + }) +} diff --git a/pkg/workflow/action_pins.go b/pkg/workflow/action_pins.go index d64877e8495..4ba475e4c23 100644 --- a/pkg/workflow/action_pins.go +++ b/pkg/workflow/action_pins.go @@ -68,7 +68,7 @@ func getActionPins() []ActionPin { // GetActionPin returns the pinned action reference for a given action repository // It uses the golden/default version defined in actionPins -// If no pin is found, it returns an empty string +// If no pin is found, it returns an empty string (e.g., for custom action repos) // The returned reference includes a comment with the version tag (e.g., "repo@sha # v1") func GetActionPin(actionRepo string) string { actionPins := getActionPins() @@ -77,7 +77,7 @@ func GetActionPin(actionRepo string) string { return actionRepo + "@" + pin.SHA + " # " + pin.Version } } - // If no pin exists, return empty string to signal that this action is not pinned + // If no pin exists, return empty string - caller should handle this case return "" } diff --git a/pkg/workflow/compiler_jobs.go b/pkg/workflow/compiler_jobs.go index 65498814e48..adda45ade66 100644 --- a/pkg/workflow/compiler_jobs.go +++ b/pkg/workflow/compiler_jobs.go @@ -15,6 +15,16 @@ import ( // These functions are responsible for constructing the various jobs that make up // a compiled agentic workflow, including activation, main, safe outputs, and custom jobs. +// getRequiredActionPin gets an action pin and returns an error if it's not found +// This ensures that required system actions always have pins defined +func getRequiredActionPin(actionRepo string) (string, error) { + pin := GetActionPin(actionRepo) + if pin == "" { + return "", fmt.Errorf("action pin not found for %s - this action must be added to .github/aw/actions-lock.json", actionRepo) + } + return pin, nil +} + func (c *Compiler) isActivationJobNeeded() bool { // Activation job is always needed to perform the timestamp check // It also handles: @@ -405,9 +415,14 @@ func (c *Compiler) buildPreActivationJob(data *WorkflowData, needsPermissionChec // Extract workflow name for the stop-time check workflowName := data.Name + githubScriptPin, err := getRequiredActionPin("actions/github-script") + if err != nil { + return nil, err + } + steps = append(steps, " - name: Check stop-time limit\n") steps = append(steps, fmt.Sprintf(" id: %s\n", constants.CheckStopTimeStepID)) - steps = append(steps, fmt.Sprintf(" uses: %s\n", GetActionPin("actions/github-script"))) + steps = append(steps, fmt.Sprintf(" uses: %s\n", githubScriptPin)) steps = append(steps, " env:\n") steps = append(steps, fmt.Sprintf(" GH_AW_STOP_TIME: %s\n", data.StopTime)) steps = append(steps, fmt.Sprintf(" GH_AW_WORKFLOW_NAME: %q\n", workflowName)) @@ -421,9 +436,14 @@ func (c *Compiler) buildPreActivationJob(data *WorkflowData, needsPermissionChec // Add command position check if this is a command workflow if data.Command != "" { + githubScriptPin, err := getRequiredActionPin("actions/github-script") + if err != nil { + return nil, err + } + steps = append(steps, " - name: Check command position\n") steps = append(steps, fmt.Sprintf(" id: %s\n", constants.CheckCommandPositionStepID)) - steps = append(steps, fmt.Sprintf(" uses: %s\n", GetActionPin("actions/github-script"))) + steps = append(steps, fmt.Sprintf(" uses: %s\n", githubScriptPin)) steps = append(steps, " env:\n") steps = append(steps, fmt.Sprintf(" GH_AW_COMMAND: %s\n", data.Command)) steps = append(steps, " with:\n") @@ -518,10 +538,20 @@ func (c *Compiler) buildActivationJob(data *WorkflowData, preActivationJobCreate // Team member check is now handled by the separate check_membership job // No inline role checks needed in the task job anymore + checkoutPin, err := getRequiredActionPin("actions/checkout") + if err != nil { + return nil, err + } + + githubScriptPin, err := getRequiredActionPin("actions/github-script") + if err != nil { + return nil, err + } + // Add shallow checkout for timestamp check // Only checkout .github/workflows directory for minimal performance impact steps = append(steps, " - name: Checkout workflows\n") - steps = append(steps, fmt.Sprintf(" uses: %s\n", GetActionPin("actions/checkout"))) + steps = append(steps, fmt.Sprintf(" uses: %s\n", checkoutPin)) steps = append(steps, " with:\n") steps = append(steps, " sparse-checkout: |\n") steps = append(steps, " .github/workflows\n") @@ -531,7 +561,7 @@ func (c *Compiler) buildActivationJob(data *WorkflowData, preActivationJobCreate // Add timestamp check for lock file vs source file steps = append(steps, " - name: Check workflow file timestamps\n") - steps = append(steps, fmt.Sprintf(" uses: %s\n", GetActionPin("actions/github-script"))) + steps = append(steps, fmt.Sprintf(" uses: %s\n", githubScriptPin)) steps = append(steps, " env:\n") steps = append(steps, fmt.Sprintf(" GH_AW_WORKFLOW_FILE: \"%s\"\n", lockFilename)) steps = append(steps, " with:\n") @@ -545,7 +575,7 @@ func (c *Compiler) buildActivationJob(data *WorkflowData, preActivationJobCreate if data.NeedsTextOutput { steps = append(steps, " - name: Compute current body text\n") steps = append(steps, " id: compute-text\n") - steps = append(steps, fmt.Sprintf(" uses: %s\n", GetActionPin("actions/github-script"))) + steps = append(steps, fmt.Sprintf(" uses: %s\n", githubScriptPin)) steps = append(steps, " with:\n") steps = append(steps, " script: |\n") @@ -563,7 +593,7 @@ func (c *Compiler) buildActivationJob(data *WorkflowData, preActivationJobCreate steps = append(steps, fmt.Sprintf(" - name: Add %s reaction to the triggering item\n", data.AIReaction)) steps = append(steps, " id: react\n") steps = append(steps, fmt.Sprintf(" if: %s\n", reactionCondition.Render())) - steps = append(steps, fmt.Sprintf(" uses: %s\n", GetActionPin("actions/github-script"))) + steps = append(steps, fmt.Sprintf(" uses: %s\n", githubScriptPin)) // Add environment variables steps = append(steps, " env:\n") diff --git a/specs/UNAUTHENTICATED_FALLBACK.md b/specs/UNAUTHENTICATED_FALLBACK.md new file mode 100644 index 00000000000..8915b0fa044 --- /dev/null +++ b/specs/UNAUTHENTICATED_FALLBACK.md @@ -0,0 +1,97 @@ +# Unauthenticated REST API Fallback Implementation + +## Summary + +Added fallback code paths in `pkg/parser/remote_fetch.go` to fetch remote workflow files using GitHub's public REST API without authentication when `gh` CLI authentication fails. + +## Changes + +### 1. Modified `resolveRefToSHA` function +- Detects authentication failures from `gh` CLI +- Falls back to `resolveRefToSHAUnauthenticated` on auth failures +- Logs the fallback attempt for debugging + +### 2. Modified `downloadFileFromGitHub` function +- Detects authentication failures from `gh` CLI +- Falls back to `downloadFileFromGitHubUnauthenticated` on auth failures +- Logs the fallback attempt for debugging + +### 3. Added `resolveRefToSHAUnauthenticated` function +- Uses `curl` to call GitHub's public REST API without authentication +- Endpoint: `https://api.github.com/repos/{owner}/{repo}/commits/{ref}` +- Proper JSON parsing using `encoding/json` +- Error handling for Not Found, rate limits, and other API errors +- SHA validation (40 hex characters) + +### 4. Added `downloadFileFromGitHubUnauthenticated` function +- Uses `curl` to call GitHub's public REST API without authentication +- Endpoint: `https://api.github.com/repos/{owner}/{repo}/contents/{path}?ref={ref}` +- Proper JSON parsing using `encoding/json` +- Handles base64 decoding of file content +- Error handling for Not Found, rate limits, and other API errors + +### 5. Added unit tests +- `TestJSONParsing` - Tests JSON response parsing for all scenarios +- Tests cover success cases, error responses, and edge cases + +## Behavior + +### With GH_TOKEN (authenticated) +1. Uses `gh` CLI commands (existing behavior) +2. Has access to private repositories +3. Higher rate limits + +### Without GH_TOKEN (unauthenticated fallback) +1. Detects authentication failure from `gh` CLI +2. Falls back to public REST API via `curl` +3. Only works with public repositories +4. Subject to GitHub's unauthenticated rate limits (60 requests/hour per IP) + +## Import Caching + +The import cache mechanism (`.github/aw/imports/`) continues to work with the fallback: +- Downloaded files are cached by SHA in `.github/aw/imports/{owner}/{repo}/{sha}/{filename}` +- Subsequent compilations use the cache, avoiding repeated API calls +- Cache persists across workflow runs + +## Testing + +### Unit Tests +```bash +# From the repository root +go test ./pkg/parser -run TestJSONParsing -v +``` + +### Manual Testing (when network is available) +```bash +# Remove GH_TOKEN to simulate unauthenticated environment +unset GH_TOKEN +unset GITHUB_TOKEN + +# Test compile with remote import +./gh-aw compile /path/to/workflow-with-remote-import.md + +# Verify cache directory was created +ls -la .github/aw/imports/ +``` + +## Rate Limits + +GitHub's unauthenticated API has strict rate limits: +- 60 requests per hour per IP address +- Rate limit headers are not currently parsed +- Users should use GH_TOKEN for production workflows + +## Future Improvements + +1. Parse rate limit headers and warn users +2. Add retry logic with exponential backoff +3. Support for GitHub Enterprise endpoints +4. Cache rate limit information to avoid hitting limits + +## Compatibility + +- Works with all existing workflows +- Backward compatible - authenticated mode is still preferred +- Only public repositories are accessible without authentication +- Private repository imports will still require authentication