Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions pkg/cli/add_current_repo_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ func TestAddWorkflowsFromCurrentRepository(t *testing.T) {
if !strings.Contains(err.Error(), tt.errorContains) {
t.Errorf("Expected error to contain %q, got: %v", tt.errorContains, err)
}
if !strings.Contains(err.Error(), "Expected a workflow from another repository") || !strings.Contains(err.Error(), "Example: gh aw add octo-org/agentic-workflows/example-workflow") {
t.Errorf("Expected actionable current repository error, got: %v", err)
}
} else {
// For "allow" case, we expect a different error (workflow not found, not current repo error)
if err != nil && strings.Contains(err.Error(), "cannot add workflows from the current repository") {
Expand Down Expand Up @@ -164,6 +167,9 @@ func TestAddWorkflowsFromCurrentRepositoryMultiple(t *testing.T) {
if !strings.Contains(err.Error(), tt.errorContains) {
t.Errorf("Expected error to contain %q, got: %v", tt.errorContains, err)
}
if !strings.Contains(err.Error(), "Expected a workflow from another repository") || !strings.Contains(err.Error(), "Example: gh aw add octo-org/agentic-workflows/example-workflow") {
t.Errorf("Expected actionable current repository error, got: %v", err)
}
} else {
// For "allow" case, we expect a different error (workflow not found, not current repo error)
if err != nil && strings.Contains(err.Error(), "cannot add workflows from the current repository") {
Expand Down
10 changes: 5 additions & 5 deletions pkg/cli/add_interactive_orchestrator.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func RunAddInteractive(ctx context.Context, config *AddInteractiveConfig) error
// are treated consistently across test and automation environments, while
// IsRunningInCI centralizes the broader CI environment detection logic.
if envutil.GetBoolFromEnv("GO_TEST_MODE", false, addInteractiveLog) || IsRunningInCI() {
return errors.New("interactive add cannot be used in automated tests or CI environments")
return errors.New("interactive add is unavailable in automated tests or CI environments. Expected an interactive terminal outside automation. Example: run `gh aw add-wizard` from a local terminal")
}

// Set context on the config
Expand Down Expand Up @@ -248,11 +248,11 @@ func (c *AddInteractiveConfig) determineFilesToAdd() (workflowFiles []string, in
return nil, nil, fmt.Errorf("resolved workflow at position %d from %q is nil", i+1, workflowSpecsForError)
}
if rw.Spec == nil {
return nil, nil, fmt.Errorf("resolved workflow at position %d from %q is missing its specification", i+1, workflowSpecsForError)
return nil, nil, fmt.Errorf("resolved workflow at position %d from %q has no specification. Expected a resolved workflow specification. Example: github/gh-aw/example-workflow", i+1, workflowSpecsForError)
}
workflowName := strings.TrimSpace(rw.Spec.WorkflowName)
if workflowName == "" {
return nil, nil, fmt.Errorf("resolved workflow at position %d from %q is missing its workflow name", i+1, workflowSpecsForError)
return nil, nil, fmt.Errorf("resolved workflow at position %d from %q has no workflow name. Expected a named resolved workflow. Example: github/gh-aw/example-workflow", i+1, workflowSpecsForError)
}
if rw.IsActionWorkflow {
// Raw GitHub Actions YAML files are installed as-is; no .lock.yml is produced.
Expand Down Expand Up @@ -293,11 +293,11 @@ func (c *AddInteractiveConfig) workflowNamesForInteractiveAdd() ([]string, error
return nil, fmt.Errorf("resolved manifest workflow at position %d from %q is nil", i+1, workflowSpecsForError)
}
if resolvedWorkflow.Spec == nil {
return nil, fmt.Errorf("resolved manifest workflow at position %d from %q is missing its specification", i+1, workflowSpecsForError)
return nil, fmt.Errorf("resolved manifest workflow at position %d from %q has no specification. Expected a resolved workflow specification. Example: github/gh-aw/example-workflow", i+1, workflowSpecsForError)
}
workflowName := strings.TrimSpace(resolvedWorkflow.Spec.WorkflowName)
if workflowName == "" {
return nil, fmt.Errorf("resolved manifest workflow at position %d from %q is missing its workflow name", i+1, workflowSpecsForError)
return nil, fmt.Errorf("resolved manifest workflow at position %d from %q has no workflow name. Expected a named resolved workflow. Example: github/gh-aw/example-workflow", i+1, workflowSpecsForError)
}
workflowNames = append(workflowNames, workflowName)
}
Expand Down
21 changes: 21 additions & 0 deletions pkg/cli/add_interactive_orchestrator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,33 @@
package cli

import (
"context"
"os"
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestRunAddInteractive_InAutomatedEnvironment(t *testing.T) {
origTestMode := os.Getenv("GO_TEST_MODE")
os.Setenv("GO_TEST_MODE", "true")
t.Cleanup(func() {
if origTestMode != "" {
os.Setenv("GO_TEST_MODE", origTestMode)
} else {
os.Unsetenv("GO_TEST_MODE")
}
})

err := RunAddInteractive(context.Background(), &AddInteractiveConfig{})
require.Error(t, err)
if !strings.Contains(err.Error(), "Expected an interactive terminal outside automation") || !strings.Contains(err.Error(), "Example: run `gh aw add-wizard` from a local terminal") {
t.Errorf("Expected actionable error message, got %q", err.Error())
}
}

func TestAddInteractiveConfig_determineFilesToAdd(t *testing.T) {
t.Parallel()
tests := []struct {
Expand Down
3 changes: 3 additions & 0 deletions pkg/cli/add_package_manifest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -846,6 +846,9 @@ func TestResolveWorkflows_RepositoryPackageRejectsPrivateTrue(t *testing.T) {
_, err := ResolveWorkflows(context.Background(), []string{"owner/repo"}, false)
require.Error(t, err)
require.ErrorContains(t, err, `workflow "workflows/review.md" sets private: true`)
require.ErrorContains(t, err, "Expected an installable workflow with private: false")
require.ErrorContains(t, err, "Example: private: false")
require.NotContains(t, err.Error(), "\n")
}

func TestResolveWorkflows_NestedRepositoryPackage(t *testing.T) {
Expand Down
12 changes: 6 additions & 6 deletions pkg/cli/add_workflow_resolution.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,11 @@ type specResolutionResult struct {

func validateResolveWorkflowsInput(workflows []string) error {
if len(workflows) == 0 {
return errors.New("at least one workflow name is required")
return errors.New("no workflow names were supplied. Expected at least one workflow name. Example: gh aw add github/gh-aw/example-workflow")
}
for i, workflow := range workflows {
if workflow == "" {
return fmt.Errorf("workflow name cannot be empty (workflow %d)", i+1)
return fmt.Errorf("workflow name at position %d is empty. Expected a workflow name. Example: github/gh-aw/example-workflow", i+1)
}
}
return nil
Expand Down Expand Up @@ -260,7 +260,7 @@ func validateCurrentRepositorySpecs(parsedSpecs []*WorkflowSpec) error {
continue
}
if spec.RepoSlug == currentRepoSlug {
return fmt.Errorf("cannot add workflows from the current repository (%s). The 'add' command is for installing workflows from other repositories", currentRepoSlug)
return fmt.Errorf("cannot add workflows from the current repository %q. Expected a workflow from another repository. Example: gh aw add octo-org/agentic-workflows/example-workflow", currentRepoSlug)
}
}
return nil
Expand Down Expand Up @@ -369,7 +369,7 @@ func resolveStandardWorkflow(spec, resolvedSpec *WorkflowSpec, fetched *FetchedW
}

if ExtractWorkflowPrivate(content) {
return nil, fmt.Errorf("workflow '%s' is private and cannot be added to other repositories", spec.String())
return nil, fmt.Errorf("workflow %q is private. Expected a workflow that can be added to another repository. Example: set private: false before running gh aw add", spec.String())
}

workflowHasDispatch := checkWorkflowHasDispatchFromContent(content)
Expand Down Expand Up @@ -408,9 +408,9 @@ func validateManifestWorkflowPrivateSetting(spec, resolvedSpec *WorkflowSpec, co
}
manifestPath := joinRepositoryPackagePath(spec.PackagePath, repositoryPackageManifestFileName)
return fmt.Errorf(
"invalid Agentic Workflow manifest %q: workflow %q sets private: true and cannot be included because private workflows cannot be added",
manifestPath,
"workflow %q sets private: true in agentic workflow manifest %q. Expected an installable workflow with private: false. Example: private: false",
resolvedSpec.WorkflowPath,
manifestPath,
)
}

Expand Down
10 changes: 5 additions & 5 deletions pkg/cli/audit_diff_command.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,21 +50,21 @@ analyzes their data, and produces a diff showing:
RunE: func(cmd *cobra.Command, args []string) error {
baseRunID, err := strconv.ParseInt(args[0], 10, 64)
if err != nil {
return fmt.Errorf("invalid base run ID %q: must be a numeric run ID", args[0])
return fmt.Errorf("base run ID %q is not numeric. Expected a numeric GitHub Actions run ID. Example: gh aw audit diff 12345 12346", args[0])
}

compareRunIDs := make([]int64, 0, len(args)-1)
seen := make(map[int64]bool)
for _, arg := range args[1:] {
id, err := strconv.ParseInt(arg, 10, 64)
if err != nil {
return fmt.Errorf("invalid run ID %q: must be a numeric run ID", arg)
return fmt.Errorf("comparison run ID %q is not numeric. Expected a numeric GitHub Actions run ID. Example: gh aw audit diff 12345 12346", arg)
}
if id == baseRunID {
return fmt.Errorf("comparison run ID %d is the same as the base run ID: cannot diff a run against itself", id)
return fmt.Errorf("comparison run ID %d matches the base run ID. Expected a different run ID for comparison. Example: gh aw audit diff 12345 12346", id)
}
if seen[id] {
return fmt.Errorf("duplicate comparison run ID %d: each run ID must appear only once", id)
return fmt.Errorf("comparison run ID %d appears more than once. Expected each comparison run ID once. Example: gh aw audit diff 12345 12346", id)
}
seen[id] = true
compareRunIDs = append(compareRunIDs, id)
Expand All @@ -81,7 +81,7 @@ analyzes their data, and produces a diff showing:
if repoFlag != "" {
parts := strings.SplitN(repoFlag, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
return fmt.Errorf("invalid repository format '%s': expected 'owner/repo'", repoFlag)
return fmt.Errorf("repository %q is not in owner/repo format. Expected an owner and repository name separated by '/'. Example: --repo github/gh-aw", repoFlag)
}
owner = parts[0]
repo = parts[1]
Expand Down
42 changes: 42 additions & 0 deletions pkg/cli/audit_diff_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,55 @@ package cli

import (
"encoding/json"
"strings"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestAuditDiffValidationErrorsAreActionable(t *testing.T) {
tests := []struct {
name string
args []string
want string
}{
{
name: "base run ID is non-numeric",
args: []string{"abc", "12346"},
want: "Expected a numeric GitHub Actions run ID",
},
{
name: "comparison run ID matches base",
args: []string{"12345", "12345"},
want: "Expected a different run ID for comparison",
},
{
name: "comparison run ID repeats",
args: []string{"12345", "12346", "12346"},
want: "Expected each comparison run ID once",
},
{
name: "repository is not owner repo",
args: []string{"12345", "12346", "--repo", "github"},
want: "Expected an owner and repository name separated by '/'",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cmd := NewAuditDiffSubcommand()
cmd.SetArgs(tt.args)
err := cmd.Execute()
require.Error(t, err)
if !strings.Contains(err.Error(), tt.want) || !strings.Contains(err.Error(), "Example:") {
t.Errorf("Expected actionable error containing %q and Example:, got %q", tt.want, err.Error())
}
})
}
}

func TestComputeFirewallDiff_NewDomains(t *testing.T) {
run1 := &FirewallAnalysis{
AnalysisBase: AnalysisBase{TotalRequests: 5, AllowedRequests: 5},
Expand Down
10 changes: 5 additions & 5 deletions pkg/cli/interactive.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func CreateWorkflowInteractively(ctx context.Context, workflowName string, verbo
// are treated consistently across test and automation environments, while
// IsRunningInCI centralizes the broader CI environment detection logic.
if envutil.GetBoolFromEnv("GO_TEST_MODE", false, interactiveLog) || IsRunningInCI() {
return errors.New("interactive workflow creation cannot be used in automated tests or CI environments")
return errors.New("interactive workflow creation is unavailable in automated tests or CI environments. Expected an interactive terminal outside automation. Example: run `gh aw new` from a local terminal")
}

if verbose {
Expand Down Expand Up @@ -400,7 +400,7 @@ func promptNonInteractiveSelect(scanner *bufio.Scanner, title string, options []
// Accept a numeric index
if idx, err := strconv.Atoi(input); err == nil {
if idx < 1 || idx > len(options) {
return "", fmt.Errorf("selection out of range (must be 1-%d)", len(options))
return "", fmt.Errorf("selection is out of range. Expected a number from 1 to %d. Example: 1", len(options))

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[/tdd] The Example: 1 hint is not contextual — the actual valid range varies per call, but the example always shows 1. This is fine for the single-select path, but the multi-select path (promptNonInteractiveMultiSelect) allows comma-separated values; the example should reflect that.

💡 Suggested fix

For the multi-select error at line ~456 and ~475, a comma-separated example would be more instructive:

return nil, fmt.Errorf("selection %d is out of range. Expected a number from 1 to %d. Example: 1,2", idx, len(options))

@copilot please address this.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed in 61891c4e78: multi-select range and unknown-value errors now use comma-separated examples (1,2).

}
return options[idx-1].value, nil
}
Expand All @@ -411,7 +411,7 @@ func promptNonInteractiveSelect(scanner *bufio.Scanner, title string, options []
return opt.value, nil
}
}
return "", fmt.Errorf("invalid selection %q", input)
return "", fmt.Errorf("selection %q is not available. Expected a number or option value. Example: 1", input)
}

// promptNonInteractiveMultiSelect prints a numbered list and reads comma-separated selections.
Expand Down Expand Up @@ -453,7 +453,7 @@ func promptNonInteractiveMultiSelect(scanner *bufio.Scanner, title string, optio
// Try numeric index
if idx, err := strconv.Atoi(tok); err == nil {
if idx < 1 || idx > len(options) {
return nil, fmt.Errorf("selection %d out of range (must be 1-%d)", idx, len(options))
return nil, fmt.Errorf("selection %d is out of range. Expected a number from 1 to %d. Example: 1,2", idx, len(options))
}
val := options[idx-1].value
if _, dup := seen[val]; !dup {
Expand All @@ -472,7 +472,7 @@ func promptNonInteractiveMultiSelect(scanner *bufio.Scanner, title string, optio
continue
}

return nil, fmt.Errorf("unknown option %q", tok)
return nil, fmt.Errorf("option %q is not available. Expected comma-separated numbers or option values. Example: 1,2", tok)
}
return selected, nil
}
Expand Down
14 changes: 13 additions & 1 deletion pkg/cli/interactive_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -438,10 +438,13 @@ func TestCreateWorkflowInteractively_InAutomatedEnvironment(t *testing.T) {
t.Error("Expected error in automated environment, got nil")
}

expectedErrMsg := "interactive workflow creation cannot be used in automated tests or CI environments"
expectedErrMsg := "interactive workflow creation is unavailable in automated tests or CI environments"
if !strings.Contains(err.Error(), expectedErrMsg) {
t.Errorf("Expected error containing %q, got %q", expectedErrMsg, err.Error())
}
if !strings.Contains(err.Error(), "Expected an interactive terminal outside automation") || !strings.Contains(err.Error(), "Example: run `gh aw new` from a local terminal") {
t.Errorf("Expected actionable error message, got %q", err.Error())
}
}

func TestCreateWorkflowInteractively_WithForceFlag(t *testing.T) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[/tdd] Test coverage is extended only for interactive.go — the five other changed files have no new or updated tests.

The structured error pattern (state, expected, example) is now enforced in one place but unverified in add_workflow_resolution.go, audit_diff_command.go, add_interactive_orchestrator.go, and org_runner.go. A regression in any of those files will go undetected.

💡 Suggested approach

Add a parallel assertion for each changed file that checks both an Expected and an Example: fragment:

if !strings.Contains(err.Error(), "Expected") || !strings.Contains(err.Error(), "Example:") {
    t.Errorf("Expected actionable error message, got %q", err.Error())
}

@copilot please address this.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed in 61891c4e78: added focused assertions for the other changed validation paths, including org runner, audit diff, current-repository resolution, private workflow resolution, and add-wizard automation errors.

Expand Down Expand Up @@ -795,6 +798,9 @@ func TestPromptNonInteractiveSelect_InvalidValue(t *testing.T) {
if err == nil {
t.Fatal("expected error for unknown value, got nil")
}
if !strings.Contains(err.Error(), "Expected a number or option value") || !strings.Contains(err.Error(), "Example: 1") {
t.Errorf("Expected actionable select error message, got %q", err.Error())
}
}

func TestPromptNonInteractiveSelect_EOF(t *testing.T) {
Expand Down Expand Up @@ -874,6 +880,9 @@ func TestPromptNonInteractiveMultiSelect_OutOfRange(t *testing.T) {
if err == nil {
t.Fatal("expected error for out-of-range index, got nil")
}
if !strings.Contains(err.Error(), "Expected a number from 1 to 1") || !strings.Contains(err.Error(), "Example: 1,2") {
t.Errorf("Expected actionable multi-select range error message, got %q", err.Error())
}
}

func TestPromptNonInteractiveMultiSelect_UnknownValue(t *testing.T) {
Expand All @@ -883,6 +892,9 @@ func TestPromptNonInteractiveMultiSelect_UnknownValue(t *testing.T) {
if err == nil {
t.Fatal("expected error for unknown value, got nil")
}
if !strings.Contains(err.Error(), "Expected comma-separated numbers or option values") || !strings.Contains(err.Error(), "Example: 1,2") {
t.Errorf("Expected actionable multi-select value error message, got %q", err.Error())
}
}

func TestPromptForWorkflowNameFrom_Valid(t *testing.T) {
Expand Down
10 changes: 5 additions & 5 deletions pkg/cli/org_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,19 +146,19 @@ func runCommandForOrg(ctx context.Context, org string, repoGlobs []string, cbs o
return errors.New("createPR and createIssue are mutually exclusive")
}
if cbs.SearchFn == nil {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These callback validation errors are internal programming errors (not user-facing CLI messages). The new Example: values expose raw Go struct literal syntax (orgRunCallbacks{SearchFn: searchFn}) which is not actionable to CLI users.

Consider either omitting Example: for developer-facing internal errors, or replacing with a user-meaningful CLI example (e.g., gh aw run --org octo-org).

@copilot please address this.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed in 61891c4e78: the org callback examples are now developer-oriented and no longer expose raw Go struct literal syntax.

return errors.New("orgRunCallbacks.SearchFn is required")
return errors.New("organization search callback is not configured. Expected orgRunCallbacks.SearchFn to search organization repositories. Example: configure SearchFn before calling runCommandForOrg")
}
if cbs.ReportFn == nil {
return errors.New("orgRunCallbacks.ReportFn is required")
return errors.New("organization report callback is not configured. Expected orgRunCallbacks.ReportFn to display the run summary. Example: configure ReportFn before calling runCommandForOrg")
}
if createPR && cbs.ApplyFn == nil {
return errors.New("orgRunCallbacks.ApplyFn is required when createPR is true")
return errors.New("pull request callback is not configured. Expected orgRunCallbacks.ApplyFn when createPR is enabled. Example: configure ApplyFn before calling runCommandForOrg with createPR")
}
if createIssue && cbs.IssueFn == nil {
return errors.New("orgRunCallbacks.IssueFn is required when createIssue is true")
return errors.New("issue callback is not configured. Expected orgRunCallbacks.IssueFn when createIssue is enabled. Example: configure IssueFn before calling runCommandForOrg with createIssue")
}
if (createPR || createIssue) && !cbs.AutoYes && isRunningInCIFn() {
return errors.New("confirmation is required for --org create operations in CI; re-run with --yes to auto-accept")
return errors.New("organization create operations in CI need confirmation. Expected --yes to auto-accept in non-interactive environments. Example: gh aw update --org octo-org --create-pull-request --yes")
}

// Handle Ctrl-C / SIGTERM so an interrupted run still renders the report
Expand Down
Loading