Skip to content

Add missing test patterns to CI - cover all 353 untested functions #12371

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pelikhan merged 4 commits into from
Jan 28, 2026
Merged

Add missing test patterns to CI - cover all 353 untested functions #12371

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b3f975f
Initial plan
Copilot Jan 28, 2026
387d198
Add missing test patterns to CI workflow
Copilot Jan 28, 2026
98b0deb
Add remaining missing test patterns for complete coverage
Copilot Jan 28, 2026
9985000
Fix error handling in CI: ensure test failures propagate correctly
Copilot Jan 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 76 additions & 17 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ jobs:

- name: Run unit tests with coverage
run: |
set -o pipefail
go test -v -parallel=8 -timeout=3m -run='^Test' -coverprofile=coverage.out -json ./... | tee test-result-unit.json
go tool cover -html=coverage.out -o coverage.html

Expand Down Expand Up @@ -71,7 +72,7 @@ jobs:
test-group:
- name: "CLI Compile & Poutine"
packages: "./pkg/cli"
pattern: "^TestCompile[^W]|TestPoutine" # Exclude TestCompileWorkflows to avoid duplicates
pattern: "^TestCompile[^W]|TestPoutine|TestBuildCommandString|TestBuildSourceString|TestBuildWorkflowDescription|TestPrintCompilationSummary|TestCompilationStats|TestModifyWorkflowForTrialMode" # Exclude TestCompileWorkflows to avoid duplicates
- name: "CLI MCP Playwright"
packages: "./pkg/cli"
pattern: "TestMCPInspectPlaywright"
Expand All @@ -80,7 +81,7 @@ jobs:
pattern: "TestMCPGateway"
- name: "CLI MCP Other"
packages: "./pkg/cli"
pattern: "TestMCPAdd|TestMCPInspectGitHub|TestMCPServer|TestMCPConfig"
pattern: "TestMCPAdd|TestMCPInspectGitHub|TestMCPServer|TestMCPConfig|TestMCPCommand|TestIsRunningAsMCPServer|TestHeaderRoundTripper"
- name: "CLI Logs & Firewall"
packages: "./pkg/cli"
pattern: "TestLogs|TestFirewall|TestNoStopTime|TestLocalWorkflow"
Expand All @@ -92,10 +93,10 @@ jobs:
pattern: "TestConnectHTTPMCPServer"
- name: "CLI Compile Workflows" # Isolate slow workflow compilation test
packages: "./pkg/cli"
pattern: "TestCompileWorkflows_EmptyMarkdown"
pattern: "TestCompileWorkflows|TestCollectWorkflowFiles|TestFilterWorkflowFiles|TestNormalizeWorkflowFile"
- name: "CLI Security Tools" # Group security tool compilation tests
packages: "./pkg/cli"
pattern: "TestCompileWithZizmor|TestCompileWithPoutine|TestCompileWithPoutineAndZizmor"
pattern: "TestCompileWithZizmor|TestCompileWithPoutine|TestCompileWithPoutineAndZizmor|TestParseAndDisplayZizmorOutput|TestSecurityToolsIndependentOfValidate|TestSecurity"
- name: "CLI Add & List Commands"
packages: "./pkg/cli"
pattern: "^TestAdd|^TestList"
Expand All @@ -107,11 +108,65 @@ jobs:
pattern: "^TestAudit|^TestInspect"
- name: "CLI Docker Build" # Isolate slow Docker build test (~38s)
packages: "./pkg/cli"
pattern: "TestDockerBuild"
pattern: "TestDockerBuild|TestDockerfile|TestDockerImage"
- name: "CLI Shell Completion" # Shell completion tests
packages: "./pkg/cli"
pattern: "TestCompletion|TestUninstall.*Completion"
- name: "CLI Secrets & Encryption" # Secrets and encryption tests
packages: "./pkg/cli"
pattern: "TestSecret|TestEncrypt|TestCheckSecretsAvailability|TestResolveSecretValueForSet|TestExtractSecretName|TestExtractSecretsFromConfig|TestNewSecretsCommand"
- name: "CLI Status & Versioning" # Status and version command tests
packages: "./pkg/cli"
pattern: "TestStatus|TestVersion|TestWorkflowStatus|TestWorkflowRunInfo|TestShellTypeString"
- name: "CLI Update & Upgrade" # Update and upgrade command tests
packages: "./pkg/cli"
pattern: "TestUpdate|TestUpgrade|TestCheckForUpdates|TestShowUpdateSummary|TestMajorVersionPreference|TestPreciseVersionPreference|TestActionKeyVersionConsistency|TestMarshalActionsLockSorted|TestShouldCheckForUpdate|TestGetLastCheckFilePath|TestEnsureUpgradeAgenticWorkflowAgent|TestEnsureUpgradeAgenticWorkflowsPrompt"
- name: "CLI Workflows Path & Discovery" # Workflow path resolution and discovery tests
packages: "./pkg/cli"
pattern: "TestFindRunnableWorkflows|TestFindWorkflowsWithSource|TestGetAvailableWorkflowNames|TestGetWorkflowInputs|TestGetLockFilePath|TestGetMarkdownWorkflowFilesExcludesREADME|TestReadWorkflowFile|TestResolveWorkflowPath|TestIsWorkflowFile|TestWorkflowCounting|TestWorkflowResolutionWindowsCompatibility|TestSuggestWorkflowNames|TestValidateWorkflowName|TestValidateWorkflowIntent|TestValidateWorkflowInputs|TestWorkflowSpecString"
- name: "CLI Trial & Interactive" # Trial mode and interactive tests
packages: "./pkg/cli"
pattern: "TestTrial|TestLipglossImportPresent|TestSectionCompositionPattern"
- name: "CLI Run Command" # Run command tests
packages: "./pkg/cli"
pattern: "TestRun|TestPoll|TestTrackWorkflowFailure|TestInputValidation|TestInputFlagSignature|TestRefFlagSignature"
- name: "CLI Repo & Git" # Repository and git related tests
packages: "./pkg/cli"
pattern: "TestRepo|TestGet.*RepoSlug|TestExtractBaseRepo|TestExtractIssueNumberFromURL|TestHostRepoSlugProcessing|TestHasLocalModifications|TestPushWorkflowFiles|TestCloneRepoWithVersion|TestClearCurrentRepoSlugCache|TestIsAuthenticationError|TestRemoveWorkflows_KeepOrphansFlag"
- name: "CLI Spec Parsing" # Spec parsing tests
packages: "./pkg/cli"
pattern: "TestParse.*Spec|TestResolveImportPathLocal|TestIsWorkflowSpecFormatLocal|TestResolveLatestRef|TestParseAwInfo|TestParseVersion"
- name: "CLI Validators & Semver" # Validator and semver tests
packages: "./pkg/cli"
pattern: "TestValidat|TestIsCommitSHA|TestIsPreciseVersion|TestIsSemanticVersionTag|TestIsRunnable"
- name: "CLI Tool Graph" # Tool graph tests
packages: "./pkg/cli"
pattern: "TestToolGraph"
- name: "CLI Signal Handling" # Signal handling tests
packages: "./pkg/cli"
pattern: "TestPollWithSignalHandling"
- name: "CLI Makefile & Setup" # Makefile and setup action tests
packages: "./pkg/cli"
pattern: "TestMakefile|TestSetupCLIAction"
- name: "CLI Merge Content" # Workflow content merging tests
packages: "./pkg/cli"
pattern: "TestMergeWorkflowContent"
- name: "CLI Workflows Actions" # Workflow actions and action extraction tests
packages: "./pkg/cli"
pattern: "TestGetActionSHAForTag|TestExtractNpxPackages"
- name: "CLI Codespace Integration" # Codespace specific tests
packages: "./pkg/cli"
pattern: "TestCodespace|TestDetectShell"
- name: "CLI Safe Inputs" # Safe inputs tests
packages: "./pkg/cli"
pattern: "TestSafeInputsMCPServerCompilation"
- name: "CMD Main & Entry" # CMD tests for main entry point
packages: "./cmd/gh-aw"
pattern: "TestMain|TestCommandGroup|TestCommandLine|TestCommand|TestArgument|TestInitFunction|TestVersion|TestMCPCommand|TestMCPSubcommand|TestCampaignSubcommand|TestPRSubcommand|TestNoCommandsInAdditionalCommandsWithGroups"
- name: "CLI Completion & Other" # Remaining catch-all (reduced from original)
packages: "./pkg/cli"
pattern: "" # Catch-all for tests not matched by other CLI patterns
skip_pattern: "^TestCompile[^W]|TestPoutine|TestMCPInspectPlaywright|TestMCPGateway|TestMCPAdd|TestMCPInspectGitHub|TestMCPServer|TestMCPConfig|TestLogs|TestFirewall|TestNoStopTime|TestLocalWorkflow|TestProgressFlagSignature|TestConnectHTTPMCPServer|TestCompileWorkflows_EmptyMarkdown|TestCompileWithZizmor|TestCompileWithPoutine|TestCompileWithPoutineAndZizmor|^TestAdd|^TestList|^TestUpdate|^TestAudit|^TestInspect|TestDockerBuild"
skip_pattern: "^TestCompile|TestPoutine|TestMCPInspectPlaywright|TestMCPGateway|TestMCPAdd|TestMCPInspectGitHub|TestMCPServer|TestMCPConfig|TestMCPCommand|TestIsRunningAsMCPServer|TestHeaderRoundTripper|TestLogs|TestFirewall|TestNoStopTime|TestLocalWorkflow|TestProgressFlagSignature|TestConnectHTTPMCPServer|TestCompileWorkflows|TestCollectWorkflowFiles|TestFilterWorkflowFiles|TestNormalizeWorkflowFile|TestParseAndDisplayZizmorOutput|TestSecurityToolsIndependentOfValidate|TestSecurity|^TestAdd|^TestList|^TestUpdate|^TestUpgrade|TestCheckForUpdates|TestShowUpdateSummary|TestMajorVersionPreference|TestPreciseVersionPreference|TestActionKeyVersionConsistency|TestMarshalActionsLockSorted|^TestAudit|^TestInspect|TestDockerBuild|TestDockerfile|TestDockerImage|TestCompletion|TestUninstall.*Completion|TestSecret|TestEncrypt|TestCheckSecretsAvailability|TestResolveSecretValueForSet|TestStatus|TestVersion|TestWorkflowStatus|TestWorkflowRunInfo|TestShellTypeString|TestFindRunnableWorkflows|TestFindWorkflowsWithSource|TestGetAvailableWorkflowNames|TestGetWorkflowInputs|TestGetLockFilePath|TestGetMarkdownWorkflowFilesExcludesREADME|TestReadWorkflowFile|TestResolveWorkflowPath|TestIsWorkflowFile|TestWorkflowCounting|TestWorkflowResolutionWindowsCompatibility|TestSuggestWorkflowNames|TestValidateWorkflowName|TestValidateWorkflowIntent|TestValidateWorkflowInputs|TestWorkflowSpecString|TestTrial|TestLipglossImportPresent|TestSectionCompositionPattern|TestRun|TestPoll|TestTrackWorkflowFailure|TestInputValidation|TestInputFlagSignature|TestRefFlagSignature|TestRepo|TestGet.*RepoSlug|TestExtractBaseRepo|TestExtractIssueNumberFromURL|TestHostRepoSlugProcessing|TestHasLocalModifications|TestPushWorkflowFiles|TestCloneRepoWithVersion|TestParse.*Spec|TestResolveImportPathLocal|TestIsWorkflowSpecFormatLocal|TestResolveLatestRef|TestValidat|TestIsCommitSHA|TestIsPreciseVersion|TestIsSemanticVersionTag|TestIsRunnable|TestToolGraph|TestPollWithSignalHandling|TestMakefile|TestSetupCLIAction|TestMergeWorkflowContent|TestGetActionSHAForTag|TestExtractNpxPackages|TestCodespace|TestDetectShell|TestSafeInputsMCPServerCompilation|TestBuildCommandString|TestBuildSourceString|TestBuildWorkflowDescription|TestPrintCompilationSummary|TestCompilationStats|TestModifyWorkflowForTrialMode"
- name: "Workflow Compiler"
packages: "./pkg/workflow"
pattern: "TestCompile|TestWorkflow|TestGenerate|TestParse"
Expand All @@ -120,13 +175,13 @@ jobs:
pattern: "TestMCP|TestTool|TestSkill|TestPlaywright|TestFirewall"
- name: "Workflow Validation"
packages: "./pkg/workflow"
pattern: "TestValidat|TestLock|TestError|TestWarning"
pattern: "TestValidat|TestLock|TestError|TestWarning|TestRepositoryFeaturesValidation|TestBackwardCompatibilityWithClaudeFormat"
- name: "Workflow Safe Outputs"
packages: "./pkg/workflow"
pattern: "SafeOutputs|CreatePullRequest|OutputLabel|HasSafeOutputs"
pattern: "SafeOutputs|SafeInputs|CreatePullRequest|OutputLabel|HasSafeOutputs|ConsolidatedSafeOutputs|SafeOutputJobs"
- name: "Workflow GitHub & Git"
packages: "./pkg/workflow"
pattern: "GitHub|Git|PushToPullRequest|BuildFromAllowed"
pattern: "GitHub|Git|PushToPullRequest|BuildFromAllowed|AdditionalClaudeTools|CopilotGitCommands"
- name: "Workflow Rendering & Bundling"
packages: "./pkg/workflow"
pattern: "Render|Bundle|Script|WritePromptText"
Expand All @@ -145,30 +200,30 @@ jobs:
- name: "CMD Tests" # All cmd/gh-aw integration tests
packages: "./cmd/gh-aw"
pattern: ""
skip_pattern: "" # No other groups cover cmd tests
skip_pattern: "TestMain|TestCommandGroup|TestCommandLine|TestCommand|TestArgument|TestInitFunction|TestVersion|TestMCPCommand|TestMCPSubcommand|TestCampaignSubcommand|TestPRSubcommand|TestNoCommandsInAdditionalCommandsWithGroups" # These are in the "CMD Main & Entry" group above
- name: "Parser Remote Fetch & Cache"
packages: "./pkg/parser"
pattern: "TestDownloadFileFromGitHub|TestResolveIncludePath|TestDownloadIncludeFromWorkflowSpec|TestImportCache"
pattern: "TestDownloadFileFromGitHub|TestResolveIncludePath|TestDownloadIncludeFromWorkflowSpec|TestImportCache|TestFrontmatterLocation|TestFrontmatterOffsetCalculation|TestImprovementComparison"
- name: "Parser Location & Validation"
packages: "./pkg/parser"
pattern: "" # Catch-all for tests not matched by other Parser patterns
skip_pattern: "TestDownloadFileFromGitHub|TestResolveIncludePath|TestDownloadIncludeFromWorkflowSpec|TestImportCache"
skip_pattern: "TestDownloadFileFromGitHub|TestResolveIncludePath|TestDownloadIncludeFromWorkflowSpec|TestImportCache|TestFrontmatterLocation|TestFrontmatterOffsetCalculation|TestImprovementComparison"
- name: "Workflow Permissions"
packages: "./pkg/workflow"
pattern: "TestPermissions|TestPackageExtractor|TestCollectPackagesFromWorkflow"
pattern: "TestPermissions|TestPackageExtractor|TestCollectPackagesFromWorkflow|TestExtractNpx"
- name: "Workflow Misc Part 1" # Split large catch-all into two balanced groups
packages: "./pkg/workflow"
pattern: "TestAgent|TestCopilot|TestCustom|TestEngine|TestModel|TestNetwork|TestOpenAI|TestProvider"
pattern: "TestAgent|TestCopilot|TestCustomEngine|TestEngine|TestModel|TestNetwork|TestOpenAI|TestProvider|TestManualApprovalEnvironmentInActivationJob|TestNeutralToolsIntegration|TestIndividualGitHubToken|TestTopLevelGitHubTokenPrecedence"
- name: "Workflow String & Sanitization"
packages: "./pkg/workflow"
pattern: "String|Sanitize|Normalize|Trim|Clean|Format"
pattern: "String|Sanitize|Normalize|Trim|Clean|Format|SingleQuoteEscaping|WorkflowTimestampCheckUsesJavaScript"
- name: "Workflow Runtime & Setup"
packages: "./pkg/workflow"
pattern: "Runtime|Setup|Install|Download|Version|Binary"
pattern: "Runtime|Setup|Install|Download|Version|Binary|StopTimeResolution"
- name: "Workflow Misc Part 2" # Remaining workflow tests
packages: "./pkg/workflow"
pattern: ""
skip_pattern: "TestCompile|TestWorkflow|TestGenerate|TestParse|TestMCP|TestTool|TestSkill|TestPlaywright|TestFirewall|TestValidat|TestLock|TestError|TestWarning|SafeOutputs|CreatePullRequest|OutputLabel|HasSafeOutputs|GitHub|Git|PushToPullRequest|BuildFromAllowed|Render|Bundle|Script|WritePromptText|^TestCache|TestCacheDependencies|TestCacheKey|TestValidateCache|^TestActionPinSHAsMatchVersionTags|^TestAction[^P]|Container|Dependabot|Security|PII|TestPermissions|TestPackageExtractor|TestCollectPackagesFromWorkflow|TestAgent|TestCopilot|TestCustom|TestEngine|TestModel|TestNetwork|TestOpenAI|TestProvider|String|Sanitize|Normalize|Trim|Clean|Format|Runtime|Setup|Install|Download|Version|Binary"
skip_pattern: "TestCompile|TestWorkflow|TestGenerate|TestParse|TestMCP|TestTool|TestSkill|TestPlaywright|TestFirewall|TestValidat|TestLock|TestError|TestWarning|TestRepositoryFeaturesValidation|TestBackwardCompatibilityWithClaudeFormat|SafeOutputs|SafeInputs|CreatePullRequest|OutputLabel|HasSafeOutputs|ConsolidatedSafeOutputs|GitHub|Git|PushToPullRequest|BuildFromAllowed|AdditionalClaudeTools|CopilotGitCommands|Render|Bundle|Script|WritePromptText|^TestCache|TestCacheDependencies|TestCacheKey|TestValidateCache|^TestActionPinSHAsMatchVersionTags|^TestAction[^P]|Container|Dependabot|Security|PII|TestPermissions|TestPackageExtractor|TestCollectPackagesFromWorkflow|TestExtractNpx|TestAgent|TestCopilot|TestCustomEngine|TestEngine|TestModel|TestNetwork|TestOpenAI|TestProvider|TestManualApprovalEnvironmentInActivationJob|TestNeutralToolsIntegration|TestIndividualGitHubToken|TestTopLevelGitHubTokenPrecedence|String|Sanitize|Normalize|Trim|Clean|Format|SingleQuoteEscaping|WorkflowTimestampCheckUsesJavaScript|Runtime|Setup|Install|Download|Version|Binary|StopTimeResolution"
concurrency:
group: ci-${{ github.ref }}-integration-${{ matrix.test-group.name }}
cancel-in-progress: true
Expand Down Expand Up @@ -197,6 +252,7 @@ jobs:

- name: Run integration tests - ${{ matrix.test-group.name }}
run: |
set -o pipefail
# Sanitize the test group name for use in filename
SAFE_NAME=$(echo "${{ matrix.test-group.name }}" | sed 's/[^a-zA-Z0-9]/-/g' | sed 's/--*/-/g')

Expand Down Expand Up @@ -230,6 +286,7 @@ jobs:

- name: List all tests in codebase
run: |
set -euo pipefail
echo "Extracting all test function names from source files..."
./scripts/list-all-tests.sh > all-tests.txt
echo "Found $(wc -l < all-tests.txt) tests in codebase"
Expand All @@ -243,13 +300,15 @@ jobs:

- name: List downloaded artifacts
run: |
set -euo pipefail
echo "Downloaded test result artifacts:"
find test-results -type f -name "*.json" | sort
echo ""
echo "Total JSON files: $(find test-results -type f -name "*.json" | wc -l)"

- name: Extract executed tests from artifacts
run: |
set -euo pipefail
echo "Extracting test names from JSON artifacts..."
./scripts/extract-executed-tests.sh test-results > executed-tests.txt
echo "Found $(wc -l < executed-tests.txt) executed tests"
Expand Down
25 changes: 20 additions & 5 deletions scripts/extract-executed-tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,24 @@ fi

# Find all JSON test result files and extract test names
# Look for lines with "Action":"run" and extract the "Test" field
# Use grep with || true to prevent exit on no matches
# Process each file separately to handle cases where files might be empty or have no matches
temp_file=$(mktemp)
find "$TEST_RESULT_DIR" -name "*.json" -type f | while read -r file; do
grep '"Action":"run"' "$file" 2>/dev/null | \
grep -o '"Test":"[^"]*"' | \
sed 's/"Test":"\([^"]*\)"/\1/' || true
done | sort -u
if [ -s "$file" ]; then
# File exists and is not empty
grep '"Action":"run"' "$file" 2>/dev/null | \
grep -o '"Test":"[^"]*"' | \
sed 's/"Test":"\([^"]*\)"/\1/' >> "$temp_file" || true
fi
done

# Sort and deduplicate the results
if [ -s "$temp_file" ]; then
sort -u "$temp_file"
rm -f "$temp_file"
else
# No tests found - this is an error condition
rm -f "$temp_file"
echo "Error: No test execution records found in $TEST_RESULT_DIR" >&2
exit 1
fi