diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19b2703c47e..ee11faaec64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,7 @@ jobs: - name: Run unit tests with coverage run: | + set -o pipefail go test -v -parallel=8 -timeout=3m -run='^Test' -coverprofile=coverage.out -json ./... | tee test-result-unit.json go tool cover -html=coverage.out -o coverage.html @@ -71,7 +72,7 @@ jobs: test-group: - name: "CLI Compile & Poutine" packages: "./pkg/cli" - pattern: "^TestCompile[^W]|TestPoutine" # Exclude TestCompileWorkflows to avoid duplicates + pattern: "^TestCompile[^W]|TestPoutine|TestBuildCommandString|TestBuildSourceString|TestBuildWorkflowDescription|TestPrintCompilationSummary|TestCompilationStats|TestModifyWorkflowForTrialMode" # Exclude TestCompileWorkflows to avoid duplicates - name: "CLI MCP Playwright" packages: "./pkg/cli" pattern: "TestMCPInspectPlaywright" @@ -80,7 +81,7 @@ jobs: pattern: "TestMCPGateway" - name: "CLI MCP Other" packages: "./pkg/cli" - pattern: "TestMCPAdd|TestMCPInspectGitHub|TestMCPServer|TestMCPConfig" + pattern: "TestMCPAdd|TestMCPInspectGitHub|TestMCPServer|TestMCPConfig|TestMCPCommand|TestIsRunningAsMCPServer|TestHeaderRoundTripper" - name: "CLI Logs & Firewall" packages: "./pkg/cli" pattern: "TestLogs|TestFirewall|TestNoStopTime|TestLocalWorkflow" @@ -92,10 +93,10 @@ jobs: pattern: "TestConnectHTTPMCPServer" - name: "CLI Compile Workflows" # Isolate slow workflow compilation test packages: "./pkg/cli" - pattern: "TestCompileWorkflows_EmptyMarkdown" + pattern: "TestCompileWorkflows|TestCollectWorkflowFiles|TestFilterWorkflowFiles|TestNormalizeWorkflowFile" - name: "CLI Security Tools" # Group security tool compilation tests packages: "./pkg/cli" - pattern: "TestCompileWithZizmor|TestCompileWithPoutine|TestCompileWithPoutineAndZizmor" + pattern: "TestCompileWithZizmor|TestCompileWithPoutine|TestCompileWithPoutineAndZizmor|TestParseAndDisplayZizmorOutput|TestSecurityToolsIndependentOfValidate|TestSecurity" - name: "CLI Add & List Commands" packages: "./pkg/cli" pattern: "^TestAdd|^TestList" @@ -107,11 +108,65 @@ jobs: pattern: "^TestAudit|^TestInspect" - name: "CLI Docker Build" # Isolate slow Docker build test (~38s) packages: "./pkg/cli" - pattern: "TestDockerBuild" + pattern: "TestDockerBuild|TestDockerfile|TestDockerImage" + - name: "CLI Shell Completion" # Shell completion tests + packages: "./pkg/cli" + pattern: "TestCompletion|TestUninstall.*Completion" + - name: "CLI Secrets & Encryption" # Secrets and encryption tests + packages: "./pkg/cli" + pattern: "TestSecret|TestEncrypt|TestCheckSecretsAvailability|TestResolveSecretValueForSet|TestExtractSecretName|TestExtractSecretsFromConfig|TestNewSecretsCommand" + - name: "CLI Status & Versioning" # Status and version command tests + packages: "./pkg/cli" + pattern: "TestStatus|TestVersion|TestWorkflowStatus|TestWorkflowRunInfo|TestShellTypeString" + - name: "CLI Update & Upgrade" # Update and upgrade command tests + packages: "./pkg/cli" + pattern: "TestUpdate|TestUpgrade|TestCheckForUpdates|TestShowUpdateSummary|TestMajorVersionPreference|TestPreciseVersionPreference|TestActionKeyVersionConsistency|TestMarshalActionsLockSorted|TestShouldCheckForUpdate|TestGetLastCheckFilePath|TestEnsureUpgradeAgenticWorkflowAgent|TestEnsureUpgradeAgenticWorkflowsPrompt" + - name: "CLI Workflows Path & Discovery" # Workflow path resolution and discovery tests + packages: "./pkg/cli" + pattern: "TestFindRunnableWorkflows|TestFindWorkflowsWithSource|TestGetAvailableWorkflowNames|TestGetWorkflowInputs|TestGetLockFilePath|TestGetMarkdownWorkflowFilesExcludesREADME|TestReadWorkflowFile|TestResolveWorkflowPath|TestIsWorkflowFile|TestWorkflowCounting|TestWorkflowResolutionWindowsCompatibility|TestSuggestWorkflowNames|TestValidateWorkflowName|TestValidateWorkflowIntent|TestValidateWorkflowInputs|TestWorkflowSpecString" + - name: "CLI Trial & Interactive" # Trial mode and interactive tests + packages: "./pkg/cli" + pattern: "TestTrial|TestLipglossImportPresent|TestSectionCompositionPattern" + - name: "CLI Run Command" # Run command tests + packages: "./pkg/cli" + pattern: "TestRun|TestPoll|TestTrackWorkflowFailure|TestInputValidation|TestInputFlagSignature|TestRefFlagSignature" + - name: "CLI Repo & Git" # Repository and git related tests + packages: "./pkg/cli" + pattern: "TestRepo|TestGet.*RepoSlug|TestExtractBaseRepo|TestExtractIssueNumberFromURL|TestHostRepoSlugProcessing|TestHasLocalModifications|TestPushWorkflowFiles|TestCloneRepoWithVersion|TestClearCurrentRepoSlugCache|TestIsAuthenticationError|TestRemoveWorkflows_KeepOrphansFlag" + - name: "CLI Spec Parsing" # Spec parsing tests + packages: "./pkg/cli" + pattern: "TestParse.*Spec|TestResolveImportPathLocal|TestIsWorkflowSpecFormatLocal|TestResolveLatestRef|TestParseAwInfo|TestParseVersion" + - name: "CLI Validators & Semver" # Validator and semver tests + packages: "./pkg/cli" + pattern: "TestValidat|TestIsCommitSHA|TestIsPreciseVersion|TestIsSemanticVersionTag|TestIsRunnable" + - name: "CLI Tool Graph" # Tool graph tests + packages: "./pkg/cli" + pattern: "TestToolGraph" + - name: "CLI Signal Handling" # Signal handling tests + packages: "./pkg/cli" + pattern: "TestPollWithSignalHandling" + - name: "CLI Makefile & Setup" # Makefile and setup action tests + packages: "./pkg/cli" + pattern: "TestMakefile|TestSetupCLIAction" + - name: "CLI Merge Content" # Workflow content merging tests + packages: "./pkg/cli" + pattern: "TestMergeWorkflowContent" + - name: "CLI Workflows Actions" # Workflow actions and action extraction tests + packages: "./pkg/cli" + pattern: "TestGetActionSHAForTag|TestExtractNpxPackages" + - name: "CLI Codespace Integration" # Codespace specific tests + packages: "./pkg/cli" + pattern: "TestCodespace|TestDetectShell" + - name: "CLI Safe Inputs" # Safe inputs tests + packages: "./pkg/cli" + pattern: "TestSafeInputsMCPServerCompilation" + - name: "CMD Main & Entry" # CMD tests for main entry point + packages: "./cmd/gh-aw" + pattern: "TestMain|TestCommandGroup|TestCommandLine|TestCommand|TestArgument|TestInitFunction|TestVersion|TestMCPCommand|TestMCPSubcommand|TestCampaignSubcommand|TestPRSubcommand|TestNoCommandsInAdditionalCommandsWithGroups" - name: "CLI Completion & Other" # Remaining catch-all (reduced from original) packages: "./pkg/cli" pattern: "" # Catch-all for tests not matched by other CLI patterns - skip_pattern: "^TestCompile[^W]|TestPoutine|TestMCPInspectPlaywright|TestMCPGateway|TestMCPAdd|TestMCPInspectGitHub|TestMCPServer|TestMCPConfig|TestLogs|TestFirewall|TestNoStopTime|TestLocalWorkflow|TestProgressFlagSignature|TestConnectHTTPMCPServer|TestCompileWorkflows_EmptyMarkdown|TestCompileWithZizmor|TestCompileWithPoutine|TestCompileWithPoutineAndZizmor|^TestAdd|^TestList|^TestUpdate|^TestAudit|^TestInspect|TestDockerBuild" + skip_pattern: "^TestCompile|TestPoutine|TestMCPInspectPlaywright|TestMCPGateway|TestMCPAdd|TestMCPInspectGitHub|TestMCPServer|TestMCPConfig|TestMCPCommand|TestIsRunningAsMCPServer|TestHeaderRoundTripper|TestLogs|TestFirewall|TestNoStopTime|TestLocalWorkflow|TestProgressFlagSignature|TestConnectHTTPMCPServer|TestCompileWorkflows|TestCollectWorkflowFiles|TestFilterWorkflowFiles|TestNormalizeWorkflowFile|TestParseAndDisplayZizmorOutput|TestSecurityToolsIndependentOfValidate|TestSecurity|^TestAdd|^TestList|^TestUpdate|^TestUpgrade|TestCheckForUpdates|TestShowUpdateSummary|TestMajorVersionPreference|TestPreciseVersionPreference|TestActionKeyVersionConsistency|TestMarshalActionsLockSorted|^TestAudit|^TestInspect|TestDockerBuild|TestDockerfile|TestDockerImage|TestCompletion|TestUninstall.*Completion|TestSecret|TestEncrypt|TestCheckSecretsAvailability|TestResolveSecretValueForSet|TestStatus|TestVersion|TestWorkflowStatus|TestWorkflowRunInfo|TestShellTypeString|TestFindRunnableWorkflows|TestFindWorkflowsWithSource|TestGetAvailableWorkflowNames|TestGetWorkflowInputs|TestGetLockFilePath|TestGetMarkdownWorkflowFilesExcludesREADME|TestReadWorkflowFile|TestResolveWorkflowPath|TestIsWorkflowFile|TestWorkflowCounting|TestWorkflowResolutionWindowsCompatibility|TestSuggestWorkflowNames|TestValidateWorkflowName|TestValidateWorkflowIntent|TestValidateWorkflowInputs|TestWorkflowSpecString|TestTrial|TestLipglossImportPresent|TestSectionCompositionPattern|TestRun|TestPoll|TestTrackWorkflowFailure|TestInputValidation|TestInputFlagSignature|TestRefFlagSignature|TestRepo|TestGet.*RepoSlug|TestExtractBaseRepo|TestExtractIssueNumberFromURL|TestHostRepoSlugProcessing|TestHasLocalModifications|TestPushWorkflowFiles|TestCloneRepoWithVersion|TestParse.*Spec|TestResolveImportPathLocal|TestIsWorkflowSpecFormatLocal|TestResolveLatestRef|TestValidat|TestIsCommitSHA|TestIsPreciseVersion|TestIsSemanticVersionTag|TestIsRunnable|TestToolGraph|TestPollWithSignalHandling|TestMakefile|TestSetupCLIAction|TestMergeWorkflowContent|TestGetActionSHAForTag|TestExtractNpxPackages|TestCodespace|TestDetectShell|TestSafeInputsMCPServerCompilation|TestBuildCommandString|TestBuildSourceString|TestBuildWorkflowDescription|TestPrintCompilationSummary|TestCompilationStats|TestModifyWorkflowForTrialMode" - name: "Workflow Compiler" packages: "./pkg/workflow" pattern: "TestCompile|TestWorkflow|TestGenerate|TestParse" @@ -120,13 +175,13 @@ jobs: pattern: "TestMCP|TestTool|TestSkill|TestPlaywright|TestFirewall" - name: "Workflow Validation" packages: "./pkg/workflow" - pattern: "TestValidat|TestLock|TestError|TestWarning" + pattern: "TestValidat|TestLock|TestError|TestWarning|TestRepositoryFeaturesValidation|TestBackwardCompatibilityWithClaudeFormat" - name: "Workflow Safe Outputs" packages: "./pkg/workflow" - pattern: "SafeOutputs|CreatePullRequest|OutputLabel|HasSafeOutputs" + pattern: "SafeOutputs|SafeInputs|CreatePullRequest|OutputLabel|HasSafeOutputs|ConsolidatedSafeOutputs|SafeOutputJobs" - name: "Workflow GitHub & Git" packages: "./pkg/workflow" - pattern: "GitHub|Git|PushToPullRequest|BuildFromAllowed" + pattern: "GitHub|Git|PushToPullRequest|BuildFromAllowed|AdditionalClaudeTools|CopilotGitCommands" - name: "Workflow Rendering & Bundling" packages: "./pkg/workflow" pattern: "Render|Bundle|Script|WritePromptText" @@ -145,30 +200,30 @@ jobs: - name: "CMD Tests" # All cmd/gh-aw integration tests packages: "./cmd/gh-aw" pattern: "" - skip_pattern: "" # No other groups cover cmd tests + skip_pattern: "TestMain|TestCommandGroup|TestCommandLine|TestCommand|TestArgument|TestInitFunction|TestVersion|TestMCPCommand|TestMCPSubcommand|TestCampaignSubcommand|TestPRSubcommand|TestNoCommandsInAdditionalCommandsWithGroups" # These are in the "CMD Main & Entry" group above - name: "Parser Remote Fetch & Cache" packages: "./pkg/parser" - pattern: "TestDownloadFileFromGitHub|TestResolveIncludePath|TestDownloadIncludeFromWorkflowSpec|TestImportCache" + pattern: "TestDownloadFileFromGitHub|TestResolveIncludePath|TestDownloadIncludeFromWorkflowSpec|TestImportCache|TestFrontmatterLocation|TestFrontmatterOffsetCalculation|TestImprovementComparison" - name: "Parser Location & Validation" packages: "./pkg/parser" pattern: "" # Catch-all for tests not matched by other Parser patterns - skip_pattern: "TestDownloadFileFromGitHub|TestResolveIncludePath|TestDownloadIncludeFromWorkflowSpec|TestImportCache" + skip_pattern: "TestDownloadFileFromGitHub|TestResolveIncludePath|TestDownloadIncludeFromWorkflowSpec|TestImportCache|TestFrontmatterLocation|TestFrontmatterOffsetCalculation|TestImprovementComparison" - name: "Workflow Permissions" packages: "./pkg/workflow" - pattern: "TestPermissions|TestPackageExtractor|TestCollectPackagesFromWorkflow" + pattern: "TestPermissions|TestPackageExtractor|TestCollectPackagesFromWorkflow|TestExtractNpx" - name: "Workflow Misc Part 1" # Split large catch-all into two balanced groups packages: "./pkg/workflow" - pattern: "TestAgent|TestCopilot|TestCustom|TestEngine|TestModel|TestNetwork|TestOpenAI|TestProvider" + pattern: "TestAgent|TestCopilot|TestCustomEngine|TestEngine|TestModel|TestNetwork|TestOpenAI|TestProvider|TestManualApprovalEnvironmentInActivationJob|TestNeutralToolsIntegration|TestIndividualGitHubToken|TestTopLevelGitHubTokenPrecedence" - name: "Workflow String & Sanitization" packages: "./pkg/workflow" - pattern: "String|Sanitize|Normalize|Trim|Clean|Format" + pattern: "String|Sanitize|Normalize|Trim|Clean|Format|SingleQuoteEscaping|WorkflowTimestampCheckUsesJavaScript" - name: "Workflow Runtime & Setup" packages: "./pkg/workflow" - pattern: "Runtime|Setup|Install|Download|Version|Binary" + pattern: "Runtime|Setup|Install|Download|Version|Binary|StopTimeResolution" - name: "Workflow Misc Part 2" # Remaining workflow tests packages: "./pkg/workflow" pattern: "" - skip_pattern: "TestCompile|TestWorkflow|TestGenerate|TestParse|TestMCP|TestTool|TestSkill|TestPlaywright|TestFirewall|TestValidat|TestLock|TestError|TestWarning|SafeOutputs|CreatePullRequest|OutputLabel|HasSafeOutputs|GitHub|Git|PushToPullRequest|BuildFromAllowed|Render|Bundle|Script|WritePromptText|^TestCache|TestCacheDependencies|TestCacheKey|TestValidateCache|^TestActionPinSHAsMatchVersionTags|^TestAction[^P]|Container|Dependabot|Security|PII|TestPermissions|TestPackageExtractor|TestCollectPackagesFromWorkflow|TestAgent|TestCopilot|TestCustom|TestEngine|TestModel|TestNetwork|TestOpenAI|TestProvider|String|Sanitize|Normalize|Trim|Clean|Format|Runtime|Setup|Install|Download|Version|Binary" + skip_pattern: "TestCompile|TestWorkflow|TestGenerate|TestParse|TestMCP|TestTool|TestSkill|TestPlaywright|TestFirewall|TestValidat|TestLock|TestError|TestWarning|TestRepositoryFeaturesValidation|TestBackwardCompatibilityWithClaudeFormat|SafeOutputs|SafeInputs|CreatePullRequest|OutputLabel|HasSafeOutputs|ConsolidatedSafeOutputs|GitHub|Git|PushToPullRequest|BuildFromAllowed|AdditionalClaudeTools|CopilotGitCommands|Render|Bundle|Script|WritePromptText|^TestCache|TestCacheDependencies|TestCacheKey|TestValidateCache|^TestActionPinSHAsMatchVersionTags|^TestAction[^P]|Container|Dependabot|Security|PII|TestPermissions|TestPackageExtractor|TestCollectPackagesFromWorkflow|TestExtractNpx|TestAgent|TestCopilot|TestCustomEngine|TestEngine|TestModel|TestNetwork|TestOpenAI|TestProvider|TestManualApprovalEnvironmentInActivationJob|TestNeutralToolsIntegration|TestIndividualGitHubToken|TestTopLevelGitHubTokenPrecedence|String|Sanitize|Normalize|Trim|Clean|Format|SingleQuoteEscaping|WorkflowTimestampCheckUsesJavaScript|Runtime|Setup|Install|Download|Version|Binary|StopTimeResolution" concurrency: group: ci-${{ github.ref }}-integration-${{ matrix.test-group.name }} cancel-in-progress: true @@ -197,6 +252,7 @@ jobs: - name: Run integration tests - ${{ matrix.test-group.name }} run: | + set -o pipefail # Sanitize the test group name for use in filename SAFE_NAME=$(echo "${{ matrix.test-group.name }}" | sed 's/[^a-zA-Z0-9]/-/g' | sed 's/--*/-/g') @@ -230,6 +286,7 @@ jobs: - name: List all tests in codebase run: | + set -euo pipefail echo "Extracting all test function names from source files..." ./scripts/list-all-tests.sh > all-tests.txt echo "Found $(wc -l < all-tests.txt) tests in codebase" @@ -243,6 +300,7 @@ jobs: - name: List downloaded artifacts run: | + set -euo pipefail echo "Downloaded test result artifacts:" find test-results -type f -name "*.json" | sort echo "" @@ -250,6 +308,7 @@ jobs: - name: Extract executed tests from artifacts run: | + set -euo pipefail echo "Extracting test names from JSON artifacts..." ./scripts/extract-executed-tests.sh test-results > executed-tests.txt echo "Found $(wc -l < executed-tests.txt) executed tests" diff --git a/scripts/extract-executed-tests.sh b/scripts/extract-executed-tests.sh index 0b000ddc48d..a71d387e69e 100755 --- a/scripts/extract-executed-tests.sh +++ b/scripts/extract-executed-tests.sh @@ -19,9 +19,24 @@ fi # Find all JSON test result files and extract test names # Look for lines with "Action":"run" and extract the "Test" field -# Use grep with || true to prevent exit on no matches +# Process each file separately to handle cases where files might be empty or have no matches +temp_file=$(mktemp) find "$TEST_RESULT_DIR" -name "*.json" -type f | while read -r file; do - grep '"Action":"run"' "$file" 2>/dev/null | \ - grep -o '"Test":"[^"]*"' | \ - sed 's/"Test":"\([^"]*\)"/\1/' || true -done | sort -u + if [ -s "$file" ]; then + # File exists and is not empty + grep '"Action":"run"' "$file" 2>/dev/null | \ + grep -o '"Test":"[^"]*"' | \ + sed 's/"Test":"\([^"]*\)"/\1/' >> "$temp_file" || true + fi +done + +# Sort and deduplicate the results +if [ -s "$temp_file" ]; then + sort -u "$temp_file" + rm -f "$temp_file" +else + # No tests found - this is an error condition + rm -f "$temp_file" + echo "Error: No test execution records found in $TEST_RESULT_DIR" >&2 + exit 1 +fi