From 8aa83c95026543cb1f7c42085eef49d8c257c73c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 10 Jul 2026 05:01:44 +0000 Subject: [PATCH 1/4] Initial plan From 5db13614a9836cd932affc431398c0a56385d726 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 10 Jul 2026 05:22:29 +0000 Subject: [PATCH 2/4] refactor(workflow): split compiler_yaml_main_job.go into focused modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits the 1265-line compiler_yaml_main_job.go into smaller, focused files: - compiler_yaml_main_job.go: reduced to 42-line orchestrator only - compiler_yaml_checkout.go (new, 302 lines): Phase 1 – OTLP masking, pre-steps, workspace checkout, dev-mode CLI build, repository import checkouts, legacy agent import checkout, and merge-remote-.github-folder step - compiler_yaml_runtime_setup.go (new, 431 lines): Phase 2 – runtime detection, ARC/DinD adapters, activation artifact download, comment-memory setup, custom step emission and sanitization - compiler_yaml_post_agent.go (new, 267 lines): Phase 5 – artifact path accumulation, step-summary generation, post-agent upload and cleanup - compiler_yaml_ai_execution.go (extended, 497 lines): Phases 3 & 4 – generateEngineInstallAndPreAgentSteps and generateAgentRunSteps added All files remain in package workflow; no imports changed; all tests pass. Closes #44549 Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/compiler_yaml_ai_execution.go | 243 ++++ pkg/workflow/compiler_yaml_checkout.go | 302 +++++ pkg/workflow/compiler_yaml_main_job.go | 1223 ------------------- pkg/workflow/compiler_yaml_post_agent.go | 267 ++++ pkg/workflow/compiler_yaml_runtime_setup.go | 431 +++++++ 5 files changed, 1243 insertions(+), 1223 deletions(-) create mode 100644 pkg/workflow/compiler_yaml_checkout.go create mode 100644 pkg/workflow/compiler_yaml_post_agent.go create mode 100644 pkg/workflow/compiler_yaml_runtime_setup.go diff --git a/pkg/workflow/compiler_yaml_ai_execution.go b/pkg/workflow/compiler_yaml_ai_execution.go index 3e2fda0f248..fe3794bed64 100644 --- a/pkg/workflow/compiler_yaml_ai_execution.go +++ b/pkg/workflow/compiler_yaml_ai_execution.go @@ -252,3 +252,246 @@ func (c *Compiler) generateDetectAgentErrorsStep(yaml *strings.Builder, data *Wo yaml.WriteString(" continue-on-error: true\n") fmt.Fprintf(yaml, " run: node \"${RUNNER_TEMP}/gh-aw/actions/%s.cjs\"\n", scriptId) } + +// generateEngineInstallAndPreAgentSteps emits git credential configuration, the PR-ready-for-review +// checkout, engine installation steps, GitHub MCP app token minting, MCP lockdown detection, guard +// variable parsing, DIFC proxy stop, base-.github-folder restore, pre-agent steps, MCP gateway +// setup, and MCP CLI mount. +// The activation artifact download and comment-memory file preparation are emitted earlier (in +// generateActivationArtifactAndCommentMemorySteps) so that user steps: can access prior +// comment-memory state. +// It returns the resolved CodingAgentEngine for use in subsequent phases. +func (c *Compiler) generateEngineInstallAndPreAgentSteps(yaml *strings.Builder, data *WorkflowData, needsGitConfig bool) (CodingAgentEngine, error) { + // Configure git credentials for agentic workflows. + // Git credential configuration requires a .git directory in the workspace, which is only + // present when the repository was checked out. Skip these steps when checkout is disabled + // and no custom steps perform a checkout, since git remote set-url origin would fail + // with "fatal: not a git repository" otherwise. + compilerYamlLog.Printf("Git credential configuration needed: %t", needsGitConfig) + if needsGitConfig { + gitConfigSteps := c.generateGitConfigurationSteps() + for _, line := range gitConfigSteps { + yaml.WriteString(line) + } + } + + // Add step to checkout PR branch if the event is pull_request + c.generatePRReadyForReviewCheckout(yaml, data) + + // Add Node.js setup if the engine requires it and it's not already set up in custom steps + engine, err := c.getAgenticEngine(data.AI) + if err != nil { + return nil, fmt.Errorf("failed to resolve agentic engine from AI configuration: %w", err) + } + + // Ensure MCP gateway defaults are set before generating aw_info.json + // This is needed so that awmg_version is populated correctly + if HasMCPServers(data) { + ensureDefaultMCPGatewayConfig(data) + } + + // Add engine-specific installation steps (includes Node.js setup and secret validation for npm-based engines) + installSteps := engine.GetInstallationSteps(data) + compilerYamlLog.Printf("Adding %d engine installation steps for %s", len(installSteps), engine.GetID()) + for _, step := range installSteps { + for _, line := range step { + yaml.WriteString(line) + yaml.WriteByte('\n') + } + } + + // Add Playwright CLI install steps when playwright is configured in CLI mode. + // These run after Node.js is available (set up by the engine install steps above). + for _, step := range generatePlaywrightCLIInstallSteps(data) { + for _, line := range step { + yaml.WriteString(line) + yaml.WriteByte('\n') + } + } + + // GH_AW_SAFE_OUTPUTS is now set at job level, no setup step needed + + // Mint the GitHub MCP App token directly in the agent job. + // The token cannot be passed via job outputs from the activation job because + // actions/create-github-app-token calls ::add-mask:: on the token, and the + // GitHub Actions runner silently drops masked values in job outputs (runner v2.308+). + // By minting the token here, the app-id / private-key secrets are accessed only + // within this job and the minted token is available as steps.github-mcp-app-token.outputs.token. + for _, step := range c.generateGitHubMCPAppTokenMintingSteps(data) { + yaml.WriteString(step) + } + + // Add GitHub MCP lockdown detection step if needed + c.generateGitHubMCPLockdownDetectionStep(yaml, data) + + // Add step to parse blocked-users and approval-labels guard variables into JSON arrays + c.generateParseGuardVarsStep(yaml, data) + + // Stop DIFC proxy before starting the MCP gateway. The proxy must be stopped first + // to avoid double-filtering: the gateway uses the same guard policy for the agent phase. + c.generateStopDIFCProxyStep(yaml, data) + + // Stop-time safety checks are now handled by a dedicated job (stop_time_check) + // No longer generated in the main job steps + + // Restore agent config folders from the base branch snapshot in the activation artifact. + // The activation job saved these before the PR checkout ran, so this step overwrites any + // PR-branch-injected files (e.g. forked skill/instruction files) with trusted base content. + // The .github/mcp.json file is also removed since it may come from the PR branch. + // The folder and file lists match those used in the save step (derived from engine registry). + // + // IMPORTANT: This must run BEFORE pre-agent-steps (below) so that APM-restored skills + // placed in .github/skills/ by pre-agent-steps are not clobbered by this restore. + if ShouldGeneratePRCheckoutStep(data) { + registry := GetGlobalEngineRegistry() + generateRestoreBaseGitHubFoldersStep(yaml, + registry.GetAllAgentManifestFolders(), + registry.GetAllAgentManifestFiles(), + ) + } + + // Restore inline sub-agents written during the activation job. + // This step runs AFTER the base-branch restore so the engine-specific agent directory + // is not clobbered. Inline sub-agents are enabled by default. + if isFeatureEnabled(constants.FeatureFlag("inline-agents"), data) { + generateRestoreInlineSubAgentsStep(yaml, data) + } + // Restore the engine-specific skills directory when inline skills are enabled or when + // explicit frontmatter skills were installed during activation. + if isFeatureEnabled(constants.FeatureFlag("inline-agents"), data) || len(data.Skills) > 0 { + generateRestoreInlineSkillsStep(yaml, data) + } + + // Add pre-agent-steps (if any) after base-branch restore but before MCP setup. + // Running after base restore ensures APM-restored skills (.github/skills/) are not + // overwritten by the restore step above in PR context. + // Running before MCP setup ensures pre-agent-steps can install/configure MCP + // dependencies that the gateway may reference when it starts. + c.generatePreAgentSteps(yaml, data) + + // Add MCP setup + if err := c.generateMCPSetup(yaml, data.Tools, engine, data); err != nil { + return nil, fmt.Errorf("failed to generate MCP setup: %w", err) + } + + // Mount MCP servers as CLI tools (runs after gateway is started) + c.generateMCPCLIMountStep(yaml, data) + + return engine, nil +} + +// generateAgentRunSteps emits the git credentials cleaner, engine config steps, CLI proxy start, +// AI execution, CLI proxy stop, Copilot error detection, agent-execution-complete marker, +// post-agent git credential regeneration, firewall log collection, engine pre-bundle steps, +// MCP gateway stop, secret redaction, agent step summary append, and output collection. +// It returns the initial set of artifact paths (to be extended by the caller) and the +// agent stdio log path constant. +func (c *Compiler) generateAgentRunSteps(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine, needsGitConfig bool) ([]string, string, error) { + // Collect artifact paths for unified upload at the end + var artifactPaths []string + artifactPaths = append(artifactPaths, constants.AwPromptsFile) + + logFileFull := constants.AgentStdioLogPath + + // Clean credentials before executing the agentic engine. + // This removes git credentials from .git/config and, when known credential-leaking + // actions were detected, also removes cloud-provider / registry credentials. + credentialsCleanerSteps := c.generateCredentialsCleanerStep(data.KnownActionCredentialEnvVars) + for _, line := range credentialsCleanerSteps { + yaml.WriteString(line) + } + + // Emit an audit step after credentials have been cleaned but before the agent begins + // execution. This captures a file listing of agent-related directories so the final + // pre-agent state (including any config written by MCP setup and engine config steps) + // is visible in the agent artifact without exposing raw credentials. + c.generatePreAgentAuditStep(yaml) + + // Emit engine config steps (from RenderConfig) before the AI execution step. + // These steps write runtime config files to disk (e.g. provider/model config files). + // Most engines return no steps here; only engines that require config files use this. + if len(data.EngineConfigSteps) > 0 { + compilerYamlLog.Printf("Adding %d engine config steps for %s", len(data.EngineConfigSteps), engine.GetID()) + for _, step := range data.EngineConfigSteps { + stepYAML, err := ConvertStepToYAML(step) + if err != nil { + return nil, "", fmt.Errorf("failed to render engine config step: %w", err) + } + yaml.WriteString(stepYAML) + } + } + + // Start CLI proxy on the host before AWF execution. When features.cli-proxy is enabled, + // the compiler starts a difc-proxy container on the host that AWF's cli-proxy sidecar + // connects to via host.docker.internal:18443. + c.generateStartCliProxyStep(yaml, data) + + // Add AI execution step using the agentic engine + compilerYamlLog.Printf("Generating engine execution steps for %s", engine.GetID()) + c.generateEngineExecutionSteps(yaml, data, engine, logFileFull) + + // Stop CLI proxy after AWF execution (always runs to ensure cleanup) + c.generateStopCliProxyStep(yaml, data) + + // Detect agent errors on the host runner immediately after the AWF container exits. + // GITHUB_OUTPUT is not accessible inside the AWF sandbox, so this step must run here + // (on the host runner) rather than from within the container. Engines that provide a + // detection script via GetErrorDetectionScriptId will emit this step. + c.generateDetectAgentErrorsStep(yaml, data, engine) + + // Mark that we've completed agent execution - step order validation starts from here + compilerYamlLog.Print("Marking agent execution as complete for step order tracking") + c.stepOrderTracker.MarkAgentExecutionComplete() + + // Regenerate git credentials after agent execution + // This allows safe-outputs operations (like create_pull_request) to work properly + // We regenerate the credentials rather than restoring from backup. + // Only emit these steps when a checkout was performed (requires a .git directory). + if needsGitConfig { + gitConfigStepsAfterAgent := c.generateGitConfigurationSteps() + for _, line := range gitConfigStepsAfterAgent { + yaml.WriteString(line) + } + } + + // Collect firewall logs BEFORE secret redaction so secrets in logs can be redacted + for _, step := range engine.GetFirewallLogsCollectionStep(data) { + for _, line := range step { + yaml.WriteString(line) + yaml.WriteByte('\n') + } + } + + // Run engine pre-bundle steps to relocate files before secret redaction. + // This ensures all artifact paths share a common ancestor under /tmp/gh-aw/. + for _, step := range engine.GetPreBundleSteps(data) { + for _, line := range step { + yaml.WriteString(line) + yaml.WriteByte('\n') + } + } + + // Stop MCP gateway after agent execution and before secret redaction + // This ensures the gateway process is properly cleaned up + // The MCP gateway is always enabled, even when agent sandbox is disabled + c.generateStopMCPGateway(yaml, data) + + // Add secret redaction step BEFORE any artifact uploads + // This ensures all artifacts are scanned for secrets before being uploaded + c.generateSecretRedactionStep(yaml, yaml.String(), data) + + // Append the agent step summary to the real $GITHUB_STEP_SUMMARY after secrets are redacted. + // The agent writes its GITHUB_STEP_SUMMARY content to AgentStepSummaryPath (a file inside + // /tmp/gh-aw/ that is reachable in both AWF sandbox and non-sandbox modes). + // secret redaction already scanned this file, so it is safe to append. + c.generateAgentStepSummaryAppend(yaml) + + // Add output collection step only if safe-outputs feature is used (GH_AW_SAFE_OUTPUTS functionality) + if data.SafeOutputs != nil { + if err := c.generateOutputCollectionStep(yaml, data); err != nil { + return nil, "", err + } + } + + return artifactPaths, logFileFull, nil +} diff --git a/pkg/workflow/compiler_yaml_checkout.go b/pkg/workflow/compiler_yaml_checkout.go new file mode 100644 index 00000000000..85b26e9acc0 --- /dev/null +++ b/pkg/workflow/compiler_yaml_checkout.go @@ -0,0 +1,302 @@ +package workflow + +import ( + "encoding/json" + "fmt" + "strings" +) + +// generateInitialAndCheckoutSteps emits the OTLP mask step, pre-steps, all checkout steps +// (default workspace checkout, dev-mode CLI build, additional checkouts), repository import +// checkouts, legacy agent import checkout, and the merge-.github-folder step. +// It returns the CheckoutManager (needed later for token invalidation and dev-mode restore) +// and a flag indicating whether the default workspace checkout was emitted. +func (c *Compiler) generateInitialAndCheckoutSteps(yaml *strings.Builder, data *WorkflowData) (*CheckoutManager, bool, error) { + // Mask OTLP telemetry headers early so authentication tokens cannot leak in runner + // debug logs. The workflow-level OTEL_EXPORTER_OTLP_HEADERS env var is available + // from the very first step, so masking can happen before any other work. + if isOTLPHeadersPresent(data) { + yaml.WriteString(generateOTLPHeadersMaskStep()) + } + // Mask custom OTLP attribute values so user-supplied values cannot leak into runner logs. + if isOTLPAttributesPresent(data) { + yaml.WriteString(generateOTLPAttributesMaskStep()) + } + + // Add pre-steps before checkout and the subsequent built-in steps in this agent job. + // This allows users to mint short-lived tokens (via custom actions) in the same + // job as checkout, so the tokens are never dropped by the GitHub Actions runner's + // add-mask behaviour that silently suppresses masked values across job boundaries. + // Step outputs are available as ${{ steps..outputs. }} and can be + // referenced directly in checkout.token. Some compiler-injected setup steps may + // still be emitted earlier than these pre-steps. + c.generatePreSteps(yaml, data) + + // Determine if we need to add a checkout step + needsCheckout := c.shouldAddCheckoutStep(data) + compilerYamlLog.Printf("Checkout step needed: %t", needsCheckout) + + // Build a CheckoutManager with any user-configured checkouts + checkoutMgr := NewCheckoutManager(data.CheckoutConfigs) + + // Propagate the platform (host) repo resolved by the activation job so that + // checkout steps in this job and in safe_outputs can use the correct repository + // for .github/.agents sparse checkouts when called cross-repo. + // The activation job exposes this as needs.activation.outputs.target_repo. + if hasWorkflowCallTrigger(data.On) && !data.InlinedImports { + checkoutMgr.SetCrossRepoTargetRepo("${{ needs.activation.outputs.target_repo }}") + } + + // Mint checkout app tokens directly in the agent job before checkout steps are executed. + // Tokens cannot be passed via job outputs from the activation job because + // actions/create-github-app-token calls ::add-mask:: on the token, and the GitHub Actions + // runner silently drops masked values when used as job outputs (runner v2.308+). + // By minting here, the token is available as steps.checkout-app-token-{index}.outputs.token + // within the same job, just like the github-mcp-app-token pattern. + if checkoutMgr.HasAppAuth() { + compilerYamlLog.Print("Generating checkout app token minting steps in agent job") + for _, step := range checkoutMgr.GenerateCheckoutAppTokenSteps(c, resolveCheckoutPermissions(data)) { + yaml.WriteString(step) + } + } + + // Add checkout step first if needed + if needsCheckout { + // Emit the default workspace checkout, applying any user-supplied overrides + defaultLines := checkoutMgr.GenerateDefaultCheckoutStep( + c.trialMode, + c.trialLogicalRepoSlug, + c.getActionPin, + ) + for _, line := range defaultLines { + yaml.WriteString(line) + } + + // Add CLI build steps in dev mode (after automatic checkout, before other steps) + // This builds the gh-aw CLI and Docker image for use by the agentic-workflows MCP server + // Only generate build steps if agentic-workflows tool is enabled + if c.actionMode.IsDev() { + if _, hasAgenticWorkflows := data.Tools["agentic-workflows"]; hasAgenticWorkflows { + compilerYamlLog.Printf("Generating CLI build steps for dev mode (agentic-workflows tool enabled)") + c.generateDevModeCLIBuildSteps(yaml) + } else { + compilerYamlLog.Printf("Skipping CLI build steps in dev mode (agentic-workflows tool not enabled)") + } + } + } + + // Emit additional (non-default) user-configured checkouts + additionalLines := checkoutMgr.GenerateAdditionalCheckoutSteps(c.getActionPin) + for _, line := range additionalLines { + yaml.WriteString(line) + } + + // Emit a manifest step that records the path and resolved default branch for each + // non-default cross-repo checkout. The safe-outputs MCP server reads this file to + // resolve base branches without making any credentialed network calls. + for _, line := range checkoutMgr.GenerateCheckoutManifestStep(c.getActionPin) { + yaml.WriteString(line) + } + + // Add checkout steps for repository imports + // Each repository import needs to be checked out into a temporary folder + // so the merge script can copy files from it + if len(data.RepositoryImports) > 0 { + compilerYamlLog.Printf("Adding checkout steps for %d repository imports", len(data.RepositoryImports)) + c.generateRepositoryImportCheckouts(yaml, data.RepositoryImports) + } + + // Add checkout step for legacy agent import (if present) + // This handles the older import format where a specific agent file is imported + if data.AgentFile != "" && data.AgentImportSpec != "" { + compilerYamlLog.Printf("Adding checkout step for legacy agent import: %s", data.AgentImportSpec) + c.generateLegacyAgentImportCheckout(yaml, data.AgentImportSpec) + } + + // Add merge remote .github folder step for repository imports or agent imports + needsGithubMerge := (len(data.RepositoryImports) > 0) || (data.AgentFile != "" && data.AgentImportSpec != "") + if needsGithubMerge { + compilerYamlLog.Printf("Adding merge remote .github folder step") + yaml.WriteString(" - name: Merge remote .github folder\n") + fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/github-script", data)) + yaml.WriteString(" env:\n") + + // Set repository imports if present + if len(data.RepositoryImports) > 0 { + // Convert to JSON array for the script + repoImportsJSON, err := json.Marshal(data.RepositoryImports) + if err != nil { + return nil, false, fmt.Errorf("failed to marshal repository imports for merge step: %w", err) + } + writeYAMLEnv(yaml, " ", "GH_AW_REPOSITORY_IMPORTS", string(repoImportsJSON)) + } + + // Set agent import spec if present (legacy path) + if data.AgentFile != "" && data.AgentImportSpec != "" { + writeYAMLEnv(yaml, " ", "GH_AW_AGENT_FILE", data.AgentFile) + writeYAMLEnv(yaml, " ", "GH_AW_AGENT_IMPORT_SPEC", data.AgentImportSpec) + } + + yaml.WriteString(" with:\n") + yaml.WriteString(" script: |\n") + yaml.WriteString(" const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');\n") + yaml.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") + yaml.WriteString(" const { main } = require('${{ runner.temp }}/gh-aw/actions/merge_remote_agent_github_folder.cjs');\n") + yaml.WriteString(" await main();\n") + } + + return checkoutMgr, needsCheckout, nil +} + +// generateRepositoryImportCheckouts generates checkout steps for repository imports +// Each repository is checked out into a temporary folder at .github/aw/imports/-- +// relative to GITHUB_WORKSPACE. This allows the merge script to copy files from pre-checked-out folders instead of doing git operations +func (c *Compiler) generateRepositoryImportCheckouts(yaml *strings.Builder, repositoryImports []string) { + for _, repoImport := range repositoryImports { + compilerYamlLog.Printf("Generating checkout step for repository import: %s", repoImport) + + // Parse the import spec to extract owner, repo, and ref + // Format: owner/repo@ref or owner/repo + owner, repo, ref := parseRepositoryImportSpec(repoImport) + if owner == "" || repo == "" { + compilerYamlLog.Printf("Warning: failed to parse repository import: %s", repoImport) + continue + } + + // Generate a sanitized directory name for the checkout + // Use a consistent format: owner-repo-ref + // NOTE: Path must be relative to GITHUB_WORKSPACE for actions/checkout@v6 + sanitizedRef := sanitizeRefForPath(ref) + checkoutPath := fmt.Sprintf(".github/aw/imports/%s-%s-%s", owner, repo, sanitizedRef) + + // Generate the checkout step + fmt.Fprintf(yaml, " - name: Checkout repository import %s/%s@%s\n", owner, repo, ref) + fmt.Fprintf(yaml, " uses: %s\n", getActionPin("actions/checkout")) + yaml.WriteString(" with:\n") + fmt.Fprintf(yaml, " repository: %s/%s\n", owner, repo) + fmt.Fprintf(yaml, " ref: %s\n", ref) + fmt.Fprintf(yaml, " path: %s\n", checkoutPath) + yaml.WriteString(" sparse-checkout: |\n") + yaml.WriteString(" .github/\n") + yaml.WriteString(" persist-credentials: false\n") + + compilerYamlLog.Printf("Added checkout step: %s/%s@%s -> %s", owner, repo, ref, checkoutPath) + } +} + +// parseRepositoryImportSpec parses a repository import specification +// Format: owner/repo@ref or owner/repo (defaults to "main" if no ref) +// Returns: owner, repo, ref +func parseRepositoryImportSpec(importSpec string) (owner, repo, ref string) { + // Remove section reference if present (file.md#Section) + cleanSpec := importSpec + if before, _, ok := strings.Cut(importSpec, "#"); ok { + cleanSpec = before + } + + // Split on @ to get path and ref + parts := strings.Split(cleanSpec, "@") + pathPart := parts[0] + ref = "main" // default ref + if len(parts) > 1 { + ref = parts[1] + } + + // Parse path: owner/repo + slashParts := strings.Split(pathPart, "/") + if len(slashParts) != 2 { + return "", "", "" + } + + owner = slashParts[0] + repo = slashParts[1] + + return owner, repo, ref +} + +// generateLegacyAgentImportCheckout generates a checkout step for legacy agent imports +// Legacy format: owner/repo/path/to/file.md@ref +// This checks out the entire repository (not just .github folder) since the file could be anywhere +func (c *Compiler) generateLegacyAgentImportCheckout(yaml *strings.Builder, agentImportSpec string) { + compilerYamlLog.Printf("Generating checkout step for legacy agent import: %s", agentImportSpec) + + // Parse the import spec to extract owner, repo, and ref + owner, repo, ref := parseRepositoryImportSpec(agentImportSpec) + if owner == "" || repo == "" { + compilerYamlLog.Printf("Warning: failed to parse legacy agent import spec: %s", agentImportSpec) + return + } + + // Generate a sanitized directory name for the checkout + sanitizedRef := sanitizeRefForPath(ref) + checkoutPath := fmt.Sprintf("/tmp/gh-aw/repo-imports/%s-%s-%s", owner, repo, sanitizedRef) + + // Generate the checkout step + fmt.Fprintf(yaml, " - name: Checkout agent import %s/%s@%s\n", owner, repo, ref) + fmt.Fprintf(yaml, " uses: %s\n", getActionPin("actions/checkout")) + yaml.WriteString(" with:\n") + fmt.Fprintf(yaml, " repository: %s/%s\n", owner, repo) + fmt.Fprintf(yaml, " ref: %s\n", ref) + fmt.Fprintf(yaml, " path: %s\n", checkoutPath) + yaml.WriteString(" sparse-checkout: |\n") + yaml.WriteString(" .github/\n") + yaml.WriteString(" persist-credentials: false\n") + + compilerYamlLog.Printf("Added legacy agent checkout step: %s/%s@%s -> %s", owner, repo, ref, checkoutPath) +} + +// generateDevModeCLIBuildSteps generates the steps needed to build the gh-aw CLI and Docker image in dev mode +// These steps are injected after checkout in dev mode to create a locally built Docker image that includes +// the gh-aw binary and all dependencies. The agentic-workflows MCP server uses this image instead of alpine:latest. +// +// The build process: +// 1. Setup Go using go.mod version +// 2. Build the gh-aw CLI binary for linux/amd64 (since it runs in a Linux container) +// 3. Setup Docker Buildx for advanced build features +// 4. Build Docker image and tag it as localhost/gh-aw:dev +// +// The built image is used by the agentic-workflows MCP server configuration (see mcp_config_builtin.go) +func (c *Compiler) generateDevModeCLIBuildSteps(yaml *strings.Builder) { + compilerYamlLog.Print("Generating dev mode CLI build steps") + + // Step 1: Setup Go for building the CLI + yaml.WriteString(" - name: Setup Go for CLI build\n") + fmt.Fprintf(yaml, " uses: %s\n", getActionPin("actions/setup-go")) + yaml.WriteString(" with:\n") + yaml.WriteString(" go-version-file: go.mod\n") + yaml.WriteString(" cache: true\n") + + // Step 2: Build CLI binary for linux/amd64 + // Use the standard build command from CI/Makefile (not release build) + // CGO_ENABLED=0 for static linking (required for Alpine containers) + yaml.WriteString(" - name: Build gh-aw CLI\n") + yaml.WriteString(" run: |\n") + yaml.WriteString(" echo \"Building gh-aw CLI for linux/amd64...\"\n") + yaml.WriteString(" mkdir -p dist\n") + yaml.WriteString(" VERSION=$(git describe --tags --always --dirty)\n") + yaml.WriteString(" CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \\\n") + yaml.WriteString(" -ldflags \"-s -w -X main.version=${VERSION}\" \\\n") + yaml.WriteString(" -o dist/gh-aw-linux-amd64 \\\n") + yaml.WriteString(" ./cmd/gh-aw\n") + yaml.WriteString(" # Copy binary to root for direct execution in user-defined steps\n") + yaml.WriteString(" cp dist/gh-aw-linux-amd64 ./gh-aw\n") + yaml.WriteString(" chmod +x ./gh-aw\n") + yaml.WriteString(" echo \"✓ Built gh-aw CLI successfully\"\n") + + // Step 3: Setup Docker Buildx + yaml.WriteString(" - name: Setup Docker Buildx\n") + fmt.Fprintf(yaml, " uses: %s\n", getActionPin("docker/setup-buildx-action")) + + // Step 4: Build Docker image + // Use the Dockerfile at the repository root which expects BINARY build arg + yaml.WriteString(" - name: Build gh-aw Docker image\n") + fmt.Fprintf(yaml, " uses: %s\n", getActionPin("docker/build-push-action")) + yaml.WriteString(" with:\n") + yaml.WriteString(" context: .\n") + yaml.WriteString(" platforms: linux/amd64\n") + yaml.WriteString(" push: false\n") + yaml.WriteString(" load: true\n") + yaml.WriteString(" tags: localhost/gh-aw:dev\n") + yaml.WriteString(" build-args: |\n") + yaml.WriteString(" BINARY=dist/gh-aw-linux-amd64\n") +} diff --git a/pkg/workflow/compiler_yaml_main_job.go b/pkg/workflow/compiler_yaml_main_job.go index 79681631def..9afe475fedc 100644 --- a/pkg/workflow/compiler_yaml_main_job.go +++ b/pkg/workflow/compiler_yaml_main_job.go @@ -1,13 +1,7 @@ package workflow import ( - "encoding/json" - "fmt" - "os" "strings" - - "github.com/github/gh-aw/pkg/console" - "github.com/github/gh-aw/pkg/constants" ) // generateMainJobSteps generates the complete sequence of steps for the main agent execution job @@ -46,1220 +40,3 @@ func (c *Compiler) generateMainJobSteps(yaml *strings.Builder, data *WorkflowDat // Phase 5: Artifact collection, log parsing, upload, and cleanup return c.generatePostAgentCollectionAndUpload(yaml, data, engine, artifactPaths, logFileFull, checkoutMgr) } - -// generateInitialAndCheckoutSteps emits the OTLP mask step, pre-steps, all checkout steps -// (default workspace checkout, dev-mode CLI build, additional checkouts), repository import -// checkouts, legacy agent import checkout, and the merge-.github-folder step. -// It returns the CheckoutManager (needed later for token invalidation and dev-mode restore) -// and a flag indicating whether the default workspace checkout was emitted. -func (c *Compiler) generateInitialAndCheckoutSteps(yaml *strings.Builder, data *WorkflowData) (*CheckoutManager, bool, error) { - // Mask OTLP telemetry headers early so authentication tokens cannot leak in runner - // debug logs. The workflow-level OTEL_EXPORTER_OTLP_HEADERS env var is available - // from the very first step, so masking can happen before any other work. - if isOTLPHeadersPresent(data) { - yaml.WriteString(generateOTLPHeadersMaskStep()) - } - // Mask custom OTLP attribute values so user-supplied values cannot leak into runner logs. - if isOTLPAttributesPresent(data) { - yaml.WriteString(generateOTLPAttributesMaskStep()) - } - - // Add pre-steps before checkout and the subsequent built-in steps in this agent job. - // This allows users to mint short-lived tokens (via custom actions) in the same - // job as checkout, so the tokens are never dropped by the GitHub Actions runner's - // add-mask behaviour that silently suppresses masked values across job boundaries. - // Step outputs are available as ${{ steps..outputs. }} and can be - // referenced directly in checkout.token. Some compiler-injected setup steps may - // still be emitted earlier than these pre-steps. - c.generatePreSteps(yaml, data) - - // Determine if we need to add a checkout step - needsCheckout := c.shouldAddCheckoutStep(data) - compilerYamlLog.Printf("Checkout step needed: %t", needsCheckout) - - // Build a CheckoutManager with any user-configured checkouts - checkoutMgr := NewCheckoutManager(data.CheckoutConfigs) - - // Propagate the platform (host) repo resolved by the activation job so that - // checkout steps in this job and in safe_outputs can use the correct repository - // for .github/.agents sparse checkouts when called cross-repo. - // The activation job exposes this as needs.activation.outputs.target_repo. - if hasWorkflowCallTrigger(data.On) && !data.InlinedImports { - checkoutMgr.SetCrossRepoTargetRepo("${{ needs.activation.outputs.target_repo }}") - } - - // Mint checkout app tokens directly in the agent job before checkout steps are executed. - // Tokens cannot be passed via job outputs from the activation job because - // actions/create-github-app-token calls ::add-mask:: on the token, and the GitHub Actions - // runner silently drops masked values when used as job outputs (runner v2.308+). - // By minting here, the token is available as steps.checkout-app-token-{index}.outputs.token - // within the same job, just like the github-mcp-app-token pattern. - if checkoutMgr.HasAppAuth() { - compilerYamlLog.Print("Generating checkout app token minting steps in agent job") - for _, step := range checkoutMgr.GenerateCheckoutAppTokenSteps(c, resolveCheckoutPermissions(data)) { - yaml.WriteString(step) - } - } - - // Add checkout step first if needed - if needsCheckout { - // Emit the default workspace checkout, applying any user-supplied overrides - defaultLines := checkoutMgr.GenerateDefaultCheckoutStep( - c.trialMode, - c.trialLogicalRepoSlug, - c.getActionPin, - ) - for _, line := range defaultLines { - yaml.WriteString(line) - } - - // Add CLI build steps in dev mode (after automatic checkout, before other steps) - // This builds the gh-aw CLI and Docker image for use by the agentic-workflows MCP server - // Only generate build steps if agentic-workflows tool is enabled - if c.actionMode.IsDev() { - if _, hasAgenticWorkflows := data.Tools["agentic-workflows"]; hasAgenticWorkflows { - compilerYamlLog.Printf("Generating CLI build steps for dev mode (agentic-workflows tool enabled)") - c.generateDevModeCLIBuildSteps(yaml) - } else { - compilerYamlLog.Printf("Skipping CLI build steps in dev mode (agentic-workflows tool not enabled)") - } - } - } - - // Emit additional (non-default) user-configured checkouts - additionalLines := checkoutMgr.GenerateAdditionalCheckoutSteps(c.getActionPin) - for _, line := range additionalLines { - yaml.WriteString(line) - } - - // Emit a manifest step that records the path and resolved default branch for each - // non-default cross-repo checkout. The safe-outputs MCP server reads this file to - // resolve base branches without making any credentialed network calls. - for _, line := range checkoutMgr.GenerateCheckoutManifestStep(c.getActionPin) { - yaml.WriteString(line) - } - - // Add checkout steps for repository imports - // Each repository import needs to be checked out into a temporary folder - // so the merge script can copy files from it - if len(data.RepositoryImports) > 0 { - compilerYamlLog.Printf("Adding checkout steps for %d repository imports", len(data.RepositoryImports)) - c.generateRepositoryImportCheckouts(yaml, data.RepositoryImports) - } - - // Add checkout step for legacy agent import (if present) - // This handles the older import format where a specific agent file is imported - if data.AgentFile != "" && data.AgentImportSpec != "" { - compilerYamlLog.Printf("Adding checkout step for legacy agent import: %s", data.AgentImportSpec) - c.generateLegacyAgentImportCheckout(yaml, data.AgentImportSpec) - } - - // Add merge remote .github folder step for repository imports or agent imports - needsGithubMerge := (len(data.RepositoryImports) > 0) || (data.AgentFile != "" && data.AgentImportSpec != "") - if needsGithubMerge { - compilerYamlLog.Printf("Adding merge remote .github folder step") - yaml.WriteString(" - name: Merge remote .github folder\n") - fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/github-script", data)) - yaml.WriteString(" env:\n") - - // Set repository imports if present - if len(data.RepositoryImports) > 0 { - // Convert to JSON array for the script - repoImportsJSON, err := json.Marshal(data.RepositoryImports) - if err != nil { - return nil, false, fmt.Errorf("failed to marshal repository imports for merge step: %w", err) - } - writeYAMLEnv(yaml, " ", "GH_AW_REPOSITORY_IMPORTS", string(repoImportsJSON)) - } - - // Set agent import spec if present (legacy path) - if data.AgentFile != "" && data.AgentImportSpec != "" { - writeYAMLEnv(yaml, " ", "GH_AW_AGENT_FILE", data.AgentFile) - writeYAMLEnv(yaml, " ", "GH_AW_AGENT_IMPORT_SPEC", data.AgentImportSpec) - } - - yaml.WriteString(" with:\n") - yaml.WriteString(" script: |\n") - yaml.WriteString(" const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');\n") - yaml.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") - yaml.WriteString(" const { main } = require('${{ runner.temp }}/gh-aw/actions/merge_remote_agent_github_folder.cjs');\n") - yaml.WriteString(" await main();\n") - } - - return checkoutMgr, needsCheckout, nil -} - -// generateRuntimeAndWorkspaceSetupSteps emits runtime setup steps, the gh-aw temp directory -// creation step, GitHub Enterprise CLI configuration, DIFC proxy start, activation artifact -// download, comment-memory file preparation, cache-memory steps, repo-memory steps, the user's -// custom steps, and cache steps. -// Memory restore steps (comment-memory, cache-memory, repo-memory) intentionally run before -// custom steps so that deterministic steps: code can read prior state without requiring an LLM -// turn. -// It mutates data.CustomSteps (via deduplication) and returns whether the custom steps -// themselves contain a checkout action (used by the caller to compute needsGitConfig). -func (c *Compiler) generateRuntimeAndWorkspaceSetupSteps(yaml *strings.Builder, data *WorkflowData, needsCheckout bool) bool { - runtimeSetupSteps, customStepsContainCheckout := c.prepareRuntimeSetupAndCheckoutInfo(data) - compilerYamlLog.Printf("Custom steps contain checkout: %t (len(customSteps)=%d)", customStepsContainCheckout, len(data.CustomSteps)) - - c.emitRuntimeSetupPrelude(yaml, data, needsCheckout, customStepsContainCheckout, runtimeSetupSteps) - - // Create /tmp/gh-aw/ base directory for all temporary files - // This must be created before custom steps so they can use the temp directory - yaml.WriteString(" - name: Create gh-aw temp directory\n") - yaml.WriteString(" run: bash \"${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh\"\n") - - // Configure gh CLI for GitHub Enterprise hosts (*.ghe.com / GHES). - // This step runs configure_gh_for_ghe.sh which: - // 1. Detects the GitHub host from GITHUB_SERVER_URL - // 2. For github.com: exits immediately (no-op) - // 3. For GHE/GHES: authenticates gh CLI with the enterprise host and sets - // GH_HOST= in GITHUB_ENV so every subsequent step in this job - // picks up the correct host without manual per-step configuration. - // Must run after the setup action (so the script is available at ${RUNNER_TEMP}/gh-aw/actions/) - // and before any custom steps that invoke gh CLI commands. - yaml.WriteString(" - name: Configure gh CLI for GitHub Enterprise\n") - yaml.WriteString(" run: bash \"${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh\"\n") - yaml.WriteString(" env:\n") - yaml.WriteString(" GH_TOKEN: ${{ github.token }}\n") - - // Start DIFC proxy for pre-agent gh CLI calls (only when guard policies are configured - // and pre-agent steps with GH_TOKEN are present). The proxy routes gh CLI calls through - // integrity filtering before the agent runs. Must start before custom steps. - c.generateStartDIFCProxyStep(yaml, data) - - // Download the activation artifact and prepare comment-memory files BEFORE user steps - // so that deterministic steps: blocks can read prior comment-memory state without an LLM - // turn. Unlike cache-memory/repo-memory (pure restores), comment-memory fetches comment - // content via the GitHub API, which is available at this point in the job. - c.generateActivationArtifactAndCommentMemorySteps(yaml, data) - - // Add cache-memory steps before custom steps so that user steps: code can read - // /tmp/gh-aw/cache-memory// without an LLM turn. - compilerYamlLog.Printf("Generating cache-memory steps for workflow") - generateCacheMemorySteps(yaml, data) - - // Add repo-memory clone steps before custom steps so that user steps: code can read - // /tmp/gh-aw/repo-memory// without an LLM turn. - compilerYamlLog.Printf("Generating repo-memory steps for workflow") - generateRepoMemorySteps(yaml, data) - - c.emitCustomSteps(yaml, data, customStepsContainCheckout, runtimeSetupSteps) - - // Add cache steps if cache configuration is present. Keep workspace caches after user - // steps so a user-provided checkout step cannot wipe restored repository paths. - compilerYamlLog.Printf("Generating cache steps for workflow") - generateCacheSteps(yaml, data, c.verbose) - - return customStepsContainCheckout -} - -func (c *Compiler) prepareRuntimeSetupAndCheckoutInfo(data *WorkflowData) ([]GitHubActionStep, bool) { - // Add automatic runtime setup steps if needed - // This detects runtimes from custom steps and MCP configs - runtimeRequirements := detectRuntimeRequirementsCached(data) - - // Deduplicate runtime setup steps from custom steps - // This removes any runtime setup action steps (like actions/setup-go) from custom steps - // since we're adding them. It also preserves user-customized setup actions and - // filters those runtimes from requirements so we don't generate duplicates. - if len(runtimeRequirements) > 0 && data.CustomSteps != "" { - deduplicatedCustomSteps, filteredRequirements, err := DeduplicateRuntimeSetupStepsFromCustomSteps(data.CustomSteps, runtimeRequirements) - if err != nil { - compilerYamlLog.Printf("Warning: failed to deduplicate runtime setup steps: %v", err) - } else { - data.CustomSteps = deduplicatedCustomSteps - runtimeRequirements = filteredRequirements - } - } - - // Generate runtime setup steps (after filtering out user-customized ones) - runtimeSetupSteps := GenerateRuntimeSetupSteps(runtimeRequirements, data) - compilerYamlLog.Printf("Detected runtime requirements: %d runtimes, %d setup steps", len(runtimeRequirements), len(runtimeSetupSteps)) - - customStepsContainCheckout := data.CustomSteps != "" && ContainsCheckout(data.CustomSteps) - - return runtimeSetupSteps, customStepsContainCheckout -} - -func (c *Compiler) emitRuntimeSetupPrelude(yaml *strings.Builder, data *WorkflowData, needsCheckout bool, customStepsContainCheckout bool, runtimeSetupSteps []GitHubActionStep) { - // Redirect tool cache for ARC/DinD runners BEFORE runtime setup steps. - // On ARC, the standard RUNNER_TOOL_CACHE=/opt/hostedtoolcache is invisible to the DinD - // daemon's filesystem. Redirecting to ${RUNNER_TEMP}/gh-aw/tool-cache ensures the cache - // lives on the daemon-visible shared workspace volume. - // ensures setup-* actions install to a path visible to both runner and DinD containers. - // This step must run before any runtime setup steps (setup-go, setup-node, etc.) so that - // those actions pick up the redirected path when they write into RUNNER_TOOL_CACHE. - if isArcDindTopology(data) { - c.generateArcDindToolCacheRedirectStep(yaml) - } - - runtimeStepsEmittedEarly := needsCheckout || !customStepsContainCheckout - if runtimeStepsEmittedEarly { - // Case 1 or 3: Add runtime steps before custom steps - // This ensures checkout -> runtime -> custom steps order - compilerYamlLog.Printf("Adding %d runtime steps before custom steps (needsCheckout=%t, !customStepsContainCheckout=%t)", len(runtimeSetupSteps), needsCheckout, !customStepsContainCheckout) - for _, step := range runtimeSetupSteps { - for _, line := range step { - yaml.WriteString(line) - yaml.WriteByte('\n') - } - } - } - - // ARC/DinD: ensure Node.js is at a daemon-visible path. - // On ARC runners, setup-node may find a pre-cached node at the original tool cache - // (e.g. /home/runner/_work/_tool/node/...) which is NOT under RUNNER_TEMP and therefore - // not bind-mounted into the AWF container. This step copies node to the redirected - // tool cache if needed and sets GH_AW_NODE_BIN for the AWF entrypoint. - // Only emit when runtime steps (including setup-node) were already emitted above; - // when they are deferred to after a custom checkout, this step would run before - // setup-node and could relocate an absent or wrong node binary. - if isArcDindTopology(data) && runtimeStepsEmittedEarly { - c.generateArcDindNodePathStep(yaml) - } -} - -func (c *Compiler) generateArcDindToolCacheRedirectStep(yaml *strings.Builder) { - yaml.WriteString(" - name: Redirect tool cache and install paths for ARC/DinD\n") - yaml.WriteString(" run: |\n") - yaml.WriteString(" mkdir -p \"${RUNNER_TEMP}/gh-aw/tool-cache\"\n") - yaml.WriteString(" echo \"RUNNER_TOOL_CACHE=${RUNNER_TEMP}/gh-aw/tool-cache\" >> \"$GITHUB_ENV\"\n") - yaml.WriteString(" echo \"DOTNET_INSTALL_DIR=${RUNNER_TEMP}/gh-aw/tool-cache/dotnet\" >> \"$GITHUB_ENV\"\n") - yaml.WriteString(" echo \"GOPATH=${RUNNER_TEMP}/gh-aw/tool-cache/go\" >> \"$GITHUB_ENV\"\n") -} - -func (c *Compiler) generateArcDindNodePathStep(yaml *strings.Builder) { - yaml.WriteString(" - name: Ensure Node.js is at daemon-visible path\n") - yaml.WriteString(" run: |\n") - yaml.WriteString(" NODE_BIN=\"$(command -v node)\"\n") - yaml.WriteString(" NODE_PREFIX=\"$(dirname \"$(dirname \"$NODE_BIN\")\")\"\n") - yaml.WriteString(" TOOL_DEST=\"${RUNNER_TEMP}/gh-aw/tool-cache/node\"\n") - yaml.WriteString(" if [[ \"$NODE_PREFIX\" != \"${RUNNER_TEMP}\"/* ]]; then\n") - yaml.WriteString(" echo \"Node at $NODE_PREFIX is not under RUNNER_TEMP, copying to $TOOL_DEST\"\n") - yaml.WriteString(" mkdir -p \"$TOOL_DEST\"\n") - yaml.WriteString(" cp -a \"$NODE_PREFIX\"/. \"$TOOL_DEST\"/\n") - yaml.WriteString(" echo \"${TOOL_DEST}/bin\" >> \"$GITHUB_PATH\"\n") - yaml.WriteString(" echo \"GH_AW_NODE_BIN=${TOOL_DEST}/bin/node\" >> \"$GITHUB_ENV\"\n") - yaml.WriteString(" fi\n") -} - -func (c *Compiler) emitCustomSteps(yaml *strings.Builder, data *WorkflowData, customStepsContainCheckout bool, runtimeSetupSteps []GitHubActionStep) { - // Add custom steps if present - if data.CustomSteps == "" { - return - } - - // When the DIFC proxy is active, inject proxy routing env vars as step-level env - // on each custom step. Step-level env takes precedence over $GITHUB_ENV without - // mutating it, so GHE host values are preserved for non-proxied steps. - customStepsToEmit := data.CustomSteps - if hasDIFCProxyNeeded(data) { - customStepsToEmit = injectProxyEnvIntoCustomSteps(customStepsToEmit) - } - if customStepsContainCheckout && len(runtimeSetupSteps) > 0 { - // Custom steps contain checkout and we have runtime steps to insert - // Insert runtime steps after the first checkout step - compilerYamlLog.Printf("Calling addCustomStepsWithRuntimeInsertion: %d runtime steps to insert after checkout", len(runtimeSetupSteps)) - c.addCustomStepsWithRuntimeInsertion(yaml, customStepsToEmit, runtimeSetupSteps, data.ParsedTools) - } else { - // No checkout in custom steps or no runtime steps, just add custom steps as-is - compilerYamlLog.Printf("Calling addCustomStepsAsIs (customStepsContainCheckout=%t, runtimeStepsCount=%d)", customStepsContainCheckout, len(runtimeSetupSteps)) - c.addCustomStepsAsIs(yaml, customStepsToEmit) - } -} - -// generateActivationArtifactAndCommentMemorySteps emits the activation artifact download and, -// when comment-memory is configured, the minimal config write and comment-memory file preparation -// steps. These steps are placed BEFORE the user's custom steps: block so that deterministic -// steps can read prior comment-memory state without an LLM turn. -// -// The activation artifact (aw-prompts/prompt.txt, base/ snapshot, etc.) is downloaded here so -// the comment-memory setup can inject prompt guidance into prompt.txt. The rest of the agent -// setup (git config, engine install, MCP) still runs in generateEngineInstallAndPreAgentSteps. -func (c *Compiler) generateActivationArtifactAndCommentMemorySteps(yaml *strings.Builder, data *WorkflowData) { - // Download activation artifact from activation job (contains aw_info.json, prompt.txt, - // base/ snapshot and engine-specific sub-agent/skill dirs). - // In workflow_call context, apply the per-invocation prefix to avoid name clashes. - // Must happen before comment-memory preparation (needs prompt.txt for injection) and - // before the base-branch restore in generateEngineInstallAndPreAgentSteps. - compilerYamlLog.Print("Adding activation artifact download step") - activationArtifactName := artifactPrefixExprForDownstreamJob(data) + constants.ActivationArtifactName - yaml.WriteString(" - name: Download activation artifact\n") - fmt.Fprintf(yaml, " uses: %s\n", c.getActionPin("actions/download-artifact")) - yaml.WriteString(" with:\n") - fmt.Fprintf(yaml, " name: %s\n", activationArtifactName) - yaml.WriteString(" path: /tmp/gh-aw\n") - - // Materialize comment-memory safe outputs as editable markdown files BEFORE user steps. - // This prepares /tmp/gh-aw/comment-memory/*.md from prior comment history and injects - // prompt guidance so the agent can update files directly and persist them via the - // comment_memory safe output. - if data.SafeOutputs == nil || data.SafeOutputs.CommentMemory == nil { - return - } - - // Write a minimal comment-memory config so setup_comment_memory_files.cjs can locate the - // comment_memory section. The full safeoutputs config (generated in MCP setup) is written - // later; this early write provides only what the read step needs. - if !c.generateCommentMemoryEarlyConfigStep(yaml, data) { - return - } - - yaml.WriteString(" - name: Prepare comment memory files\n") - fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/github-script", data)) - yaml.WriteString(" with:\n") - fmt.Fprintf(yaml, " github-token: %s\n", getEffectiveSafeOutputGitHubToken(data.SafeOutputs.CommentMemory.GitHubToken)) - yaml.WriteString(" script: |\n") - yaml.WriteString(" const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');\n") - yaml.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") - yaml.WriteString(" const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_comment_memory_files.cjs');\n") - yaml.WriteString(" await main();\n") -} - -// generateCommentMemoryEarlyConfigStep emits a step that writes a minimal comment-memory -// configuration to ${RUNNER_TEMP}/gh-aw/safeoutputs/config.json so that the -// "Prepare comment memory files" step can read the comment_memory handler config before the -// full "Generate Safe Outputs Config" step runs later in MCP setup. -// The full safeoutputs config written by MCP setup will overwrite this file. -// Returns true if the step was emitted, false if the step was skipped (e.g. handler missing). -func (c *Compiler) generateCommentMemoryEarlyConfigStep(yaml *strings.Builder, data *WorkflowData) bool { - lines, ok := c.generateCommentMemoryEarlyConfigLines(data) - if !ok { - return false - } - for _, line := range lines { - yaml.WriteString(line) - } - return true -} - -// generateCommentMemoryEarlyConfigLines returns the formatted YAML lines for the -// early comment-memory config write step. This is shared by the agent job, custom -// jobs, and pre-activation memory restore so all deterministic paths can prepare -// comment-memory files before the full safe-outputs config exists. -func (c *Compiler) generateCommentMemoryEarlyConfigLines(data *WorkflowData) ([]string, bool) { - builder := handlerRegistry[commentMemoryHandlerKey] - if builder == nil { - compilerYamlLog.Printf("Warning: %s handler not found in registry; skipping early config write", commentMemoryHandlerKey) - return nil, false - } - cfg := builder(data.SafeOutputs) - if cfg == nil { - return nil, false - } - // INTENTIONALLY MINIMAL: this config contains only the comment_memory section and - // deliberately omits workspace-path injections and checkout mappings, which are not - // needed by setup_comment_memory_files.cjs. The full safeoutputs config (generated by - // generateSafeOutputsConfig later in MCP setup) will overwrite this file. Do not add - // handler-registry-wide iterations here. - // github-token is stripped from the early config: the github-script step supplies the - // token directly via its `github-token:` input, so embedding it here would be redundant - // and could embed a secret or context expression into the YAML shell script body. - delete(cfg, "github-token") - configMap := map[string]any{commentMemoryHandlerKey: cfg} - jsonBytes, err := json.Marshal(configMap) - if err != nil { - compilerYamlLog.Printf("Warning: failed to marshal comment-memory config: %v", err) - return nil, false - } - configJSON := string(jsonBytes) - delimiter := GenerateHeredocDelimiterFromContent("COMMENT_MEMORY_CONFIG", configJSON) - if err := ValidateHeredocContent(configJSON, delimiter); err != nil { - compilerYamlLog.Printf("Warning: comment-memory config contains heredoc delimiter; skipping early config write: %v", err) - return nil, false - } - var lines []string - lines = append(lines, " - name: Write comment-memory configuration\n") - lines = append(lines, " run: |\n") - lines = append(lines, " mkdir -p \"${RUNNER_TEMP}/gh-aw/safeoutputs\"\n") - lines = append(lines, fmt.Sprintf(" cat > \"${RUNNER_TEMP}/gh-aw/safeoutputs/config.json\" << '%s'\n", delimiter)) - // The 10-space YAML block-scalar indentation is stripped by the YAML parser before the - // shell script is executed, so the JSON content lands at column 0 inside the heredoc. - // This matches the pattern used by mcp_setup_generator.go for the full config write. - lines = append(lines, fmt.Sprintf(" %s\n", configJSON)) - lines = append(lines, fmt.Sprintf(" %s\n", delimiter)) - return lines, true -} - -// generateEngineInstallAndPreAgentSteps emits git credential configuration, the PR-ready-for-review -// checkout, engine installation steps, GitHub MCP app token minting, MCP lockdown detection, guard -// variable parsing, DIFC proxy stop, base-.github-folder restore, pre-agent steps, MCP gateway -// setup, and MCP CLI mount. -// The activation artifact download and comment-memory file preparation are emitted earlier (in -// generateActivationArtifactAndCommentMemorySteps) so that user steps: can access prior -// comment-memory state. -// It returns the resolved CodingAgentEngine for use in subsequent phases. -func (c *Compiler) generateEngineInstallAndPreAgentSteps(yaml *strings.Builder, data *WorkflowData, needsGitConfig bool) (CodingAgentEngine, error) { - // Configure git credentials for agentic workflows. - // Git credential configuration requires a .git directory in the workspace, which is only - // present when the repository was checked out. Skip these steps when checkout is disabled - // and no custom steps perform a checkout, since git remote set-url origin would fail - // with "fatal: not a git repository" otherwise. - compilerYamlLog.Printf("Git credential configuration needed: %t", needsGitConfig) - if needsGitConfig { - gitConfigSteps := c.generateGitConfigurationSteps() - for _, line := range gitConfigSteps { - yaml.WriteString(line) - } - } - - // Add step to checkout PR branch if the event is pull_request - c.generatePRReadyForReviewCheckout(yaml, data) - - // Add Node.js setup if the engine requires it and it's not already set up in custom steps - engine, err := c.getAgenticEngine(data.AI) - if err != nil { - return nil, fmt.Errorf("failed to resolve agentic engine from AI configuration: %w", err) - } - - // Ensure MCP gateway defaults are set before generating aw_info.json - // This is needed so that awmg_version is populated correctly - if HasMCPServers(data) { - ensureDefaultMCPGatewayConfig(data) - } - - // Add engine-specific installation steps (includes Node.js setup and secret validation for npm-based engines) - installSteps := engine.GetInstallationSteps(data) - compilerYamlLog.Printf("Adding %d engine installation steps for %s", len(installSteps), engine.GetID()) - for _, step := range installSteps { - for _, line := range step { - yaml.WriteString(line) - yaml.WriteByte('\n') - } - } - - // Add Playwright CLI install steps when playwright is configured in CLI mode. - // These run after Node.js is available (set up by the engine install steps above). - for _, step := range generatePlaywrightCLIInstallSteps(data) { - for _, line := range step { - yaml.WriteString(line) - yaml.WriteByte('\n') - } - } - - // GH_AW_SAFE_OUTPUTS is now set at job level, no setup step needed - - // Mint the GitHub MCP App token directly in the agent job. - // The token cannot be passed via job outputs from the activation job because - // actions/create-github-app-token calls ::add-mask:: on the token, and the - // GitHub Actions runner silently drops masked values in job outputs (runner v2.308+). - // By minting the token here, the app-id / private-key secrets are accessed only - // within this job and the minted token is available as steps.github-mcp-app-token.outputs.token. - for _, step := range c.generateGitHubMCPAppTokenMintingSteps(data) { - yaml.WriteString(step) - } - - // Add GitHub MCP lockdown detection step if needed - c.generateGitHubMCPLockdownDetectionStep(yaml, data) - - // Add step to parse blocked-users and approval-labels guard variables into JSON arrays - c.generateParseGuardVarsStep(yaml, data) - - // Stop DIFC proxy before starting the MCP gateway. The proxy must be stopped first - // to avoid double-filtering: the gateway uses the same guard policy for the agent phase. - c.generateStopDIFCProxyStep(yaml, data) - - // Stop-time safety checks are now handled by a dedicated job (stop_time_check) - // No longer generated in the main job steps - - // Restore agent config folders from the base branch snapshot in the activation artifact. - // The activation job saved these before the PR checkout ran, so this step overwrites any - // PR-branch-injected files (e.g. forked skill/instruction files) with trusted base content. - // The .github/mcp.json file is also removed since it may come from the PR branch. - // The folder and file lists match those used in the save step (derived from engine registry). - // - // IMPORTANT: This must run BEFORE pre-agent-steps (below) so that APM-restored skills - // placed in .github/skills/ by pre-agent-steps are not clobbered by this restore. - if ShouldGeneratePRCheckoutStep(data) { - registry := GetGlobalEngineRegistry() - generateRestoreBaseGitHubFoldersStep(yaml, - registry.GetAllAgentManifestFolders(), - registry.GetAllAgentManifestFiles(), - ) - } - - // Restore inline sub-agents written during the activation job. - // This step runs AFTER the base-branch restore so the engine-specific agent directory - // is not clobbered. Inline sub-agents are enabled by default. - if isFeatureEnabled(constants.FeatureFlag("inline-agents"), data) { - generateRestoreInlineSubAgentsStep(yaml, data) - } - // Restore the engine-specific skills directory when inline skills are enabled or when - // explicit frontmatter skills were installed during activation. - if isFeatureEnabled(constants.FeatureFlag("inline-agents"), data) || len(data.Skills) > 0 { - generateRestoreInlineSkillsStep(yaml, data) - } - - // Add pre-agent-steps (if any) after base-branch restore but before MCP setup. - // Running after base restore ensures APM-restored skills (.github/skills/) are not - // overwritten by the restore step above in PR context. - // Running before MCP setup ensures pre-agent-steps can install/configure MCP - // dependencies that the gateway may reference when it starts. - c.generatePreAgentSteps(yaml, data) - - // Add MCP setup - if err := c.generateMCPSetup(yaml, data.Tools, engine, data); err != nil { - return nil, fmt.Errorf("failed to generate MCP setup: %w", err) - } - - // Mount MCP servers as CLI tools (runs after gateway is started) - c.generateMCPCLIMountStep(yaml, data) - - return engine, nil -} - -// generateAgentRunSteps emits the git credentials cleaner, engine config steps, CLI proxy start, -// AI execution, CLI proxy stop, Copilot error detection, agent-execution-complete marker, -// post-agent git credential regeneration, firewall log collection, engine pre-bundle steps, -// MCP gateway stop, secret redaction, agent step summary append, and output collection. -// It returns the initial set of artifact paths (to be extended by the caller) and the -// agent stdio log path constant. -func (c *Compiler) generateAgentRunSteps(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine, needsGitConfig bool) ([]string, string, error) { - // Collect artifact paths for unified upload at the end - var artifactPaths []string - artifactPaths = append(artifactPaths, constants.AwPromptsFile) - - logFileFull := constants.AgentStdioLogPath - - // Clean credentials before executing the agentic engine. - // This removes git credentials from .git/config and, when known credential-leaking - // actions were detected, also removes cloud-provider / registry credentials. - credentialsCleanerSteps := c.generateCredentialsCleanerStep(data.KnownActionCredentialEnvVars) - for _, line := range credentialsCleanerSteps { - yaml.WriteString(line) - } - - // Emit an audit step after credentials have been cleaned but before the agent begins - // execution. This captures a file listing of agent-related directories so the final - // pre-agent state (including any config written by MCP setup and engine config steps) - // is visible in the agent artifact without exposing raw credentials. - c.generatePreAgentAuditStep(yaml) - - // Emit engine config steps (from RenderConfig) before the AI execution step. - // These steps write runtime config files to disk (e.g. provider/model config files). - // Most engines return no steps here; only engines that require config files use this. - if len(data.EngineConfigSteps) > 0 { - compilerYamlLog.Printf("Adding %d engine config steps for %s", len(data.EngineConfigSteps), engine.GetID()) - for _, step := range data.EngineConfigSteps { - stepYAML, err := ConvertStepToYAML(step) - if err != nil { - return nil, "", fmt.Errorf("failed to render engine config step: %w", err) - } - yaml.WriteString(stepYAML) - } - } - - // Start CLI proxy on the host before AWF execution. When features.cli-proxy is enabled, - // the compiler starts a difc-proxy container on the host that AWF's cli-proxy sidecar - // connects to via host.docker.internal:18443. - c.generateStartCliProxyStep(yaml, data) - - // Add AI execution step using the agentic engine - compilerYamlLog.Printf("Generating engine execution steps for %s", engine.GetID()) - c.generateEngineExecutionSteps(yaml, data, engine, logFileFull) - - // Stop CLI proxy after AWF execution (always runs to ensure cleanup) - c.generateStopCliProxyStep(yaml, data) - - // Detect agent errors on the host runner immediately after the AWF container exits. - // GITHUB_OUTPUT is not accessible inside the AWF sandbox, so this step must run here - // (on the host runner) rather than from within the container. Engines that provide a - // detection script via GetErrorDetectionScriptId will emit this step. - c.generateDetectAgentErrorsStep(yaml, data, engine) - - // Mark that we've completed agent execution - step order validation starts from here - compilerYamlLog.Print("Marking agent execution as complete for step order tracking") - c.stepOrderTracker.MarkAgentExecutionComplete() - - // Regenerate git credentials after agent execution - // This allows safe-outputs operations (like create_pull_request) to work properly - // We regenerate the credentials rather than restoring from backup. - // Only emit these steps when a checkout was performed (requires a .git directory). - if needsGitConfig { - gitConfigStepsAfterAgent := c.generateGitConfigurationSteps() - for _, line := range gitConfigStepsAfterAgent { - yaml.WriteString(line) - } - } - - // Collect firewall logs BEFORE secret redaction so secrets in logs can be redacted - for _, step := range engine.GetFirewallLogsCollectionStep(data) { - for _, line := range step { - yaml.WriteString(line) - yaml.WriteByte('\n') - } - } - - // Run engine pre-bundle steps to relocate files before secret redaction. - // This ensures all artifact paths share a common ancestor under /tmp/gh-aw/. - for _, step := range engine.GetPreBundleSteps(data) { - for _, line := range step { - yaml.WriteString(line) - yaml.WriteByte('\n') - } - } - - // Stop MCP gateway after agent execution and before secret redaction - // This ensures the gateway process is properly cleaned up - // The MCP gateway is always enabled, even when agent sandbox is disabled - c.generateStopMCPGateway(yaml, data) - - // Add secret redaction step BEFORE any artifact uploads - // This ensures all artifacts are scanned for secrets before being uploaded - c.generateSecretRedactionStep(yaml, yaml.String(), data) - - // Append the agent step summary to the real $GITHUB_STEP_SUMMARY after secrets are redacted. - // The agent writes its GITHUB_STEP_SUMMARY content to AgentStepSummaryPath (a file inside - // /tmp/gh-aw/ that is reachable in both AWF sandbox and non-sandbox modes). - // secret redaction already scanned this file, so it is safe to append. - c.generateAgentStepSummaryAppend(yaml) - - // Add output collection step only if safe-outputs feature is used (GH_AW_SAFE_OUTPUTS functionality) - if data.SafeOutputs != nil { - if err := c.generateOutputCollectionStep(yaml, data); err != nil { - return nil, "", err - } - } - - return artifactPaths, logFileFull, nil -} - -// collectArtifactPaths gathers all paths for the unified artifact upload. -// It starts from the initial paths already accumulated by generateAgentRunSteps and appends -// engine-declared output paths, log directories, observability files, safe-outputs files, -// patch/bundle paths, and firewall audit paths. -func (c *Compiler) collectArtifactPaths(data *WorkflowData, engine CodingAgentEngine, logFileFull string, initialPaths []string) []string { - paths := initialPaths - - // Merge engine-declared output files into the unified artifact instead of creating a - // separate agent_outputs artifact. - paths = append(paths, getEngineArtifactPaths(engine)...) - - // Collect MCP logs. - paths = append(paths, constants.TmpMcpLogsDir) - - // Collect DIFC proxy logs (proxy-tls certs + container stderr) when proxy was injected - paths = append(paths, difcProxyLogPaths(data)...) - - // Collect MCPScripts logs path if mcp-scripts is enabled - if IsMCPScriptsEnabled(data.MCPScripts) { - paths = append(paths, constants.TmpMcpScriptsLogsDir) - } - - // Include the aggregated agent_usage.json in the agent artifact so third-party - // tools can consume structured token data without parsing the step summary. - // Requires AWF v0.25.8+ - if isFirewallEnabled(data) { - paths = append(paths, constants.TmpGhAwDirSlash+constants.TokenUsageFilename) - } - - // Collect agent stdio logs path for unified upload - paths = append(paths, logFileFull) - - // Include the pre-agent audit file (file listing of agent-related directories captured - // before agent execution) so it is available in the agent artifact for post-run inspection. - paths = append(paths, constants.PreAgentAuditFilePath) - - // Collect agent-generated files path for unified upload - // This directory is used by workflows that instruct the agent to write files - // (e.g., smoke-claude status summaries) - paths = append(paths, constants.TmpGhAwAgentDir) - - // Collect GitHub API rate-limit log for observability. - // Written by github_rate_limit_logger.cjs during REST API calls. - paths = append(paths, constants.TmpGhAwDirSlash+constants.GithubRateLimitsFilename) - - // Collect OTLP span mirror — enables post-hoc trace debugging without a live collector. - // Written by send_otlp_span.cjs; each line is a full OTLP/HTTP JSON traces payload. - // Only included when OTLP is configured for this workflow. - if isOTLPEnabled(data) { - paths = append(paths, constants.TmpGhAwDirSlash+constants.OtelJsonlFilename) - paths = append(paths, constants.TmpGhAwDirSlash+constants.OtlpExportErrorsFilename) - } - - // Collect safe outputs and agent output paths for the unified artifact. - // These were previously uploaded as separate safe-output and agent-output artifacts. - if data.SafeOutputs != nil { - // Raw safe-output NDJSON (copied to /tmp/gh-aw/ by generateOutputCollectionStep) - paths = append(paths, constants.TmpGhAwDirSlash+constants.SafeOutputsFilename) - // Processed agent output JSON produced by collect_ndjson_output.cjs - paths = append(paths, constants.TmpGhAwDirSlash+constants.AgentOutputFilename) - if data.SafeOutputs.CommentMemory != nil { - paths = append(paths, constants.TmpCommentMemoryDir) - } - } - - // Collect git patch path if safe-outputs with PR operations is configured. - // NOTE: Git patch generation has been moved to the safe-outputs MCP server. - // The patch is now generated when create_pull_request or push_to_pull_request_branch - // tools are called, providing immediate error feedback if no changes are present. - // Include patches in the artifact when: - // 1. Safe outputs needs them for checkout (non-staged create_pull_request/push_to_pull_request_branch) - // 2. Threat detection is enabled (detection job needs patches for security analysis, even when the - // safe-output handler is staged and doesn't need checkout itself) - threatDetectionNeedsPatches := IsDetectionJobEnabled(data.SafeOutputs) - if usesPatchesAndCheckouts(data.SafeOutputs) || threatDetectionNeedsPatches { - paths = append(paths, constants.TmpAwPatchGlob) - // Bundle files are generated when patch-format: bundle is configured. - // Both formats use the same download path in the safe_outputs job, so - // include the bundle glob unconditionally alongside the patch glob. - // The artifact upload step already sets if-no-files-found: ignore, so - // this is safe even when no bundle files exist. - paths = append(paths, constants.TmpAwBundleGlob) - } - - // Include firewall audit/observability logs in the unified agent artifact - // so all agent job outputs ship as a single artifact (AWF v0.25.0+). - if isFirewallEnabled(data) { - if isArcDindTopology(data) { - // On ARC/DinD, logs are under ${{ runner.temp }}/gh-aw (daemon-visible path). - // Use ${{ runner.temp }} because `with:` blocks expand Actions expressions, not shell vars. - paths = append(paths, constants.AWFConfigFilePathExpr) - paths = append(paths, constants.AWFProxyLogsDirExpr+"/") - paths = append(paths, constants.AWFAuditDirExpr+"/") - paths = append(paths, constants.AWFReflectFilePathExpr) - } else { - paths = append(paths, constants.AWFConfigFilePath) - paths = append(paths, constants.AWFProxyLogsDir+"/") - paths = append(paths, constants.AWFAuditDir+"/") - // Include the AWF /reflect payload persisted by the agent harness. - // Co-located under /tmp/gh-aw/sandbox/firewall/ so the existing - // chmod -R a+rX step covers its permissions before upload. - paths = append(paths, constants.AWFReflectFilePath) - } - } - - // For ARC/DinD, rewrite all /tmp/gh-aw/ paths to ${{ runner.temp }}/gh-aw/ so - // the artifact upload has a single root. A consolidation step (emitted before upload) - // copies the files from /tmp/gh-aw/ to the runner.temp location. See gh-aw#34896 Bug B. - if isArcDindTopology(data) { - paths = rewriteTmpGhAwPathsForArcDind(paths) - } - - return paths -} - -// generateSummarySteps emits all GITHUB_STEP_SUMMARY log-parsing steps for the agent job. -// It covers agent log parsing, MCP scripts, MCP gateway, firewall logs, token usage, -// AWF reflect summary, and observability summary. -func (c *Compiler) generateSummarySteps(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine) { - // Parse agent logs for GITHUB_STEP_SUMMARY - c.generateLogParsing(yaml, data, engine) - - // Parse mcp-scripts logs for GITHUB_STEP_SUMMARY (if mcp-scripts is enabled) - if IsMCPScriptsEnabled(data.MCPScripts) { - c.generateMCPScriptsLogParsing(yaml, data) - } - - // Parse MCP gateway logs for GITHUB_STEP_SUMMARY. - // The MCP gateway is always enabled, even when agent sandbox is disabled. - c.generateMCPGatewayLogParsing(yaml, data) - - // Add firewall log parsing for all firewall-enabled engines. - // This replaces the previous per-engine blocks (Copilot, Codex, Claude) and extends - // support to all engines (including Gemini) so every agentic workflow uploads audit logs. - if isFirewallEnabled(data) { - firewallLogParsing := generateFirewallLogParsingStep(data.Name, data) - for _, line := range firewallLogParsing { - yaml.WriteString(line) - yaml.WriteByte('\n') - } - } - - // Parse token-usage.jsonl and append to step summary (requires AWF v0.25.8+) - if isFirewallEnabled(data) { - c.generateTokenUsageSummary(yaml, data) - } - - // Append AWF API proxy reflection data (available endpoints and models) to step summary. - // This data is fetched from the /reflect endpoint by copilot_harness.cjs before the - // agent exits and persisted to /tmp/gh-aw/awf-reflect.json. - if isFirewallEnabled(data) { - c.generateAWFReflectSummary(yaml, data) - } - - // Synthesize a compact observability section from runtime artifacts when OTLP is enabled. - c.generateObservabilitySummary(yaml, data) -} - -// generatePostAgentCollectionAndUpload orchestrates the post-agent phase: -// engine output cleanup, access log collection, artifact path accumulation via collectArtifactPaths, -// step-summary generation via generateSummarySteps, safe-outputs/memory/staging artifact uploads, -// post-steps, the unified artifact upload, token invalidation, dev-mode actions restore, -// and step-order validation. -func (c *Compiler) generatePostAgentCollectionAndUpload(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine, artifactPaths []string, logFileFull string, checkoutMgr *CheckoutManager) error { - // Generate engine output cleanup step so workspace files are removed after collection. - // The engine-declared output paths are gathered by collectArtifactPaths below. - if len(getEngineArtifactPaths(engine)) > 0 { - c.generateEngineOutputCleanup(yaml, engine) - } - - // Extract and upload squid access logs (if any proxy tools were used) - c.generateExtractAccessLogs(yaml, data.Tools) - c.generateUploadAccessLogs(yaml, data.Tools) - - // Collect all artifact paths for the unified upload. - artifactPaths = c.collectArtifactPaths(data, engine, logFileFull, artifactPaths) - - // Emit all GITHUB_STEP_SUMMARY log-parsing steps. - c.generateSummarySteps(yaml, data, engine) - - // Write a minimal agent_output.json placeholder when the engine fails before - // producing any safe outputs, so downstream safe_outputs and conclusion jobs - // receive a valid (empty) JSON file instead of an ENOENT error. - // The placeholder is only written if the engine did not already write the file. - if data.SafeOutputs != nil { - c.generateAgentOutputPlaceholderStep(yaml) - } - - // Add post-execution cleanup step for Copilot engine - if copilotEngine, ok := engine.(*CopilotEngine); ok { - cleanupStep := copilotEngine.GetCleanupStep(data) - for _, line := range cleanupStep { - yaml.WriteString(line) - yaml.WriteByte('\n') - } - } - - // Add repo-memory artifact upload to save state for push job - generateRepoMemoryArtifactUpload(yaml, data, c.getActionPin) - - // Add cache-memory git commit steps (after agent execution, before validation) - // This commits agent-written changes to the current integrity branch. - generateCacheMemoryGitCommitSteps(yaml, data) - - // Add cache-memory validation (after agent execution) - // This validates file types before cache is saved or uploaded - generateCacheMemoryValidation(yaml, data) - - // Add cache-memory artifact upload (after agent execution) - // This ensures artifacts are uploaded after the agent has finished modifying the cache - generateCacheMemoryArtifactUpload(yaml, data, c.getActionPin) - - // Add safe-outputs assets artifact upload (after agent execution) - // This creates a separate artifact for assets that will be downloaded by upload_assets job - generateSafeOutputsAssetsArtifactUpload(yaml, data, c.getActionPin) - - // Add safe-outputs upload-artifact staging upload (after agent execution) - // This creates a separate artifact for files the model staged for artifact upload, - // to be downloaded and processed by the upload_artifact job - generateSafeOutputsArtifactStagingUpload(yaml, data, c.getActionPin) - - // Add post-steps (if any) after AI execution - c.generatePostSteps(yaml, data) - - // For ARC/DinD, consolidate all artifact files under ${{ runner.temp }}/gh-aw/ - // before upload. Without this, upload-artifact receives paths from two roots - // (/tmp/gh-aw/ and ${{ runner.temp }}/gh-aw/), computes "/" as the common ancestor, - // and creates a nested directory layout that breaks downstream artifact downloads. - // See gh-aw#34896 Bug B. - if isArcDindTopology(data) { - c.generateArcDindArtifactConsolidationStep(yaml) - } - - // Generate single unified artifact upload with all collected paths. - // In workflow_call context, apply the per-invocation prefix to avoid name clashes. - agentArtifactPrefix := artifactPrefixExprForDownstreamJob(data) - c.generateUnifiedArtifactUpload(yaml, artifactPaths, agentArtifactPrefix) - - // In dev mode the setup action is referenced via a local path (./actions/setup), so its files - // live in the workspace. When a checkout: entry targets an external repository without a path - // (e.g. "checkout: [{repository: owner/other-repo}]"), actions/checkout replaces the workspace - // root with the external repository content, removing the actions/setup directory. - // Without restoring it, the runner's post-step for Setup Scripts would fail with - // "Can't find 'action.yml', 'action.yaml' or 'Dockerfile' under .../actions/setup". - // We add a restore checkout step (if: always()) as the final step so the post-step - // can always find action.yml and complete its /tmp/gh-aw cleanup. - if c.actionMode.IsDev() && checkoutMgr.HasExternalRootCheckout() { - yaml.WriteString(c.generateRestoreActionsSetupStep()) - compilerYamlLog.Print("Added restore actions folder step to agent job (dev mode with external root checkout)") - } - - // Validate step ordering - this is a compiler check to ensure security - if err := c.stepOrderTracker.ValidateStepOrdering(); err != nil { - // This is a compiler bug if validation fails - return fmt.Errorf("step ordering validation failed: %w", err) - } - return nil -} - -// addCustomStepsAsIs adds custom steps after sanitizing any GitHub Actions expressions -// found directly in run: fields. Any ${{ ... }} expression in a run: script is -// extracted into an env: variable to prevent shell injection attacks; a compiler -// warning is emitted for every such extraction. -func (c *Compiler) addCustomStepsAsIs(yaml *strings.Builder, customSteps string) { - customSteps = c.sanitizeAndWarnCustomSteps(customSteps) - // Remove "steps:" line and adjust indentation - lines := strings.Split(customSteps, "\n") - if len(lines) > 1 { - for _, line := range lines[1:] { - // Skip empty lines - if strings.TrimSpace(line) == "" { - yaml.WriteString("\n") - continue - } - - // Simply add 6 spaces for job context indentation - yaml.WriteString(" " + line + "\n") - } - } -} - -// addCustomStepsWithRuntimeInsertion adds custom steps and inserts runtime steps after the first checkout. -// Like addCustomStepsAsIs it sanitizes any ${{ ... }} expressions found in run: fields before writing. -func (c *Compiler) addCustomStepsWithRuntimeInsertion(yaml *strings.Builder, customSteps string, runtimeSetupSteps []GitHubActionStep, tools *ToolsConfig) { - customSteps = c.sanitizeAndWarnCustomSteps(customSteps) - // Remove "steps:" line and adjust indentation - lines := strings.Split(customSteps, "\n") - if len(lines) <= 1 { - return - } - - insertedRuntime := false - i := 1 // Start from index 1 to skip "steps:" line - - for i < len(lines) { - line := lines[i] - - // Skip empty lines - if strings.TrimSpace(line) == "" { - yaml.WriteString("\n") - i++ - continue - } - - // Add the line with proper indentation - yaml.WriteString(" " + line + "\n") - - // Check if this line starts a step with "- name:" or "- uses:" - trimmed := strings.TrimSpace(line) - isStepStart := strings.HasPrefix(trimmed, "- name:") || strings.HasPrefix(trimmed, "- uses:") - - if isStepStart && !insertedRuntime { - // This is the start of a step, check if it's a checkout step - isCheckoutStep := false - - // Look ahead to find "uses:" line with "checkout" - for j := i + 1; j < len(lines); j++ { - nextLine := lines[j] - nextTrimmed := strings.TrimSpace(nextLine) - - // Stop if we hit the next step - if strings.HasPrefix(nextTrimmed, "- name:") || strings.HasPrefix(nextTrimmed, "- uses:") { - break - } - - // Check if this is a uses line with checkout - if strings.Contains(nextTrimmed, "uses:") && strings.Contains(nextTrimmed, "checkout") { - isCheckoutStep = true - break - } - } - - if isCheckoutStep { - // This is a checkout step, copy all its lines until the next step - i++ - for i < len(lines) { - nextLine := lines[i] - nextTrimmed := strings.TrimSpace(nextLine) - - // Stop if we hit the next step - if strings.HasPrefix(nextTrimmed, "- name:") || strings.HasPrefix(nextTrimmed, "- uses:") { - break - } - - // Add the line - if nextTrimmed == "" { - yaml.WriteString("\n") - } else { - yaml.WriteString(" " + nextLine + "\n") - } - i++ - } - - // Now insert runtime steps after the checkout step - compilerYamlLog.Printf("Inserting %d runtime setup steps after checkout in custom steps", len(runtimeSetupSteps)) - for _, step := range runtimeSetupSteps { - for _, stepLine := range step { - yaml.WriteString(stepLine + "\n") - } - } - - insertedRuntime = true - continue // Continue with the next iteration (i is already advanced) - } - } - - i++ - } -} - -// generateRepositoryImportCheckouts generates checkout steps for repository imports -// Each repository is checked out into a temporary folder at .github/aw/imports/-- -// relative to GITHUB_WORKSPACE. This allows the merge script to copy files from pre-checked-out folders instead of doing git operations -func (c *Compiler) generateRepositoryImportCheckouts(yaml *strings.Builder, repositoryImports []string) { - for _, repoImport := range repositoryImports { - compilerYamlLog.Printf("Generating checkout step for repository import: %s", repoImport) - - // Parse the import spec to extract owner, repo, and ref - // Format: owner/repo@ref or owner/repo - owner, repo, ref := parseRepositoryImportSpec(repoImport) - if owner == "" || repo == "" { - compilerYamlLog.Printf("Warning: failed to parse repository import: %s", repoImport) - continue - } - - // Generate a sanitized directory name for the checkout - // Use a consistent format: owner-repo-ref - // NOTE: Path must be relative to GITHUB_WORKSPACE for actions/checkout@v6 - sanitizedRef := sanitizeRefForPath(ref) - checkoutPath := fmt.Sprintf(".github/aw/imports/%s-%s-%s", owner, repo, sanitizedRef) - - // Generate the checkout step - fmt.Fprintf(yaml, " - name: Checkout repository import %s/%s@%s\n", owner, repo, ref) - fmt.Fprintf(yaml, " uses: %s\n", getActionPin("actions/checkout")) - yaml.WriteString(" with:\n") - fmt.Fprintf(yaml, " repository: %s/%s\n", owner, repo) - fmt.Fprintf(yaml, " ref: %s\n", ref) - fmt.Fprintf(yaml, " path: %s\n", checkoutPath) - yaml.WriteString(" sparse-checkout: |\n") - yaml.WriteString(" .github/\n") - yaml.WriteString(" persist-credentials: false\n") - - compilerYamlLog.Printf("Added checkout step: %s/%s@%s -> %s", owner, repo, ref, checkoutPath) - } -} - -// parseRepositoryImportSpec parses a repository import specification -// Format: owner/repo@ref or owner/repo (defaults to "main" if no ref) -// Returns: owner, repo, ref -func parseRepositoryImportSpec(importSpec string) (owner, repo, ref string) { - // Remove section reference if present (file.md#Section) - cleanSpec := importSpec - if before, _, ok := strings.Cut(importSpec, "#"); ok { - cleanSpec = before - } - - // Split on @ to get path and ref - parts := strings.Split(cleanSpec, "@") - pathPart := parts[0] - ref = "main" // default ref - if len(parts) > 1 { - ref = parts[1] - } - - // Parse path: owner/repo - slashParts := strings.Split(pathPart, "/") - if len(slashParts) != 2 { - return "", "", "" - } - - owner = slashParts[0] - repo = slashParts[1] - - return owner, repo, ref -} - -// generateLegacyAgentImportCheckout generates a checkout step for legacy agent imports -// Legacy format: owner/repo/path/to/file.md@ref -// This checks out the entire repository (not just .github folder) since the file could be anywhere -func (c *Compiler) generateLegacyAgentImportCheckout(yaml *strings.Builder, agentImportSpec string) { - compilerYamlLog.Printf("Generating checkout step for legacy agent import: %s", agentImportSpec) - - // Parse the import spec to extract owner, repo, and ref - owner, repo, ref := parseRepositoryImportSpec(agentImportSpec) - if owner == "" || repo == "" { - compilerYamlLog.Printf("Warning: failed to parse legacy agent import spec: %s", agentImportSpec) - return - } - - // Generate a sanitized directory name for the checkout - sanitizedRef := sanitizeRefForPath(ref) - checkoutPath := fmt.Sprintf("/tmp/gh-aw/repo-imports/%s-%s-%s", owner, repo, sanitizedRef) - - // Generate the checkout step - fmt.Fprintf(yaml, " - name: Checkout agent import %s/%s@%s\n", owner, repo, ref) - fmt.Fprintf(yaml, " uses: %s\n", getActionPin("actions/checkout")) - yaml.WriteString(" with:\n") - fmt.Fprintf(yaml, " repository: %s/%s\n", owner, repo) - fmt.Fprintf(yaml, " ref: %s\n", ref) - fmt.Fprintf(yaml, " path: %s\n", checkoutPath) - yaml.WriteString(" sparse-checkout: |\n") - yaml.WriteString(" .github/\n") - yaml.WriteString(" persist-credentials: false\n") - - compilerYamlLog.Printf("Added legacy agent checkout step: %s/%s@%s -> %s", owner, repo, ref, checkoutPath) -} - -// generateDevModeCLIBuildSteps generates the steps needed to build the gh-aw CLI and Docker image in dev mode -// These steps are injected after checkout in dev mode to create a locally built Docker image that includes -// the gh-aw binary and all dependencies. The agentic-workflows MCP server uses this image instead of alpine:latest. -// -// The build process: -// 1. Setup Go using go.mod version -// 2. Build the gh-aw CLI binary for linux/amd64 (since it runs in a Linux container) -// 3. Setup Docker Buildx for advanced build features -// 4. Build Docker image and tag it as localhost/gh-aw:dev -// -// The built image is used by the agentic-workflows MCP server configuration (see mcp_config_builtin.go) -func (c *Compiler) generateDevModeCLIBuildSteps(yaml *strings.Builder) { - compilerYamlLog.Print("Generating dev mode CLI build steps") - - // Step 1: Setup Go for building the CLI - yaml.WriteString(" - name: Setup Go for CLI build\n") - fmt.Fprintf(yaml, " uses: %s\n", getActionPin("actions/setup-go")) - yaml.WriteString(" with:\n") - yaml.WriteString(" go-version-file: go.mod\n") - yaml.WriteString(" cache: true\n") - - // Step 2: Build CLI binary for linux/amd64 - // Use the standard build command from CI/Makefile (not release build) - // CGO_ENABLED=0 for static linking (required for Alpine containers) - yaml.WriteString(" - name: Build gh-aw CLI\n") - yaml.WriteString(" run: |\n") - yaml.WriteString(" echo \"Building gh-aw CLI for linux/amd64...\"\n") - yaml.WriteString(" mkdir -p dist\n") - yaml.WriteString(" VERSION=$(git describe --tags --always --dirty)\n") - yaml.WriteString(" CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \\\n") - yaml.WriteString(" -ldflags \"-s -w -X main.version=${VERSION}\" \\\n") - yaml.WriteString(" -o dist/gh-aw-linux-amd64 \\\n") - yaml.WriteString(" ./cmd/gh-aw\n") - yaml.WriteString(" # Copy binary to root for direct execution in user-defined steps\n") - yaml.WriteString(" cp dist/gh-aw-linux-amd64 ./gh-aw\n") - yaml.WriteString(" chmod +x ./gh-aw\n") - yaml.WriteString(" echo \"✓ Built gh-aw CLI successfully\"\n") - - // Step 3: Setup Docker Buildx - yaml.WriteString(" - name: Setup Docker Buildx\n") - fmt.Fprintf(yaml, " uses: %s\n", getActionPin("docker/setup-buildx-action")) - - // Step 4: Build Docker image - // Use the Dockerfile at the repository root which expects BINARY build arg - yaml.WriteString(" - name: Build gh-aw Docker image\n") - fmt.Fprintf(yaml, " uses: %s\n", getActionPin("docker/build-push-action")) - yaml.WriteString(" with:\n") - yaml.WriteString(" context: .\n") - yaml.WriteString(" platforms: linux/amd64\n") - yaml.WriteString(" push: false\n") - yaml.WriteString(" load: true\n") - yaml.WriteString(" tags: localhost/gh-aw:dev\n") - yaml.WriteString(" build-args: |\n") - yaml.WriteString(" BINARY=dist/gh-aw-linux-amd64\n") -} - -// sanitizeAndWarnCustomSteps applies sanitizeCustomStepsYAML to the custom steps string, -// emits a compiler warning for every expression that was extracted, and returns the -// sanitized string. If sanitization fails or produces no changes the original is returned. -func (c *Compiler) sanitizeAndWarnCustomSteps(customSteps string) string { - sanitized, warnings, err := sanitizeCustomStepsYAML(customSteps) - if err != nil { - compilerYamlLog.Printf("Failed to sanitize custom steps YAML: %v", err) - return customSteps - } - for _, w := range warnings { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(w)) - c.IncrementWarningCount() - } - return sanitized -} diff --git a/pkg/workflow/compiler_yaml_post_agent.go b/pkg/workflow/compiler_yaml_post_agent.go new file mode 100644 index 00000000000..03c03208d63 --- /dev/null +++ b/pkg/workflow/compiler_yaml_post_agent.go @@ -0,0 +1,267 @@ +package workflow + +import ( + "fmt" + "strings" + + "github.com/github/gh-aw/pkg/constants" +) + +// collectArtifactPaths gathers all paths for the unified artifact upload. +// It starts from the initial paths already accumulated by generateAgentRunSteps and appends +// engine-declared output paths, log directories, observability files, safe-outputs files, +// patch/bundle paths, and firewall audit paths. +func (c *Compiler) collectArtifactPaths(data *WorkflowData, engine CodingAgentEngine, logFileFull string, initialPaths []string) []string { + paths := initialPaths + + // Merge engine-declared output files into the unified artifact instead of creating a + // separate agent_outputs artifact. + paths = append(paths, getEngineArtifactPaths(engine)...) + + // Collect MCP logs. + paths = append(paths, constants.TmpMcpLogsDir) + + // Collect DIFC proxy logs (proxy-tls certs + container stderr) when proxy was injected + paths = append(paths, difcProxyLogPaths(data)...) + + // Collect MCPScripts logs path if mcp-scripts is enabled + if IsMCPScriptsEnabled(data.MCPScripts) { + paths = append(paths, constants.TmpMcpScriptsLogsDir) + } + + // Include the aggregated agent_usage.json in the agent artifact so third-party + // tools can consume structured token data without parsing the step summary. + // Requires AWF v0.25.8+ + if isFirewallEnabled(data) { + paths = append(paths, constants.TmpGhAwDirSlash+constants.TokenUsageFilename) + } + + // Collect agent stdio logs path for unified upload + paths = append(paths, logFileFull) + + // Include the pre-agent audit file (file listing of agent-related directories captured + // before agent execution) so it is available in the agent artifact for post-run inspection. + paths = append(paths, constants.PreAgentAuditFilePath) + + // Collect agent-generated files path for unified upload + // This directory is used by workflows that instruct the agent to write files + // (e.g., smoke-claude status summaries) + paths = append(paths, constants.TmpGhAwAgentDir) + + // Collect GitHub API rate-limit log for observability. + // Written by github_rate_limit_logger.cjs during REST API calls. + paths = append(paths, constants.TmpGhAwDirSlash+constants.GithubRateLimitsFilename) + + // Collect OTLP span mirror — enables post-hoc trace debugging without a live collector. + // Written by send_otlp_span.cjs; each line is a full OTLP/HTTP JSON traces payload. + // Only included when OTLP is configured for this workflow. + if isOTLPEnabled(data) { + paths = append(paths, constants.TmpGhAwDirSlash+constants.OtelJsonlFilename) + paths = append(paths, constants.TmpGhAwDirSlash+constants.OtlpExportErrorsFilename) + } + + // Collect safe outputs and agent output paths for the unified artifact. + // These were previously uploaded as separate safe-output and agent-output artifacts. + if data.SafeOutputs != nil { + // Raw safe-output NDJSON (copied to /tmp/gh-aw/ by generateOutputCollectionStep) + paths = append(paths, constants.TmpGhAwDirSlash+constants.SafeOutputsFilename) + // Processed agent output JSON produced by collect_ndjson_output.cjs + paths = append(paths, constants.TmpGhAwDirSlash+constants.AgentOutputFilename) + if data.SafeOutputs.CommentMemory != nil { + paths = append(paths, constants.TmpCommentMemoryDir) + } + } + + // Collect git patch path if safe-outputs with PR operations is configured. + // NOTE: Git patch generation has been moved to the safe-outputs MCP server. + // The patch is now generated when create_pull_request or push_to_pull_request_branch + // tools are called, providing immediate error feedback if no changes are present. + // Include patches in the artifact when: + // 1. Safe outputs needs them for checkout (non-staged create_pull_request/push_to_pull_request_branch) + // 2. Threat detection is enabled (detection job needs patches for security analysis, even when the + // safe-output handler is staged and doesn't need checkout itself) + threatDetectionNeedsPatches := IsDetectionJobEnabled(data.SafeOutputs) + if usesPatchesAndCheckouts(data.SafeOutputs) || threatDetectionNeedsPatches { + paths = append(paths, constants.TmpAwPatchGlob) + // Bundle files are generated when patch-format: bundle is configured. + // Both formats use the same download path in the safe_outputs job, so + // include the bundle glob unconditionally alongside the patch glob. + // The artifact upload step already sets if-no-files-found: ignore, so + // this is safe even when no bundle files exist. + paths = append(paths, constants.TmpAwBundleGlob) + } + + // Include firewall audit/observability logs in the unified agent artifact + // so all agent job outputs ship as a single artifact (AWF v0.25.0+). + if isFirewallEnabled(data) { + if isArcDindTopology(data) { + // On ARC/DinD, logs are under ${{ runner.temp }}/gh-aw (daemon-visible path). + // Use ${{ runner.temp }} because `with:` blocks expand Actions expressions, not shell vars. + paths = append(paths, constants.AWFConfigFilePathExpr) + paths = append(paths, constants.AWFProxyLogsDirExpr+"/") + paths = append(paths, constants.AWFAuditDirExpr+"/") + paths = append(paths, constants.AWFReflectFilePathExpr) + } else { + paths = append(paths, constants.AWFConfigFilePath) + paths = append(paths, constants.AWFProxyLogsDir+"/") + paths = append(paths, constants.AWFAuditDir+"/") + // Include the AWF /reflect payload persisted by the agent harness. + // Co-located under /tmp/gh-aw/sandbox/firewall/ so the existing + // chmod -R a+rX step covers its permissions before upload. + paths = append(paths, constants.AWFReflectFilePath) + } + } + + // For ARC/DinD, rewrite all /tmp/gh-aw/ paths to ${{ runner.temp }}/gh-aw/ so + // the artifact upload has a single root. A consolidation step (emitted before upload) + // copies the files from /tmp/gh-aw/ to the runner.temp location. See gh-aw#34896 Bug B. + if isArcDindTopology(data) { + paths = rewriteTmpGhAwPathsForArcDind(paths) + } + + return paths +} + +// generateSummarySteps emits all GITHUB_STEP_SUMMARY log-parsing steps for the agent job. +// It covers agent log parsing, MCP scripts, MCP gateway, firewall logs, token usage, +// AWF reflect summary, and observability summary. +func (c *Compiler) generateSummarySteps(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine) { + // Parse agent logs for GITHUB_STEP_SUMMARY + c.generateLogParsing(yaml, data, engine) + + // Parse mcp-scripts logs for GITHUB_STEP_SUMMARY (if mcp-scripts is enabled) + if IsMCPScriptsEnabled(data.MCPScripts) { + c.generateMCPScriptsLogParsing(yaml, data) + } + + // Parse MCP gateway logs for GITHUB_STEP_SUMMARY. + // The MCP gateway is always enabled, even when agent sandbox is disabled. + c.generateMCPGatewayLogParsing(yaml, data) + + // Add firewall log parsing for all firewall-enabled engines. + // This replaces the previous per-engine blocks (Copilot, Codex, Claude) and extends + // support to all engines (including Gemini) so every agentic workflow uploads audit logs. + if isFirewallEnabled(data) { + firewallLogParsing := generateFirewallLogParsingStep(data.Name, data) + for _, line := range firewallLogParsing { + yaml.WriteString(line) + yaml.WriteByte('\n') + } + } + + // Parse token-usage.jsonl and append to step summary (requires AWF v0.25.8+) + if isFirewallEnabled(data) { + c.generateTokenUsageSummary(yaml, data) + } + + // Append AWF API proxy reflection data (available endpoints and models) to step summary. + // This data is fetched from the /reflect endpoint by copilot_harness.cjs before the + // agent exits and persisted to /tmp/gh-aw/awf-reflect.json. + if isFirewallEnabled(data) { + c.generateAWFReflectSummary(yaml, data) + } + + // Synthesize a compact observability section from runtime artifacts when OTLP is enabled. + c.generateObservabilitySummary(yaml, data) +} + +// generatePostAgentCollectionAndUpload orchestrates the post-agent phase: +// engine output cleanup, access log collection, artifact path accumulation via collectArtifactPaths, +// step-summary generation via generateSummarySteps, safe-outputs/memory/staging artifact uploads, +// post-steps, the unified artifact upload, token invalidation, dev-mode actions restore, +// and step-order validation. +func (c *Compiler) generatePostAgentCollectionAndUpload(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine, artifactPaths []string, logFileFull string, checkoutMgr *CheckoutManager) error { + // Generate engine output cleanup step so workspace files are removed after collection. + // The engine-declared output paths are gathered by collectArtifactPaths below. + if len(getEngineArtifactPaths(engine)) > 0 { + c.generateEngineOutputCleanup(yaml, engine) + } + + // Extract and upload squid access logs (if any proxy tools were used) + c.generateExtractAccessLogs(yaml, data.Tools) + c.generateUploadAccessLogs(yaml, data.Tools) + + // Collect all artifact paths for the unified upload. + artifactPaths = c.collectArtifactPaths(data, engine, logFileFull, artifactPaths) + + // Emit all GITHUB_STEP_SUMMARY log-parsing steps. + c.generateSummarySteps(yaml, data, engine) + + // Write a minimal agent_output.json placeholder when the engine fails before + // producing any safe outputs, so downstream safe_outputs and conclusion jobs + // receive a valid (empty) JSON file instead of an ENOENT error. + // The placeholder is only written if the engine did not already write the file. + if data.SafeOutputs != nil { + c.generateAgentOutputPlaceholderStep(yaml) + } + + // Add post-execution cleanup step for Copilot engine + if copilotEngine, ok := engine.(*CopilotEngine); ok { + cleanupStep := copilotEngine.GetCleanupStep(data) + for _, line := range cleanupStep { + yaml.WriteString(line) + yaml.WriteByte('\n') + } + } + + // Add repo-memory artifact upload to save state for push job + generateRepoMemoryArtifactUpload(yaml, data, c.getActionPin) + + // Add cache-memory git commit steps (after agent execution, before validation) + // This commits agent-written changes to the current integrity branch. + generateCacheMemoryGitCommitSteps(yaml, data) + + // Add cache-memory validation (after agent execution) + // This validates file types before cache is saved or uploaded + generateCacheMemoryValidation(yaml, data) + + // Add cache-memory artifact upload (after agent execution) + // This ensures artifacts are uploaded after the agent has finished modifying the cache + generateCacheMemoryArtifactUpload(yaml, data, c.getActionPin) + + // Add safe-outputs assets artifact upload (after agent execution) + // This creates a separate artifact for assets that will be downloaded by upload_assets job + generateSafeOutputsAssetsArtifactUpload(yaml, data, c.getActionPin) + + // Add safe-outputs upload-artifact staging upload (after agent execution) + // This creates a separate artifact for files the model staged for artifact upload, + // to be downloaded and processed by the upload_artifact job + generateSafeOutputsArtifactStagingUpload(yaml, data, c.getActionPin) + + // Add post-steps (if any) after AI execution + c.generatePostSteps(yaml, data) + + // For ARC/DinD, consolidate all artifact files under ${{ runner.temp }}/gh-aw/ + // before upload. Without this, upload-artifact receives paths from two roots + // (/tmp/gh-aw/ and ${{ runner.temp }}/gh-aw/), computes "/" as the common ancestor, + // and creates a nested directory layout that breaks downstream artifact downloads. + // See gh-aw#34896 Bug B. + if isArcDindTopology(data) { + c.generateArcDindArtifactConsolidationStep(yaml) + } + + // Generate single unified artifact upload with all collected paths. + // In workflow_call context, apply the per-invocation prefix to avoid name clashes. + agentArtifactPrefix := artifactPrefixExprForDownstreamJob(data) + c.generateUnifiedArtifactUpload(yaml, artifactPaths, agentArtifactPrefix) + + // In dev mode the setup action is referenced via a local path (./actions/setup), so its files + // live in the workspace. When a checkout: entry targets an external repository without a path + // (e.g. "checkout: [{repository: owner/other-repo}]"), actions/checkout replaces the workspace + // root with the external repository content, removing the actions/setup directory. + // Without restoring it, the runner's post-step for Setup Scripts would fail with + // "Can't find 'action.yml', 'action.yaml' or 'Dockerfile' under .../actions/setup". + // We add a restore checkout step (if: always()) as the final step so the post-step + // can always find action.yml and complete its /tmp/gh-aw cleanup. + if c.actionMode.IsDev() && checkoutMgr.HasExternalRootCheckout() { + yaml.WriteString(c.generateRestoreActionsSetupStep()) + compilerYamlLog.Print("Added restore actions folder step to agent job (dev mode with external root checkout)") + } + + // Validate step ordering - this is a compiler check to ensure security + if err := c.stepOrderTracker.ValidateStepOrdering(); err != nil { + // This is a compiler bug if validation fails + return fmt.Errorf("step ordering validation failed: %w", err) + } + return nil +} diff --git a/pkg/workflow/compiler_yaml_runtime_setup.go b/pkg/workflow/compiler_yaml_runtime_setup.go new file mode 100644 index 00000000000..1331aa4d6cf --- /dev/null +++ b/pkg/workflow/compiler_yaml_runtime_setup.go @@ -0,0 +1,431 @@ +package workflow + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/github/gh-aw/pkg/console" + "github.com/github/gh-aw/pkg/constants" +) + +// generateRuntimeAndWorkspaceSetupSteps emits runtime setup steps, the gh-aw temp directory +// creation step, GitHub Enterprise CLI configuration, DIFC proxy start, activation artifact +// download, comment-memory file preparation, cache-memory steps, repo-memory steps, the user's +// custom steps, and cache steps. +// Memory restore steps (comment-memory, cache-memory, repo-memory) intentionally run before +// custom steps so that deterministic steps: code can read prior state without requiring an LLM +// turn. +// It mutates data.CustomSteps (via deduplication) and returns whether the custom steps +// themselves contain a checkout action (used by the caller to compute needsGitConfig). +func (c *Compiler) generateRuntimeAndWorkspaceSetupSteps(yaml *strings.Builder, data *WorkflowData, needsCheckout bool) bool { + runtimeSetupSteps, customStepsContainCheckout := c.prepareRuntimeSetupAndCheckoutInfo(data) + compilerYamlLog.Printf("Custom steps contain checkout: %t (len(customSteps)=%d)", customStepsContainCheckout, len(data.CustomSteps)) + + c.emitRuntimeSetupPrelude(yaml, data, needsCheckout, customStepsContainCheckout, runtimeSetupSteps) + + // Create /tmp/gh-aw/ base directory for all temporary files + // This must be created before custom steps so they can use the temp directory + yaml.WriteString(" - name: Create gh-aw temp directory\n") + yaml.WriteString(" run: bash \"${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh\"\n") + + // Configure gh CLI for GitHub Enterprise hosts (*.ghe.com / GHES). + // This step runs configure_gh_for_ghe.sh which: + // 1. Detects the GitHub host from GITHUB_SERVER_URL + // 2. For github.com: exits immediately (no-op) + // 3. For GHE/GHES: authenticates gh CLI with the enterprise host and sets + // GH_HOST= in GITHUB_ENV so every subsequent step in this job + // picks up the correct host without manual per-step configuration. + // Must run after the setup action (so the script is available at ${RUNNER_TEMP}/gh-aw/actions/) + // and before any custom steps that invoke gh CLI commands. + yaml.WriteString(" - name: Configure gh CLI for GitHub Enterprise\n") + yaml.WriteString(" run: bash \"${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh\"\n") + yaml.WriteString(" env:\n") + yaml.WriteString(" GH_TOKEN: ${{ github.token }}\n") + + // Start DIFC proxy for pre-agent gh CLI calls (only when guard policies are configured + // and pre-agent steps with GH_TOKEN are present). The proxy routes gh CLI calls through + // integrity filtering before the agent runs. Must start before custom steps. + c.generateStartDIFCProxyStep(yaml, data) + + // Download the activation artifact and prepare comment-memory files BEFORE user steps + // so that deterministic steps: blocks can read prior comment-memory state without an LLM + // turn. Unlike cache-memory/repo-memory (pure restores), comment-memory fetches comment + // content via the GitHub API, which is available at this point in the job. + c.generateActivationArtifactAndCommentMemorySteps(yaml, data) + + // Add cache-memory steps before custom steps so that user steps: code can read + // /tmp/gh-aw/cache-memory// without an LLM turn. + compilerYamlLog.Printf("Generating cache-memory steps for workflow") + generateCacheMemorySteps(yaml, data) + + // Add repo-memory clone steps before custom steps so that user steps: code can read + // /tmp/gh-aw/repo-memory// without an LLM turn. + compilerYamlLog.Printf("Generating repo-memory steps for workflow") + generateRepoMemorySteps(yaml, data) + + c.emitCustomSteps(yaml, data, customStepsContainCheckout, runtimeSetupSteps) + + // Add cache steps if cache configuration is present. Keep workspace caches after user + // steps so a user-provided checkout step cannot wipe restored repository paths. + compilerYamlLog.Printf("Generating cache steps for workflow") + generateCacheSteps(yaml, data, c.verbose) + + return customStepsContainCheckout +} + +func (c *Compiler) prepareRuntimeSetupAndCheckoutInfo(data *WorkflowData) ([]GitHubActionStep, bool) { + // Add automatic runtime setup steps if needed + // This detects runtimes from custom steps and MCP configs + runtimeRequirements := detectRuntimeRequirementsCached(data) + + // Deduplicate runtime setup steps from custom steps + // This removes any runtime setup action steps (like actions/setup-go) from custom steps + // since we're adding them. It also preserves user-customized setup actions and + // filters those runtimes from requirements so we don't generate duplicates. + if len(runtimeRequirements) > 0 && data.CustomSteps != "" { + deduplicatedCustomSteps, filteredRequirements, err := DeduplicateRuntimeSetupStepsFromCustomSteps(data.CustomSteps, runtimeRequirements) + if err != nil { + compilerYamlLog.Printf("Warning: failed to deduplicate runtime setup steps: %v", err) + } else { + data.CustomSteps = deduplicatedCustomSteps + runtimeRequirements = filteredRequirements + } + } + + // Generate runtime setup steps (after filtering out user-customized ones) + runtimeSetupSteps := GenerateRuntimeSetupSteps(runtimeRequirements, data) + compilerYamlLog.Printf("Detected runtime requirements: %d runtimes, %d setup steps", len(runtimeRequirements), len(runtimeSetupSteps)) + + customStepsContainCheckout := data.CustomSteps != "" && ContainsCheckout(data.CustomSteps) + + return runtimeSetupSteps, customStepsContainCheckout +} + +func (c *Compiler) emitRuntimeSetupPrelude(yaml *strings.Builder, data *WorkflowData, needsCheckout bool, customStepsContainCheckout bool, runtimeSetupSteps []GitHubActionStep) { + // Redirect tool cache for ARC/DinD runners BEFORE runtime setup steps. + // On ARC, the standard RUNNER_TOOL_CACHE=/opt/hostedtoolcache is invisible to the DinD + // daemon's filesystem. Redirecting to ${RUNNER_TEMP}/gh-aw/tool-cache ensures the cache + // lives on the daemon-visible shared workspace volume. + // ensures setup-* actions install to a path visible to both runner and DinD containers. + // This step must run before any runtime setup steps (setup-go, setup-node, etc.) so that + // those actions pick up the redirected path when they write into RUNNER_TOOL_CACHE. + if isArcDindTopology(data) { + c.generateArcDindToolCacheRedirectStep(yaml) + } + + runtimeStepsEmittedEarly := needsCheckout || !customStepsContainCheckout + if runtimeStepsEmittedEarly { + // Case 1 or 3: Add runtime steps before custom steps + // This ensures checkout -> runtime -> custom steps order + compilerYamlLog.Printf("Adding %d runtime steps before custom steps (needsCheckout=%t, !customStepsContainCheckout=%t)", len(runtimeSetupSteps), needsCheckout, !customStepsContainCheckout) + for _, step := range runtimeSetupSteps { + for _, line := range step { + yaml.WriteString(line) + yaml.WriteByte('\n') + } + } + } + + // ARC/DinD: ensure Node.js is at a daemon-visible path. + // On ARC runners, setup-node may find a pre-cached node at the original tool cache + // (e.g. /home/runner/_work/_tool/node/...) which is NOT under RUNNER_TEMP and therefore + // not bind-mounted into the AWF container. This step copies node to the redirected + // tool cache if needed and sets GH_AW_NODE_BIN for the AWF entrypoint. + // Only emit when runtime steps (including setup-node) were already emitted above; + // when they are deferred to after a custom checkout, this step would run before + // setup-node and could relocate an absent or wrong node binary. + if isArcDindTopology(data) && runtimeStepsEmittedEarly { + c.generateArcDindNodePathStep(yaml) + } +} + +func (c *Compiler) generateArcDindToolCacheRedirectStep(yaml *strings.Builder) { + yaml.WriteString(" - name: Redirect tool cache and install paths for ARC/DinD\n") + yaml.WriteString(" run: |\n") + yaml.WriteString(" mkdir -p \"${RUNNER_TEMP}/gh-aw/tool-cache\"\n") + yaml.WriteString(" echo \"RUNNER_TOOL_CACHE=${RUNNER_TEMP}/gh-aw/tool-cache\" >> \"$GITHUB_ENV\"\n") + yaml.WriteString(" echo \"DOTNET_INSTALL_DIR=${RUNNER_TEMP}/gh-aw/tool-cache/dotnet\" >> \"$GITHUB_ENV\"\n") + yaml.WriteString(" echo \"GOPATH=${RUNNER_TEMP}/gh-aw/tool-cache/go\" >> \"$GITHUB_ENV\"\n") +} + +func (c *Compiler) generateArcDindNodePathStep(yaml *strings.Builder) { + yaml.WriteString(" - name: Ensure Node.js is at daemon-visible path\n") + yaml.WriteString(" run: |\n") + yaml.WriteString(" NODE_BIN=\"$(command -v node)\"\n") + yaml.WriteString(" NODE_PREFIX=\"$(dirname \"$(dirname \"$NODE_BIN\")\")\"\n") + yaml.WriteString(" TOOL_DEST=\"${RUNNER_TEMP}/gh-aw/tool-cache/node\"\n") + yaml.WriteString(" if [[ \"$NODE_PREFIX\" != \"${RUNNER_TEMP}\"/* ]]; then\n") + yaml.WriteString(" echo \"Node at $NODE_PREFIX is not under RUNNER_TEMP, copying to $TOOL_DEST\"\n") + yaml.WriteString(" mkdir -p \"$TOOL_DEST\"\n") + yaml.WriteString(" cp -a \"$NODE_PREFIX\"/. \"$TOOL_DEST\"/\n") + yaml.WriteString(" echo \"${TOOL_DEST}/bin\" >> \"$GITHUB_PATH\"\n") + yaml.WriteString(" echo \"GH_AW_NODE_BIN=${TOOL_DEST}/bin/node\" >> \"$GITHUB_ENV\"\n") + yaml.WriteString(" fi\n") +} + +func (c *Compiler) emitCustomSteps(yaml *strings.Builder, data *WorkflowData, customStepsContainCheckout bool, runtimeSetupSteps []GitHubActionStep) { + // Add custom steps if present + if data.CustomSteps == "" { + return + } + + // When the DIFC proxy is active, inject proxy routing env vars as step-level env + // on each custom step. Step-level env takes precedence over $GITHUB_ENV without + // mutating it, so GHE host values are preserved for non-proxied steps. + customStepsToEmit := data.CustomSteps + if hasDIFCProxyNeeded(data) { + customStepsToEmit = injectProxyEnvIntoCustomSteps(customStepsToEmit) + } + if customStepsContainCheckout && len(runtimeSetupSteps) > 0 { + // Custom steps contain checkout and we have runtime steps to insert + // Insert runtime steps after the first checkout step + compilerYamlLog.Printf("Calling addCustomStepsWithRuntimeInsertion: %d runtime steps to insert after checkout", len(runtimeSetupSteps)) + c.addCustomStepsWithRuntimeInsertion(yaml, customStepsToEmit, runtimeSetupSteps, data.ParsedTools) + } else { + // No checkout in custom steps or no runtime steps, just add custom steps as-is + compilerYamlLog.Printf("Calling addCustomStepsAsIs (customStepsContainCheckout=%t, runtimeStepsCount=%d)", customStepsContainCheckout, len(runtimeSetupSteps)) + c.addCustomStepsAsIs(yaml, customStepsToEmit) + } +} + +// generateActivationArtifactAndCommentMemorySteps emits the activation artifact download and, +// when comment-memory is configured, the minimal config write and comment-memory file preparation +// steps. These steps are placed BEFORE the user's custom steps: block so that deterministic +// steps can read prior comment-memory state without an LLM turn. +// +// The activation artifact (aw-prompts/prompt.txt, base/ snapshot, etc.) is downloaded here so +// the comment-memory setup can inject prompt guidance into prompt.txt. The rest of the agent +// setup (git config, engine install, MCP) still runs in generateEngineInstallAndPreAgentSteps. +func (c *Compiler) generateActivationArtifactAndCommentMemorySteps(yaml *strings.Builder, data *WorkflowData) { + // Download activation artifact from activation job (contains aw_info.json, prompt.txt, + // base/ snapshot and engine-specific sub-agent/skill dirs). + // In workflow_call context, apply the per-invocation prefix to avoid name clashes. + // Must happen before comment-memory preparation (needs prompt.txt for injection) and + // before the base-branch restore in generateEngineInstallAndPreAgentSteps. + compilerYamlLog.Print("Adding activation artifact download step") + activationArtifactName := artifactPrefixExprForDownstreamJob(data) + constants.ActivationArtifactName + yaml.WriteString(" - name: Download activation artifact\n") + fmt.Fprintf(yaml, " uses: %s\n", c.getActionPin("actions/download-artifact")) + yaml.WriteString(" with:\n") + fmt.Fprintf(yaml, " name: %s\n", activationArtifactName) + yaml.WriteString(" path: /tmp/gh-aw\n") + + // Materialize comment-memory safe outputs as editable markdown files BEFORE user steps. + // This prepares /tmp/gh-aw/comment-memory/*.md from prior comment history and injects + // prompt guidance so the agent can update files directly and persist them via the + // comment_memory safe output. + if data.SafeOutputs == nil || data.SafeOutputs.CommentMemory == nil { + return + } + + // Write a minimal comment-memory config so setup_comment_memory_files.cjs can locate the + // comment_memory section. The full safeoutputs config (generated in MCP setup) is written + // later; this early write provides only what the read step needs. + if !c.generateCommentMemoryEarlyConfigStep(yaml, data) { + return + } + + yaml.WriteString(" - name: Prepare comment memory files\n") + fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/github-script", data)) + yaml.WriteString(" with:\n") + fmt.Fprintf(yaml, " github-token: %s\n", getEffectiveSafeOutputGitHubToken(data.SafeOutputs.CommentMemory.GitHubToken)) + yaml.WriteString(" script: |\n") + yaml.WriteString(" const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');\n") + yaml.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") + yaml.WriteString(" const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_comment_memory_files.cjs');\n") + yaml.WriteString(" await main();\n") +} + +// generateCommentMemoryEarlyConfigStep emits a step that writes a minimal comment-memory +// configuration to ${RUNNER_TEMP}/gh-aw/safeoutputs/config.json so that the +// "Prepare comment memory files" step can read the comment_memory handler config before the +// full "Generate Safe Outputs Config" step runs later in MCP setup. +// The full safeoutputs config written by MCP setup will overwrite this file. +// Returns true if the step was emitted, false if the step was skipped (e.g. handler missing). +func (c *Compiler) generateCommentMemoryEarlyConfigStep(yaml *strings.Builder, data *WorkflowData) bool { + lines, ok := c.generateCommentMemoryEarlyConfigLines(data) + if !ok { + return false + } + for _, line := range lines { + yaml.WriteString(line) + } + return true +} + +// generateCommentMemoryEarlyConfigLines returns the formatted YAML lines for the +// early comment-memory config write step. This is shared by the agent job, custom +// jobs, and pre-activation memory restore so all deterministic paths can prepare +// comment-memory files before the full safe-outputs config exists. +func (c *Compiler) generateCommentMemoryEarlyConfigLines(data *WorkflowData) ([]string, bool) { + builder := handlerRegistry[commentMemoryHandlerKey] + if builder == nil { + compilerYamlLog.Printf("Warning: %s handler not found in registry; skipping early config write", commentMemoryHandlerKey) + return nil, false + } + cfg := builder(data.SafeOutputs) + if cfg == nil { + return nil, false + } + // INTENTIONALLY MINIMAL: this config contains only the comment_memory section and + // deliberately omits workspace-path injections and checkout mappings, which are not + // needed by setup_comment_memory_files.cjs. The full safeoutputs config (generated by + // generateSafeOutputsConfig later in MCP setup) will overwrite this file. Do not add + // handler-registry-wide iterations here. + // github-token is stripped from the early config: the github-script step supplies the + // token directly via its `github-token:` input, so embedding it here would be redundant + // and could embed a secret or context expression into the YAML shell script body. + delete(cfg, "github-token") + configMap := map[string]any{commentMemoryHandlerKey: cfg} + jsonBytes, err := json.Marshal(configMap) + if err != nil { + compilerYamlLog.Printf("Warning: failed to marshal comment-memory config: %v", err) + return nil, false + } + configJSON := string(jsonBytes) + delimiter := GenerateHeredocDelimiterFromContent("COMMENT_MEMORY_CONFIG", configJSON) + if err := ValidateHeredocContent(configJSON, delimiter); err != nil { + compilerYamlLog.Printf("Warning: comment-memory config contains heredoc delimiter; skipping early config write: %v", err) + return nil, false + } + var lines []string + lines = append(lines, " - name: Write comment-memory configuration\n") + lines = append(lines, " run: |\n") + lines = append(lines, " mkdir -p \"${RUNNER_TEMP}/gh-aw/safeoutputs\"\n") + lines = append(lines, fmt.Sprintf(" cat > \"${RUNNER_TEMP}/gh-aw/safeoutputs/config.json\" << '%s'\n", delimiter)) + // The 10-space YAML block-scalar indentation is stripped by the YAML parser before the + // shell script is executed, so the JSON content lands at column 0 inside the heredoc. + // This matches the pattern used by mcp_setup_generator.go for the full config write. + lines = append(lines, fmt.Sprintf(" %s\n", configJSON)) + lines = append(lines, fmt.Sprintf(" %s\n", delimiter)) + return lines, true +} + +// addCustomStepsAsIs adds custom steps after sanitizing any GitHub Actions expressions +// found directly in run: fields. Any ${{ ... }} expression in a run: script is +// extracted into an env: variable to prevent shell injection attacks; a compiler +// warning is emitted for every such extraction. +func (c *Compiler) addCustomStepsAsIs(yaml *strings.Builder, customSteps string) { + customSteps = c.sanitizeAndWarnCustomSteps(customSteps) + // Remove "steps:" line and adjust indentation + lines := strings.Split(customSteps, "\n") + if len(lines) > 1 { + for _, line := range lines[1:] { + // Skip empty lines + if strings.TrimSpace(line) == "" { + yaml.WriteString("\n") + continue + } + + // Simply add 6 spaces for job context indentation + yaml.WriteString(" " + line + "\n") + } + } +} + +// addCustomStepsWithRuntimeInsertion adds custom steps and inserts runtime steps after the first checkout. +// Like addCustomStepsAsIs it sanitizes any ${{ ... }} expressions found in run: fields before writing. +func (c *Compiler) addCustomStepsWithRuntimeInsertion(yaml *strings.Builder, customSteps string, runtimeSetupSteps []GitHubActionStep, tools *ToolsConfig) { + customSteps = c.sanitizeAndWarnCustomSteps(customSteps) + // Remove "steps:" line and adjust indentation + lines := strings.Split(customSteps, "\n") + if len(lines) <= 1 { + return + } + + insertedRuntime := false + i := 1 // Start from index 1 to skip "steps:" line + + for i < len(lines) { + line := lines[i] + + // Skip empty lines + if strings.TrimSpace(line) == "" { + yaml.WriteString("\n") + i++ + continue + } + + // Add the line with proper indentation + yaml.WriteString(" " + line + "\n") + + // Check if this line starts a step with "- name:" or "- uses:" + trimmed := strings.TrimSpace(line) + isStepStart := strings.HasPrefix(trimmed, "- name:") || strings.HasPrefix(trimmed, "- uses:") + + if isStepStart && !insertedRuntime { + // This is the start of a step, check if it's a checkout step + isCheckoutStep := false + + // Look ahead to find "uses:" line with "checkout" + for j := i + 1; j < len(lines); j++ { + nextLine := lines[j] + nextTrimmed := strings.TrimSpace(nextLine) + + // Stop if we hit the next step + if strings.HasPrefix(nextTrimmed, "- name:") || strings.HasPrefix(nextTrimmed, "- uses:") { + break + } + + // Check if this is a uses line with checkout + if strings.Contains(nextTrimmed, "uses:") && strings.Contains(nextTrimmed, "checkout") { + isCheckoutStep = true + break + } + } + + if isCheckoutStep { + // This is a checkout step, copy all its lines until the next step + i++ + for i < len(lines) { + nextLine := lines[i] + nextTrimmed := strings.TrimSpace(nextLine) + + // Stop if we hit the next step + if strings.HasPrefix(nextTrimmed, "- name:") || strings.HasPrefix(nextTrimmed, "- uses:") { + break + } + + // Add the line + if nextTrimmed == "" { + yaml.WriteString("\n") + } else { + yaml.WriteString(" " + nextLine + "\n") + } + i++ + } + + // Now insert runtime steps after the checkout step + compilerYamlLog.Printf("Inserting %d runtime setup steps after checkout in custom steps", len(runtimeSetupSteps)) + for _, step := range runtimeSetupSteps { + for _, stepLine := range step { + yaml.WriteString(stepLine + "\n") + } + } + + insertedRuntime = true + continue // Continue with the next iteration (i is already advanced) + } + } + + i++ + } +} + +// sanitizeAndWarnCustomSteps applies sanitizeCustomStepsYAML to the custom steps string, +// emits a compiler warning for every expression that was extracted, and returns the +// sanitized string. If sanitization fails or produces no changes the original is returned. +func (c *Compiler) sanitizeAndWarnCustomSteps(customSteps string) string { + sanitized, warnings, err := sanitizeCustomStepsYAML(customSteps) + if err != nil { + compilerYamlLog.Printf("Failed to sanitize custom steps YAML: %v", err) + return customSteps + } + for _, w := range warnings { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(w)) + c.IncrementWarningCount() + } + return sanitized +} From 4f509c225e6529f0399de4ab5586245d40b108b3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 05:40:41 +0000 Subject: [PATCH 3/4] docs(adr): add draft ADR-44651 for splitting compiler_yaml_main_job.go into phase modules Co-Authored-By: Claude Sonnet 4.6 --- ...mpiler-yaml-main-job-into-phase-modules.md | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 docs/adr/44651-split-compiler-yaml-main-job-into-phase-modules.md diff --git a/docs/adr/44651-split-compiler-yaml-main-job-into-phase-modules.md b/docs/adr/44651-split-compiler-yaml-main-job-into-phase-modules.md new file mode 100644 index 00000000000..2d2eb947eb9 --- /dev/null +++ b/docs/adr/44651-split-compiler-yaml-main-job-into-phase-modules.md @@ -0,0 +1,44 @@ +# ADR-44651: Split compiler_yaml_main_job.go into Phase-Focused Modules + +**Date**: 2026-07-10 +**Status**: Draft +**Deciders**: pelikhan, copilot-swe-agent + +--- + +### Context + +`pkg/workflow/compiler_yaml_main_job.go` had grown to 1265 lines spanning 23 methods across 5 distinct compilation phases: checkout, runtime setup, AI execution, post-agent cleanup, and orchestration. The single-file layout made targeted edits expensive because readers had to mentally filter across phases to locate a specific method, and navigating to a phase-specific bug required scrolling past unrelated code. The file had become a maintenance liability as the compiler's feature set expanded. + +### Decision + +We will split the monolithic `compiler_yaml_main_job.go` into four phase-aligned files (`compiler_yaml_checkout.go`, `compiler_yaml_runtime_setup.go`, `compiler_yaml_ai_execution.go`, `compiler_yaml_post_agent.go`) within the same `package workflow`, reducing the orchestrator to a ~42-line phase dispatcher. No logic is changed and no imports are rewired — the split is purely mechanical to improve code navigability and reduce per-file cognitive load. + +### Alternatives Considered + +#### Alternative 1: Add region comments and IDE navigation hints within the single file + +Add structured comments (e.g. `// --- Phase 1: Checkout ---`) and rely on IDE symbol navigation to improve discoverability within the existing file. This avoids any file system change and keeps the full compilation flow visible in one scroll. It was rejected because it doesn't reduce file size or parallel-edit conflicts, and region comments are not a Go convention — they would require team discipline to maintain consistently. + +#### Alternative 2: Extract into sub-packages (e.g. `workflow/phases/checkout`) + +Move each phase into its own sub-package to enforce encapsulation at the language level. This would grant each phase a private namespace and prevent accidental cross-phase coupling. It was rejected because the methods share many unexported helpers, types, and the `Compiler` receiver from `package workflow`; extracting them would require either a large interface refactor or circular imports, making this a significantly larger and riskier change than a pure file split. + +### Consequences + +#### Positive +- Each phase now lives in its own file, matching file name to responsibility and reducing per-file line count from ~1265 to ~250–500 lines. +- Parallel edits to different phases no longer require touching the same file, reducing merge conflicts in active development. +- Onboarding is faster: a developer investigating checkout behaviour can open `compiler_yaml_checkout.go` directly. + +#### Negative +- The full end-to-end compilation flow is no longer readable in a single file; understanding the sequence now requires reading `compiler_yaml_main_job.go` plus the four phase files. +- All files remain in `package workflow`, so no encapsulation boundary is enforced — cross-phase coupling can still accumulate silently. + +#### Neutral +- The `Compiler` method set grows across files, which is idiomatic Go but can be unexpected for developers coming from languages where a type's methods must be co-located. +- The `compiler_yaml_ai_execution.go` file absorbs two phases (3 and 4) and remains ~497 lines; a future split may be warranted if it grows further. + +--- + +*ADR created by [adr-writer agent]. Review and finalize before changing status from Draft to Accepted.* From 4ae943f73f93dae41dd7781ded897883d93984e7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 10 Jul 2026 06:02:49 +0000 Subject: [PATCH 4/4] fix(compiler_yaml_checkout): correct doc comment for generateLegacyAgentImportCheckout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous comment incorrectly described: - The accepted format as 'owner/repo/path/to/file.md@ref' — specs with extra path segments are rejected by parseRepositoryImportSpec. - The checkout scope as 'entire repository' — only .github/ is checked out via sparse-checkout. Update the comment to match actual parser behavior and step output. Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- pkg/workflow/compiler_yaml_checkout.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/workflow/compiler_yaml_checkout.go b/pkg/workflow/compiler_yaml_checkout.go index 85b26e9acc0..67266d579da 100644 --- a/pkg/workflow/compiler_yaml_checkout.go +++ b/pkg/workflow/compiler_yaml_checkout.go @@ -215,8 +215,9 @@ func parseRepositoryImportSpec(importSpec string) (owner, repo, ref string) { } // generateLegacyAgentImportCheckout generates a checkout step for legacy agent imports -// Legacy format: owner/repo/path/to/file.md@ref -// This checks out the entire repository (not just .github folder) since the file could be anywhere +// Accepted format: owner/repo@ref or owner/repo (defaults to ref "main") +// Specs with extra path segments are rejected by parseRepositoryImportSpec. +// Only the .github/ folder is checked out via sparse-checkout. func (c *Compiler) generateLegacyAgentImportCheckout(yaml *strings.Builder, agentImportSpec string) { compilerYamlLog.Printf("Generating checkout step for legacy agent import: %s", agentImportSpec)