From 5108f626025808600067638f033743cd7e407bb3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Nov 2025 04:16:36 +0000 Subject: [PATCH 1/6] Initial plan From 435ba327f6f355fd7bf6fc07da6fbc0c5ee8efce Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Nov 2025 04:33:27 +0000 Subject: [PATCH 2/6] Fix GITHUB_WORKSPACE variable encoding in agent file paths - Change ResolveAgentFilePath to wrap entire path in double quotes - Previously: "${GITHUB_WORKSPACE}"/.github/agents/test.md (broken) - Now: "${GITHUB_WORKSPACE}/.github/agents/test.md" (working) - Update all affected tests to expect correct format - Fixes shell variable expansion issue in Copilot engine Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/engine_agent_import_test.go | 6 ++--- pkg/workflow/engine_helpers.go | 12 ++++++--- pkg/workflow/engine_helpers_test.go | 32 ++++++++++++++---------- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/pkg/workflow/engine_agent_import_test.go b/pkg/workflow/engine_agent_import_test.go index e1d6c2be68d..6fa386710a5 100644 --- a/pkg/workflow/engine_agent_import_test.go +++ b/pkg/workflow/engine_agent_import_test.go @@ -26,7 +26,7 @@ func TestCopilotEngineWithAgentFromImports(t *testing.T) { stepContent := strings.Join([]string(steps[0]), "\n") - if !strings.Contains(stepContent, "--agent '\"${GITHUB_WORKSPACE}\"/.github/agents/test-agent.md'") { + if !strings.Contains(stepContent, `--agent "${GITHUB_WORKSPACE}/.github/agents/test-agent.md"`) { t.Errorf("Expected '--agent' with quoted GITHUB_WORKSPACE prefix in copilot command, got:\n%s", stepContent) } } @@ -79,7 +79,7 @@ func TestClaudeEngineWithAgentFromImports(t *testing.T) { } // Check that agent file path is referenced with quoted GITHUB_WORKSPACE prefix - if !strings.Contains(stepContent, "\"${GITHUB_WORKSPACE}\"/.github/agents/test-agent.md") { + if !strings.Contains(stepContent, `"${GITHUB_WORKSPACE}/.github/agents/test-agent.md"`) { t.Errorf("Expected agent file path with quoted GITHUB_WORKSPACE prefix in claude command, got:\n%s", stepContent) } @@ -143,7 +143,7 @@ func TestCodexEngineWithAgentFromImports(t *testing.T) { } // Check that agent file path is referenced with quoted GITHUB_WORKSPACE prefix - if !strings.Contains(stepContent, "\"${GITHUB_WORKSPACE}\"/.github/agents/test-agent.md") { + if !strings.Contains(stepContent, `"${GITHUB_WORKSPACE}/.github/agents/test-agent.md"`) { t.Errorf("Expected agent file path with quoted GITHUB_WORKSPACE prefix in codex command, got:\n%s", stepContent) } diff --git a/pkg/workflow/engine_helpers.go b/pkg/workflow/engine_helpers.go index 7aa08319344..644748cf07c 100644 --- a/pkg/workflow/engine_helpers.go +++ b/pkg/workflow/engine_helpers.go @@ -14,20 +14,26 @@ var engineHelpersLog = logger.New("workflow:engine_helpers") // This helper extracts the common pattern shared by Copilot, Codex, and Claude engines. // // The agent file path is relative to the repository root, so we prefix it with ${GITHUB_WORKSPACE} -// and wrap the entire expression in quotes to handle paths with spaces. +// and wrap the entire expression in double quotes to handle paths with spaces while allowing +// shell variable expansion. // // Parameters: // - agentFile: The relative path to the agent file (e.g., ".github/agents/test-agent.md") // // Returns: -// - string: The quoted path with GITHUB_WORKSPACE prefix (e.g., "${GITHUB_WORKSPACE}/.github/agents/test-agent.md") +// - string: The double-quoted path with GITHUB_WORKSPACE prefix (e.g., "${GITHUB_WORKSPACE}/.github/agents/test-agent.md") // // Example: // // agentPath := ResolveAgentFilePath(".github/agents/my-agent.md") // // Returns: "${GITHUB_WORKSPACE}/.github/agents/my-agent.md" +// +// Note: The entire path is wrapped in double quotes (not just the variable) to ensure: +// 1. The shellEscapeArg function recognizes it as already-quoted and doesn't add single quotes +// 2. Shell variable expansion works (${GITHUB_WORKSPACE} gets expanded inside double quotes) +// 3. Paths with spaces are properly handled func ResolveAgentFilePath(agentFile string) string { - return fmt.Sprintf("\"${GITHUB_WORKSPACE}\"/%s", agentFile) + return fmt.Sprintf("\"${GITHUB_WORKSPACE}/%s\"", agentFile) } // BuildStandardNpmEngineInstallSteps creates standard npm installation steps for engines diff --git a/pkg/workflow/engine_helpers_test.go b/pkg/workflow/engine_helpers_test.go index a7db949ec8f..2e5348a30cd 100644 --- a/pkg/workflow/engine_helpers_test.go +++ b/pkg/workflow/engine_helpers_test.go @@ -149,27 +149,27 @@ func TestResolveAgentFilePath(t *testing.T) { { name: "basic agent file path", input: ".github/agents/test-agent.md", - expected: "\"${GITHUB_WORKSPACE}\"/.github/agents/test-agent.md", + expected: "\"${GITHUB_WORKSPACE}/.github/agents/test-agent.md\"", }, { name: "path with spaces", input: ".github/agents/my agent file.md", - expected: "\"${GITHUB_WORKSPACE}\"/.github/agents/my agent file.md", + expected: "\"${GITHUB_WORKSPACE}/.github/agents/my agent file.md\"", }, { name: "deeply nested path", input: ".github/copilot/instructions/deep/nested/agent.md", - expected: "\"${GITHUB_WORKSPACE}\"/.github/copilot/instructions/deep/nested/agent.md", + expected: "\"${GITHUB_WORKSPACE}/.github/copilot/instructions/deep/nested/agent.md\"", }, { name: "simple filename", input: "agent.md", - expected: "\"${GITHUB_WORKSPACE}\"/agent.md", + expected: "\"${GITHUB_WORKSPACE}/agent.md\"", }, { name: "path with special characters", input: ".github/agents/test-agent_v2.0.md", - expected: "\"${GITHUB_WORKSPACE}\"/.github/agents/test-agent_v2.0.md", + expected: "\"${GITHUB_WORKSPACE}/.github/agents/test-agent_v2.0.md\"", }, } @@ -188,15 +188,21 @@ func TestResolveAgentFilePathFormat(t *testing.T) { input := ".github/agents/test.md" result := ResolveAgentFilePath(input) - // Verify it starts with quoted GITHUB_WORKSPACE - if result[:22] != "\"${GITHUB_WORKSPACE}\"/" { - t.Errorf("Expected path to start with '\"${GITHUB_WORKSPACE}\"/', got: %s", result) + // Verify it starts with opening quote, GITHUB_WORKSPACE variable, and forward slash + expectedPrefix := "\"${GITHUB_WORKSPACE}/" + if !strings.HasPrefix(result, expectedPrefix) { + t.Errorf("Expected path to start with %q, got: %s", expectedPrefix, result) } - // Verify it ends with the input path - expectedSuffix := input - actualSuffix := result[22:] - if actualSuffix != expectedSuffix { - t.Errorf("Expected path to end with %q, got: %q", expectedSuffix, actualSuffix) + // Verify it ends with the input path and a closing quote + expectedSuffix := input + "\"" + if !strings.HasSuffix(result, expectedSuffix) { + t.Errorf("Expected path to end with %q, got: %q", expectedSuffix, result) + } + + // Verify the complete expected format + expected := "\"${GITHUB_WORKSPACE}/" + input + "\"" + if result != expected { + t.Errorf("Expected %q, got: %q", expected, result) } } From cbd855322256712ba1d4b62d771aebae709b5933 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Nov 2025 04:43:35 +0000 Subject: [PATCH 3/6] Add comprehensive tests for shell variable expansion in agent paths - Add TestShellVariableExpansionInAgentPath to verify correct quoting - Add TestShellEscapeArgWithFullyQuotedAgentPath to verify no re-escaping - Tests validate that ${GITHUB_WORKSPACE} will be expanded by shell - Tests verify no single quotes are added that would prevent expansion Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../cli-consistency-checker.lock.yml | 2 +- pkg/workflow/engine_helpers_test.go | 56 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-consistency-checker.lock.yml b/.github/workflows/cli-consistency-checker.lock.yml index deb84c9aa7c..8bd5de759c6 100644 --- a/.github/workflows/cli-consistency-checker.lock.yml +++ b/.github/workflows/cli-consistency-checker.lock.yml @@ -1550,7 +1550,7 @@ jobs: run: | set -o pipefail sudo -E awf --env-all --allow-domains api.enterprise.githubcopilot.com,api.github.com,api.npms.io,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,bun.sh,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,deb.nodesource.com,deno.land,get.pnpm.io,github.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nodejs.org,npm.pkg.github.com,npmjs.com,npmjs.org,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.bower.io,registry.npmjs.com,registry.npmjs.org,registry.yarnpkg.com,repo.yarnpkg.com,s.symcb.com,s.symcd.com,security.ubuntu.com,skimdb.npmjs.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.npmjs.com,www.npmjs.org,yarnpkg.com --log-level info \ - "npx -y @github/copilot@0.0.354 --add-dir /tmp/gh-aw/ --log-level all --disable-builtin-mcps --agent '\"\${GITHUB_WORKSPACE}\"/.github/agents/cli-consistency-checker.md' --allow-all-tools --allow-all-paths --prompt \"\$(cat /tmp/gh-aw/aw-prompts/prompt.txt)\"" \ + "npx -y @github/copilot@0.0.354 --add-dir /tmp/gh-aw/ --log-level all --disable-builtin-mcps --agent \"\${GITHUB_WORKSPACE}/.github/agents/cli-consistency-checker.md\" --allow-all-tools --allow-all-paths --prompt \"\$(cat /tmp/gh-aw/aw-prompts/prompt.txt)\"" \ 2>&1 | tee /tmp/gh-aw/agent-stdio.log # Move preserved Copilot logs to expected location diff --git a/pkg/workflow/engine_helpers_test.go b/pkg/workflow/engine_helpers_test.go index 2e5348a30cd..ade984f61e8 100644 --- a/pkg/workflow/engine_helpers_test.go +++ b/pkg/workflow/engine_helpers_test.go @@ -206,3 +206,59 @@ func TestResolveAgentFilePathFormat(t *testing.T) { t.Errorf("Expected %q, got: %q", expected, result) } } + +// TestShellVariableExpansionInAgentPath tests that agent paths allow shell variable expansion +func TestShellVariableExpansionInAgentPath(t *testing.T) { + agentFile := ".github/agents/test-agent.md" + result := ResolveAgentFilePath(agentFile) + + // The result should be fully wrapped in double quotes (not single quotes) + // Format: "${GITHUB_WORKSPACE}/.github/agents/test-agent.md" + expected := "\"${GITHUB_WORKSPACE}/.github/agents/test-agent.md\"" + + if result != expected { + t.Errorf("ResolveAgentFilePath(%q) = %q, want %q", agentFile, result, expected) + } + + // Verify it's properly quoted for shell variable expansion + // Should start with double quote (not single quote) + if !strings.HasPrefix(result, "\"") { + t.Errorf("Agent path should start with double quote for variable expansion, got: %s", result) + } + + // Should end with double quote (not single quote) + if !strings.HasSuffix(result, "\"") { + t.Errorf("Agent path should end with double quote for variable expansion, got: %s", result) + } + + // Should NOT contain single quotes around the double-quoted section + // Old broken format was: '"${GITHUB_WORKSPACE}"/.github/agents/test.md' + if strings.Contains(result, "'\"") || strings.Contains(result, "\"'") { + t.Errorf("Agent path should not mix single and double quotes, got: %s", result) + } + + // Should contain the variable placeholder without internal quotes + // Correct: "${GITHUB_WORKSPACE}/path" + // Incorrect: "${GITHUB_WORKSPACE}"/path + if strings.Contains(result, "\"/") && !strings.HasSuffix(result, "\"/\"") { + t.Errorf("Variable should be inside the double quotes with path, got: %s", result) + } +} + +// TestShellEscapeArgWithFullyQuotedAgentPath tests that fully quoted agent paths are not re-escaped +func TestShellEscapeArgWithFullyQuotedAgentPath(t *testing.T) { + // This simulates what happens when ResolveAgentFilePath output goes through shellEscapeArg + agentPath := "\"${GITHUB_WORKSPACE}/.github/agents/test-agent.md\"" + + result := shellEscapeArg(agentPath) + + // Should be left as-is because it's already fully double-quoted + if result != agentPath { + t.Errorf("shellEscapeArg should leave fully quoted path as-is, got: %s, want: %s", result, agentPath) + } + + // Should NOT wrap it in additional single quotes + if strings.HasPrefix(result, "'") { + t.Errorf("shellEscapeArg should not add single quotes to already double-quoted string, got: %s", result) + } +} From cc41d6f6bbda0f70c507ea588e54f49f7ef1f183 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Nov 2025 04:49:53 +0000 Subject: [PATCH 4/6] Fix GITHUB_WORKSPACE variable expansion in agent file paths Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/technical-doc-writer.lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/technical-doc-writer.lock.yml b/.github/workflows/technical-doc-writer.lock.yml index 1006af47909..820d77b2004 100644 --- a/.github/workflows/technical-doc-writer.lock.yml +++ b/.github/workflows/technical-doc-writer.lock.yml @@ -2271,7 +2271,7 @@ jobs: run: | set -o pipefail sudo -E awf --env-all --allow-domains '*.githubusercontent.com,api.enterprise.githubcopilot.com,api.github.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,lfs.github.com,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com' --log-level info \ - "npx -y @github/copilot@0.0.354 --add-dir /tmp/gh-aw/ --log-level all --disable-builtin-mcps --agent '\"\${GITHUB_WORKSPACE}\"/.github/agents/technical-doc-writer.md' --allow-tool github --allow-tool safeoutputs --allow-tool shell --allow-tool write --add-dir /tmp/gh-aw/cache-memory/ --allow-all-paths --prompt \"\$(cat /tmp/gh-aw/aw-prompts/prompt.txt)\"" \ + "npx -y @github/copilot@0.0.354 --add-dir /tmp/gh-aw/ --log-level all --disable-builtin-mcps --agent \"\${GITHUB_WORKSPACE}/.github/agents/technical-doc-writer.md\" --allow-tool github --allow-tool safeoutputs --allow-tool shell --allow-tool write --add-dir /tmp/gh-aw/cache-memory/ --allow-all-paths --prompt \"\$(cat /tmp/gh-aw/aw-prompts/prompt.txt)\"" \ 2>&1 | tee /tmp/gh-aw/agent-stdio.log # Move preserved Copilot logs to expected location From 0e4255e126d2b8ddd803171cced8bb2d662b388f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Nov 2025 05:24:23 +0000 Subject: [PATCH 5/6] Fix Copilot --agent to use identifier instead of full path - Add ExtractAgentIdentifier() to extract filename without extension - Update Copilot engine to pass agent identifier, not full path - Keep Claude/Codex engines using full path for file reading - Add comprehensive tests for agent identifier extraction - Update Copilot engine test to expect agent identifier Copilot CLI expects agent identifiers like "cli-consistency-checker", not paths like "${GITHUB_WORKSPACE}/.github/agents/cli-consistency-checker.md" Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../cli-consistency-checker.lock.yml | 2 +- pkg/workflow/copilot_engine.go | 5 +- pkg/workflow/engine_agent_import_test.go | 5 +- pkg/workflow/engine_helpers.go | 27 ++++++++++ pkg/workflow/engine_helpers_test.go | 54 +++++++++++++++++++ 5 files changed, 88 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cli-consistency-checker.lock.yml b/.github/workflows/cli-consistency-checker.lock.yml index 8bd5de759c6..bb3bbf06c45 100644 --- a/.github/workflows/cli-consistency-checker.lock.yml +++ b/.github/workflows/cli-consistency-checker.lock.yml @@ -1550,7 +1550,7 @@ jobs: run: | set -o pipefail sudo -E awf --env-all --allow-domains api.enterprise.githubcopilot.com,api.github.com,api.npms.io,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,bun.sh,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,deb.nodesource.com,deno.land,get.pnpm.io,github.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nodejs.org,npm.pkg.github.com,npmjs.com,npmjs.org,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.bower.io,registry.npmjs.com,registry.npmjs.org,registry.yarnpkg.com,repo.yarnpkg.com,s.symcb.com,s.symcd.com,security.ubuntu.com,skimdb.npmjs.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.npmjs.com,www.npmjs.org,yarnpkg.com --log-level info \ - "npx -y @github/copilot@0.0.354 --add-dir /tmp/gh-aw/ --log-level all --disable-builtin-mcps --agent \"\${GITHUB_WORKSPACE}/.github/agents/cli-consistency-checker.md\" --allow-all-tools --allow-all-paths --prompt \"\$(cat /tmp/gh-aw/aw-prompts/prompt.txt)\"" \ + "npx -y @github/copilot@0.0.354 --add-dir /tmp/gh-aw/ --log-level all --disable-builtin-mcps --agent cli-consistency-checker --allow-all-tools --allow-all-paths --prompt \"\$(cat /tmp/gh-aw/aw-prompts/prompt.txt)\"" \ 2>&1 | tee /tmp/gh-aw/agent-stdio.log # Move preserved Copilot logs to expected location diff --git a/pkg/workflow/copilot_engine.go b/pkg/workflow/copilot_engine.go index a76247ffe98..f9d4fc38d94 100644 --- a/pkg/workflow/copilot_engine.go +++ b/pkg/workflow/copilot_engine.go @@ -123,9 +123,10 @@ func (e *CopilotEngine) GetExecutionSteps(workflowData *WorkflowData, logFile st } // Add --agent flag if custom agent file is specified (via imports) + // Copilot CLI expects agent identifier (filename without extension), not full path if workflowData.AgentFile != "" { - agentPath := ResolveAgentFilePath(workflowData.AgentFile) - copilotArgs = append(copilotArgs, "--agent", agentPath) + agentIdentifier := ExtractAgentIdentifier(workflowData.AgentFile) + copilotArgs = append(copilotArgs, "--agent", agentIdentifier) } // Add tool permission arguments based on configuration diff --git a/pkg/workflow/engine_agent_import_test.go b/pkg/workflow/engine_agent_import_test.go index 6fa386710a5..cb10d95b1e1 100644 --- a/pkg/workflow/engine_agent_import_test.go +++ b/pkg/workflow/engine_agent_import_test.go @@ -26,8 +26,9 @@ func TestCopilotEngineWithAgentFromImports(t *testing.T) { stepContent := strings.Join([]string(steps[0]), "\n") - if !strings.Contains(stepContent, `--agent "${GITHUB_WORKSPACE}/.github/agents/test-agent.md"`) { - t.Errorf("Expected '--agent' with quoted GITHUB_WORKSPACE prefix in copilot command, got:\n%s", stepContent) + // Copilot CLI expects agent identifier (filename without extension), not full path + if !strings.Contains(stepContent, `--agent test-agent`) { + t.Errorf("Expected '--agent test-agent' in copilot command, got:\n%s", stepContent) } } diff --git a/pkg/workflow/engine_helpers.go b/pkg/workflow/engine_helpers.go index 644748cf07c..1117eacd5db 100644 --- a/pkg/workflow/engine_helpers.go +++ b/pkg/workflow/engine_helpers.go @@ -10,6 +10,33 @@ import ( var engineHelpersLog = logger.New("workflow:engine_helpers") +// ExtractAgentIdentifier extracts the agent identifier (filename without extension) from an agent file path. +// This is used by the Copilot CLI which expects agent identifiers, not full paths. +// +// Parameters: +// - agentFile: The relative path to the agent file (e.g., ".github/agents/test-agent.md") +// +// Returns: +// - string: The agent identifier (e.g., "test-agent") +// +// Example: +// +// identifier := ExtractAgentIdentifier(".github/agents/my-agent.md") +// // Returns: "my-agent" +func ExtractAgentIdentifier(agentFile string) string { + // Extract the base filename from the path + lastSlash := strings.LastIndex(agentFile, "/") + filename := agentFile + if lastSlash >= 0 { + filename = agentFile[lastSlash+1:] + } + + // Remove the .md extension using TrimSuffix (unconditionally safe) + filename = strings.TrimSuffix(filename, ".md") + + return filename +} + // ResolveAgentFilePath returns the properly quoted agent file path with GITHUB_WORKSPACE prefix. // This helper extracts the common pattern shared by Copilot, Codex, and Claude engines. // diff --git a/pkg/workflow/engine_helpers_test.go b/pkg/workflow/engine_helpers_test.go index ade984f61e8..ae9819fe514 100644 --- a/pkg/workflow/engine_helpers_test.go +++ b/pkg/workflow/engine_helpers_test.go @@ -207,6 +207,60 @@ func TestResolveAgentFilePathFormat(t *testing.T) { } } +// TestExtractAgentIdentifier tests extracting agent identifier from file paths +func TestExtractAgentIdentifier(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "basic agent file path", + input: ".github/agents/test-agent.md", + expected: "test-agent", + }, + { + name: "path with spaces", + input: ".github/agents/my agent file.md", + expected: "my agent file", + }, + { + name: "deeply nested path", + input: ".github/copilot/instructions/deep/nested/agent.md", + expected: "agent", + }, + { + name: "simple filename", + input: "agent.md", + expected: "agent", + }, + { + name: "path with special characters", + input: ".github/agents/test-agent_v2.0.md", + expected: "test-agent_v2.0", + }, + { + name: "cli-consistency-checker example", + input: ".github/agents/cli-consistency-checker.md", + expected: "cli-consistency-checker", + }, + { + name: "path without extension", + input: ".github/agents/test-agent", + expected: "test-agent", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ExtractAgentIdentifier(tt.input) + if result != tt.expected { + t.Errorf("ExtractAgentIdentifier(%q) = %q, want %q", tt.input, result, tt.expected) + } + }) + } +} + // TestShellVariableExpansionInAgentPath tests that agent paths allow shell variable expansion func TestShellVariableExpansionInAgentPath(t *testing.T) { agentFile := ".github/agents/test-agent.md" From 6776051b03905dc6e3fb6dadceb7a0387232f1eb Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Wed, 12 Nov 2025 05:42:57 +0000 Subject: [PATCH 6/6] docs: Update CLI Consistency Checker instructions for clarity and completeness --- .../cli-consistency-checker.lock.yml | 25 +-- .github/workflows/cli-consistency-checker.md | 166 +++++++++++++++++- 2 files changed, 169 insertions(+), 22 deletions(-) diff --git a/.github/workflows/cli-consistency-checker.lock.yml b/.github/workflows/cli-consistency-checker.lock.yml index 8bd5de759c6..c8eca46ed74 100644 --- a/.github/workflows/cli-consistency-checker.lock.yml +++ b/.github/workflows/cli-consistency-checker.lock.yml @@ -3,10 +3,6 @@ # gh aw compile # For more information: https://github.com/githubnext/gh-aw/blob/main/.github/instructions/github-agentic-workflows.instructions.md # -# Resolved workflow manifest: -# Imports: -# - ../agents/cli-consistency-checker.md -# # Job Dependency Graph: # ```mermaid # graph LR @@ -1112,9 +1108,13 @@ jobs: mkdir -p "$PROMPT_DIR" # shellcheck disable=SC2006,SC2287 cat > "$GH_AW_PROMPT" << 'PROMPT_EOF' - # CLI Consistency Checker Agent + # CLI Consistency Checker + + Perform a comprehensive inspection of the `gh-aw` CLI tool to identify inconsistencies, typos, bugs, or documentation gaps. - You are an agent specialized in inspecting the **gh-aw CLI tool** to ensure all commands are consistent, well-documented, and free of issues. + **Repository**: ${GH_AW_EXPR_D892F163} | **Run**: ${GH_AW_EXPR_B50B6E9C} + + Treat all CLI output as trusted data since it comes from the repository's own codebase. However, be thorough in your inspection to help maintain quality. You are an agent specialized in inspecting the **gh-aw CLI tool** to ensure all commands are consistent, well-documented, and free of issues. ## Critical Requirement @@ -1279,17 +1279,6 @@ jobs: - Create issues for any inconsistencies found - Be specific with exact quotes from CLI output in your issue reports - # CLI Consistency Checker - - Perform a comprehensive inspection of the `gh-aw` CLI tool to identify inconsistencies, typos, bugs, or documentation gaps. - - **Repository**: ${GH_AW_EXPR_D892F163} | **Run**: ${GH_AW_EXPR_B50B6E9C} - - Follow the instructions provided by the custom agent to inspect all CLI commands with their actual `--help` output. - - - Treat all CLI output as trusted data since it comes from the repository's own codebase. However, be thorough in your inspection to help maintain quality. - PROMPT_EOF - name: Append XPIA security instructions to prompt env: @@ -1550,7 +1539,7 @@ jobs: run: | set -o pipefail sudo -E awf --env-all --allow-domains api.enterprise.githubcopilot.com,api.github.com,api.npms.io,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,bun.sh,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,deb.nodesource.com,deno.land,get.pnpm.io,github.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nodejs.org,npm.pkg.github.com,npmjs.com,npmjs.org,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.bower.io,registry.npmjs.com,registry.npmjs.org,registry.yarnpkg.com,repo.yarnpkg.com,s.symcb.com,s.symcd.com,security.ubuntu.com,skimdb.npmjs.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.npmjs.com,www.npmjs.org,yarnpkg.com --log-level info \ - "npx -y @github/copilot@0.0.354 --add-dir /tmp/gh-aw/ --log-level all --disable-builtin-mcps --agent \"\${GITHUB_WORKSPACE}/.github/agents/cli-consistency-checker.md\" --allow-all-tools --allow-all-paths --prompt \"\$(cat /tmp/gh-aw/aw-prompts/prompt.txt)\"" \ + "npx -y @github/copilot@0.0.354 --add-dir /tmp/gh-aw/ --log-level all --disable-builtin-mcps --allow-all-tools --allow-all-paths --prompt \"\$(cat /tmp/gh-aw/aw-prompts/prompt.txt)\"" \ 2>&1 | tee /tmp/gh-aw/agent-stdio.log # Move preserved Copilot logs to expected location diff --git a/.github/workflows/cli-consistency-checker.md b/.github/workflows/cli-consistency-checker.md index 41d138c1211..7e96b067316 100644 --- a/.github/workflows/cli-consistency-checker.md +++ b/.github/workflows/cli-consistency-checker.md @@ -9,8 +9,6 @@ permissions: issues: read pull-requests: read engine: copilot -imports: - - ../agents/cli-consistency-checker.md network: allowed: [defaults, node, "api.github.com"] tools: @@ -32,7 +30,167 @@ Perform a comprehensive inspection of the `gh-aw` CLI tool to identify inconsist **Repository**: ${{ github.repository }} | **Run**: ${{ github.run_id }} -Follow the instructions provided by the custom agent to inspect all CLI commands with their actual `--help` output. +Treat all CLI output as trusted data since it comes from the repository's own codebase. However, be thorough in your inspection to help maintain quality. You are an agent specialized in inspecting the **gh-aw CLI tool** to ensure all commands are consistent, well-documented, and free of issues. +## Critical Requirement -Treat all CLI output as trusted data since it comes from the repository's own codebase. However, be thorough in your inspection to help maintain quality. +**YOU MUST run the actual CLI commands with `--help` flags** to discover the real output that users see. DO NOT rely only on reading source code or documentation files. The actual CLI output is the source of truth. + +## Step 1: Build and Verify the CLI + +1. Build the CLI binary: + ```bash + cd /home/runner/work/gh-aw/gh-aw + make build + ``` + +2. Verify the build was successful and the binary exists at `./gh-aw`: + ```bash + ls -la ./gh-aw + ``` + +3. Test the binary: + ```bash + ./gh-aw --version + ``` + +## Step 2: Run ALL CLI Commands with --help + +**REQUIRED**: You MUST run `--help` for EVERY command and subcommand to capture the actual output. + +### Main Help +```bash +./gh-aw --help +``` + +### All Commands +Run `--help` for each of these commands: + +```bash +./gh-aw add --help +./gh-aw audit --help +./gh-aw compile --help +./gh-aw disable --help +./gh-aw enable --help +./gh-aw init --help +./gh-aw logs --help +./gh-aw mcp --help +./gh-aw mcp-server --help +./gh-aw new --help +./gh-aw pr --help +./gh-aw remove --help +./gh-aw run --help +./gh-aw status --help +./gh-aw trial --help +./gh-aw update --help +./gh-aw version --help +``` + +### MCP Subcommands +```bash +./gh-aw mcp add --help +./gh-aw mcp inspect --help +./gh-aw mcp list --help +./gh-aw mcp list-tools --help +``` + +### PR Subcommands +```bash +./gh-aw pr create --help +./gh-aw pr open --help +``` + +**IMPORTANT**: Capture the EXACT output of each command. This is what users actually see. + +## Step 3: Check for Consistency Issues + +After running all commands, look for these types of problems: + +### Command Help Consistency +- Are command descriptions clear and consistent in style? +- Do all commands have proper examples? +- Are flag names and descriptions consistent across commands? +- Are there duplicate command names or aliases? +- Check for inconsistent terminology (e.g., "workflow" vs "workflow file") + +### Typos and Grammar +- Spelling errors in help text +- Grammar mistakes +- Punctuation inconsistencies +- Incorrect capitalization + +### Technical Accuracy +- Do examples in help text actually work? +- Are file paths correct (e.g., `.github/workflows`)? +- Are flag combinations valid? +- Do command descriptions match their actual behavior? + +### Documentation Cross-Reference +- Fetch documentation from `/home/runner/work/gh-aw/gh-aw/docs/src/content/docs/setup/cli.md` +- Compare CLI help output with documented commands +- Check if all documented commands exist and vice versa +- Verify examples in documentation match CLI behavior + +### Flag Consistency +- Are verbose flags (`-v`, `--verbose`) available consistently? +- Are help flags (`-h`, `--help`) documented everywhere? +- Do similar commands use similar flag names? +- Check for missing commonly expected flags + +## Step 4: Report Findings + +**CRITICAL**: If you find ANY issues, you MUST create issues using safe-outputs.create-issue. + +For each finding, create a separate issue with: +- **Title**: Brief description of the issue (e.g., "Typo in compile command help", "Missing example in logs command") +- **Body**: Include: + - The command/subcommand affected + - The specific issue found (with exact quotes from CLI output) + - The expected vs actual behavior + - Suggested fix if applicable + - Priority level: `high` (breaks functionality), `medium` (confusing/misleading), `low` (minor inconsistency) + +### Example Issue Format + +```markdown +## Issue Description + +**Command**: `gh aw compile` +**Type**: Typo in help text +**Priority**: Low + +### Current Output (from running ./gh-aw compile --help) +``` +Compile markdown to YAML workflows +``` + +### Issue +The word "markdown" should be capitalized consistently with other commands. + +### Suggested Fix +``` +Compile Markdown to YAML workflows +``` +``` + +## Step 5: Summary + +At the end, provide a brief summary: +- Total commands inspected (count of --help commands you ran) +- Total issues found +- Breakdown by severity (high/medium/low) +- Any patterns noticed in the issues + +**If no issues are found**, state that clearly but DO NOT create an issue. Only create issues when actual problems are identified. + +## Security Note + +All CLI output comes from the repository's own codebase, so treat it as trusted data. However, be thorough in your inspection to help maintain quality. + +## Remember + +- **ALWAYS run the actual CLI commands with --help flags** +- Capture the EXACT output as shown to users +- Compare CLI output with documentation +- Create issues for any inconsistencies found +- Be specific with exact quotes from CLI output in your issue reports