diff --git a/.github/workflows/smoke-gemini.lock.yml b/.github/workflows/smoke-gemini.lock.yml index b05b3d0d377..c2fdd1135b0 100644 --- a/.github/workflows/smoke-gemini.lock.yml +++ b/.github/workflows/smoke-gemini.lock.yml @@ -28,7 +28,7 @@ # - shared/gh.md # - shared/reporting.md # -# gh-aw-metadata: {"schema_version":"v1","frontmatter_hash":"1c7e9095a4960a70d4be26e98a3da1fb9a7a3526fc8e60d0614b182f9fbf0904"} +# gh-aw-metadata: {"schema_version":"v1","frontmatter_hash":"0cbff208fe0695987e6a089673e8346206c6dca1fdb91f20648b52de072e50c6"} name: "Smoke Gemini" "on": @@ -359,7 +359,7 @@ jobs: const awInfo = { engine_id: "gemini", engine_name: "Google Gemini CLI", - model: process.env.GH_AW_MODEL_AGENT_CUSTOM || "", + model: "gemini-2.0-flash-lite", version: "", agent_version: "", workflow_name: "Smoke Gemini", @@ -900,12 +900,12 @@ jobs: id: agentic_execution run: | set -o pipefail - sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains '*.githubusercontent.com,*.googleapis.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,generativelanguage.googleapis.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.com,github.githubassets.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,lfs.github.com,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.20.2 --skip-pull \ - -- /bin/bash -c 'export PATH="$(find /opt/hostedtoolcache -maxdepth 4 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && gemini --yolo --output-format json --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains '*.githubusercontent.com,*.googleapis.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,generativelanguage.googleapis.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.com,github.githubassets.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,lfs.github.com,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.20.2 --skip-pull --enable-api-proxy \ + -- /bin/bash -c 'export PATH="$(find /opt/hostedtoolcache -maxdepth 4 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && gemini --model gemini-2.0-flash-lite --yolo --output-format stream-json --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: + GEMINI_API_BASE_URL: http://host.docker.internal:10003 GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} GH_AW_MCP_CONFIG: ${{ github.workspace }}/.gemini/settings.json - GH_AW_MODEL_AGENT_GEMINI: ${{ vars.GH_AW_MODEL_AGENT_GEMINI || '' }} GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} GITHUB_WORKSPACE: ${{ github.workspace }} @@ -1197,10 +1197,9 @@ jobs: id: agentic_execution run: | set -o pipefail - gemini --yolo --output-format json --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" 2>&1 | tee /tmp/gh-aw/threat-detection/detection.log + gemini --model gemini-2.0-flash-lite --yolo --output-format stream-json --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log env: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - GH_AW_MODEL_DETECTION_GEMINI: ${{ vars.GH_AW_MODEL_DETECTION_GEMINI || '' }} GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GITHUB_WORKSPACE: ${{ github.workspace }} - name: Parse threat detection results @@ -1267,6 +1266,7 @@ jobs: timeout-minutes: 15 env: GH_AW_ENGINE_ID: "gemini" + GH_AW_ENGINE_MODEL: "gemini-2.0-flash-lite" GH_AW_SAFE_OUTPUT_MESSAGES: "{\"footer\":\"\\u003e ✨ *[{workflow_name}]({run_url}) — Powered by Gemini*\",\"runStarted\":\"✨ Gemini awakens... [{workflow_name}]({run_url}) begins its journey on this {event_type}...\",\"runSuccess\":\"🚀 [{workflow_name}]({run_url}) **MISSION COMPLETE!** Gemini has spoken. ✨\",\"runFailure\":\"⚠️ [{workflow_name}]({run_url}) {status}. Gemini encountered unexpected challenges...\"}" GH_AW_WORKFLOW_ID: "smoke-gemini" GH_AW_WORKFLOW_NAME: "Smoke Gemini" diff --git a/.github/workflows/smoke-gemini.md b/.github/workflows/smoke-gemini.md index 0f03d4dd91d..e5ae75ea644 100644 --- a/.github/workflows/smoke-gemini.md +++ b/.github/workflows/smoke-gemini.md @@ -11,7 +11,9 @@ permissions: issues: read pull-requests: read name: Smoke Gemini -engine: gemini +engine: + id: gemini + model: gemini-2.0-flash-lite strict: true imports: - shared/gh.md diff --git a/actions/setup/js/parse_gemini_log.cjs b/actions/setup/js/parse_gemini_log.cjs new file mode 100644 index 00000000000..3d2760ce385 --- /dev/null +++ b/actions/setup/js/parse_gemini_log.cjs @@ -0,0 +1,98 @@ +// @ts-check +/// + +const { createEngineLogParser } = require("./log_parser_shared.cjs"); + +const main = createEngineLogParser({ + parserName: "Gemini", + parseFunction: parseGeminiLog, + supportsDirectories: false, +}); + +/** + * Parse Gemini CLI streaming JSON log output and format as markdown. + * Gemini CLI outputs one JSON object per line when using --output-format stream-json (JSONL). + * @param {string} logContent - The raw log content to parse + * @returns {{markdown: string, logEntries: Array, mcpFailures: Array, maxTurnsHit: boolean}} Parsed log data + */ +function parseGeminiLog(logContent) { + if (!logContent) { + return { + markdown: "## 🤖 Gemini\n\nNo log content provided.\n\n", + logEntries: [], + mcpFailures: [], + maxTurnsHit: false, + }; + } + + let markdown = ""; + let totalInputTokens = 0; + let totalOutputTokens = 0; + let lastResponse = ""; + + const lines = logContent.split("\n"); + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + + // Try to parse each line as a JSON object (Gemini --output-format json output) + try { + const parsed = JSON.parse(trimmed); + + if (parsed.response) { + lastResponse = parsed.response; + } + + // Aggregate token usage from stats + if (parsed.stats && parsed.stats.models) { + for (const modelStats of Object.values(parsed.stats.models)) { + if (modelStats && typeof modelStats === "object") { + if (typeof modelStats.input_tokens === "number") { + totalInputTokens += modelStats.input_tokens; + } + if (typeof modelStats.output_tokens === "number") { + totalOutputTokens += modelStats.output_tokens; + } + } + } + } + } catch (_e) { + // Not JSON - skip non-JSON lines + } + } + + // Build markdown output + if (lastResponse) { + markdown += "## 🤖 Reasoning\n\n"; + markdown += lastResponse + "\n\n"; + } + + markdown += "## 📊 Information\n\n"; + const totalTokens = totalInputTokens + totalOutputTokens; + if (totalTokens > 0) { + markdown += `**Total Tokens Used:** ${totalTokens.toLocaleString()}\n\n`; + if (totalInputTokens > 0) { + markdown += `**Input Tokens:** ${totalInputTokens.toLocaleString()}\n\n`; + } + if (totalOutputTokens > 0) { + markdown += `**Output Tokens:** ${totalOutputTokens.toLocaleString()}\n\n`; + } + } + + return { + markdown, + logEntries: [], + mcpFailures: [], + maxTurnsHit: false, + }; +} + +// Export for testing +if (typeof module !== "undefined" && module.exports) { + module.exports = { + main, + parseGeminiLog, + }; +} diff --git a/pkg/workflow/awf_helpers.go b/pkg/workflow/awf_helpers.go index 8028d8677d6..a4943736826 100644 --- a/pkg/workflow/awf_helpers.go +++ b/pkg/workflow/awf_helpers.go @@ -170,7 +170,8 @@ func BuildAWFArgs(config AWFCommandConfig) []string { awfArgs = append(awfArgs, "--proxy-logs-dir", string(constants.AWFProxyLogsDir)) // Add --enable-host-access when MCP servers are configured (gateway is used) - if HasMCPServers(config.WorkflowData) { + // OR when the API proxy sidecar is enabled (needs to reach host.docker.internal:) + if HasMCPServers(config.WorkflowData) || config.UsesAPIProxy { awfArgs = append(awfArgs, "--enable-host-access") awfHelpersLog.Print("Added --enable-host-access for MCP gateway communication") } diff --git a/pkg/workflow/enable_api_proxy_test.go b/pkg/workflow/enable_api_proxy_test.go index d49dd5be7b2..6d86cc0a8f5 100644 --- a/pkg/workflow/enable_api_proxy_test.go +++ b/pkg/workflow/enable_api_proxy_test.go @@ -88,4 +88,31 @@ func TestEngineAWFEnableApiProxy(t *testing.T) { t.Error("Expected Codex AWF command to contain '--enable-api-proxy' flag") } }) + + t.Run("Gemini AWF command includes enable-api-proxy flag (supports LLM gateway)", func(t *testing.T) { + workflowData := &WorkflowData{ + Name: "test-workflow", + EngineConfig: &EngineConfig{ + ID: "gemini", + }, + NetworkPermissions: &NetworkPermissions{ + Firewall: &FirewallConfig{ + Enabled: true, + }, + }, + } + + engine := NewGeminiEngine() + steps := engine.GetExecutionSteps(workflowData, "test.log") + + if len(steps) == 0 { + t.Fatal("Expected at least one execution step") + } + + stepContent := strings.Join(steps[0], "\n") + + if !strings.Contains(stepContent, "--enable-api-proxy") { + t.Error("Expected Gemini AWF command to contain '--enable-api-proxy' flag") + } + }) } diff --git a/pkg/workflow/gemini_engine.go b/pkg/workflow/gemini_engine.go index 8f8f7e52c1e..a393bcf216e 100644 --- a/pkg/workflow/gemini_engine.go +++ b/pkg/workflow/gemini_engine.go @@ -179,8 +179,8 @@ func (e *GeminiEngine) GetExecutionSteps(workflowData *WorkflowData, logFile str // Without this, Gemini CLI's default approval mode rejects tool calls with "Tool execution denied by policy" geminiArgs = append(geminiArgs, "--yolo") - // Add headless mode with JSON output - geminiArgs = append(geminiArgs, "--output-format", "json") + // Add streaming JSON output (JSONL format, compatible with the log parser) + geminiArgs = append(geminiArgs, "--output-format", "stream-json") // Add prompt argument geminiArgs = append(geminiArgs, "--prompt", "\"$(cat /tmp/gh-aw/aw-prompts/prompt.txt)\"") @@ -206,18 +206,22 @@ func (e *GeminiEngine) GetExecutionSteps(workflowData *WorkflowData, logFile str npmPathSetup := GetNpmBinPathSetup() geminiCommandWithPath := fmt.Sprintf("%s && %s", npmPathSetup, geminiCommand) + // Enable API proxy sidecar if this engine supports LLM gateway + llmGatewayPort := e.SupportsLLMGateway() + usesAPIProxy := llmGatewayPort > 0 + command = BuildAWFCommand(AWFCommandConfig{ EngineName: "gemini", EngineCommand: geminiCommandWithPath, LogFile: logFile, WorkflowData: workflowData, UsesTTY: false, - UsesAPIProxy: false, + UsesAPIProxy: usesAPIProxy, AllowedDomains: allowedDomains, }) } else { command = fmt.Sprintf(`set -o pipefail -%s 2>&1 | tee %s`, geminiCommand, logFile) +%s 2>&1 | tee -a %s`, geminiCommand, logFile) } // Build environment variables @@ -232,6 +236,12 @@ func (e *GeminiEngine) GetExecutionSteps(workflowData *WorkflowData, logFile str env["GH_AW_MCP_CONFIG"] = "${{ github.workspace }}/.gemini/settings.json" } + // When the firewall (AWF) is enabled with --enable-api-proxy, point Gemini CLI at the + // LLM gateway sidecar instead of the real googleapis.com endpoint. + if firewallEnabled { + env["GEMINI_API_BASE_URL"] = fmt.Sprintf("http://host.docker.internal:%d", constants.GeminiLLMGatewayPort) + } + // Add safe outputs env applySafeOutputEnvToMap(env, workflowData) diff --git a/pkg/workflow/gemini_engine_test.go b/pkg/workflow/gemini_engine_test.go index bd85bf7473c..65ef85b6d7c 100644 --- a/pkg/workflow/gemini_engine_test.go +++ b/pkg/workflow/gemini_engine_test.go @@ -152,7 +152,7 @@ func TestGeminiEngineExecution(t *testing.T) { assert.Contains(t, stepContent, "id: agentic_execution", "Should have agentic_execution ID") assert.Contains(t, stepContent, "gemini", "Should invoke gemini command") assert.Contains(t, stepContent, "--yolo", "Should include --yolo flag for auto-approving tool executions") - assert.Contains(t, stepContent, "--output-format json", "Should use JSON output format") + assert.Contains(t, stepContent, "--output-format stream-json", "Should use streaming JSON output format") assert.Contains(t, stepContent, `--prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"`, "Should include prompt argument with correct shell quoting") assert.Contains(t, stepContent, "/tmp/test.log", "Should include log file") assert.Contains(t, stepContent, "GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}", "Should set GEMINI_API_KEY env var") @@ -273,6 +273,8 @@ func TestGeminiEngineFirewallIntegration(t *testing.T) { // Should use AWF command assert.Contains(t, stepContent, "awf", "Should use AWF when firewall is enabled") assert.Contains(t, stepContent, "--allow-domains", "Should include allow-domains flag") + assert.Contains(t, stepContent, "--enable-api-proxy", "Should include --enable-api-proxy flag") + assert.Contains(t, stepContent, "GEMINI_API_BASE_URL: http://host.docker.internal:10003", "Should set GEMINI_API_BASE_URL to LLM gateway URL") }) t.Run("firewall disabled", func(t *testing.T) { @@ -293,5 +295,6 @@ func TestGeminiEngineFirewallIntegration(t *testing.T) { // Should use simple command without AWF assert.Contains(t, stepContent, "set -o pipefail", "Should use simple command with pipefail") assert.NotContains(t, stepContent, "awf", "Should not use AWF when firewall is disabled") + assert.NotContains(t, stepContent, "GEMINI_API_BASE_URL", "Should not set GEMINI_API_BASE_URL when firewall is disabled") }) } diff --git a/pkg/workflow/strict_mode_llm_gateway_test.go b/pkg/workflow/strict_mode_llm_gateway_test.go index 065ff3a24c4..2cdd042e6df 100644 --- a/pkg/workflow/strict_mode_llm_gateway_test.go +++ b/pkg/workflow/strict_mode_llm_gateway_test.go @@ -287,6 +287,11 @@ func TestSupportsLLMGateway(t *testing.T) { expectedPort: constants.CopilotLLMGatewayPort, description: "Copilot engine uses dedicated port for LLM gateway", }, + { + engineID: "gemini", + expectedPort: constants.GeminiLLMGatewayPort, + description: "Gemini engine uses dedicated port for LLM gateway", + }, } for _, tt := range tests { diff --git a/smoke_test_push_22239107636.txt b/smoke_test_push_22239107636.txt new file mode 100644 index 00000000000..df22b6a0443 --- /dev/null +++ b/smoke_test_push_22239107636.txt @@ -0,0 +1 @@ +Test file for PR push - smoke test run 22239107636