From a7ea4ffe3ab0d2d17967e45aebbf602f222b4508 Mon Sep 17 00:00:00 2001 From: Landon Cox Date: Fri, 10 Apr 2026 11:08:15 -0700 Subject: [PATCH] refactor: use gh aw logs for token analysis workflows Replace manual artifact downloading (gh run download --name firewall-audit-logs) with gh aw logs --json in all 4 token analysis workflows. This aligns with the approach used by gh-aw's own token-audit and token-optimizer workflows. Benefits: - Eliminates dependency on knowing the correct artifact name (the root cause of the bug fixed in PR #1883) - Gets pre-aggregated structured data (token counts, costs, turns) instead of parsing raw token-usage.jsonl - Handles artifact naming backward/forward compat automatically - Pre-downloads data in steps block (faster, more reliable) Changes: - copilot-token-usage-analyzer.md: replaced manual run discovery + artifact download with gh aw logs --engine copilot --json pre-step - claude-token-usage-analyzer.md: same for --engine claude - copilot-token-optimizer.md: added gh aw logs --engine copilot pre-step for 7-day data, removed manual artifact download - claude-token-optimizer.md: same for --engine claude - shared/mcp/gh-aw.md: new shared component to install gh-aw CLI (mirroring upstream gh-aw's shared/mcp/gh-aw.md) Refs: #1883, github/gh-aw#25683 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../workflows/claude-token-optimizer.lock.yml | 42 ++-- .github/workflows/claude-token-optimizer.md | 118 ++++----- .../claude-token-usage-analyzer.lock.yml | 41 +-- .../workflows/claude-token-usage-analyzer.md | 233 +++++++++--------- .../copilot-token-optimizer.lock.yml | 42 ++-- .github/workflows/copilot-token-optimizer.md | 111 +++++---- .../copilot-token-usage-analyzer.lock.yml | 41 +-- .../workflows/copilot-token-usage-analyzer.md | 232 ++++++++--------- .github/workflows/shared/mcp/gh-aw.md | 31 +++ 9 files changed, 494 insertions(+), 397 deletions(-) create mode 100644 .github/workflows/shared/mcp/gh-aw.md diff --git a/.github/workflows/claude-token-optimizer.lock.yml b/.github/workflows/claude-token-optimizer.lock.yml index bbdbe5cd6..dc4bfa766 100644 --- a/.github/workflows/claude-token-optimizer.lock.yml +++ b/.github/workflows/claude-token-optimizer.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"25ef9aa68f70272b23b0f780d7ec8a7e982a15cf11f5312ccf1c875275619a43","compiler_version":"v0.68.0","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"c48be8ef2bd2db971a34c0c4357a10457b1ac7cad40a8d10edc6e717cca912fa","compiler_version":"v0.68.0","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"ed597411d8f924073f98dfc5c65a23a2325f34cd","version":"v8"},{"repo":"actions/upload-artifact","sha":"bbbca2ddaa5d8feaa63e36b76fdaad77386f024f","version":"v7"},{"repo":"github/gh-aw-actions/setup","sha":"0acfb4a691fe207cd8bc982ea5cb9d750d57a702","version":"v0.68.0"}]} # ___ _ _ # / _ \ | | (_) @@ -24,6 +24,10 @@ # # Daily Claude token optimization advisor — reads the latest token usage report and creates actionable recommendations to reduce token consumption for the most expensive workflow # +# Resolved workflow manifest: +# Imports: +# - shared/mcp/gh-aw.md +# # Secrets used: # - COPILOT_GITHUB_TOKEN # - GH_AW_GITHUB_MCP_SERVER_TOKEN @@ -161,19 +165,18 @@ jobs: GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - GH_AW_STEPS_X_OUTPUTS_Y: ${{ steps.X.outputs.Y }} # poutine:ignore untrusted_checkout_exec run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_c020825fa23291cb_EOF' + cat << 'GH_AW_PROMPT_e08fbc295449b2be_EOF' - GH_AW_PROMPT_c020825fa23291cb_EOF + GH_AW_PROMPT_e08fbc295449b2be_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_c020825fa23291cb_EOF' + cat << 'GH_AW_PROMPT_e08fbc295449b2be_EOF' Tools: create_issue, missing_tool, missing_data, noop @@ -205,18 +208,18 @@ jobs: {{/if}} - GH_AW_PROMPT_c020825fa23291cb_EOF + GH_AW_PROMPT_e08fbc295449b2be_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_c020825fa23291cb_EOF' + cat << 'GH_AW_PROMPT_e08fbc295449b2be_EOF' + {{#runtime-import .github/workflows/shared/mcp/gh-aw.md}} {{#runtime-import .github/workflows/claude-token-optimizer.md}} - GH_AW_PROMPT_c020825fa23291cb_EOF + GH_AW_PROMPT_e08fbc295449b2be_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_STEPS_X_OUTPUTS_Y: ${{ steps.X.outputs.Y }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); @@ -236,7 +239,6 @@ jobs: GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: ${{ needs.pre_activation.outputs.activated }} - GH_AW_STEPS_X_OUTPUTS_Y: ${{ steps.X.outputs.Y }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); @@ -256,8 +258,7 @@ jobs: GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, - GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED, - GH_AW_STEPS_X_OUTPUTS_Y: process.env.GH_AW_STEPS_X_OUTPUTS_Y + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED } }); - name: Validate prompt placeholders @@ -332,6 +333,15 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + - env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + name: Install gh-aw extension + run: "# Install gh-aw if not already available\nif ! gh aw --version >/dev/null 2>&1; then\n echo \"Installing gh-aw extension...\"\n curl -fsSL https://raw.githubusercontent.com/github/gh-aw/refs/heads/main/install-gh-aw.sh | bash\nfi\ngh aw --version\n# Copy the gh-aw binary to RUNNER_TEMP for MCP server containerization\nmkdir -p \"${RUNNER_TEMP}/gh-aw\"\nGH_AW_BIN=$(which gh-aw 2>/dev/null || find ~/.local/share/gh/extensions/gh-aw -name 'gh-aw' -type f 2>/dev/null | head -1)\nif [ -n \"$GH_AW_BIN\" ] && [ -f \"$GH_AW_BIN\" ]; then\n cp \"$GH_AW_BIN\" \"${RUNNER_TEMP}/gh-aw/gh-aw\"\n chmod +x \"${RUNNER_TEMP}/gh-aw/gh-aw\"\n echo \"Copied gh-aw binary to ${RUNNER_TEMP}/gh-aw/gh-aw\"\nelse\n echo \"::error::Failed to find gh-aw binary for MCP server\"\n exit 1\nfi" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Download recent Claude workflow logs + run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/token-audit\n\necho \"\\U0001F4E5 Downloading Claude workflow logs (last 7 days)...\"\n\nLOGS_EXIT=0\ngh aw logs \\\n --engine claude \\\n --start-date -7d \\\n --json \\\n -c 50 \\\n > /tmp/gh-aw/token-audit/claude-logs.json || LOGS_EXIT=$?\n\nif [ -s /tmp/gh-aw/token-audit/claude-logs.json ]; then\n TOTAL=$(jq '.runs | length' /tmp/gh-aw/token-audit/claude-logs.json)\n echo \"\\u2705 Downloaded $TOTAL Claude workflow runs (last 7 days)\"\n if [ \"$LOGS_EXIT\" -ne 0 ]; then\n echo \"\\u26a0\\ufe0f gh aw logs exited with code $LOGS_EXIT (partial results)\"\n fi\nelse\n echo \"\\u274c No log data downloaded (exit code $LOGS_EXIT)\"\n echo '{\"runs\":[],\"summary\":{}}' > /tmp/gh-aw/token-audit/claude-logs.json\nfi" + - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} @@ -405,9 +415,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_6189d385a13beb27_EOF' + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_a7b12cdddc382995_EOF' {"create_issue":{"close_older_issues":true,"labels":["claude-token-optimization"],"max":1,"title_prefix":"⚡ Claude Token Optimization"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_6189d385a13beb27_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_a7b12cdddc382995_EOF - name: Write Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -602,7 +612,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.17' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_0245fd0fb89bc5e8_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" + cat << GH_AW_MCP_CONFIG_7f9a00f95a76718c_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" { "mcpServers": { "github": { @@ -643,7 +653,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_0245fd0fb89bc5e8_EOF + GH_AW_MCP_CONFIG_7f9a00f95a76718c_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: diff --git a/.github/workflows/claude-token-optimizer.md b/.github/workflows/claude-token-optimizer.md index c50b4a1c9..c755368ee 100644 --- a/.github/workflows/claude-token-optimizer.md +++ b/.github/workflows/claude-token-optimizer.md @@ -14,6 +14,8 @@ permissions: actions: read issues: read pull-requests: read +imports: + - uses: shared/mcp/gh-aw.md network: allowed: - github @@ -23,11 +25,39 @@ tools: bash: true safe-outputs: create-issue: - title-prefix: "⚡ Claude Token Optimization" + title-prefix: "\u26a1 Claude Token Optimization" labels: [claude-token-optimization] close-older-issues: true timeout-minutes: 10 strict: true +steps: + - name: Download recent Claude workflow logs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + mkdir -p /tmp/gh-aw/token-audit + + echo "\U0001F4E5 Downloading Claude workflow logs (last 7 days)..." + + LOGS_EXIT=0 + gh aw logs \ + --engine claude \ + --start-date -7d \ + --json \ + -c 50 \ + > /tmp/gh-aw/token-audit/claude-logs.json || LOGS_EXIT=$? + + if [ -s /tmp/gh-aw/token-audit/claude-logs.json ]; then + TOTAL=$(jq '.runs | length' /tmp/gh-aw/token-audit/claude-logs.json) + echo "\u2705 Downloaded $TOTAL Claude workflow runs (last 7 days)" + if [ "$LOGS_EXIT" -ne 0 ]; then + echo "\u26a0\ufe0f gh aw logs exited with code $LOGS_EXIT (partial results)" + fi + else + echo "\u274c No log data downloaded (exit code $LOGS_EXIT)" + echo '{"runs":[],"summary":{}}' > /tmp/gh-aw/token-audit/claude-logs.json + fi --- # Daily Claude Token Optimization Advisor @@ -57,7 +87,7 @@ From the report's **Workflow Summary** table, identify the workflow with: Extract these key metrics for the target workflow: - Total tokens per run -- Cache hit rate (read and write separately — Anthropic exposes both) +- Cache hit rate (read and write separately \u2014 Anthropic exposes both) - Cache write rate - Input/output ratio - Number of LLM turns (request count) @@ -81,53 +111,32 @@ cat "$WORKFLOW_FILE" ``` Analyze: -- **Tools loaded** — List all tools in the `tools:` section. Flag any that may not be needed. -- **Network groups** — List network groups in `network.allowed:`. Flag unused ones. -- **Prompt length** — Estimate the markdown body size. Is it verbose? -- **Pre-agent steps** — Does it use `steps:` to pre-compute deterministic work? -- **Post-agent steps** — Does it use `post-steps:` for validation? +- **Tools loaded** \u2014 List all tools in the `tools:` section. Flag any that may not be needed. +- **Network groups** \u2014 List network groups in `network.allowed:`. Flag unused ones. +- **Prompt length** \u2014 Estimate the markdown body size. Is it verbose? +- **Pre-agent steps** \u2014 Does it use `steps:` to pre-compute deterministic work? +- **Post-agent steps** \u2014 Does it use `post-steps:` for validation? -## Step 4: Analyze Recent Run Artifacts +## Step 4: Analyze Recent Run Data -Download the most recent successful run's artifacts to understand actual tool usage: +The pre-agent step downloaded the last 7 days of Claude workflow logs to `/tmp/gh-aw/token-audit/claude-logs.json`. Filter this data for the target workflow: ```bash -# Find the latest successful run using the resolved workflow file -LOCK_FILE="$(basename "$WORKFLOW_FILE" .md).lock.yml" -RUN_ID=$(gh run list --repo "$GITHUB_REPOSITORY" \ - --workflow "$LOCK_FILE" \ - --status success --limit 1 \ - --json databaseId --jq '.[0].databaseId') - -if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then - echo "No successful runs found for $LOCK_FILE — skipping artifact analysis" -else - # Download artifacts - TMPDIR=$(mktemp -d) - gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ - --name agent-artifacts --dir "$TMPDIR" 2>/dev/null || \ - gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ - --name agent --dir "$TMPDIR" 2>/dev/null - - # Check token usage - find "$TMPDIR" -name "token-usage.jsonl" -exec cat {} \; - - # Check agent stdio log for tool calls (|| true to handle no matches) - find "$TMPDIR" -name "agent-stdio.log" -exec grep -h "^●" {} \; || true - - # Check prompt size - find "$TMPDIR" -name "prompt.txt" -exec wc -c {} \; -fi +# Extract runs for the target workflow +cat /tmp/gh-aw/token-audit/claude-logs.json | \ + jq --arg name "$WORKFLOW_NAME" '[.runs[] | select(.workflow_name == $name)]' ``` -From the artifacts, determine: -- **Which tools were actually called** vs which are loaded -- **How many LLM turns** were used -- **Per-turn token breakdown** (first turn is usually the most expensive) -- **Cache write vs cache read ratio** — Anthropic charges 12.5x more for cache writes than reads -- **Whether cache writes are amortized** — Are they reused across subsequent turns? +From the run data, determine: +- **Per-run token breakdown** (token_usage, estimated_cost per run) +- **Average turns** per run +- **Error/warning patterns** +- **Token usage summary** (per-model breakdown from `token_usage_summary` if available) +- **Cache write vs read ratio** \u2014 Anthropic charges 12.5x more for cache writes than reads + +Also check the `tool_usage` and `mcp_tool_usage` fields in the JSON to identify which tools are actually being used vs loaded. -Clean up: `rm -rf "$TMPDIR"` +Clean up is not needed \u2014 data is pre-downloaded to /tmp. ## Step 5: Generate Optimization Recommendations @@ -137,18 +146,16 @@ Produce **specific, implementable recommendations** based on these patterns: If many tools are loaded but few are used: - List which tools to remove from `tools:` in the workflow `.md` - Estimate token savings (each tool schema is ~500-700 tokens) -- Example: "Remove `agentic-workflows:`, `web-fetch:` — saves ~15K tokens/turn" +- Example: "Remove `agentic-workflows:`, `web-fetch:` \u2014 saves ~15K tokens/turn" ### Pre-Agent Steps If the workflow does deterministic work (API calls, file creation, data fetching) inside the agent: - Identify which operations could move to `steps:` (pre-agent) - Show example `steps:` configuration -- Example: "Move `gh pr list` to a pre-step, inject results via `${{ steps.X.outputs.Y }}`" ### Prompt Optimization If the prompt is verbose or contains data the agent doesn't need: - Suggest specific cuts or rewrites -- Example: "Replace 15-line test instructions with 3-line summary referencing pre-computed results" ### GitHub Toolset Restriction If `github:` tools are loaded without `toolsets:` restriction: @@ -163,8 +170,7 @@ If unused network groups are configured (e.g., `node`, `playwright`): Anthropic charges significantly more for cache writes than reads: - Cache write: $3.75/M tokens (Sonnet), cache read: $0.30/M tokens - If cache writes are high but not reused across turns, the caching cost may exceed the benefit -- Check if `cache_write_tokens` from Turn 1 are reflected as `cache_read_tokens` in Turn 2+ -- If prompts change substantially between turns, caching provides no benefit +- Check if prompts change substantially between turns ### Cache Read Optimization If cache hit rate is low (<50%): @@ -174,7 +180,7 @@ If cache hit rate is low (<50%): ## Step 6: Create the Optimization Issue -Create an issue with title: `YYYY-MM-DD — ` +Create an issue with title: `YYYY-MM-DD \u2014 ` Body structure: @@ -248,11 +254,11 @@ Body structure: ## Important Guidelines -- **Be concrete** — Every recommendation must include specific file changes, not just "reduce tools" -- **Estimate savings** — Quantify each recommendation in tokens and percentage -- **Prioritize by impact** — Order recommendations from highest to lowest token savings -- **Include implementation steps** — Someone should be able to follow your recommendations without additional research -- **Reference the report** — Link back to the source token usage report issue -- **One workflow per issue** — Focus on the single most expensive workflow -- **Anthropic-specific insights** — Leverage cache write data that Copilot workflows don't expose -- **Clean up** temporary files after analysis +- **Be concrete** \u2014 Every recommendation must include specific file changes, not just "reduce tools" +- **Estimate savings** \u2014 Quantify each recommendation in tokens and percentage +- **Prioritize by impact** \u2014 Order recommendations from highest to lowest token savings +- **Include implementation steps** \u2014 Someone should be able to follow your recommendations without additional research +- **Reference the report** \u2014 Link back to the source token usage report issue +- **One workflow per issue** \u2014 Focus on the single most expensive workflow +- **Anthropic-specific insights** \u2014 Leverage cache write data that Copilot workflows don't expose +- **Use pre-downloaded data** \u2014 All run data is at `/tmp/gh-aw/token-audit/claude-logs.json`. Do not download artifacts manually. diff --git a/.github/workflows/claude-token-usage-analyzer.lock.yml b/.github/workflows/claude-token-usage-analyzer.lock.yml index 0d59084c5..a69e06ebd 100644 --- a/.github/workflows/claude-token-usage-analyzer.lock.yml +++ b/.github/workflows/claude-token-usage-analyzer.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"050995a1e51e6fb4c47feea8c91a3c9d39a180db586fdc519ded957b64e97b67","compiler_version":"v0.68.0","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"80c9975720eba4e76d1f98ed481ac20fadeeda3f0c6c8d41da26215f8ac5fe42","compiler_version":"v0.68.0","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"ed597411d8f924073f98dfc5c65a23a2325f34cd","version":"v8"},{"repo":"actions/upload-artifact","sha":"bbbca2ddaa5d8feaa63e36b76fdaad77386f024f","version":"v7"},{"repo":"github/gh-aw-actions/setup","sha":"0acfb4a691fe207cd8bc982ea5cb9d750d57a702","version":"v0.68.0"}]} # ___ _ _ # / _ \ | | (_) @@ -27,6 +27,7 @@ # Resolved workflow manifest: # Imports: # - shared/mcp-pagination.md +# - shared/mcp/gh-aw.md # - shared/reporting.md # # Secrets used: @@ -95,7 +96,7 @@ jobs: GH_AW_INFO_EXPERIMENTAL: "false" GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" GH_AW_INFO_STAGED: "false" - GH_AW_INFO_ALLOWED_DOMAINS: '["github","*.blob.core.windows.net"]' + GH_AW_INFO_ALLOWED_DOMAINS: '["github"]' GH_AW_INFO_FIREWALL_ENABLED: "true" GH_AW_INFO_AWF_VERSION: "v0.25.18" GH_AW_INFO_AWMG_VERSION: "" @@ -156,14 +157,14 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_4ba4a957792355f1_EOF' + cat << 'GH_AW_PROMPT_fb7136c754fca430_EOF' - GH_AW_PROMPT_4ba4a957792355f1_EOF + GH_AW_PROMPT_fb7136c754fca430_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_4ba4a957792355f1_EOF' + cat << 'GH_AW_PROMPT_fb7136c754fca430_EOF' Tools: create_issue, missing_tool, missing_data, noop @@ -195,14 +196,15 @@ jobs: {{/if}} - GH_AW_PROMPT_4ba4a957792355f1_EOF + GH_AW_PROMPT_fb7136c754fca430_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_4ba4a957792355f1_EOF' + cat << 'GH_AW_PROMPT_fb7136c754fca430_EOF' + {{#runtime-import .github/workflows/shared/mcp/gh-aw.md}} {{#runtime-import .github/workflows/shared/mcp-pagination.md}} {{#runtime-import .github/workflows/shared/reporting.md}} {{#runtime-import .github/workflows/claude-token-usage-analyzer.md}} - GH_AW_PROMPT_4ba4a957792355f1_EOF + GH_AW_PROMPT_fb7136c754fca430_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 @@ -319,6 +321,15 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + - env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + name: Install gh-aw extension + run: "# Install gh-aw if not already available\nif ! gh aw --version >/dev/null 2>&1; then\n echo \"Installing gh-aw extension...\"\n curl -fsSL https://raw.githubusercontent.com/github/gh-aw/refs/heads/main/install-gh-aw.sh | bash\nfi\ngh aw --version\n# Copy the gh-aw binary to RUNNER_TEMP for MCP server containerization\nmkdir -p \"${RUNNER_TEMP}/gh-aw\"\nGH_AW_BIN=$(which gh-aw 2>/dev/null || find ~/.local/share/gh/extensions/gh-aw -name 'gh-aw' -type f 2>/dev/null | head -1)\nif [ -n \"$GH_AW_BIN\" ] && [ -f \"$GH_AW_BIN\" ]; then\n cp \"$GH_AW_BIN\" \"${RUNNER_TEMP}/gh-aw/gh-aw\"\n chmod +x \"${RUNNER_TEMP}/gh-aw/gh-aw\"\n echo \"Copied gh-aw binary to ${RUNNER_TEMP}/gh-aw/gh-aw\"\nelse\n echo \"::error::Failed to find gh-aw binary for MCP server\"\n exit 1\nfi" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Download Claude workflow logs + run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/token-audit\n\necho \"\\U0001F4E5 Downloading Claude workflow logs (last 24 hours)...\"\n\nLOGS_EXIT=0\ngh aw logs \\\n --engine claude \\\n --start-date -1d \\\n --json \\\n -c 50 \\\n > /tmp/gh-aw/token-audit/claude-logs.json || LOGS_EXIT=$?\n\nif [ -s /tmp/gh-aw/token-audit/claude-logs.json ]; then\n TOTAL=$(jq '.runs | length' /tmp/gh-aw/token-audit/claude-logs.json)\n echo \"\\u2705 Downloaded $TOTAL Claude workflow runs (last 24 hours)\"\n if [ \"$LOGS_EXIT\" -ne 0 ]; then\n echo \"\\u26a0\\ufe0f gh aw logs exited with code $LOGS_EXIT (partial results \\u2014 likely API rate limit)\"\n fi\nelse\n echo \"\\u274c No log data downloaded (exit code $LOGS_EXIT)\"\n echo '{\"runs\":[],\"summary\":{}}' > /tmp/gh-aw/token-audit/claude-logs.json\nfi" + - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} @@ -392,9 +403,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_81db5b566ffeb845_EOF' + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_01a63805a9b17f6c_EOF' {"create_issue":{"close_older_issues":true,"labels":["claude-token-usage-report"],"max":1,"title_prefix":"📊 Claude Token Usage Report"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_81db5b566ffeb845_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_01a63805a9b17f6c_EOF - name: Write Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -589,7 +600,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.17' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_5bc0ccc553ab0d50_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" + cat << GH_AW_MCP_CONFIG_e02ff4eea2f95761_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" { "mcpServers": { "github": { @@ -630,7 +641,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_5bc0ccc553ab0d50_EOF + GH_AW_MCP_CONFIG_e02ff4eea2f95761_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -647,7 +658,7 @@ jobs: set -o pipefail touch /tmp/gh-aw/agent-step-summary.md # shellcheck disable=SC1003 - sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --allow-domains '*.blob.core.windows.net,*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,codeload.github.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,lfs.github.com,objects.githubusercontent.com,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --build-local --enable-api-proxy \ + sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --allow-domains '*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,codeload.github.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,lfs.github.com,objects.githubusercontent.com,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --build-local --enable-api-proxy \ -- /bin/bash -c 'node ${RUNNER_TEMP}/gh-aw/actions/copilot_driver.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE @@ -733,7 +744,7 @@ jobs: uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "*.blob.core.windows.net,*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,codeload.github.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,lfs.github.com,objects.githubusercontent.com,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com" + GH_AW_ALLOWED_DOMAINS: "*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,codeload.github.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,lfs.github.com,objects.githubusercontent.com,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} with: @@ -1177,7 +1188,7 @@ jobs: uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_ALLOWED_DOMAINS: "*.blob.core.windows.net,*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,codeload.github.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,lfs.github.com,objects.githubusercontent.com,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com" + GH_AW_ALLOWED_DOMAINS: "*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,codeload.github.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,lfs.github.com,objects.githubusercontent.com,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"close_older_issues\":true,\"labels\":[\"claude-token-usage-report\"],\"max\":1,\"title_prefix\":\"📊 Claude Token Usage Report\"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" diff --git a/.github/workflows/claude-token-usage-analyzer.md b/.github/workflows/claude-token-usage-analyzer.md index 78d6994e4..d96bf8444 100644 --- a/.github/workflows/claude-token-usage-analyzer.md +++ b/.github/workflows/claude-token-usage-analyzer.md @@ -9,22 +9,50 @@ permissions: issues: read pull-requests: read imports: + - uses: shared/mcp/gh-aw.md - shared/mcp-pagination.md - shared/reporting.md network: allowed: - github - - "*.blob.core.windows.net" tools: github: toolsets: [default, actions] bash: true safe-outputs: create-issue: - title-prefix: "📊 Claude Token Usage Report" + title-prefix: "\U0001F4CA Claude Token Usage Report" labels: [claude-token-usage-report] close-older-issues: true timeout-minutes: 45 +steps: + - name: Download Claude workflow logs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + mkdir -p /tmp/gh-aw/token-audit + + echo "\U0001F4E5 Downloading Claude workflow logs (last 24 hours)..." + + LOGS_EXIT=0 + gh aw logs \ + --engine claude \ + --start-date -1d \ + --json \ + -c 50 \ + > /tmp/gh-aw/token-audit/claude-logs.json || LOGS_EXIT=$? + + if [ -s /tmp/gh-aw/token-audit/claude-logs.json ]; then + TOTAL=$(jq '.runs | length' /tmp/gh-aw/token-audit/claude-logs.json) + echo "\u2705 Downloaded $TOTAL Claude workflow runs (last 24 hours)" + if [ "$LOGS_EXIT" -ne 0 ]; then + echo "\u26a0\ufe0f gh aw logs exited with code $LOGS_EXIT (partial results \u2014 likely API rate limit)" + fi + else + echo "\u274c No log data downloaded (exit code $LOGS_EXIT)" + echo '{"runs":[],"summary":{}}' > /tmp/gh-aw/token-audit/claude-logs.json + fi --- # Daily Claude Token Usage Analyzer @@ -33,114 +61,99 @@ You are an AI agent that analyzes Claude token usage across agentic workflow run ## Background -This repository uses the **Agent Workflow Firewall (AWF)** with an api-proxy sidecar that tracks token usage for LLM API calls. Each workflow run with `--enable-api-proxy` produces a `token-usage.jsonl` file captured in the `agent-artifacts` upload artifact. +This repository uses the **Agent Workflow Firewall (AWF)** with an api-proxy sidecar that tracks token usage for LLM API calls. The pre-agent step has already downloaded structured run data using `gh aw logs --json`, which includes per-run token usage, cost estimates, and run metadata. -**Token usage tracking is a new feature** — many older runs won't have this data. Handle missing data gracefully. +**Note:** Copilot-engine and Codex-engine workflows are excluded \u2014 they are covered by the separate Copilot Token Usage Analyzer. -### Token Usage Record Format +## Data Sources + +### Pre-downloaded logs + +The file `/tmp/gh-aw/token-audit/claude-logs.json` contains structured JSON output from `gh aw logs --engine claude --json` with this shape: -Each line in `token-usage.jsonl` is a JSON object: ```json { - "timestamp": "2026-04-01T17:38:12.486Z", - "request_id": "uuid", - "provider": "anthropic", - "model": "claude-sonnet-4-6", - "path": "/v1/messages?beta=true", - "status": 200, - "streaming": true, - "input_tokens": 3, - "output_tokens": 418, - "cache_read_tokens": 14044, - "cache_write_tokens": 26042, - "duration_ms": 5858, - "response_bytes": 2800 + "summary": { + "run_count": N, + "total_tokens": N, + "avg_tokens": N, + "total_cost": F, + "avg_cost": F, + "total_turns": N, + "avg_turns": F, + "total_action_minutes": F, + "error_count": N, + "warning_count": N + }, + "runs": [ ... ], + "tool_usage": [ ... ], + "mcp_tool_usage": { ... } } ``` -## Your Mission - -### Step 1: Discover Recent Workflow Runs +Each element of `.runs` includes: + +| Field | Type | Notes | +|---|---|---| +| `workflow_name` | string | Human-readable name | +| `workflow_path` | string | `.github/workflows/....lock.yml` | +| `token_usage` | int | Total tokens (treat missing/null as 0) | +| `effective_tokens` | int | Cost-normalized tokens | +| `estimated_cost` | float | USD cost (treat missing/null as 0) | +| `action_minutes` | float | Billable GitHub Actions minutes | +| `turns` | int | Number of agent turns | +| `duration` | string | Human-readable duration | +| `created_at` | ISO 8601 | Run creation time | +| `database_id` | int64 | Unique run ID | +| `url` | string | Link to the run | +| `status` | string | `completed`, `in_progress`, etc. | +| `conclusion` | string | `success`, `failure`, etc. | +| `error_count` | int | Errors encountered | +| `warning_count` | int | Warnings encountered | +| `token_usage_summary` | object or null | Firewall-level breakdown by model (includes Anthropic cache read/write tokens) | -Use `gh run list` via bash to find completed agentic workflow runs from the past 24 hours (or since the last token usage report issue). Focus on **Claude-engine workflows** that use the api-proxy: - -- `smoke-claude` -- `secret-digger-claude` -- `security-review`, `security-guard` -- Any other Claude-engine workflow with `agent-artifacts` - -**Note:** Copilot-engine and Codex-engine workflows (e.g., `smoke-copilot`, `smoke-chroot`, `smoke-services`, `build-test`, `smoke-codex`, `secret-digger-copilot`) are excluded from this analysis — they are covered by the separate Copilot Token Usage Analyzer. - -Use bash to run: -```bash -# Find runs from the last 24 hours -CUTOFF="$(date -u -Iseconds -d '24 hours ago')" - -gh run list --repo "$GITHUB_REPOSITORY" --limit 50 \ - --created ">=$CUTOFF" \ - --json databaseId,name,status,conclusion,createdAt,workflowName \ - --jq '[.[] | select(.conclusion == "success" or .conclusion == "failure")]' -``` - -### Step 2: Download and Parse Token Usage Data +## Your Mission -For each discovered run, attempt to download the `agent-artifacts` artifact and extract `token-usage.jsonl`. +### Step 1: Load and Parse Pre-Downloaded Data -**IMPORTANT:** Always use `gh run download` via bash — this is much faster than using MCP `get_job_logs` and the network is configured to allow artifact blob storage access. +Read the pre-downloaded logs: ```bash -# Create temp directory -TMPDIR=$(mktemp -d) - -# Try to download artifacts for a run -gh run download --repo "$GITHUB_REPOSITORY" --name agent-artifacts --dir "$TMPDIR/run-" 2>/dev/null - -# Look for token-usage.jsonl (may be nested under sandbox/firewall/logs/api-proxy-logs/) -find "$TMPDIR/run-" -name "token-usage.jsonl" 2>/dev/null +cat /tmp/gh-aw/token-audit/claude-logs.json | jq '.summary' +cat /tmp/gh-aw/token-audit/claude-logs.json | jq '.runs | length' ``` -**Filter for Claude/Anthropic requests:** When parsing `token-usage.jsonl`, only include records where `provider` is `"anthropic"`. This ensures we're analyzing Claude-specific usage even if a workflow makes calls to multiple providers. - -**Graceful degradation:** -- If artifact download fails → skip run, note it as "no artifacts" -- If `token-usage.jsonl` is missing → skip run, note it as "no token logs" -- If the file is empty → skip run, note it as "empty token logs" -- Track which workflows have instrumentation vs which don't +If `.runs` is empty, create a minimal report noting that no Claude workflow runs were found in the last 24 hours. -### Step 3: Compute Per-Workflow Statistics +### Step 2: Compute Per-Workflow Statistics -For each workflow that has token data, calculate: +Group `.runs` by `workflow_name` and compute per-workflow aggregates: -1. **Total tokens**: `input_tokens + output_tokens + cache_read_tokens + cache_write_tokens` -2. **Full-price tokens**: `input_tokens + output_tokens + cache_write_tokens` (excludes discounted cache reads; use **Estimated cost** below for true billed amount) -3. **Input/output ratio**: `(input_tokens + cache_read_tokens) / output_tokens` (if `output_tokens == 0`, treat the ratio as `∞`/`N/A` and exclude that request from ratio averages to avoid division by zero) -4. **Cache hit rate**: `cache_read_tokens / (cache_read_tokens + input_tokens) * 100` -5. **Cache write rate**: `cache_write_tokens / (cache_read_tokens + input_tokens + cache_write_tokens) * 100` -6. **Request count**: Number of records in the JSONL -7. **Average latency**: Mean `duration_ms` per request -8. **Model distribution**: Count of requests per model -9. **Estimated cost** (sum all four token types at their respective rates — cache reads are discounted, not free): - - Anthropic Sonnet: input $3/M, output $15/M, cache_read $0.30/M, cache_write $3.75/M - - Anthropic Haiku: input $0.80/M, output $4/M, cache_read $0.08/M, cache_write $1/M - - Anthropic Opus: input $15/M, output $75/M, cache_read $1.50/M, cache_write $18.75/M +1. **Run count**: Number of runs per workflow +2. **Total tokens**: Sum of `token_usage` across runs +3. **Avg tokens/run**: Mean `token_usage` +4. **Total cost**: Sum of `estimated_cost` +5. **Avg cost/run**: Mean `estimated_cost` +6. **Total turns**: Sum of `turns` +7. **Error/warning counts**: Sum of `error_count`, `warning_count` +8. **Model & cache breakdown**: Extract from `token_usage_summary` where available (Anthropic provides cache_read_tokens and cache_write_tokens) -Use bash with a Python or jq script to process the JSONL files efficiently. +Use bash with jq or a Python script to process efficiently. Handle null/missing `token_usage` and `estimated_cost` by treating them as 0. -### Step 4: Identify Optimization Opportunities +### Step 3: Identify Optimization Opportunities Flag workflows with these patterns: | Pattern | Threshold | Recommendation | |---------|-----------|----------------| -| Zero cache hits | cache_hit_rate = 0% | Enable prompt caching | -| Low cache hits | cache_hit_rate < 50% | Review cache breakpoints | -| High cache write vs read | cache_write > cache_read | Cache is being written but not reused — check if conversation turns are too short | -| High input/output ratio | ratio > 100:1 | Reduce system prompt or MCP tool surface | -| Many small requests | >10 requests, <50 output tokens each | Batch requests or combine tool calls | | High total cost | >$1.00 per run | Review if workflow is doing too much | +| High token count | >100K tokens/run | Reduce system prompt or MCP tool surface | +| Many turns | >15 turns/run | Consider pre-computing deterministic work in steps | +| High cache write vs read | cache_write > cache_read in token_usage_summary | Cache is being written but not reused \u2014 check if conversation turns are too short | +| High error rate | >30% of runs with errors | Investigate reliability issues | | Increasing trend | >20% increase vs last report | Investigate what changed | -### Step 5: Check for Historical Trends +### Step 4: Check for Historical Trends Search for previous token usage report issues: ```bash @@ -150,14 +163,14 @@ gh issue list --repo "$GITHUB_REPOSITORY" --label claude-token-usage-report --st If previous reports exist, compare current metrics to identify: - Workflows with increasing token consumption - Workflows that gained or lost prompt caching -- New workflows that started using the api-proxy +- New workflows that appeared - Cost trend over time -### Step 6: Create the Summary Issue +### Step 5: Create the Summary Issue Create an issue with the following structure: -#### Title: `YYYY-MM-DD` (safe-outputs will automatically prefix this with "📊 Claude Token Usage Report") +#### Title: `YYYY-MM-DD` (safe-outputs will automatically prefix this with "\U0001F4CA Claude Token Usage Report") #### Body structure: @@ -165,21 +178,22 @@ Create an issue with the following structure: ### Overview **Period**: [start date] to [end date] -**Runs analyzed**: X of Y (Z had token data) +**Runs analyzed**: X (Y had token data) **Total tokens**: N across all workflows **Estimated total cost**: $X.XX +**Total Actions minutes**: X.X min ### Workflow Summary -| Workflow | Runs | Total Tokens | Cost | Cache Rate | I/O Ratio | Top Model | -|----------|------|-------------|------|------------|-----------|-----------| -| smoke-claude | 2 | 395K | $0.46 | 99.5% | 0.6:1 | sonnet-4.6 | -| security-review | 1 | 120K | $0.22 | 85% | 2.3:1 | sonnet-4.6 | +| Workflow | Runs | Total Tokens | Avg Tokens | Cost | Avg Cost | Turns | +|----------|------|-------------|------------|------|----------|-------| +| smoke-claude | 2 | 395K | 197K | $0.46 | $0.23 | 12 | +| security-review | 1 | 120K | 120K | $0.22 | $0.22 | 5 | | ... | | | | | | | -### 🔍 Optimization Opportunities +### \U0001F50D Optimization Opportunities -1. **secret-digger-claude** — 45% cache hit rate, high cache writes +1. **security-review** \u2014 $0.22/run, high cache writes - Cache is being created but not fully reused across turns - Consider restructuring the prompt to maximize cache prefix reuse @@ -189,44 +203,37 @@ Create an issue with the following structure: Per-Workflow Details #### smoke-claude -- **Runs**: 2 (run 123, run 456) -- **Requests**: 12 total (avg 6/run) -- **Models**: claude-haiku-4.5 (4 reqs), claude-sonnet-4.6 (8 reqs) -- **Tokens**: 395K total (1.5K input, 2.5K output, 304K cache_read, 87K cache_write) -- **Cache hit rate**: 99.5% -- **Cache write rate**: 22.3% -- **Avg latency**: 3,800ms/request -- **Estimated cost**: $0.46 - -#### security-review -... +- **Runs**: 2 (links to each run) +- **Total tokens**: 395K (avg 197K/run) +- **Estimated cost**: $0.46 (avg $0.23/run) +- **Turns**: 12 total (avg 6/run) +- **Model breakdown**: [from token_usage_summary if available] +- **Cache analysis**: [cache read/write breakdown if available] +- **Error rate**: 0/2 runs
Workflows Without Token Data -The following workflows either don't use `--enable-api-proxy` or ran before token tracking was enabled: -- secret-digger-claude (1 run — no agent-artifacts) +Runs where `token_usage` was null or 0 \u2014 these may not have the api-proxy enabled: +- [list workflows with missing data]
### Historical Trend -[If previous reports exist, show comparison. Otherwise note: "This is the first Claude token usage report. Historical trends will be available in future reports."] +[If previous reports exist, show comparison. Otherwise: "This is the first Claude token usage report."] ### Previous Report -[Link to previous report issue if one exists, otherwise omit this section] +[Link to previous report issue if one exists] ``` ## Important Guidelines -- **Time budget** — You have 15 minutes total. Spend at most 8 minutes on data collection (steps 1-2) and reserve the rest for analysis and issue creation. If artifact downloads are slow, limit to the 5 most recent runs. -- **Prefer bash over MCP** for data collection — `gh run download` via bash is much faster than MCP `get_job_logs` for retrieving artifacts. -- **Do NOT fail** if no token data is available. Create a minimal report explaining that token tracking is new and which workflows need instrumentation. -- **Clean up** temporary directories after processing. -- **Respect rate limits** — download artifacts one at a time, not in parallel. -- **Use `--perPage` parameters** when listing runs to avoid token limits on MCP responses. +- **Do NOT fail** if no token data is available. Create a minimal report explaining which workflows need instrumentation. +- **All data is pre-downloaded** \u2014 do not attempt to download artifacts or run `gh run download`. Use only the JSON at `/tmp/gh-aw/token-audit/claude-logs.json`. +- **Anthropic-specific insights** \u2014 Leverage cache write/read data from `token_usage_summary` when available. Anthropic charges 12.5x more for cache writes than reads. - **Wrap verbose output** in `
` blocks for progressive disclosure. - **Round costs** to 2 decimal places, token counts to nearest thousand for readability. - **Sort workflows** by estimated cost (highest first) in the summary table. diff --git a/.github/workflows/copilot-token-optimizer.lock.yml b/.github/workflows/copilot-token-optimizer.lock.yml index bc4851367..a77aba54f 100644 --- a/.github/workflows/copilot-token-optimizer.lock.yml +++ b/.github/workflows/copilot-token-optimizer.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"07ab79c2261e86a524d123719639ace4840309660b54947929cd4889518842ab","compiler_version":"v0.68.0","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"c6b0a29297428b9305c4906573a2a5c295dba22c0699f675dad14d08bc7f5d4e","compiler_version":"v0.68.0","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"ed597411d8f924073f98dfc5c65a23a2325f34cd","version":"v8"},{"repo":"actions/upload-artifact","sha":"bbbca2ddaa5d8feaa63e36b76fdaad77386f024f","version":"v7"},{"repo":"github/gh-aw-actions/setup","sha":"0acfb4a691fe207cd8bc982ea5cb9d750d57a702","version":"v0.68.0"}]} # ___ _ _ # / _ \ | | (_) @@ -24,6 +24,10 @@ # # Daily Copilot token optimization advisor — reads the latest token usage report and creates actionable recommendations to reduce token consumption for the most expensive workflow # +# Resolved workflow manifest: +# Imports: +# - shared/mcp/gh-aw.md +# # Secrets used: # - COPILOT_GITHUB_TOKEN # - GH_AW_GITHUB_MCP_SERVER_TOKEN @@ -161,19 +165,18 @@ jobs: GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - GH_AW_STEPS_X_OUTPUTS_Y: ${{ steps.X.outputs.Y }} # poutine:ignore untrusted_checkout_exec run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_9f92f479cd9fd538_EOF' + cat << 'GH_AW_PROMPT_b452c772a73b52cb_EOF' - GH_AW_PROMPT_9f92f479cd9fd538_EOF + GH_AW_PROMPT_b452c772a73b52cb_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_9f92f479cd9fd538_EOF' + cat << 'GH_AW_PROMPT_b452c772a73b52cb_EOF' Tools: create_issue, missing_tool, missing_data, noop @@ -205,18 +208,18 @@ jobs: {{/if}} - GH_AW_PROMPT_9f92f479cd9fd538_EOF + GH_AW_PROMPT_b452c772a73b52cb_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_9f92f479cd9fd538_EOF' + cat << 'GH_AW_PROMPT_b452c772a73b52cb_EOF' + {{#runtime-import .github/workflows/shared/mcp/gh-aw.md}} {{#runtime-import .github/workflows/copilot-token-optimizer.md}} - GH_AW_PROMPT_9f92f479cd9fd538_EOF + GH_AW_PROMPT_b452c772a73b52cb_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_STEPS_X_OUTPUTS_Y: ${{ steps.X.outputs.Y }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); @@ -236,7 +239,6 @@ jobs: GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: ${{ needs.pre_activation.outputs.activated }} - GH_AW_STEPS_X_OUTPUTS_Y: ${{ steps.X.outputs.Y }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); @@ -256,8 +258,7 @@ jobs: GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, - GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED, - GH_AW_STEPS_X_OUTPUTS_Y: process.env.GH_AW_STEPS_X_OUTPUTS_Y + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED } }); - name: Validate prompt placeholders @@ -332,6 +333,15 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + - env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + name: Install gh-aw extension + run: "# Install gh-aw if not already available\nif ! gh aw --version >/dev/null 2>&1; then\n echo \"Installing gh-aw extension...\"\n curl -fsSL https://raw.githubusercontent.com/github/gh-aw/refs/heads/main/install-gh-aw.sh | bash\nfi\ngh aw --version\n# Copy the gh-aw binary to RUNNER_TEMP for MCP server containerization\nmkdir -p \"${RUNNER_TEMP}/gh-aw\"\nGH_AW_BIN=$(which gh-aw 2>/dev/null || find ~/.local/share/gh/extensions/gh-aw -name 'gh-aw' -type f 2>/dev/null | head -1)\nif [ -n \"$GH_AW_BIN\" ] && [ -f \"$GH_AW_BIN\" ]; then\n cp \"$GH_AW_BIN\" \"${RUNNER_TEMP}/gh-aw/gh-aw\"\n chmod +x \"${RUNNER_TEMP}/gh-aw/gh-aw\"\n echo \"Copied gh-aw binary to ${RUNNER_TEMP}/gh-aw/gh-aw\"\nelse\n echo \"::error::Failed to find gh-aw binary for MCP server\"\n exit 1\nfi" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Download recent Copilot workflow logs + run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/token-audit\n\necho \"\\U0001F4E5 Downloading Copilot workflow logs (last 7 days)...\"\n\nLOGS_EXIT=0\ngh aw logs \\\n --engine copilot \\\n --start-date -7d \\\n --json \\\n -c 50 \\\n > /tmp/gh-aw/token-audit/copilot-logs.json || LOGS_EXIT=$?\n\nif [ -s /tmp/gh-aw/token-audit/copilot-logs.json ]; then\n TOTAL=$(jq '.runs | length' /tmp/gh-aw/token-audit/copilot-logs.json)\n echo \"\\u2705 Downloaded $TOTAL Copilot workflow runs (last 7 days)\"\n if [ \"$LOGS_EXIT\" -ne 0 ]; then\n echo \"\\u26a0\\ufe0f gh aw logs exited with code $LOGS_EXIT (partial results)\"\n fi\nelse\n echo \"\\u274c No log data downloaded (exit code $LOGS_EXIT)\"\n echo '{\"runs\":[],\"summary\":{}}' > /tmp/gh-aw/token-audit/copilot-logs.json\nfi" + - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} @@ -405,9 +415,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_5dcbacbe421377bf_EOF' + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_b37d5d7a570f7975_EOF' {"create_issue":{"close_older_issues":true,"labels":["copilot-token-optimization"],"max":1,"title_prefix":"⚡ Copilot Token Optimization"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_5dcbacbe421377bf_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_b37d5d7a570f7975_EOF - name: Write Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -602,7 +612,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.17' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_cb1185ae78386888_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" + cat << GH_AW_MCP_CONFIG_38380a78c084e8f4_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" { "mcpServers": { "github": { @@ -643,7 +653,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_cb1185ae78386888_EOF + GH_AW_MCP_CONFIG_38380a78c084e8f4_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: diff --git a/.github/workflows/copilot-token-optimizer.md b/.github/workflows/copilot-token-optimizer.md index b6e5e64f4..efc7ccf0c 100644 --- a/.github/workflows/copilot-token-optimizer.md +++ b/.github/workflows/copilot-token-optimizer.md @@ -14,6 +14,8 @@ permissions: actions: read issues: read pull-requests: read +imports: + - uses: shared/mcp/gh-aw.md network: allowed: - github @@ -23,11 +25,39 @@ tools: bash: true safe-outputs: create-issue: - title-prefix: "⚡ Copilot Token Optimization" + title-prefix: "\u26a1 Copilot Token Optimization" labels: [copilot-token-optimization] close-older-issues: true timeout-minutes: 10 strict: true +steps: + - name: Download recent Copilot workflow logs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + mkdir -p /tmp/gh-aw/token-audit + + echo "\U0001F4E5 Downloading Copilot workflow logs (last 7 days)..." + + LOGS_EXIT=0 + gh aw logs \ + --engine copilot \ + --start-date -7d \ + --json \ + -c 50 \ + > /tmp/gh-aw/token-audit/copilot-logs.json || LOGS_EXIT=$? + + if [ -s /tmp/gh-aw/token-audit/copilot-logs.json ]; then + TOTAL=$(jq '.runs | length' /tmp/gh-aw/token-audit/copilot-logs.json) + echo "\u2705 Downloaded $TOTAL Copilot workflow runs (last 7 days)" + if [ "$LOGS_EXIT" -ne 0 ]; then + echo "\u26a0\ufe0f gh aw logs exited with code $LOGS_EXIT (partial results)" + fi + else + echo "\u274c No log data downloaded (exit code $LOGS_EXIT)" + echo '{"runs":[],"summary":{}}' > /tmp/gh-aw/token-audit/copilot-logs.json + fi --- # Daily Copilot Token Optimization Advisor @@ -80,52 +110,31 @@ cat "$WORKFLOW_FILE" ``` Analyze: -- **Tools loaded** — List all tools in the `tools:` section. Flag any that may not be needed. -- **Network groups** — List network groups in `network.allowed:`. Flag unused ones. -- **Prompt length** — Estimate the markdown body size. Is it verbose? -- **Pre-agent steps** — Does it use `steps:` to pre-compute deterministic work? -- **Post-agent steps** — Does it use `post-steps:` for validation? +- **Tools loaded** \u2014 List all tools in the `tools:` section. Flag any that may not be needed. +- **Network groups** \u2014 List network groups in `network.allowed:`. Flag unused ones. +- **Prompt length** \u2014 Estimate the markdown body size. Is it verbose? +- **Pre-agent steps** \u2014 Does it use `steps:` to pre-compute deterministic work? +- **Post-agent steps** \u2014 Does it use `post-steps:` for validation? -## Step 4: Analyze Recent Run Artifacts +## Step 4: Analyze Recent Run Data -Download the most recent successful run's artifacts to understand actual tool usage: +The pre-agent step downloaded the last 7 days of Copilot workflow logs to `/tmp/gh-aw/token-audit/copilot-logs.json`. Filter this data for the target workflow: ```bash -# Find the latest successful run using the resolved workflow file -LOCK_FILE="$(basename "$WORKFLOW_FILE" .md).lock.yml" -RUN_ID=$(gh run list --repo "$GITHUB_REPOSITORY" \ - --workflow "$LOCK_FILE" \ - --status success --limit 1 \ - --json databaseId --jq '.[0].databaseId') - -if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then - echo "No successful runs found for $LOCK_FILE — skipping artifact analysis" -else - # Download artifacts - TMPDIR=$(mktemp -d) - gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ - --name agent-artifacts --dir "$TMPDIR" 2>/dev/null || \ - gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ - --name agent --dir "$TMPDIR" 2>/dev/null - - # Check token usage - find "$TMPDIR" -name "token-usage.jsonl" -exec cat {} \; - - # Check agent stdio log for tool calls (|| true to handle no matches) - find "$TMPDIR" -name "agent-stdio.log" -exec grep -h "^●" {} \; || true - - # Check prompt size - find "$TMPDIR" -name "prompt.txt" -exec wc -c {} \; -fi +# Extract runs for the target workflow +cat /tmp/gh-aw/token-audit/copilot-logs.json | \ + jq --arg name "$WORKFLOW_NAME" '[.runs[] | select(.workflow_name == $name)]' ``` -From the artifacts, determine: -- **Which tools were actually called** vs which are loaded -- **How many LLM turns** were used -- **Per-turn token breakdown** (first turn is usually the most expensive) -- **Whether there's a "tool probing" pattern** (Turn 1 calls every tool once) +From the run data, determine: +- **Per-run token breakdown** (token_usage, estimated_cost per run) +- **Average turns** per run +- **Error/warning patterns** +- **Token usage summary** (per-model breakdown from `token_usage_summary` if available) + +Also check the `tool_usage` and `mcp_tool_usage` fields in the JSON to identify which tools are actually being used vs loaded. -Clean up: `rm -rf "$TMPDIR"` +Clean up is not needed \u2014 data is pre-downloaded to /tmp. ## Step 5: Generate Optimization Recommendations @@ -135,18 +144,16 @@ Produce **specific, implementable recommendations** based on these patterns: If many tools are loaded but few are used: - List which tools to remove from `tools:` in the workflow `.md` - Estimate token savings (each tool schema is ~500-700 tokens) -- Example: "Remove `playwright:`, `web-fetch:`, `edit:` — saves ~30K tokens/turn" +- Example: "Remove `playwright:`, `web-fetch:`, `edit:` \u2014 saves ~30K tokens/turn" ### Pre-Agent Steps If the workflow does deterministic work (API calls, file creation, data fetching) inside the agent: - Identify which operations could move to `steps:` (pre-agent) - Show example `steps:` configuration -- Example: "Move `gh pr list` to a pre-step, inject results via `${{ steps.X.outputs.Y }}`" ### Prompt Optimization If the prompt is verbose or contains data the agent doesn't need: - Suggest specific cuts or rewrites -- Example: "Replace 15-line test instructions with 3-line summary referencing pre-computed results" ### GitHub Toolset Restriction If `github:` tools are loaded without `toolsets:` restriction: @@ -156,17 +163,15 @@ If `github:` tools are loaded without `toolsets:` restriction: ### Network Group Trimming If unused network groups are configured (e.g., `node`, `playwright`): - List which to remove -- These don't directly affect tokens but indicate unnecessary tool/domain overhead ### Cache Optimization If cache hit rate is low (<50%): - Check if prompts vary between runs (run-specific IDs, timestamps) - Suggest moving variable content to the end of prompts (prefix caching) -- Note: cache TTL is ~5 minutes, so sequential runs within a window benefit ## Step 6: Create the Optimization Issue -Create an issue with title: `YYYY-MM-DD — ` +Create an issue with title: `YYYY-MM-DD \u2014 ` Body structure: @@ -228,10 +233,10 @@ Body structure: ## Important Guidelines -- **Be concrete** — Every recommendation must include specific file changes, not just "reduce tools" -- **Estimate savings** — Quantify each recommendation in tokens and percentage -- **Prioritize by impact** — Order recommendations from highest to lowest token savings -- **Include implementation steps** — Someone should be able to follow your recommendations without additional research -- **Reference the report** — Link back to the source token usage report issue -- **One workflow per issue** — Focus on the single most expensive workflow -- **Clean up** temporary files after analysis +- **Be concrete** \u2014 Every recommendation must include specific file changes, not just "reduce tools" +- **Estimate savings** \u2014 Quantify each recommendation in tokens and percentage +- **Prioritize by impact** \u2014 Order recommendations from highest to lowest token savings +- **Include implementation steps** \u2014 Someone should be able to follow your recommendations without additional research +- **Reference the report** \u2014 Link back to the source token usage report issue +- **One workflow per issue** \u2014 Focus on the single most expensive workflow +- **Use pre-downloaded data** \u2014 All run data is at `/tmp/gh-aw/token-audit/copilot-logs.json`. Do not download artifacts manually. diff --git a/.github/workflows/copilot-token-usage-analyzer.lock.yml b/.github/workflows/copilot-token-usage-analyzer.lock.yml index bf16b4e7a..0258a82a6 100644 --- a/.github/workflows/copilot-token-usage-analyzer.lock.yml +++ b/.github/workflows/copilot-token-usage-analyzer.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"b83d6b6c5ad848a2654eafc8675f9103879dd9ffaa54c21d4c127485ad2e48d5","compiler_version":"v0.68.0","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"62ccc40fbb4b224d3f7abc1edf772226ed67163907402af5a635d4648f250827","compiler_version":"v0.68.0","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"ed597411d8f924073f98dfc5c65a23a2325f34cd","version":"v8"},{"repo":"actions/upload-artifact","sha":"bbbca2ddaa5d8feaa63e36b76fdaad77386f024f","version":"v7"},{"repo":"github/gh-aw-actions/setup","sha":"0acfb4a691fe207cd8bc982ea5cb9d750d57a702","version":"v0.68.0"}]} # ___ _ _ # / _ \ | | (_) @@ -27,6 +27,7 @@ # Resolved workflow manifest: # Imports: # - shared/mcp-pagination.md +# - shared/mcp/gh-aw.md # - shared/reporting.md # # Secrets used: @@ -95,7 +96,7 @@ jobs: GH_AW_INFO_EXPERIMENTAL: "false" GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" GH_AW_INFO_STAGED: "false" - GH_AW_INFO_ALLOWED_DOMAINS: '["github","*.blob.core.windows.net"]' + GH_AW_INFO_ALLOWED_DOMAINS: '["github"]' GH_AW_INFO_FIREWALL_ENABLED: "true" GH_AW_INFO_AWF_VERSION: "v0.25.18" GH_AW_INFO_AWMG_VERSION: "" @@ -156,14 +157,14 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_e24bc1f76d966cfd_EOF' + cat << 'GH_AW_PROMPT_b0246fb6bdcdcbc1_EOF' - GH_AW_PROMPT_e24bc1f76d966cfd_EOF + GH_AW_PROMPT_b0246fb6bdcdcbc1_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_e24bc1f76d966cfd_EOF' + cat << 'GH_AW_PROMPT_b0246fb6bdcdcbc1_EOF' Tools: create_issue, missing_tool, missing_data, noop @@ -195,14 +196,15 @@ jobs: {{/if}} - GH_AW_PROMPT_e24bc1f76d966cfd_EOF + GH_AW_PROMPT_b0246fb6bdcdcbc1_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_e24bc1f76d966cfd_EOF' + cat << 'GH_AW_PROMPT_b0246fb6bdcdcbc1_EOF' + {{#runtime-import .github/workflows/shared/mcp/gh-aw.md}} {{#runtime-import .github/workflows/shared/mcp-pagination.md}} {{#runtime-import .github/workflows/shared/reporting.md}} {{#runtime-import .github/workflows/copilot-token-usage-analyzer.md}} - GH_AW_PROMPT_e24bc1f76d966cfd_EOF + GH_AW_PROMPT_b0246fb6bdcdcbc1_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 @@ -319,6 +321,15 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + - env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + name: Install gh-aw extension + run: "# Install gh-aw if not already available\nif ! gh aw --version >/dev/null 2>&1; then\n echo \"Installing gh-aw extension...\"\n curl -fsSL https://raw.githubusercontent.com/github/gh-aw/refs/heads/main/install-gh-aw.sh | bash\nfi\ngh aw --version\n# Copy the gh-aw binary to RUNNER_TEMP for MCP server containerization\nmkdir -p \"${RUNNER_TEMP}/gh-aw\"\nGH_AW_BIN=$(which gh-aw 2>/dev/null || find ~/.local/share/gh/extensions/gh-aw -name 'gh-aw' -type f 2>/dev/null | head -1)\nif [ -n \"$GH_AW_BIN\" ] && [ -f \"$GH_AW_BIN\" ]; then\n cp \"$GH_AW_BIN\" \"${RUNNER_TEMP}/gh-aw/gh-aw\"\n chmod +x \"${RUNNER_TEMP}/gh-aw/gh-aw\"\n echo \"Copied gh-aw binary to ${RUNNER_TEMP}/gh-aw/gh-aw\"\nelse\n echo \"::error::Failed to find gh-aw binary for MCP server\"\n exit 1\nfi" + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: Download Copilot workflow logs + run: "set -euo pipefail\nmkdir -p /tmp/gh-aw/token-audit\n\necho \"\\U0001F4E5 Downloading Copilot workflow logs (last 24 hours)...\"\n\nLOGS_EXIT=0\ngh aw logs \\\n --engine copilot \\\n --start-date -1d \\\n --json \\\n -c 50 \\\n > /tmp/gh-aw/token-audit/copilot-logs.json || LOGS_EXIT=$?\n\nif [ -s /tmp/gh-aw/token-audit/copilot-logs.json ]; then\n TOTAL=$(jq '.runs | length' /tmp/gh-aw/token-audit/copilot-logs.json)\n echo \"\\u2705 Downloaded $TOTAL Copilot workflow runs (last 24 hours)\"\n if [ \"$LOGS_EXIT\" -ne 0 ]; then\n echo \"\\u26a0\\ufe0f gh aw logs exited with code $LOGS_EXIT (partial results \\u2014 likely API rate limit)\"\n fi\nelse\n echo \"\\u274c No log data downloaded (exit code $LOGS_EXIT)\"\n echo '{\"runs\":[],\"summary\":{}}' > /tmp/gh-aw/token-audit/copilot-logs.json\nfi" + - name: Configure Git credentials env: REPO_NAME: ${{ github.repository }} @@ -392,9 +403,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_25eef300f6ef8a9d_EOF' + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_c7b1b3c8c7779849_EOF' {"create_issue":{"close_older_issues":true,"labels":["token-usage-report"],"max":1,"title_prefix":"📊 Copilot Token Usage Report"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_25eef300f6ef8a9d_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_c7b1b3c8c7779849_EOF - name: Write Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -589,7 +600,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.17' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_44cac78d3f8f4a62_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" + cat << GH_AW_MCP_CONFIG_6dff4fe0a0ab48a1_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" { "mcpServers": { "github": { @@ -630,7 +641,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_44cac78d3f8f4a62_EOF + GH_AW_MCP_CONFIG_6dff4fe0a0ab48a1_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -647,7 +658,7 @@ jobs: set -o pipefail touch /tmp/gh-aw/agent-step-summary.md # shellcheck disable=SC1003 - sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --allow-domains '*.blob.core.windows.net,*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,codeload.github.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,lfs.github.com,objects.githubusercontent.com,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --build-local --enable-api-proxy \ + sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --allow-domains '*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,codeload.github.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,lfs.github.com,objects.githubusercontent.com,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --build-local --enable-api-proxy \ -- /bin/bash -c 'node ${RUNNER_TEMP}/gh-aw/actions/copilot_driver.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE @@ -733,7 +744,7 @@ jobs: uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "*.blob.core.windows.net,*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,codeload.github.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,lfs.github.com,objects.githubusercontent.com,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com" + GH_AW_ALLOWED_DOMAINS: "*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,codeload.github.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,lfs.github.com,objects.githubusercontent.com,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} with: @@ -1177,7 +1188,7 @@ jobs: uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_ALLOWED_DOMAINS: "*.blob.core.windows.net,*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,codeload.github.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,lfs.github.com,objects.githubusercontent.com,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com" + GH_AW_ALLOWED_DOMAINS: "*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,codeload.github.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,host.docker.internal,lfs.github.com,objects.githubusercontent.com,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"close_older_issues\":true,\"labels\":[\"token-usage-report\"],\"max\":1,\"title_prefix\":\"📊 Copilot Token Usage Report\"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" diff --git a/.github/workflows/copilot-token-usage-analyzer.md b/.github/workflows/copilot-token-usage-analyzer.md index a7d24d8a5..ab2a94b21 100644 --- a/.github/workflows/copilot-token-usage-analyzer.md +++ b/.github/workflows/copilot-token-usage-analyzer.md @@ -9,22 +9,50 @@ permissions: issues: read pull-requests: read imports: + - uses: shared/mcp/gh-aw.md - shared/mcp-pagination.md - shared/reporting.md network: allowed: - github - - "*.blob.core.windows.net" tools: github: toolsets: [default, actions] bash: true safe-outputs: create-issue: - title-prefix: "📊 Copilot Token Usage Report" + title-prefix: "\U0001F4CA Copilot Token Usage Report" labels: [token-usage-report] close-older-issues: true timeout-minutes: 15 +steps: + - name: Download Copilot workflow logs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + mkdir -p /tmp/gh-aw/token-audit + + echo "\U0001F4E5 Downloading Copilot workflow logs (last 24 hours)..." + + LOGS_EXIT=0 + gh aw logs \ + --engine copilot \ + --start-date -1d \ + --json \ + -c 50 \ + > /tmp/gh-aw/token-audit/copilot-logs.json || LOGS_EXIT=$? + + if [ -s /tmp/gh-aw/token-audit/copilot-logs.json ]; then + TOTAL=$(jq '.runs | length' /tmp/gh-aw/token-audit/copilot-logs.json) + echo "\u2705 Downloaded $TOTAL Copilot workflow runs (last 24 hours)" + if [ "$LOGS_EXIT" -ne 0 ]; then + echo "\u26a0\ufe0f gh aw logs exited with code $LOGS_EXIT (partial results \u2014 likely API rate limit)" + fi + else + echo "\u274c No log data downloaded (exit code $LOGS_EXIT)" + echo '{"runs":[],"summary":{}}' > /tmp/gh-aw/token-audit/copilot-logs.json + fi --- # Daily Copilot Token Usage Analyzer @@ -33,111 +61,98 @@ You are an AI agent that analyzes Copilot token usage across agentic workflow ru ## Background -This repository uses the **Agent Workflow Firewall (AWF)** with an api-proxy sidecar that tracks token usage for LLM API calls. Each workflow run with `--enable-api-proxy` produces a `token-usage.jsonl` file captured in the `agent-artifacts` upload artifact. +This repository uses the **Agent Workflow Firewall (AWF)** with an api-proxy sidecar that tracks token usage for LLM API calls. The pre-agent step has already downloaded structured run data using `gh aw logs --json`, which includes per-run token usage, cost estimates, and run metadata. -**Token usage tracking is a new feature** — many older runs won't have this data. Handle missing data gracefully. +**Note:** Claude-engine and Codex-engine workflows are excluded \u2014 they are covered by the separate Claude Token Usage Analyzer. -### Token Usage Record Format +## Data Sources + +### Pre-downloaded logs + +The file `/tmp/gh-aw/token-audit/copilot-logs.json` contains structured JSON output from `gh aw logs --engine copilot --json` with this shape: -Each line in `token-usage.jsonl` is a JSON object: ```json { - "timestamp": "2026-04-01T17:38:12.486Z", - "request_id": "uuid", - "provider": "anthropic", - "model": "claude-sonnet-4-6", - "path": "/v1/messages?beta=true", - "status": 200, - "streaming": true, - "input_tokens": 3, - "output_tokens": 418, - "cache_read_tokens": 14044, - "cache_write_tokens": 26042, - "duration_ms": 5858, - "response_bytes": 2800 + "summary": { + "run_count": N, + "total_tokens": N, + "avg_tokens": N, + "total_cost": F, + "avg_cost": F, + "total_turns": N, + "avg_turns": F, + "total_action_minutes": F, + "error_count": N, + "warning_count": N + }, + "runs": [ ... ], + "tool_usage": [ ... ], + "mcp_tool_usage": { ... } } ``` -## Your Mission - -### Step 1: Discover Recent Workflow Runs +Each element of `.runs` includes: + +| Field | Type | Notes | +|---|---|---| +| `workflow_name` | string | Human-readable name | +| `workflow_path` | string | `.github/workflows/....lock.yml` | +| `token_usage` | int | Total tokens (treat missing/null as 0) | +| `effective_tokens` | int | Cost-normalized tokens | +| `estimated_cost` | float | USD cost (treat missing/null as 0) | +| `action_minutes` | float | Billable GitHub Actions minutes | +| `turns` | int | Number of agent turns | +| `duration` | string | Human-readable duration | +| `created_at` | ISO 8601 | Run creation time | +| `database_id` | int64 | Unique run ID | +| `url` | string | Link to the run | +| `status` | string | `completed`, `in_progress`, etc. | +| `conclusion` | string | `success`, `failure`, etc. | +| `error_count` | int | Errors encountered | +| `warning_count` | int | Warnings encountered | +| `token_usage_summary` | object or null | Firewall-level breakdown by model | -Use `gh run list` via bash to find completed agentic workflow runs from the past 24 hours (or since the last token usage report issue). Focus on **Copilot-engine workflows** that use the api-proxy: - -- `smoke-copilot`, `smoke-chroot`, `smoke-services` -- `build-test`, `ci-doctor`, `plan` -- `secret-digger-copilot` -- `cli-flag-consistency-checker`, `doc-maintainer` -- Any other Copilot-engine workflow with `agent-artifacts` - -**Note:** Claude-engine and Codex-engine workflows (e.g., `smoke-claude`, `smoke-codex`, `secret-digger-claude`, `secret-digger-codex`, `security-review`, `security-guard`) are excluded from this analysis to limit scope and keep within the time budget. - -Use bash to run: -```bash -# Find runs from the last 24 hours -CUTOFF="$(date -u -Iseconds -d '24 hours ago')" - -gh run list --repo "$GITHUB_REPOSITORY" --limit 50 \ - --created ">=$CUTOFF" \ - --json databaseId,name,status,conclusion,createdAt,workflowName \ - --jq '[.[] | select(.conclusion == "success" or .conclusion == "failure")]' -``` - -### Step 2: Download and Parse Token Usage Data +## Your Mission -For each discovered run, attempt to download the `agent-artifacts` artifact and extract `token-usage.jsonl`. +### Step 1: Load and Parse Pre-Downloaded Data -**IMPORTANT:** Always use `gh run download` via bash — this is much faster than using MCP `get_job_logs` and the network is configured to allow artifact blob storage access. +Read the pre-downloaded logs: ```bash -# Create temp directory -TMPDIR=$(mktemp -d) - -# Try to download artifacts for a run -gh run download --repo "$GITHUB_REPOSITORY" --name agent-artifacts --dir "$TMPDIR/run-" 2>/dev/null - -# Look for token-usage.jsonl (may be nested under sandbox/firewall/logs/api-proxy-logs/) -find "$TMPDIR/run-" -name "token-usage.jsonl" 2>/dev/null +cat /tmp/gh-aw/token-audit/copilot-logs.json | jq '.summary' +cat /tmp/gh-aw/token-audit/copilot-logs.json | jq '.runs | length' ``` -**Graceful degradation:** -- If artifact download fails → skip run, note it as "no artifacts" -- If `token-usage.jsonl` is missing → skip run, note it as "no token logs" -- If the file is empty → skip run, note it as "empty token logs" -- Track which workflows have instrumentation vs which don't +If `.runs` is empty, create a minimal report noting that no Copilot workflow runs were found in the last 24 hours. -### Step 3: Compute Per-Workflow Statistics +### Step 2: Compute Per-Workflow Statistics -For each workflow that has token data, calculate: +Group `.runs` by `workflow_name` and compute per-workflow aggregates: -1. **Total tokens**: `input_tokens + output_tokens + cache_read_tokens + cache_write_tokens` -2. **Billable tokens**: `input_tokens + output_tokens + cache_write_tokens` (cache reads are discounted) -3. **Input/output ratio**: `(input_tokens + cache_read_tokens) / output_tokens` (if `output_tokens == 0`, treat the ratio as `∞`/`N/A` and exclude that request from ratio averages to avoid division by zero) -4. **Cache hit rate**: `cache_read_tokens / (cache_read_tokens + input_tokens) * 100` -5. **Request count**: Number of records in the JSONL -6. **Average latency**: Mean `duration_ms` per request -7. **Model distribution**: Count of requests per model -8. **Estimated cost** (use approximate rates): - - Anthropic Sonnet: input $3/M, output $15/M, cache_read $0.30/M, cache_write $3.75/M - - Anthropic Haiku: input $0.80/M, output $4/M, cache_read $0.08/M, cache_write $1/M - - OpenAI/Copilot: input $2.50/M, output $10/M +1. **Run count**: Number of runs per workflow +2. **Total tokens**: Sum of `token_usage` across runs +3. **Avg tokens/run**: Mean `token_usage` +4. **Total cost**: Sum of `estimated_cost` +5. **Avg cost/run**: Mean `estimated_cost` +6. **Total turns**: Sum of `turns` +7. **Error/warning counts**: Sum of `error_count`, `warning_count` +8. **Model breakdown**: Extract from `token_usage_summary` where available -Use bash with a Python or jq script to process the JSONL files efficiently. +Use bash with jq or a Python script to process efficiently. Handle null/missing `token_usage` and `estimated_cost` by treating them as 0. -### Step 4: Identify Optimization Opportunities +### Step 3: Identify Optimization Opportunities Flag workflows with these patterns: | Pattern | Threshold | Recommendation | |---------|-----------|----------------| -| Zero cache hits | cache_hit_rate = 0% | Enable prompt caching | -| Low cache hits | cache_hit_rate < 50% | Review cache breakpoints | -| High input/output ratio | ratio > 100:1 | Reduce system prompt or MCP tool surface | -| Many small requests | >10 requests, <50 output tokens each | Batch requests or combine tool calls | | High total cost | >$1.00 per run | Review if workflow is doing too much | +| High token count | >100K tokens/run | Reduce system prompt or MCP tool surface | +| Many turns | >15 turns/run | Consider pre-computing deterministic work in steps | +| High error rate | >30% of runs with errors | Investigate reliability issues | | Increasing trend | >20% increase vs last report | Investigate what changed | -### Step 5: Check for Historical Trends +### Step 4: Check for Historical Trends Search for previous token usage report issues: ```bash @@ -146,15 +161,14 @@ gh issue list --repo "$GITHUB_REPOSITORY" --label token-usage-report --state all If previous reports exist, compare current metrics to identify: - Workflows with increasing token consumption -- Workflows that gained or lost prompt caching -- New workflows that started using the api-proxy +- New workflows that appeared - Cost trend over time -### Step 6: Create the Summary Issue +### Step 5: Create the Summary Issue Create an issue with the following structure: -#### Title: `YYYY-MM-DD` (safe-outputs will automatically prefix this with "📊 Copilot Token Usage Report") +#### Title: `YYYY-MM-DD` (safe-outputs will automatically prefix this with "\U0001F4CA Copilot Token Usage Report") #### Body structure: @@ -162,68 +176,60 @@ Create an issue with the following structure: ### Overview **Period**: [start date] to [end date] -**Runs analyzed**: X of Y (Z had token data) +**Runs analyzed**: X (Y had token data) **Total tokens**: N across all workflows **Estimated total cost**: $X.XX +**Total Actions minutes**: X.X min ### Workflow Summary -| Workflow | Runs | Total Tokens | Cost | Cache Rate | I/O Ratio | Top Model | -|----------|------|-------------|------|------------|-----------|-----------| -| smoke-claude | 2 | 395K | $0.46 | 99.5% | 0.6:1 | sonnet-4.6 | -| smoke-copilot | 2 | 603K | $1.20 | 0% | 184:1 | gpt-4o | +| Workflow | Runs | Total Tokens | Avg Tokens | Cost | Avg Cost | Turns | +|----------|------|-------------|------------|------|----------|-------| +| smoke-copilot | 3 | 786K | 262K | $1.56 | $0.52 | 15 | +| build-test | 2 | 450K | 225K | $0.90 | $0.45 | 8 | | ... | | | | | | | -### 🔍 Optimization Opportunities +### \U0001F50D Optimization Opportunities -1. **smoke-copilot** — 0% cache hit rate, 184:1 input/output ratio - - Enable prompt caching to reduce input costs by ~80% - - Review MCP tool surface (Playwright loads 30+ tools but barely uses them) +1. **smoke-copilot** \u2014 $0.52/run, 262K tokens/run + - Consider reducing MCP tool surface if many tools are unused + - Review prompt length for optimization 2. ...
Per-Workflow Details -#### smoke-claude -- **Runs**: 2 (run 123, run 456) -- **Requests**: 12 total (avg 6/run) -- **Models**: claude-haiku-4.5 (4 reqs), claude-sonnet-4.6 (8 reqs) -- **Tokens**: 395K total (1.5K input, 2.5K output, 304K cache_read, 87K cache_write) -- **Cache hit rate**: 99.5% -- **Avg latency**: 3,800ms/request -- **Estimated cost**: $0.46 - #### smoke-copilot -... +- **Runs**: 3 (links to each run) +- **Total tokens**: 786K (avg 262K/run) +- **Estimated cost**: $1.56 (avg $0.52/run) +- **Turns**: 15 total (avg 5/run) +- **Model breakdown**: [from token_usage_summary if available] +- **Error rate**: 0/3 runs
Workflows Without Token Data -The following workflows either don't use `--enable-api-proxy` or ran before token tracking was enabled: -- ci-doctor (3 runs — no agent-artifacts) -- issue-monster (1 run — no token-usage.jsonl) +Runs where `token_usage` was null or 0 \u2014 these may not have the api-proxy enabled: +- [list workflows with missing data]
### Historical Trend -[If previous reports exist, show comparison. Otherwise note: "This is the first token usage report. Historical trends will be available in future reports."] +[If previous reports exist, show comparison. Otherwise: "This is the first token usage report."] ### Previous Report -[Link to previous report issue if one exists, otherwise omit this section] +[Link to previous report issue if one exists] ``` ## Important Guidelines -- **Time budget** — You have 15 minutes total. Spend at most 8 minutes on data collection (steps 1-2) and reserve the rest for analysis and issue creation. If artifact downloads are slow, limit to the 5 most recent runs. -- **Prefer bash over MCP** for data collection — `gh run download` via bash is much faster than MCP `get_job_logs` for retrieving artifacts. -- **Do NOT fail** if no token data is available. Create a minimal report explaining that token tracking is new and which workflows need instrumentation. -- **Clean up** temporary directories after processing. -- **Respect rate limits** — download artifacts one at a time, not in parallel. -- **Use `--perPage` parameters** when listing runs to avoid token limits on MCP responses. +- **Do NOT fail** if no token data is available. Create a minimal report explaining which workflows need instrumentation. +- **All data is pre-downloaded** \u2014 do not attempt to download artifacts or run `gh run download`. Use only the JSON at `/tmp/gh-aw/token-audit/copilot-logs.json`. - **Wrap verbose output** in `
` blocks for progressive disclosure. - **Round costs** to 2 decimal places, token counts to nearest thousand for readability. - **Sort workflows** by estimated cost (highest first) in the summary table. diff --git a/.github/workflows/shared/mcp/gh-aw.md b/.github/workflows/shared/mcp/gh-aw.md new file mode 100644 index 000000000..5e83ec983 --- /dev/null +++ b/.github/workflows/shared/mcp/gh-aw.md @@ -0,0 +1,31 @@ +--- +# gh-aw Extension - Shared Component +# Installs the gh-aw CLI extension for use in pre-agent steps. +# +# Usage: +# imports: +# - uses: shared/mcp/gh-aw.md + +steps: + - name: Install gh-aw extension + env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + run: | + # Install gh-aw if not already available + if ! gh aw --version >/dev/null 2>&1; then + echo "Installing gh-aw extension..." + curl -fsSL https://raw.githubusercontent.com/github/gh-aw/refs/heads/main/install-gh-aw.sh | bash + fi + gh aw --version + # Copy the gh-aw binary to RUNNER_TEMP for MCP server containerization + mkdir -p "${RUNNER_TEMP}/gh-aw" + GH_AW_BIN=$(which gh-aw 2>/dev/null || find ~/.local/share/gh/extensions/gh-aw -name 'gh-aw' -type f 2>/dev/null | head -1) + if [ -n "$GH_AW_BIN" ] && [ -f "$GH_AW_BIN" ]; then + cp "$GH_AW_BIN" "${RUNNER_TEMP}/gh-aw/gh-aw" + chmod +x "${RUNNER_TEMP}/gh-aw/gh-aw" + echo "Copied gh-aw binary to ${RUNNER_TEMP}/gh-aw/gh-aw" + else + echo "::error::Failed to find gh-aw binary for MCP server" + exit 1 + fi +---