diff --git a/.github/workflows/agent-performance-analyzer.lock.yml b/.github/workflows/agent-performance-analyzer.lock.yml index 2133a42d0cb..5c103e132e3 100644 --- a/.github/workflows/agent-performance-analyzer.lock.yml +++ b/.github/workflows/agent-performance-analyzer.lock.yml @@ -1,5 +1,5 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"4717241fe4fbdab574034e352805809781b9f48cc3125e620cc8c3c23c0087f0","body_hash":"93ac963fe785605c68dd5d64dbaf9f5cd49eacd9638fd145ce1aac3838561946","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75","copilot-sdk":"1.0.8"}} -# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"53b7df96c91f9c12dcc8a07bcb9ccacbed38856a","version":"v7.3.0"},{"repo":"docker/setup-buildx-action","sha":"bb05f3f5519dd87d3ba754cc423b652a5edd6d2c","version":"v4.2.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42","digest":"sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42@sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"5a72934c2d31ff00abcd0b8397039d0d3d4854af7b2737c2b7639b6a93b27e89","body_hash":"8083eb2bd38e718b63c30e736b9409792ec10b70d67d71b55cbc54e516c20974","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75","copilot-sdk":"1.0.8"}} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"53b7df96c91f9c12dcc8a07bcb9ccacbed38856a","version":"v7.3.0"},{"repo":"docker/setup-buildx-action","sha":"bb05f3f5519dd87d3ba754cc423b652a5edd6d2c","version":"v4.2.0"}],"skills":["githubnext/rig/skills/rig/SKILL.md@0ba73e37355f92adca11f9d596eb709e77f25332"],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42","digest":"sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42@sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # # ___ _ _ @@ -115,6 +115,8 @@ jobs: setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} setup-span-id: ${{ steps.setup.outputs.span-id }} setup-trace-id: ${{ steps.setup.outputs.trace-id }} + skill_install_errors: ${{ steps.collect-skill-install-failures.outputs.errors || '' }} + skill_install_failure_count: ${{ steps.collect-skill-install-failures.outputs.failure_count || '0' }} stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} steps: - name: Checkout actions folder @@ -160,6 +162,7 @@ jobs: GH_AW_INFO_FRONTMATTER_EMOJI: "⚡" GH_AW_COMPILED_STRICT: "true" GH_AW_INFO_FEATURES: '{"gh-aw-detection":true}' + GH_AW_INFO_SKILLS: '["githubnext/rig/skills/rig/SKILL.md@0ba73e37355f92adca11f9d596eb709e77f25332"]' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | @@ -252,6 +255,32 @@ jobs: setupGlobals(core, github, context, exec, io, getOctokit); const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); await main(); + - name: Upgrade gh CLI for frontmatter skills + run: bash "${RUNNER_TEMP}/gh-aw/actions/ensure_gh_cli_min_version.sh" "2.90.0" + - name: Install frontmatter skill 1 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_GH_SKILL_AGENT_NAME: "github-copilot" + GH_AW_SKILL_DIR: ".github/skills" + GH_AW_FRONTMATTER_SKILLS: "githubnext/rig/skills/rig/SKILL.md@0ba73e37355f92adca11f9d596eb709e77f25332" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/install_frontmatter_skills.cjs'); + await main(); + - name: Collect skill install failures + id: collect-skill-install-failures + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_skill_install_failures.cjs'); + await main(); - name: Log runtime features if: ${{ contains(toJSON(vars), '"GH_AW_RUNTIME_FEATURES":') }} run: bash "${RUNNER_TEMP}/gh-aw/actions/log_runtime_features_summary.sh" @@ -1517,6 +1546,8 @@ jobs: GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} GH_AW_OAUTH_TOKEN_CHECK_FAILED: ${{ needs.activation.outputs.oauth_token_check_failed }} GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} + GH_AW_SKILL_INSTALL_FAILURE_COUNT: ${{ needs.activation.outputs.skill_install_failure_count || '0' }} + GH_AW_SKILL_INSTALL_ERRORS: ${{ needs.activation.outputs.skill_install_errors || '' }} GH_AW_DAILY_AI_CREDITS_EXCEEDED: ${{ needs.activation.outputs.daily_ai_credits_exceeded }} GH_AW_DAILY_AI_CREDITS_TOTAL_EFFECTIVE_TOKENS: ${{ needs.activation.outputs.daily_ai_credits_total_effective_tokens }} GH_AW_DAILY_AI_CREDITS_THRESHOLD: ${{ needs.activation.outputs.daily_ai_credits_threshold }} diff --git a/.github/workflows/agent-performance-analyzer.md b/.github/workflows/agent-performance-analyzer.md index 7665ee4b428..16ef76b073e 100644 --- a/.github/workflows/agent-performance-analyzer.md +++ b/.github/workflows/agent-performance-analyzer.md @@ -32,6 +32,8 @@ imports: - shared/reporting.md - shared/otlp.md - shared/default-ai-credits-pricing.md +skills: + - githubnext/rig/skills/rig/SKILL.md@0ba73e37355f92adca11f9d596eb709e77f25332 safe-outputs: create-issue: expires: 2d @@ -111,7 +113,7 @@ Treat `copilot-swe-agent` as a built-in team member in attribution/engagement fi **CI vs. agentic workflow distinction:** Workflows such as `CWI`, `CGO`, `CI`, `CJS`, and `CPI` are plain CI workflows — not agentic workflows. An `action_required` conclusion on a CI workflow means GitHub is waiting for a maintainer to approve a pull-request workflow run (a GitHub Actions permission gate), **not** an agentic activation-refused. Do not count CI-workflow `action_required` runs as agentic AR. Report them separately under "CI approval-pending" and note that the fix is to approve the Copilot-bot's workflow runs at the org level or in the PR, not an agent-side change. ### Phase 1: Data Collection (10m) -1. Load shared metrics/memory files (use `metrics-extractor` with listed paths). +1. Load shared metrics/memory files (use a Rig custom harness for metrics extraction with the listed paths). 2. Gather recent agent outputs (issues/PRs/discussions/comments + metadata). 3. Review workflow runs/logs for decisions, errors, and resource use. 4. Build per-agent profiles. @@ -122,7 +124,7 @@ Treat `copilot-swe-agent` as a built-in team member in attribution/engagement fi 7. Compare resource efficiency across agents. ### Phase 3: Pattern Detection (5m) -8. Use `pattern-detector` on profiles for behavior classification. +8. Use a Rig custom harness for behavior classification on profiles. 9. Analyze collaboration quality and conflicts. 10. Assess ecosystem coverage gaps/redundancy. @@ -755,39 +757,48 @@ Execute all phases systematically and maintain an objective, data-driven approac - Use `bash` with `gh` for GitHub reads and to inspect `/tmp/gh-aw/repo-memory/default/` contents. - If required data stays inaccessible after 1-2 materially different attempts, call `report_incomplete` with the blocker instead of ending with prose only. - If the analysis completes but there is nothing actionable to create or update, call `noop` with a short summary of what you checked. -## agent: `metrics-extractor` ---- -model: mai-code -description: Reads shared repo-memory metric files and returns structured JSON with all relevant performance data ---- -You are a metrics extraction assistant. When given a newline-separated list of file paths (one path per line), read each file using bash and return a single JSON object containing all data found. +## Rig Custom Harness Usage -For JSON files, parse and include the full content under a key matching the file's basename (without extension). For a directory path, list and read all files within it, using their basenames as keys. For markdown files, include the raw text under a key matching the filename. +Use Rig custom harnesses for the two AI-intensive analysis steps: -If a file does not exist or cannot be read, include `null` for that key. +**Phase 1 — Metrics extraction**: Discover the launcher with `find "${RUNNER_TEMP}/gh-aw" -path "*/skills/rig/rig.ts" | head -1`, then run `node ` with an inline Rig program that: +- configures `copilotEngine()` +- receives a compact JSON input listing the repo-memory file paths to read +- reads each file and returns a single structured JSON object keyed by basename -Return the result as a single valid JSON object with no additional commentary. +**Phase 3 — Pattern detection**: Run a second Rig harness invocation with an inline program that: +- configures `copilotEngine()` +- receives compact per-agent profile JSON (output counts, success rates, resource usage) +- classifies behavioral patterns and returns structured JSON -## agent: `pattern-detector` ---- -model: mai-code -description: Classifies agent behavioral patterns from profiles and returns a structured categorization of issues found ---- -You are an agent behavior classification assistant. When given a JSON object containing agent profiles (with fields such as output counts, types, success rates, and resource usage), classify each agent's behavioral patterns. +Harness guardrails: +- pass only compact structured inputs — never raw file content in full +- use a small model unless evidence shows quality loss +- one harness invocation per analysis phase (no retries without a concrete parse/runtime error) -For each agent, identify which of the following patterns apply: -- **over-creation**: Output count significantly above expected baseline -- **under-creation**: Output count significantly below expected baseline or zero -- **repetition**: Duplicate or near-duplicate outputs detected -- **scope-creep**: Outputs outside the agent's defined responsibility area -- **inconsistency**: High variance in output counts or quality across runs +## Rig Harness Output Contract -Return a JSON object where each key is the agent name and the value is an array of detected pattern strings (empty array if none detected). Example: +**Metrics extraction harness** returns: ```json { - "agent-a": ["over-creation", "inconsistency"], - "agent-b": [], - "agent-c": ["under-creation"] + "": { /* parsed file content or null */ } } +``` + +**Pattern detection harness** returns: + +```json +{ + "": ["over-creation", "inconsistency"], + "": [] +} +``` + +Recognized pattern labels: `over-creation`, `under-creation`, `repetition`, `scope-creep`, `inconsistency`. + +Rules: +- return `null` for any file that does not exist or cannot be read +- return an empty array for agents with no detected patterns +- use only provided compact evidence ``` \ No newline at end of file diff --git a/.github/workflows/architecture-guardian.lock.yml b/.github/workflows/architecture-guardian.lock.yml index 10cd49cc51d..4ae87587628 100644 --- a/.github/workflows/architecture-guardian.lock.yml +++ b/.github/workflows/architecture-guardian.lock.yml @@ -1,5 +1,5 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"60113cb4285ab662f975d66efda82ab8b6d58de4eb3ab513237be02f12e1cac6","body_hash":"3f1592b4665dade40a09240cdac8d9d9e3adb25aa410c990f321aa057b44ea72","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75","copilot-sdk":"1.0.8"}} -# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_AGENT_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"8d55990588c707140fbb6dcb6aade860f4bccf569160c0ef8934e64df36d6055","body_hash":"41e3deab9fba161a7902f739bbcf54bba57578e4586e4faeee7fa17244ee7859","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75","copilot-sdk":"1.0.8"}} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_AGENT_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"skills":["githubnext/rig/skills/rig/SKILL.md@0ba73e37355f92adca11f9d596eb709e77f25332"],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # # ___ _ _ @@ -115,6 +115,8 @@ jobs: setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} setup-span-id: ${{ steps.setup.outputs.span-id }} setup-trace-id: ${{ steps.setup.outputs.trace-id }} + skill_install_errors: ${{ steps.collect-skill-install-failures.outputs.errors || '' }} + skill_install_failure_count: ${{ steps.collect-skill-install-failures.outputs.failure_count || '0' }} stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} sub_agent_strategy: ${{ steps.pick-experiment.outputs.sub_agent_strategy }} steps: @@ -163,6 +165,7 @@ jobs: GH_AW_INFO_FRONTMATTER_EMOJI: "🏗️" GH_AW_COMPILED_STRICT: "true" GH_AW_INFO_FEATURES: '{"gh-aw-detection":true}' + GH_AW_INFO_SKILLS: '["githubnext/rig/skills/rig/SKILL.md@0ba73e37355f92adca11f9d596eb709e77f25332"]' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | @@ -255,6 +258,32 @@ jobs: setupGlobals(core, github, context, exec, io, getOctokit); const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); await main(); + - name: Upgrade gh CLI for frontmatter skills + run: bash "${RUNNER_TEMP}/gh-aw/actions/ensure_gh_cli_min_version.sh" "2.90.0" + - name: Install frontmatter skill 1 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_GH_SKILL_AGENT_NAME: "github-copilot" + GH_AW_SKILL_DIR: ".github/skills" + GH_AW_FRONTMATTER_SKILLS: "githubnext/rig/skills/rig/SKILL.md@0ba73e37355f92adca11f9d596eb709e77f25332" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/install_frontmatter_skills.cjs'); + await main(); + - name: Collect skill install failures + id: collect-skill-install-failures + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_skill_install_failures.cjs'); + await main(); - name: Log runtime features if: ${{ contains(toJSON(vars), '"GH_AW_RUNTIME_FEATURES":') }} run: bash "${RUNNER_TEMP}/gh-aw/actions/log_runtime_features_summary.sh" @@ -275,7 +304,7 @@ jobs: id: pick-experiment uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_EXPERIMENT_SPEC: '{"sub_agent_strategy":{"variants":["sub_agents","single_agent"],"description":"Test whether inlining violation classification in the main agent reduces AI credit spend without sacrificing detection accuracy","hypothesis":"H0: single_agent does not change ai_credits_spent vs sub_agents. H1: single_agent reduces ai_credits_spent by ≥15% by eliminating sub-agent model-call overhead","metric":"ai_credits_spent","secondary_metrics":["run_duration_ms","violation_count_delta"],"guardrail_metrics":[{"name":"run_failure_rate","direction":"min","threshold":"0.1"},{"name":"empty_output_rate","direction":"min","threshold":"0.05"}],"min_samples":30,"weight":[50,50],"issue":39062,"start_date":"2026-06-13"}}' + GH_AW_EXPERIMENT_SPEC: '{"sub_agent_strategy":{"variants":["sub_agents","single_agent"],"description":"Test whether inlining violation classification in the main agent reduces AI credit spend vs using a Rig custom harness","hypothesis":"H0: single_agent does not change ai_credits_spent vs sub_agents. H1: single_agent reduces ai_credits_spent by ≥15% by eliminating Rig harness overhead","metric":"ai_credits_spent","secondary_metrics":["run_duration_ms","violation_count_delta"],"guardrail_metrics":[{"name":"run_failure_rate","direction":"min","threshold":"0.1"},{"name":"empty_output_rate","direction":"min","threshold":"0.05"}],"min_samples":30,"weight":[50,50],"issue":39062,"start_date":"2026-06-13"}}' GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.json GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments with: @@ -1378,6 +1407,8 @@ jobs: GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} GH_AW_OAUTH_TOKEN_CHECK_FAILED: ${{ needs.activation.outputs.oauth_token_check_failed }} GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} + GH_AW_SKILL_INSTALL_FAILURE_COUNT: ${{ needs.activation.outputs.skill_install_failure_count || '0' }} + GH_AW_SKILL_INSTALL_ERRORS: ${{ needs.activation.outputs.skill_install_errors || '' }} GH_AW_DAILY_AI_CREDITS_EXCEEDED: ${{ needs.activation.outputs.daily_ai_credits_exceeded }} GH_AW_DAILY_AI_CREDITS_TOTAL_EFFECTIVE_TOKENS: ${{ needs.activation.outputs.daily_ai_credits_total_effective_tokens }} GH_AW_DAILY_AI_CREDITS_THRESHOLD: ${{ needs.activation.outputs.daily_ai_credits_threshold }} diff --git a/.github/workflows/architecture-guardian.md b/.github/workflows/architecture-guardian.md index 9fbac9d4766..a86b296f864 100644 --- a/.github/workflows/architecture-guardian.md +++ b/.github/workflows/architecture-guardian.md @@ -18,8 +18,8 @@ tracker-id: architecture-guardian experiments: sub_agent_strategy: variants: [sub_agents, single_agent] - description: "Test whether inlining violation classification in the main agent reduces AI credit spend without sacrificing detection accuracy" - hypothesis: "H0: single_agent does not change ai_credits_spent vs sub_agents. H1: single_agent reduces ai_credits_spent by ≥15% by eliminating sub-agent model-call overhead" + description: "Test whether inlining violation classification in the main agent reduces AI credit spend vs using a Rig custom harness" + hypothesis: "H0: single_agent does not change ai_credits_spent vs sub_agents. H1: single_agent reduces ai_credits_spent by ≥15% by eliminating Rig harness overhead" metric: ai_credits_spent secondary_metrics: [run_duration_ms, violation_count_delta] guardrail_metrics: @@ -44,6 +44,8 @@ imports: labels: [architecture, automated-analysis, cookie] assignees: [copilot] - shared/otlp.md +skills: + - githubnext/rig/skills/rig/SKILL.md@0ba73e37355f92adca11f9d596eb709e77f25332 sandbox: agent: sudo: false @@ -206,7 +208,7 @@ Read the metrics JSON already loaded in Step 1. Apply the following rules using Build `blockers`, `warnings`, and `infos` arrays from this analysis and proceed to Step 3. {{else}} -Use the `violation-classifier` agent to read `/tmp/gh-aw/agent/arch-metrics.json` and return the categorized violation list. If it returns `{"noop": true}`, skip to the noop call in Step 3. +Use a Rig custom harness to read `/tmp/gh-aw/agent/arch-metrics.json` and return the categorized violation list. Discover the launcher with `find "${RUNNER_TEMP}/gh-aw" -path "*/skills/rig/rig.ts" | head -1`, then run `node ` with an inline Rig program configured with `copilotEngine()` that applies thresholds and returns the categorized violations. If the harness returns `{"noop": true}`, skip to the noop call in Step 3. {{/if}} ## Step 3: Post Report @@ -223,7 +225,7 @@ Call the `noop` safe-output tool: Create an issue with a structured report. Only create ONE issue (the `max: 1` limit applies and an existing open issue skips the run via `skip-if-match`). -Use the `blockers`, `warnings`, and `infos` arrays returned by the `violation-classifier` agent to populate the violation rows in each section. Replace all `[PLACEHOLDER]` values with actual data, and replace `N` with actual counts. +Use the `blockers`, `warnings`, and `infos` arrays returned by the Rig harness to populate the violation rows in each section. Replace all `[PLACEHOLDER]` values with actual data, and replace `N` with actual counts. **Issue title**: Architecture Violations Detected — [DATE] @@ -288,39 +290,27 @@ Thresholds (from `.architecture.yml` or defaults): > 🏛️ *To configure thresholds, add a `.architecture.yml` file to the repository root.* ``` -## agent: `violation-classifier` ---- -description: Applies numeric thresholds to the pre-computed metrics JSON and returns a structured list of violations grouped by severity -model: kiwi ---- -You are a violation classification assistant. Read the pre-computed metrics JSON, apply the thresholds, and return a structured categorization of all findings. - -Read the file: - -```bash -cat /tmp/gh-aw/agent/arch-metrics.json -``` - -If `noop` is `true`, return immediately: - -```json -{"noop": true} -``` +## Rig Custom Harness Usage -Otherwise, apply the following rules using the values in `thresholds`: +When the `sub_agents` experiment variant is active, use a Rig custom harness for violation classification: -**BLOCKER** (critical): -- Non-empty `import_cycles` field → import cycle detected -- `files[].lines` > `thresholds.file_lines_blocker` +1. Discover the installed launcher path: + - `find "${RUNNER_TEMP}/gh-aw" -path "*/skills/rig/rig.ts" | head -1` +2. Build compact JSON input from `/tmp/gh-aw/agent/arch-metrics.json` (metrics only — do not pass raw file paths or large blobs). +3. Run the harness with `node `, feeding an inline Rig program that: + - configures `copilotEngine()` + - applies threshold rules to `files`, `import_cycles`, and `thresholds` fields + - returns categorized violations as JSON +4. If the harness returns `{"noop": true}`, proceed to the noop safe-output call. -**WARNING** (should be addressed soon): -- `files[].lines` > `thresholds.file_lines_warning` -- Any function in `files[].func_data` with line count > `thresholds.function_lines` (for Go files, each line in `func_data` is `name\tline_count`; for JS files use the presence of the entry as an indicator of a large function when line count context is available) +Harness guardrails: +- pass only the pre-computed metrics JSON — never raw source files +- use a small model unless evidence shows quality loss +- one harness invocation per run (no retries without a concrete parse/runtime error) -**INFO** (informational): -- `files[].export_count` > `thresholds.max_exports` +## Rig Harness Output Contract -Return only a JSON object with no additional commentary: +Each Rig harness invocation must return JSON only: ```json { diff --git a/.github/workflows/cli-consistency-checker.lock.yml b/.github/workflows/cli-consistency-checker.lock.yml index 01d9de29b9d..2f4d06a435d 100644 --- a/.github/workflows/cli-consistency-checker.lock.yml +++ b/.github/workflows/cli-consistency-checker.lock.yml @@ -1,5 +1,5 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"76ce42c26fa8bccded9c4de009c49cb7625800e7f10133a5499a04a7585ea61b","body_hash":"c37885bacb425ffb50bd5d957c98e916b75dc01a26200c4232cf780a2d21a713","agent_id":"copilot","engine_versions":{"copilot":"1.0.75","copilot-sdk":"1.0.8"}} -# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"ca0a0c1589b3164b273718cae97f174ae63897da2699e32741a2230849400b1c","body_hash":"8bbb1ffdcda368010d47021a4142b28af89fb23bac38f402ddcf90d0d564e665","agent_id":"copilot","engine_versions":{"copilot":"1.0.75","copilot-sdk":"1.0.8"}} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"skills":["githubnext/rig/skills/rig/SKILL.md@0ba73e37355f92adca11f9d596eb709e77f25332"],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # # ___ _ _ @@ -106,6 +106,8 @@ jobs: setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} setup-span-id: ${{ steps.setup.outputs.span-id }} setup-trace-id: ${{ steps.setup.outputs.trace-id }} + skill_install_errors: ${{ steps.collect-skill-install-failures.outputs.errors || '' }} + skill_install_failure_count: ${{ steps.collect-skill-install-failures.outputs.failure_count || '0' }} stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} steps: - name: Checkout actions folder @@ -151,6 +153,7 @@ jobs: GH_AW_INFO_FRONTMATTER_EMOJI: "✅" GH_AW_COMPILED_STRICT: "false" GH_AW_INFO_FEATURES: '{"gh-aw-detection":true}' + GH_AW_INFO_SKILLS: '["githubnext/rig/skills/rig/SKILL.md@0ba73e37355f92adca11f9d596eb709e77f25332"]' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | @@ -248,6 +251,32 @@ jobs: setupGlobals(core, github, context, exec, io, getOctokit); const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); await main(); + - name: Upgrade gh CLI for frontmatter skills + run: bash "${RUNNER_TEMP}/gh-aw/actions/ensure_gh_cli_min_version.sh" "2.90.0" + - name: Install frontmatter skill 1 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_GH_SKILL_AGENT_NAME: "github-copilot" + GH_AW_SKILL_DIR: ".github/skills" + GH_AW_FRONTMATTER_SKILLS: "githubnext/rig/skills/rig/SKILL.md@0ba73e37355f92adca11f9d596eb709e77f25332" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/install_frontmatter_skills.cjs'); + await main(); + - name: Collect skill install failures + id: collect-skill-install-failures + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_skill_install_failures.cjs'); + await main(); - name: Log runtime features if: ${{ contains(toJSON(vars), '"GH_AW_RUNTIME_FEATURES":') }} run: bash "${RUNNER_TEMP}/gh-aw/actions/log_runtime_features_summary.sh" @@ -1298,6 +1327,8 @@ jobs: GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} GH_AW_OAUTH_TOKEN_CHECK_FAILED: ${{ needs.activation.outputs.oauth_token_check_failed }} GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} + GH_AW_SKILL_INSTALL_FAILURE_COUNT: ${{ needs.activation.outputs.skill_install_failure_count || '0' }} + GH_AW_SKILL_INSTALL_ERRORS: ${{ needs.activation.outputs.skill_install_errors || '' }} GH_AW_DAILY_AI_CREDITS_EXCEEDED: ${{ needs.activation.outputs.daily_ai_credits_exceeded }} GH_AW_DAILY_AI_CREDITS_TOTAL_EFFECTIVE_TOKENS: ${{ needs.activation.outputs.daily_ai_credits_total_effective_tokens }} GH_AW_DAILY_AI_CREDITS_THRESHOLD: ${{ needs.activation.outputs.daily_ai_credits_threshold }} diff --git a/.github/workflows/cli-consistency-checker.md b/.github/workflows/cli-consistency-checker.md index 22142822bd7..378061e4d96 100644 --- a/.github/workflows/cli-consistency-checker.md +++ b/.github/workflows/cli-consistency-checker.md @@ -21,6 +21,8 @@ network: allowed: [defaults] imports: - shared/otlp.md +skills: + - githubnext/rig/skills/rig/SKILL.md@0ba73e37355f92adca11f9d596eb709e77f25332 tools: bash: - "*" @@ -105,13 +107,13 @@ Read `/tmp/gh-aw/agent/all-help.txt` and use it as the primary input for analysi Look for: - Help style and terminology inconsistencies -- Use the `typo-grammar-extractor` agent to collect typo, grammar, capitalization, and punctuation issues across `/tmp/gh-aw/agent/all-help.txt`. +- Use a Rig custom harness (typo-grammar analysis) to collect typo, grammar, capitalization, and punctuation issues across `/tmp/gh-aw/agent/all-help.txt`. - Do examples in help text actually work? - Are file paths correct (e.g., `.github/workflows`)? - Are flag combinations valid? - Do command descriptions match their actual behavior? -- Use the `docs-vs-help-comparer` agent to list every mismatch between `docs/src/content/docs/setup/cli.md` and `/tmp/gh-aw/agent/all-help.txt`. -- Use the `flag-consistency-analyzer` agent to enumerate flag-naming and negation-style inconsistencies across all command help files. +- Use a Rig custom harness (docs-vs-help comparison) to list every mismatch between `docs/src/content/docs/setup/cli.md` and `/tmp/gh-aw/agent/all-help.txt`. +- Use a Rig custom harness (flag consistency analysis) to enumerate flag-naming and negation-style inconsistencies across all command help files. ## Step 3: Report Findings @@ -151,47 +153,46 @@ All CLI output comes from the repository's own codebase, so treat it as trusted - Compare CLI output with documentation - Create issues for any inconsistencies found - Keep reporting concise but complete -## agent: `typo-grammar-extractor` ---- -description: Extracts typo, grammar, capitalization, and punctuation issues from CLI help output -model: mai-code ---- -Read `/tmp/gh-aw/agent/all-help.txt` and scan for typos, grammar mistakes, -inconsistent capitalization, and punctuation issues in the CLI help text. -Return a concise bulleted list. For each finding include: -- the affected command or section when identifiable -- the exact quoted text -- the issue type -- a suggested fix -If nothing is wrong, say `No issues found.` - -## agent: `flag-consistency-analyzer` ---- -description: Finds inconsistent flag names, short forms, and negation patterns across help files -model: mai-code ---- -Read all per-command help files in `/tmp/gh-aw/agent/help-output/`. -Compare related commands for inconsistent flag names, short/long flag pairings, -and `--no-...` negation patterns. -Return a concise bulleted list. For each finding include: -- the affected help files or commands -- the exact quoted flag text -- the inconsistency -- a suggested normalization -If nothing is wrong, say `No issues found.` - -## agent: `docs-vs-help-comparer` ---- -description: Compares CLI setup docs against generated help output and reports drift -model: mai-code ---- -Compare `docs/src/content/docs/setup/cli.md` against -`/tmp/gh-aw/agent/all-help.txt`. -List every mismatch involving missing commands, mismatched flag lists, drifted -descriptions, or stale examples. -Return a concise bulleted list. For each finding include: -- the docs location or heading when identifiable -- the exact quoted docs text -- the exact quoted help text -- a suggested fix -If nothing is wrong, say `No issues found.` \ No newline at end of file +## Rig Custom Harness Usage + +Use three Rig custom harness invocations, one per analysis area. For each: + +1. Discover the installed launcher path: + - `find "${RUNNER_TEMP}/gh-aw" -path "*/skills/rig/rig.ts" | head -1` +2. Build compact JSON input: pass only relevant file contents as strings (not full raw file dumps). +3. Run the harness with `node `, feeding an inline Rig program that: + - configures `copilotEngine()` + - receives the compact JSON input + - returns findings as a JSON array + +**Harness 1 — Typo/Grammar Analysis**: reads `/tmp/gh-aw/agent/all-help.txt` content (first 20 KB), scans for typos, grammar mistakes, inconsistent capitalization, and punctuation issues. + +**Harness 2 — Flag Consistency Analysis**: reads all per-command help file contents from `/tmp/gh-aw/agent/help-output/`, compares related commands for inconsistent flag names, short/long flag pairings, and `--no-...` negation patterns. + +**Harness 3 — Docs vs Help Comparison**: reads `docs/src/content/docs/setup/cli.md` and `/tmp/gh-aw/agent/all-help.txt` content (truncated to fit compact input), lists every mismatch involving missing commands, mismatched flag lists, drifted descriptions, or stale examples. + +Harness guardrails: +- pass only compact string snippets — never full raw file dumps +- use a small model unless evidence shows quality loss +- one harness invocation per analysis area (no retries without a concrete parse/runtime error) + +## Rig Harness Output Contract + +Each Rig harness invocation must return a JSON array only: + +```json +[ + { + "area": "typo-grammar | flag-consistency | docs-vs-help", + "location": "affected command, section, or file path", + "quoted_text": "exact quoted text", + "issue": "issue description", + "suggestion": "suggested fix" + } +] +``` + +Rules: +- return an empty array if no issues are found +- use only provided compact evidence +- keep each finding self-contained with enough context to act on it \ No newline at end of file