From 04980ea85b724e44e74aab460fd5e4e5aaae3312 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:26:29 +0000 Subject: [PATCH 1/3] Initial plan From 629a3142fae554c8b31432a8ec4676f3f83c94f3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:56:32 +0000 Subject: [PATCH 2/3] optimize: detection-analysis-report AIC savings (~45 AIC/run) - Simplify Step 1: remove redundant inline MCP tool documentation block - Merge Steps 2-4 into single 'Analyze Runs' section to reduce context duplication - Extract chart generation to .github/scripts/detection_comparison.py; replace Step 5 with direct bash invocation - Add sub-agent 'update-detection-trending' (model: small) for Step 6 historical trending - Recompile lock file Closes #45770 Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/scripts/detection_comparison.py | 94 +++++++++++++ .github/workflows/agentic-auto-upgrade.yml | 2 +- .../detection-analysis-report.lock.yml | 2 +- .../workflows/detection-analysis-report.md | 128 +++++++++--------- 4 files changed, 158 insertions(+), 68 deletions(-) create mode 100644 .github/scripts/detection_comparison.py diff --git a/.github/scripts/detection_comparison.py b/.github/scripts/detection_comparison.py new file mode 100644 index 00000000000..90454cb5e7c --- /dev/null +++ b/.github/scripts/detection_comparison.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Generate detection feature comparison chart from pre-computed metrics JSON. + +Usage: + python3 detection_comparison.py --metrics --output + +Input JSON keys: + regular_runs, detection_runs, regular_success_rate, detection_success_rate, + regular_failure_count, detection_failure_count, misconfigured_count +""" + +import argparse +import json +import os + +import matplotlib.pyplot as plt +import numpy as np +import seaborn as sns + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate detection feature comparison chart") + parser.add_argument("--metrics", required=True, help="Path to metrics JSON file") + parser.add_argument("--output", required=True, help="Output PNG file path") + args = parser.parse_args() + + with open(args.metrics) as f: + m = json.load(f) + + regular_runs = int(m.get("regular_runs", 0)) + detection_runs = int(m.get("detection_runs", 0)) + regular_success_rate = float(m.get("regular_success_rate", 0.0)) + detection_success_rate = float(m.get("detection_success_rate", 0.0)) + regular_failure = int(m.get("regular_failure_count", 0)) + detection_failure = int(m.get("detection_failure_count", 0)) + misconfigured_count = int(m.get("misconfigured_count", 0)) + + regular_success = regular_runs - regular_failure + detection_success = detection_runs - detection_failure + + sns.set_style("whitegrid") + palette = sns.color_palette("muted") + + fig, ax1 = plt.subplots(figsize=(10, 6), dpi=150) + ax2 = ax1.twinx() + + x = np.array([0, 1]) + width = 0.5 + + # Stacked bars: success (bottom) + failure (top) + ax1.bar( + x, [regular_success, detection_success], width, + label="Success", color=palette[2], alpha=0.85, + ) + ax1.bar( + x, [regular_failure, detection_failure], width, + bottom=[regular_success, detection_success], + label="Failure", color=palette[3], alpha=0.85, + ) + + # Success-rate line on secondary axis + ax2.plot( + x, [regular_success_rate, detection_success_rate], + color=palette[0], marker="o", linewidth=2, markersize=8, + label="Success Rate %", + ) + + # Annotation band when misconfigured workflows are present + if misconfigured_count > 0: + max_y = max(regular_runs, detection_runs, 1) + ax1.axhspan( + 0, max_y * 0.12, alpha=0.12, color="red", + label=f"{misconfigured_count} misconfigured workflow(s)", + ) + + ax1.set_xticks(x) + ax1.set_xticklabels(["Regular Runs", "Detection Runs"], fontsize=12) + ax1.set_ylabel("Run Count", fontsize=12) + ax1.set_title("Detection Feature Comparison — Last 24h", fontsize=14, fontweight="bold") + ax2.set_ylabel("Success Rate (%)", fontsize=12) + ax2.set_ylim(0, 110) + + handles1, labels1 = ax1.get_legend_handles_labels() + handles2, labels2 = ax2.get_legend_handles_labels() + ax1.legend(handles1 + handles2, labels1 + labels2, loc="upper right", fontsize=10) + + plt.tight_layout() + os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) + plt.savefig(args.output, dpi=150, bbox_inches="tight") + print(f"Chart saved to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/agentic-auto-upgrade.yml b/.github/workflows/agentic-auto-upgrade.yml index fe5b068f2f3..092498db4fc 100644 --- a/.github/workflows/agentic-auto-upgrade.yml +++ b/.github/workflows/agentic-auto-upgrade.yml @@ -34,7 +34,7 @@ name: Agentic Auto-Upgrade on: schedule: - - cron: "11 4 * * 6" # Weekly (auto-upgrade) + - cron: "21 3 * * 5" # Weekly (auto-upgrade) workflow_dispatch: permissions: diff --git a/.github/workflows/detection-analysis-report.lock.yml b/.github/workflows/detection-analysis-report.lock.yml index 4533250ab47..1df8222fcf1 100644 --- a/.github/workflows/detection-analysis-report.lock.yml +++ b/.github/workflows/detection-analysis-report.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"48c71de42ae39f852ce6feee58fa3607929af5e18ce42b752dafe7f24b2e7399","body_hash":"29874595fc5e095243aa1abfe2eeda22484bb36c1e0c30fe82083451136d6074","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.210"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"48c71de42ae39f852ce6feee58fa3607929af5e18ce42b752dafe7f24b2e7399","body_hash":"695fb21bfec5fee845f91072035a8887429664602fd626bccf86603751369671","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.210"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"924ae3a1cded613372ab5595356fb5720e22ba16","version":"v6.5.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/setup-python","sha":"ece7cb06caefa5fff74198d8649806c4678c61a1","version":"v6.3.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"f9f3042f7e2789586610d6e8b85c8f03e5195baf","version":"v7.2.0"},{"repo":"docker/setup-buildx-action","sha":"d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5","version":"v4.1.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.31","digest":"sha256:84d861cb6da723ac10b7a00dddf778be681b8cd74b2091f18ce1d67fe4b3e7a1","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.31@sha256:84d861cb6da723ac10b7a00dddf778be681b8cd74b2091f18ce1d67fe4b3e7a1"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.31","digest":"sha256:80d982fe7925c640d76cbbfbe94081d2d34f7657b7c37494d8d5488f5dae3c63","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.31@sha256:80d982fe7925c640d76cbbfbe94081d2d34f7657b7c37494d8d5488f5dae3c63"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.31","digest":"sha256:c05a3f086946fab0833e078f46d35571080f187ca72f038958d45aa5cc150494","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.31@sha256:c05a3f086946fab0833e078f46d35571080f187ca72f038958d45aa5cc150494"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.1","digest":"sha256:ad2a979c2cd8b50098e84938ca9c9c1580eb8e91526f101a90adfba7859b2c32","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.1@sha256:ad2a979c2cd8b50098e84938ca9c9c1580eb8e91526f101a90adfba7859b2c32"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.5.0","digest":"sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4","pinned_image":"ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/detection-analysis-report.md b/.github/workflows/detection-analysis-report.md index e6cc89a9caf..16259f77064 100644 --- a/.github/workflows/detection-analysis-report.md +++ b/.github/workflows/detection-analysis-report.md @@ -64,87 +64,57 @@ Upload the chart using the `upload_asset` safe-output tool with the absolute pat ### Step 1 — Fetch Logs -Use the `agentic-workflows` MCP `logs` tool to download workflow runs from the last 24 hours: +Call `agenticworkflows logs --start-date -1d` to download workflow run directories to `/tmp/gh-aw/aw-mcp/logs`. Each run's `aw_info.json` contains `features["gh-aw-detection"]` (boolean), `status`, `total_tokens`, and `engine_id`. Use `features.gh-aw-detection` directly — do not infer detection status from `.lock.yml` files. -``` -Use the agentic-workflows MCP tool `logs` with parameters: -- start_date: "-1d" -Output is saved to: /tmp/gh-aw/aw-mcp/logs -``` - -Each run directory contains `aw_info.json` with fields including `engine_id`, `workflow`, `status`, `tokens`, and feature flags. The `gh-aw-detection` feature flag is stored under the `features` key in `aw_info.json` (e.g., `features.gh-aw-detection: true`). Use this field directly — do not infer detection status by scanning `.lock.yml` files. - -### Step 2 — Classify Runs +### Analyze Runs -For each run, classify it as: -- **Detection-enabled**: `features.gh-aw-detection` is `true` in the run metadata -- **Regular**: `features.gh-aw-detection` is `false`, absent, or unset +In a single pass over the run directories at `/tmp/gh-aw/aw-mcp/logs`, classify and aggregate all runs: -Collect per-run: -- `workflow_name` -- `status` (success / failure / cancelled / timed_out) -- `total_tokens` (from `aw_info.json`) -- `engine_id` -- `detection_enabled` (boolean) +**Classify each run** using `features.gh-aw-detection` from `aw_info.json`: +- **Detection-enabled**: value is `true` +- **Regular**: value is `false`, absent, or unset -### Step 3 — Identify Misconfigured Workflows +Collect per-run: `workflow_name`, `status`, `total_tokens`, `engine_id`, `detection_enabled`. -Flag a workflow as **misconfigured** when any of the following apply: +**Flag a workflow as misconfigured** when any of the following apply: +1. `gh-aw-detection: false` on a workflow with >3 total runs in the last 7 days +2. Workflow name contains `audit`, `analyzer`, `report`, `detector`, `monitor`, or `inspector` but lacks `gh-aw-detection: true` +3. Run has `gh-aw-detection: true` but detection-related steps failed +4. Workflow alternates between detection-enabled and detection-disabled within the 24h window -1. **Detection explicitly disabled on an active workflow**: `gh-aw-detection: false` on a workflow that has completed more than 3 total runs (any status) in the last 7 days. -2. **Detection feature absent on a workflow that is a known audit, analysis, or report workflow**: The workflow name contains keywords such as `audit`, `analyzer`, `report`, `detector`, `monitor`, or `inspector`, but lacks `gh-aw-detection: true`. -3. **Detection job failures**: The run has `gh-aw-detection: true` but its detection-related job step failed or produced errors (check agent logs for detection-related error patterns). -4. **Inconsistent detection state**: A workflow alternates between detection-enabled and detection-disabled runs within the same 24-hour window. +For each misconfigured workflow, record: `workflow_name`, `misconfiguration_type`, `run_count`, `example_run_id`, `recommended_fix`. -For each misconfigured workflow, record: -- `workflow_name` -- `misconfiguration_type` (one of the four above) -- `run_count` -- `example_run_id` -- `recommended_fix` +**Save** aggregated metrics to `/tmp/gh-aw/python/data/metrics.json`: -### Step 4 — Compute Metrics - -Aggregate metrics for the chart and report: - -| Metric | Regular Runs | Detection Runs | -|--------|-------------|----------------| -| Total runs | count | count | -| Success rate (%) | % | % | -| Avg tokens | mean | mean | -| Failure count | count | count | -| Misconfigured count | — | count | +```json +{ + "regular_runs": , + "detection_runs": , + "regular_success_rate": , + "detection_success_rate": , + "regular_failure_count": , + "detection_failure_count": , + "regular_avg_tokens": , + "detection_avg_tokens": , + "misconfigured_count": +} +``` ### Step 5 — Generate Chart -Write a Python script to `/tmp/gh-aw/python/detection_comparison.py` that: - -- Reads aggregated metrics from `/tmp/gh-aw/python/data/metrics.json` -- Produces a grouped bar chart comparing Regular vs. Detection-enabled runs: - - Left y-axis: run counts (stacked success/failure bars) - - Right y-axis: success rate % (line with markers) - - X-axis labels: "Regular Runs", "Detection Runs" - - Title: "Detection Feature Comparison — Last 24h" - - A horizontal annotation band highlighting any workflows with detection misconfigurations -- Saves the chart to `/tmp/gh-aw/python/charts/detection_comparison.png` at 150 DPI -- Uses a clean style (seaborn `whitegrid`, muted color palette) - -### Step 6 — Historical Trending - -Append today's aggregated metrics to the cache-memory store so future runs can draw a trend line. Create the directory before writing if it does not exist: - ```bash -mkdir -p /tmp/gh-aw/cache-memory/trending/detection-metrics +python3 .github/scripts/detection_comparison.py \ + --metrics /tmp/gh-aw/python/data/metrics.json \ + --output /tmp/gh-aw/python/charts/detection_comparison.png ``` -- File: `/tmp/gh-aw/cache-memory/trending/detection-metrics/history.jsonl` -- Fields: `timestamp`, `regular_runs`, `detection_runs`, `regular_success_rate`, `detection_success_rate`, `regular_avg_tokens`, `detection_avg_tokens`, `misconfigured_count` +Upload the chart using the `upload_asset` safe-output tool. Record the returned asset URL and embed it in the discussion body. + +### Step 6 — Historical Trending -If history data exists (at least 7 days), generate a second trend chart: +Use the `update-detection-trending` agent to append today's metrics to the cache-memory trending history and optionally generate a trend chart. -- **Detection Adoption Over Time**: Line chart showing `regular_runs` vs. `detection_runs` per day for the last 30 days, with a shaded area for the detection success rate band. -- Save to: `/tmp/gh-aw/python/charts/detection_trend.png` -- Upload with `upload_asset` and include in the report. +If `/tmp/gh-aw/python/charts/detection_trend.png` was generated, upload it with `upload_asset` and embed in the report under "View Historical Trend". --- @@ -198,4 +168,30 @@ Call `noop` with a brief explanation when: - No workflow runs exist in the last 24 hours - State the evaluated window in the no-op message -Example: `noop("No workflow runs found in the last 24 full hours (YYYY-MM-DDTHH:MM:SSZ to YYYY-MM-DDTHH:MM:SSZ)")` \ No newline at end of file +Example: `noop("No workflow runs found in the last 24 full hours (YYYY-MM-DDTHH:MM:SSZ to YYYY-MM-DDTHH:MM:SSZ)")` + +--- + +## agent: `update-detection-trending` +--- +model: small +description: Appends detection metrics to cache-memory trending history and generates a trend chart when ≥7 days of data exist +--- +Append today's detection metrics to the cache-memory trending history store. + +Create the directory before writing: + +```bash +mkdir -p /tmp/gh-aw/cache-memory/trending/detection-metrics +``` + +Read `/tmp/gh-aw/python/data/metrics.json` and append one JSON line to `/tmp/gh-aw/cache-memory/trending/detection-metrics/history.jsonl`: + +```json +{"timestamp": "", "regular_runs": N, "detection_runs": N, "regular_success_rate": X.X, "detection_success_rate": X.X, "regular_avg_tokens": X, "detection_avg_tokens": X, "misconfigured_count": N} +``` + +If the history file has ≥7 entries, generate a 30-day trend chart at `/tmp/gh-aw/python/charts/detection_trend.png`: +- Line chart: `regular_runs` vs. `detection_runs` per day for the last 30 days +- Shaded area band for the detection success rate +- Style: seaborn whitegrid, 12×7 inches, 150 DPI \ No newline at end of file From 9a2ccbf18a6b2f879bc1870fb1576e2ce9f5e836 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:58:36 +0000 Subject: [PATCH 3/3] revert: restore agentic-auto-upgrade.yml cron schedule (unintentional side-effect) Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/agentic-auto-upgrade.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/agentic-auto-upgrade.yml b/.github/workflows/agentic-auto-upgrade.yml index 092498db4fc..fe5b068f2f3 100644 --- a/.github/workflows/agentic-auto-upgrade.yml +++ b/.github/workflows/agentic-auto-upgrade.yml @@ -34,7 +34,7 @@ name: Agentic Auto-Upgrade on: schedule: - - cron: "21 3 * * 5" # Weekly (auto-upgrade) + - cron: "11 4 * * 6" # Weekly (auto-upgrade) workflow_dispatch: permissions: